drivers/gpu/drm/i915/intel_pm.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *
  26  */
  27
  28 #include <linux/cpufreq.h>
  29 #include <linux/pm_runtime.h>
  30 #include <drm/drm_plane_helper.h>
  31 #include "i915_drv.h"
  32 #include "intel_drv.h"
  33 #include "../../../platform/x86/intel_ips.h"
  34 #include <linux/module.h>
  35 #include <drm/drm_atomic_helper.h>
  36
  37 /**
  38  * DOC: RC6
  39  *
  40  * RC6 is a special power stage which allows the GPU to enter an very
  41  * low-voltage mode when idle, using down to 0V while at this stage.  This
  42  * stage is entered automatically when the GPU is idle when RC6 support is
  43  * enabled, and as soon as new workload arises GPU wakes up automatically as well.
  44  *
  45  * There are different RC6 modes available in Intel GPU, which differentiate
  46  * among each other with the latency required to enter and leave RC6 and
  47  * voltage consumed by the GPU in different states.
  48  *
  49  * The combination of the following flags define which states GPU is allowed
  50  * to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
  51  * RC6pp is deepest RC6. Their support by hardware varies according to the
  52  * GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
  53  * which brings the most power savings; deeper states save more power, but
  54  * require higher latency to switch to and wake up.
  55  */
  56
  57 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
  58 {
  59         if (HAS_LLC(dev_priv)) {
  60                 /*
  61                  * WaCompressedResourceDisplayNewHashMode:skl,kbl
  62                  * Display WA #0390: skl,kbl
  63                  *
  64                  * Must match Sampler, Pixel Back End, and Media. See
  65                  * WaCompressedResourceSamplerPbeMediaNewHashMode.
  66                  */
  67                 I915_WRITE(CHICKEN_PAR1_1,
  68                            I915_READ(CHICKEN_PAR1_1) |
  69                            SKL_DE_COMPRESSED_HASH_MODE);
  70         }
  71
  72         /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
  73         I915_WRITE(CHICKEN_PAR1_1,
  74                    I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
  75
  76         /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
  77         I915_WRITE(GEN8_CHICKEN_DCPR_1,
  78                    I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
  79
  80         /* WaFbcTurnOffFbcWatermark:skl,bxt,kbl,cfl */
  81         /* WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl */
  82         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
  83                    DISP_FBC_WM_DIS |
  84                    DISP_FBC_MEMORY_WAKE);
  85
  86         /* WaFbcHighMemBwCorruptionAvoidance:skl,bxt,kbl,cfl */
  87         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
  88                    ILK_DPFC_DISABLE_DUMMY0);
  89
  90         if (IS_SKYLAKE(dev_priv)) {
  91                 /* WaDisableDopClockGating */
  92                 I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL)
  93                            & ~GEN7_DOP_CLOCK_GATE_ENABLE);
  94         }
  95 }
  96
  97 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
  98 {
  99         gen9_init_clock_gating(dev_priv);
 100
 101         /* WaDisableSDEUnitClockGating:bxt */
 102         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 103                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 104
 105         /*
 106          * FIXME:
 107          * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
 108          */
 109         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 110                    GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
 111
 112         /*
 113          * Wa: Backlight PWM may stop in the asserted state, causing backlight
 114          * to stay fully on.
 115          */
 116         I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
 117                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 118 }
 119
 120 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
 121 {
 122         gen9_init_clock_gating(dev_priv);
 123
 124         /*
 125          * WaDisablePWMClockGating:glk
 126          * Backlight PWM may stop in the asserted state, causing backlight
 127          * to stay fully on.
 128          */
 129         I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
 130                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 131
 132         /* WaDDIIOTimeout:glk */
 133         if (IS_GLK_REVID(dev_priv, 0, GLK_REVID_A1)) {
 134                 u32 val = I915_READ(CHICKEN_MISC_2);
 135                 val &= ~(GLK_CL0_PWR_DOWN |
 136                          GLK_CL1_PWR_DOWN |
 137                          GLK_CL2_PWR_DOWN);
 138                 I915_WRITE(CHICKEN_MISC_2, val);
 139         }
 140
 141 }
 142
 143 static void i915_pineview_get_mem_freq(struct drm_i915_private *dev_priv)
 144 {
 145         u32 tmp;
 146
 147         tmp = I915_READ(CLKCFG);
 148
 149         switch (tmp & CLKCFG_FSB_MASK) {
 150         case CLKCFG_FSB_533:
 151                 dev_priv->fsb_freq = 533; /* 133*4 */
 152                 break;
 153         case CLKCFG_FSB_800:
 154                 dev_priv->fsb_freq = 800; /* 200*4 */
 155                 break;
 156         case CLKCFG_FSB_667:
 157                 dev_priv->fsb_freq =  667; /* 167*4 */
 158                 break;
 159         case CLKCFG_FSB_400:
 160                 dev_priv->fsb_freq = 400; /* 100*4 */
 161                 break;
 162         }
 163
 164         switch (tmp & CLKCFG_MEM_MASK) {
 165         case CLKCFG_MEM_533:
 166                 dev_priv->mem_freq = 533;
 167                 break;
 168         case CLKCFG_MEM_667:
 169                 dev_priv->mem_freq = 667;
 170                 break;
 171         case CLKCFG_MEM_800:
 172                 dev_priv->mem_freq = 800;
 173                 break;
 174         }
 175
 176         /* detect pineview DDR3 setting */
 177         tmp = I915_READ(CSHRDDR3CTL);
 178         dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
 179 }
 180
 181 static void i915_ironlake_get_mem_freq(struct drm_i915_private *dev_priv)
 182 {
 183         u16 ddrpll, csipll;
 184
 185         ddrpll = I915_READ16(DDRMPLL1);
 186         csipll = I915_READ16(CSIPLL0);
 187
 188         switch (ddrpll & 0xff) {
 189         case 0xc:
 190                 dev_priv->mem_freq = 800;
 191                 break;
 192         case 0x10:
 193                 dev_priv->mem_freq = 1066;
 194                 break;
 195         case 0x14:
 196                 dev_priv->mem_freq = 1333;
 197                 break;
 198         case 0x18:
 199                 dev_priv->mem_freq = 1600;
 200                 break;
 201         default:
 202                 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
 203                                  ddrpll & 0xff);
 204                 dev_priv->mem_freq = 0;
 205                 break;
 206         }
 207
 208         dev_priv->ips.r_t = dev_priv->mem_freq;
 209
 210         switch (csipll & 0x3ff) {
 211         case 0x00c:
 212                 dev_priv->fsb_freq = 3200;
 213                 break;
 214         case 0x00e:
 215                 dev_priv->fsb_freq = 3733;
 216                 break;
 217         case 0x010:
 218                 dev_priv->fsb_freq = 4266;
 219                 break;
 220         case 0x012:
 221                 dev_priv->fsb_freq = 4800;
 222                 break;
 223         case 0x014:
 224                 dev_priv->fsb_freq = 5333;
 225                 break;
 226         case 0x016:
 227                 dev_priv->fsb_freq = 5866;
 228                 break;
 229         case 0x018:
 230                 dev_priv->fsb_freq = 6400;
 231                 break;
 232         default:
 233                 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
 234                                  csipll & 0x3ff);
 235                 dev_priv->fsb_freq = 0;
 236                 break;
 237         }
 238
 239         if (dev_priv->fsb_freq == 3200) {
 240                 dev_priv->ips.c_m = 0;
 241         } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
 242                 dev_priv->ips.c_m = 1;
 243         } else {
 244                 dev_priv->ips.c_m = 2;
 245         }
 246 }
 247
 248 static const struct cxsr_latency cxsr_latency_table[] = {
 249         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
 250         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
 251         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
 252         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
 253         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
 254
 255         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
 256         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
 257         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
 258         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
 259         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
 260
 261         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
 262         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
 263         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
 264         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
 265         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
 266
 267         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
 268         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
 269         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
 270         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
 271         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
 272
 273         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
 274         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
 275         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
 276         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
 277         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
 278
 279         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
 280         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
 281         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
 282         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
 283         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
 284 };
 285
 286 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
 287                                                          bool is_ddr3,
 288                                                          int fsb,
 289                                                          int mem)
 290 {
 291         const struct cxsr_latency *latency;
 292         int i;
 293
 294         if (fsb == 0 || mem == 0)
 295                 return NULL;
 296
 297         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
 298                 latency = &cxsr_latency_table[i];
 299                 if (is_desktop == latency->is_desktop &&
 300                     is_ddr3 == latency->is_ddr3 &&
 301                     fsb == latency->fsb_freq && mem == latency->mem_freq)
 302                         return latency;
 303         }
 304
 305         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 306
 307         return NULL;
 308 }
 309
 310 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
 311 {
 312         u32 val;
 313
 314         mutex_lock(&dev_priv->pcu_lock);
 315
 316         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
 317         if (enable)
 318                 val &= ~FORCE_DDR_HIGH_FREQ;
 319         else
 320                 val |= FORCE_DDR_HIGH_FREQ;
 321         val &= ~FORCE_DDR_LOW_FREQ;
 322         val |= FORCE_DDR_FREQ_REQ_ACK;
 323         vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
 324
 325         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
 326                       FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
 327                 DRM_ERROR("timed out waiting for Punit DDR DVFS request\n");
 328
 329         mutex_unlock(&dev_priv->pcu_lock);
 330 }
 331
 332 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 333 {
 334         u32 val;
 335
 336         mutex_lock(&dev_priv->pcu_lock);
 337
 338         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
 339         if (enable)
 340                 val |= DSP_MAXFIFO_PM5_ENABLE;
 341         else
 342                 val &= ~DSP_MAXFIFO_PM5_ENABLE;
 343         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
 344
 345         mutex_unlock(&dev_priv->pcu_lock);
 346 }
 347
 348 #define FW_WM(value, plane) \
 349         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
 350
 351 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 352 {
 353         bool was_enabled;
 354         u32 val;
 355
 356         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 357                 was_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
 358                 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 359                 POSTING_READ(FW_BLC_SELF_VLV);
 360         } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
 361                 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
 362                 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 363                 POSTING_READ(FW_BLC_SELF);
 364         } else if (IS_PINEVIEW(dev_priv)) {
 365                 val = I915_READ(DSPFW3);
 366                 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
 367                 if (enable)
 368                         val |= PINEVIEW_SELF_REFRESH_EN;
 369                 else
 370                         val &= ~PINEVIEW_SELF_REFRESH_EN;
 371                 I915_WRITE(DSPFW3, val);
 372                 POSTING_READ(DSPFW3);
 373         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
 374                 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
 375                 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 376                                _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 377                 I915_WRITE(FW_BLC_SELF, val);
 378                 POSTING_READ(FW_BLC_SELF);
 379         } else if (IS_I915GM(dev_priv)) {
 380                 /*
 381                  * FIXME can't find a bit like this for 915G, and
 382                  * and yet it does have the related watermark in
 383                  * FW_BLC_SELF. What's going on?
 384                  */
 385                 was_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
 386                 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 387                                _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 388                 I915_WRITE(INSTPM, val);
 389                 POSTING_READ(INSTPM);
 390         } else {
 391                 return false;
 392         }
 393
 394         trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
 395
 396         DRM_DEBUG_KMS("memory self-refresh is %s (was %s)\n",
 397                       enableddisabled(enable),
 398                       enableddisabled(was_enabled));
 399
 400         return was_enabled;
 401 }
 402
 403 /**
 404  * intel_set_memory_cxsr - Configure CxSR state
 405  * @dev_priv: i915 device
 406  * @enable: Allow vs. disallow CxSR
 407  *
 408  * Allow or disallow the system to enter a special CxSR
 409  * (C-state self refresh) state. What typically happens in CxSR mode
 410  * is that several display FIFOs may get combined into a single larger
 411  * FIFO for a particular plane (so called max FIFO mode) to allow the
 412  * system to defer memory fetches longer, and the memory will enter
 413  * self refresh.
 414  *
 415  * Note that enabling CxSR does not guarantee that the system enter
 416  * this special mode, nor does it guarantee that the system stays
 417  * in that mode once entered. So this just allows/disallows the system
 418  * to autonomously utilize the CxSR mode. Other factors such as core
 419  * C-states will affect when/if the system actually enters/exits the
 420  * CxSR mode.
 421  *
 422  * Note that on VLV/CHV this actually only controls the max FIFO mode,
 423  * and the system is free to enter/exit memory self refresh at any time
 424  * even when the use of CxSR has been disallowed.
 425  *
 426  * While the system is actually in the CxSR/max FIFO mode, some plane
 427  * control registers will not get latched on vblank. Thus in order to
 428  * guarantee the system will respond to changes in the plane registers
 429  * we must always disallow CxSR prior to making changes to those registers.
 430  * Unfortunately the system will re-evaluate the CxSR conditions at
 431  * frame start which happens after vblank start (which is when the plane
 432  * registers would get latched), so we can't proceed with the plane update
 433  * during the same frame where we disallowed CxSR.
 434  *
 435  * Certain platforms also have a deeper HPLL SR mode. Fortunately the
 436  * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
 437  * the hardware w.r.t. HPLL SR when writing to plane registers.
 438  * Disallowing just CxSR is sufficient.
 439  */
 440 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 441 {
 442         bool ret;
 443
 444         mutex_lock(&dev_priv->wm.wm_mutex);
 445         ret = _intel_set_memory_cxsr(dev_priv, enable);
 446         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 447                 dev_priv->wm.vlv.cxsr = enable;
 448         else if (IS_G4X(dev_priv))
 449                 dev_priv->wm.g4x.cxsr = enable;
 450         mutex_unlock(&dev_priv->wm.wm_mutex);
 451
 452         return ret;
 453 }
 454
 455 /*
 456  * Latency for FIFO fetches is dependent on several factors:
 457  *   - memory configuration (speed, channels)
 458  *   - chipset
 459  *   - current MCH state
 460  * It can be fairly high in some situations, so here we assume a fairly
 461  * pessimal value.  It's a tradeoff between extra memory fetches (if we
 462  * set this value too high, the FIFO will fetch frequently to stay full)
 463  * and power consumption (set it too low to save power and we might see
 464  * FIFO underruns and display "flicker").
 465  *
 466  * A value of 5us seems to be a good balance; safe for very low end
 467  * platforms but not overly aggressive on lower latency configs.
 468  */
 469 static const int pessimal_latency_ns = 5000;
 470
 471 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
 472         ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
 473
 474 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
 475 {
 476         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 477         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 478         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
 479         enum pipe pipe = crtc->pipe;
 480         int sprite0_start, sprite1_start;
 481
 482         switch (pipe) {
 483                 uint32_t dsparb, dsparb2, dsparb3;
 484         case PIPE_A:
 485                 dsparb = I915_READ(DSPARB);
 486                 dsparb2 = I915_READ(DSPARB2);
 487                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
 488                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
 489                 break;
 490         case PIPE_B:
 491                 dsparb = I915_READ(DSPARB);
 492                 dsparb2 = I915_READ(DSPARB2);
 493                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
 494                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
 495                 break;
 496         case PIPE_C:
 497                 dsparb2 = I915_READ(DSPARB2);
 498                 dsparb3 = I915_READ(DSPARB3);
 499                 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
 500                 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
 501                 break;
 502         default:
 503                 MISSING_CASE(pipe);
 504                 return;
 505         }
 506
 507         fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
 508         fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
 509         fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
 510         fifo_state->plane[PLANE_CURSOR] = 63;
 511 }
 512
 513 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
 514                               enum i9xx_plane_id i9xx_plane)
 515 {
 516         uint32_t dsparb = I915_READ(DSPARB);
 517         int size;
 518
 519         size = dsparb & 0x7f;
 520         if (i9xx_plane == PLANE_B)
 521                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 522
 523         DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
 524                       dsparb, plane_name(i9xx_plane), size);
 525
 526         return size;
 527 }
 528
 529 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
 530                               enum i9xx_plane_id i9xx_plane)
 531 {
 532         uint32_t dsparb = I915_READ(DSPARB);
 533         int size;
 534
 535         size = dsparb & 0x1ff;
 536         if (i9xx_plane == PLANE_B)
 537                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 538         size >>= 1; /* Convert to cachelines */
 539
 540         DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
 541                       dsparb, plane_name(i9xx_plane), size);
 542
 543         return size;
 544 }
 545
 546 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
 547                               enum i9xx_plane_id i9xx_plane)
 548 {
 549         uint32_t dsparb = I915_READ(DSPARB);
 550         int size;
 551
 552         size = dsparb & 0x7f;
 553         size >>= 2; /* Convert to cachelines */
 554
 555         DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
 556                       dsparb, plane_name(i9xx_plane), size);
 557
 558         return size;
 559 }
 560
 561 /* Pineview has different values for various configs */
 562 static const struct intel_watermark_params pineview_display_wm = {
 563         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 564         .max_wm = PINEVIEW_MAX_WM,
 565         .default_wm = PINEVIEW_DFT_WM,
 566         .guard_size = PINEVIEW_GUARD_WM,
 567         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 568 };
 569 static const struct intel_watermark_params pineview_display_hplloff_wm = {
 570         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 571         .max_wm = PINEVIEW_MAX_WM,
 572         .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
 573         .guard_size = PINEVIEW_GUARD_WM,
 574         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 575 };
 576 static const struct intel_watermark_params pineview_cursor_wm = {
 577         .fifo_size = PINEVIEW_CURSOR_FIFO,
 578         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 579         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 580         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 581         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 582 };
 583 static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
 584         .fifo_size = PINEVIEW_CURSOR_FIFO,
 585         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 586         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 587         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 588         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 589 };
 590 static const struct intel_watermark_params i965_cursor_wm_info = {
 591         .fifo_size = I965_CURSOR_FIFO,
 592         .max_wm = I965_CURSOR_MAX_WM,
 593         .default_wm = I965_CURSOR_DFT_WM,
 594         .guard_size = 2,
 595         .cacheline_size = I915_FIFO_LINE_SIZE,
 596 };
 597 static const struct intel_watermark_params i945_wm_info = {
 598         .fifo_size = I945_FIFO_SIZE,
 599         .max_wm = I915_MAX_WM,
 600         .default_wm = 1,
 601         .guard_size = 2,
 602         .cacheline_size = I915_FIFO_LINE_SIZE,
 603 };
 604 static const struct intel_watermark_params i915_wm_info = {
 605         .fifo_size = I915_FIFO_SIZE,
 606         .max_wm = I915_MAX_WM,
 607         .default_wm = 1,
 608         .guard_size = 2,
 609         .cacheline_size = I915_FIFO_LINE_SIZE,
 610 };
 611 static const struct intel_watermark_params i830_a_wm_info = {
 612         .fifo_size = I855GM_FIFO_SIZE,
 613         .max_wm = I915_MAX_WM,
 614         .default_wm = 1,
 615         .guard_size = 2,
 616         .cacheline_size = I830_FIFO_LINE_SIZE,
 617 };
 618 static const struct intel_watermark_params i830_bc_wm_info = {
 619         .fifo_size = I855GM_FIFO_SIZE,
 620         .max_wm = I915_MAX_WM/2,
 621         .default_wm = 1,
 622         .guard_size = 2,
 623         .cacheline_size = I830_FIFO_LINE_SIZE,
 624 };
 625 static const struct intel_watermark_params i845_wm_info = {
 626         .fifo_size = I830_FIFO_SIZE,
 627         .max_wm = I915_MAX_WM,
 628         .default_wm = 1,
 629         .guard_size = 2,
 630         .cacheline_size = I830_FIFO_LINE_SIZE,
 631 };
 632
 633 /**
 634  * intel_wm_method1 - Method 1 / "small buffer" watermark formula
 635  * @pixel_rate: Pipe pixel rate in kHz
 636  * @cpp: Plane bytes per pixel
 637  * @latency: Memory wakeup latency in 0.1us units
 638  *
 639  * Compute the watermark using the method 1 or "small buffer"
 640  * formula. The caller may additonally add extra cachelines
 641  * to account for TLB misses and clock crossings.
 642  *
 643  * This method is concerned with the short term drain rate
 644  * of the FIFO, ie. it does not account for blanking periods
 645  * which would effectively reduce the average drain rate across
 646  * a longer period. The name "small" refers to the fact the
 647  * FIFO is relatively small compared to the amount of data
 648  * fetched.
 649  *
 650  * The FIFO level vs. time graph might look something like:
 651  *
 652  *   |\   |\
 653  *   | \  | \
 654  * __---__---__ (- plane active, _ blanking)
 655  * -> time
 656  *
 657  * or perhaps like this:
 658  *
 659  *   |\|\  |\|\
 660  * __----__----__ (- plane active, _ blanking)
 661  * -> time
 662  *
 663  * Returns:
 664  * The watermark in bytes
 665  */
 666 static unsigned int intel_wm_method1(unsigned int pixel_rate,
 667                                      unsigned int cpp,
 668                                      unsigned int latency)
 669 {
 670         uint64_t ret;
 671
 672         ret = (uint64_t) pixel_rate * cpp * latency;
 673         ret = DIV_ROUND_UP_ULL(ret, 10000);
 674
 675         return ret;
 676 }
 677
 678 /**
 679  * intel_wm_method2 - Method 2 / "large buffer" watermark formula
 680  * @pixel_rate: Pipe pixel rate in kHz
 681  * @htotal: Pipe horizontal total
 682  * @width: Plane width in pixels
 683  * @cpp: Plane bytes per pixel
 684  * @latency: Memory wakeup latency in 0.1us units
 685  *
 686  * Compute the watermark using the method 2 or "large buffer"
 687  * formula. The caller may additonally add extra cachelines
 688  * to account for TLB misses and clock crossings.
 689  *
 690  * This method is concerned with the long term drain rate
 691  * of the FIFO, ie. it does account for blanking periods
 692  * which effectively reduce the average drain rate across
 693  * a longer period. The name "large" refers to the fact the
 694  * FIFO is relatively large compared to the amount of data
 695  * fetched.
 696  *
 697  * The FIFO level vs. time graph might look something like:
 698  *
 699  *    |\___       |\___
 700  *    |    \___   |    \___
 701  *    |        \  |        \
 702  * __ --__--__--__--__--__--__ (- plane active, _ blanking)
 703  * -> time
 704  *
 705  * Returns:
 706  * The watermark in bytes
 707  */
 708 static unsigned int intel_wm_method2(unsigned int pixel_rate,
 709                                      unsigned int htotal,
 710                                      unsigned int width,
 711                                      unsigned int cpp,
 712                                      unsigned int latency)
 713 {
 714         unsigned int ret;
 715
 716         /*
 717          * FIXME remove once all users are computing
 718          * watermarks in the correct place.
 719          */
 720         if (WARN_ON_ONCE(htotal == 0))
 721                 htotal = 1;
 722
 723         ret = (latency * pixel_rate) / (htotal * 10000);
 724         ret = (ret + 1) * width * cpp;
 725
 726         return ret;
 727 }
 728
 729 /**
 730  * intel_calculate_wm - calculate watermark level
 731  * @pixel_rate: pixel clock
 732  * @wm: chip FIFO params
 733  * @fifo_size: size of the FIFO buffer
 734  * @cpp: bytes per pixel
 735  * @latency_ns: memory latency for the platform
 736  *
 737  * Calculate the watermark level (the level at which the display plane will
 738  * start fetching from memory again).  Each chip has a different display
 739  * FIFO size and allocation, so the caller needs to figure that out and pass
 740  * in the correct intel_watermark_params structure.
 741  *
 742  * As the pixel clock runs, the FIFO will be drained at a rate that depends
 743  * on the pixel size.  When it reaches the watermark level, it'll start
 744  * fetching FIFO line sized based chunks from memory until the FIFO fills
 745  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 746  * will occur, and a display engine hang could result.
 747  */
 748 static unsigned int intel_calculate_wm(int pixel_rate,
 749                                        const struct intel_watermark_params *wm,
 750                                        int fifo_size, int cpp,
 751                                        unsigned int latency_ns)
 752 {
 753         int entries, wm_size;
 754
 755         /*
 756          * Note: we need to make sure we don't overflow for various clock &
 757          * latency values.
 758          * clocks go from a few thousand to several hundred thousand.
 759          * latency is usually a few thousand
 760          */
 761         entries = intel_wm_method1(pixel_rate, cpp,
 762                                    latency_ns / 100);
 763         entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
 764                 wm->guard_size;
 765         DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
 766
 767         wm_size = fifo_size - entries;
 768         DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
 769
 770         /* Don't promote wm_size to unsigned... */
 771         if (wm_size > wm->max_wm)
 772                 wm_size = wm->max_wm;
 773         if (wm_size <= 0)
 774                 wm_size = wm->default_wm;
 775
 776         /*
 777          * Bspec seems to indicate that the value shouldn't be lower than
 778          * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
 779          * Lets go for 8 which is the burst size since certain platforms
 780          * already use a hardcoded 8 (which is what the spec says should be
 781          * done).
 782          */
 783         if (wm_size <= 8)
 784                 wm_size = 8;
 785
 786         return wm_size;
 787 }
 788
 789 static bool is_disabling(int old, int new, int threshold)
 790 {
 791         return old >= threshold && new < threshold;
 792 }
 793
 794 static bool is_enabling(int old, int new, int threshold)
 795 {
 796         return old < threshold && new >= threshold;
 797 }
 798
 799 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
 800 {
 801         return dev_priv->wm.max_level + 1;
 802 }
 803
 804 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
 805                                    const struct intel_plane_state *plane_state)
 806 {
 807         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
 808
 809         /* FIXME check the 'enable' instead */
 810         if (!crtc_state->base.active)
 811                 return false;
 812
 813         /*
 814          * Treat cursor with fb as always visible since cursor updates
 815          * can happen faster than the vrefresh rate, and the current
 816          * watermark code doesn't handle that correctly. Cursor updates
 817          * which set/clear the fb or change the cursor size are going
 818          * to get throttled by intel_legacy_cursor_update() to work
 819          * around this problem with the watermark code.
 820          */
 821         if (plane->id == PLANE_CURSOR)
 822                 return plane_state->base.fb != NULL;
 823         else
 824                 return plane_state->base.visible;
 825 }
 826
 827 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
 828 {
 829         struct intel_crtc *crtc, *enabled = NULL;
 830
 831         for_each_intel_crtc(&dev_priv->drm, crtc) {
 832                 if (intel_crtc_active(crtc)) {
 833                         if (enabled)
 834                                 return NULL;
 835                         enabled = crtc;
 836                 }
 837         }
 838
 839         return enabled;
 840 }
 841
 842 static void pineview_update_wm(struct intel_crtc *unused_crtc)
 843 {
 844         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
 845         struct intel_crtc *crtc;
 846         const struct cxsr_latency *latency;
 847         u32 reg;
 848         unsigned int wm;
 849
 850         latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv),
 851                                          dev_priv->is_ddr3,
 852                                          dev_priv->fsb_freq,
 853                                          dev_priv->mem_freq);
 854         if (!latency) {
 855                 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 856                 intel_set_memory_cxsr(dev_priv, false);
 857                 return;
 858         }
 859
 860         crtc = single_enabled_crtc(dev_priv);
 861         if (crtc) {
 862                 const struct drm_display_mode *adjusted_mode =
 863                         &crtc->config->base.adjusted_mode;
 864                 const struct drm_framebuffer *fb =
 865                         crtc->base.primary->state->fb;
 866                 int cpp = fb->format->cpp[0];
 867                 int clock = adjusted_mode->crtc_clock;
 868
 869                 /* Display SR */
 870                 wm = intel_calculate_wm(clock, &pineview_display_wm,
 871                                         pineview_display_wm.fifo_size,
 872                                         cpp, latency->display_sr);
 873                 reg = I915_READ(DSPFW1);
 874                 reg &= ~DSPFW_SR_MASK;
 875                 reg |= FW_WM(wm, SR);
 876                 I915_WRITE(DSPFW1, reg);
 877                 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
 878
 879                 /* cursor SR */
 880                 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
 881                                         pineview_display_wm.fifo_size,
 882                                         4, latency->cursor_sr);
 883                 reg = I915_READ(DSPFW3);
 884                 reg &= ~DSPFW_CURSOR_SR_MASK;
 885                 reg |= FW_WM(wm, CURSOR_SR);
 886                 I915_WRITE(DSPFW3, reg);
 887
 888                 /* Display HPLL off SR */
 889                 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
 890                                         pineview_display_hplloff_wm.fifo_size,
 891                                         cpp, latency->display_hpll_disable);
 892                 reg = I915_READ(DSPFW3);
 893                 reg &= ~DSPFW_HPLL_SR_MASK;
 894                 reg |= FW_WM(wm, HPLL_SR);
 895                 I915_WRITE(DSPFW3, reg);
 896
 897                 /* cursor HPLL off SR */
 898                 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
 899                                         pineview_display_hplloff_wm.fifo_size,
 900                                         4, latency->cursor_hpll_disable);
 901                 reg = I915_READ(DSPFW3);
 902                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
 903                 reg |= FW_WM(wm, HPLL_CURSOR);
 904                 I915_WRITE(DSPFW3, reg);
 905                 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
 906
 907                 intel_set_memory_cxsr(dev_priv, true);
 908         } else {
 909                 intel_set_memory_cxsr(dev_priv, false);
 910         }
 911 }
 912
 913 /*
 914  * Documentation says:
 915  * "If the line size is small, the TLB fetches can get in the way of the
 916  *  data fetches, causing some lag in the pixel data return which is not
 917  *  accounted for in the above formulas. The following adjustment only
 918  *  needs to be applied if eight whole lines fit in the buffer at once.
 919  *  The WM is adjusted upwards by the difference between the FIFO size
 920  *  and the size of 8 whole lines. This adjustment is always performed
 921  *  in the actual pixel depth regardless of whether FBC is enabled or not."
 922  */
 923 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
 924 {
 925         int tlb_miss = fifo_size * 64 - width * cpp * 8;
 926
 927         return max(0, tlb_miss);
 928 }
 929
 930 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
 931                                 const struct g4x_wm_values *wm)
 932 {
 933         enum pipe pipe;
 934
 935         for_each_pipe(dev_priv, pipe)
 936                 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
 937
 938         I915_WRITE(DSPFW1,
 939                    FW_WM(wm->sr.plane, SR) |
 940                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 941                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 942                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 943         I915_WRITE(DSPFW2,
 944                    (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
 945                    FW_WM(wm->sr.fbc, FBC_SR) |
 946                    FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
 947                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
 948                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 949                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 950         I915_WRITE(DSPFW3,
 951                    (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
 952                    FW_WM(wm->sr.cursor, CURSOR_SR) |
 953                    FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
 954                    FW_WM(wm->hpll.plane, HPLL_SR));
 955
 956         POSTING_READ(DSPFW1);
 957 }
 958
 959 #define FW_WM_VLV(value, plane) \
 960         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
 961
 962 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
 963                                 const struct vlv_wm_values *wm)
 964 {
 965         enum pipe pipe;
 966
 967         for_each_pipe(dev_priv, pipe) {
 968                 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
 969
 970                 I915_WRITE(VLV_DDL(pipe),
 971                            (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
 972                            (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
 973                            (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
 974                            (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
 975         }
 976
 977         /*
 978          * Zero the (unused) WM1 watermarks, and also clear all the
 979          * high order bits so that there are no out of bounds values
 980          * present in the registers during the reprogramming.
 981          */
 982         I915_WRITE(DSPHOWM, 0);
 983         I915_WRITE(DSPHOWM1, 0);
 984         I915_WRITE(DSPFW4, 0);
 985         I915_WRITE(DSPFW5, 0);
 986         I915_WRITE(DSPFW6, 0);
 987
 988         I915_WRITE(DSPFW1,
 989                    FW_WM(wm->sr.plane, SR) |
 990                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 991                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 992                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 993         I915_WRITE(DSPFW2,
 994                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
 995                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 996                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 997         I915_WRITE(DSPFW3,
 998                    FW_WM(wm->sr.cursor, CURSOR_SR));
 999
1000         if (IS_CHERRYVIEW(dev_priv)) {
1001                 I915_WRITE(DSPFW7_CHV,
1002                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1003                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1004                 I915_WRITE(DSPFW8_CHV,
1005                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1006                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1007                 I915_WRITE(DSPFW9_CHV,
1008                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1009                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1010                 I915_WRITE(DSPHOWM,
1011                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1012                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1013                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1014                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1015                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1016                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1017                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1018                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1019                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1020                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1021         } else {
1022                 I915_WRITE(DSPFW7,
1023                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1024                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1025                 I915_WRITE(DSPHOWM,
1026                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1027                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1028                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1029                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1030                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1031                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1032                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1033         }
1034
1035         POSTING_READ(DSPFW1);
1036 }
1037
1038 #undef FW_WM_VLV
1039
1040 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1041 {
1042         /* all latencies in usec */
1043         dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1044         dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1045         dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1046
1047         dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1048 }
1049
1050 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1051 {
1052         /*
1053          * DSPCNTR[13] supposedly controls whether the
1054          * primary plane can use the FIFO space otherwise
1055          * reserved for the sprite plane. It's not 100% clear
1056          * what the actual FIFO size is, but it looks like we
1057          * can happily set both primary and sprite watermarks
1058          * up to 127 cachelines. So that would seem to mean
1059          * that either DSPCNTR[13] doesn't do anything, or that
1060          * the total FIFO is >= 256 cachelines in size. Either
1061          * way, we don't seem to have to worry about this
1062          * repartitioning as the maximum watermark value the
1063          * register can hold for each plane is lower than the
1064          * minimum FIFO size.
1065          */
1066         switch (plane_id) {
1067         case PLANE_CURSOR:
1068                 return 63;
1069         case PLANE_PRIMARY:
1070                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1071         case PLANE_SPRITE0:
1072                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1073         default:
1074                 MISSING_CASE(plane_id);
1075                 return 0;
1076         }
1077 }
1078
1079 static int g4x_fbc_fifo_size(int level)
1080 {
1081         switch (level) {
1082         case G4X_WM_LEVEL_SR:
1083                 return 7;
1084         case G4X_WM_LEVEL_HPLL:
1085                 return 15;
1086         default:
1087                 MISSING_CASE(level);
1088                 return 0;
1089         }
1090 }
1091
1092 static uint16_t g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1093                                const struct intel_plane_state *plane_state,
1094                                int level)
1095 {
1096         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1097         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1098         const struct drm_display_mode *adjusted_mode =
1099                 &crtc_state->base.adjusted_mode;
1100         unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1101         unsigned int clock, htotal, cpp, width, wm;
1102
1103         if (latency == 0)
1104                 return USHRT_MAX;
1105
1106         if (!intel_wm_plane_visible(crtc_state, plane_state))
1107                 return 0;
1108
1109         /*
1110          * Not 100% sure which way ELK should go here as the
1111          * spec only says CL/CTG should assume 32bpp and BW
1112          * doesn't need to. But as these things followed the
1113          * mobile vs. desktop lines on gen3 as well, let's
1114          * assume ELK doesn't need this.
1115          *
1116          * The spec also fails to list such a restriction for
1117          * the HPLL watermark, which seems a little strange.
1118          * Let's use 32bpp for the HPLL watermark as well.
1119          */
1120         if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1121             level != G4X_WM_LEVEL_NORMAL)
1122                 cpp = 4;
1123         else
1124                 cpp = plane_state->base.fb->format->cpp[0];
1125
1126         clock = adjusted_mode->crtc_clock;
1127         htotal = adjusted_mode->crtc_htotal;
1128
1129         if (plane->id == PLANE_CURSOR)
1130                 width = plane_state->base.crtc_w;
1131         else
1132                 width = drm_rect_width(&plane_state->base.dst);
1133
1134         if (plane->id == PLANE_CURSOR) {
1135                 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1136         } else if (plane->id == PLANE_PRIMARY &&
1137                    level == G4X_WM_LEVEL_NORMAL) {
1138                 wm = intel_wm_method1(clock, cpp, latency);
1139         } else {
1140                 unsigned int small, large;
1141
1142                 small = intel_wm_method1(clock, cpp, latency);
1143                 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1144
1145                 wm = min(small, large);
1146         }
1147
1148         wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1149                               width, cpp);
1150
1151         wm = DIV_ROUND_UP(wm, 64) + 2;
1152
1153         return min_t(unsigned int, wm, USHRT_MAX);
1154 }
1155
1156 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1157                                  int level, enum plane_id plane_id, u16 value)
1158 {
1159         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1160         bool dirty = false;
1161
1162         for (; level < intel_wm_num_levels(dev_priv); level++) {
1163                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1164
1165                 dirty |= raw->plane[plane_id] != value;
1166                 raw->plane[plane_id] = value;
1167         }
1168
1169         return dirty;
1170 }
1171
1172 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1173                                int level, u16 value)
1174 {
1175         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1176         bool dirty = false;
1177
1178         /* NORMAL level doesn't have an FBC watermark */
1179         level = max(level, G4X_WM_LEVEL_SR);
1180
1181         for (; level < intel_wm_num_levels(dev_priv); level++) {
1182                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1183
1184                 dirty |= raw->fbc != value;
1185                 raw->fbc = value;
1186         }
1187
1188         return dirty;
1189 }
1190
1191 static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
1192                                    const struct intel_plane_state *pstate,
1193                                    uint32_t pri_val);
1194
1195 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1196                                      const struct intel_plane_state *plane_state)
1197 {
1198         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1199         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1200         enum plane_id plane_id = plane->id;
1201         bool dirty = false;
1202         int level;
1203
1204         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1205                 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1206                 if (plane_id == PLANE_PRIMARY)
1207                         dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1208                 goto out;
1209         }
1210
1211         for (level = 0; level < num_levels; level++) {
1212                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1213                 int wm, max_wm;
1214
1215                 wm = g4x_compute_wm(crtc_state, plane_state, level);
1216                 max_wm = g4x_plane_fifo_size(plane_id, level);
1217
1218                 if (wm > max_wm)
1219                         break;
1220
1221                 dirty |= raw->plane[plane_id] != wm;
1222                 raw->plane[plane_id] = wm;
1223
1224                 if (plane_id != PLANE_PRIMARY ||
1225                     level == G4X_WM_LEVEL_NORMAL)
1226                         continue;
1227
1228                 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1229                                         raw->plane[plane_id]);
1230                 max_wm = g4x_fbc_fifo_size(level);
1231
1232                 /*
1233                  * FBC wm is not mandatory as we
1234                  * can always just disable its use.
1235                  */
1236                 if (wm > max_wm)
1237                         wm = USHRT_MAX;
1238
1239                 dirty |= raw->fbc != wm;
1240                 raw->fbc = wm;
1241         }
1242
1243         /* mark watermarks as invalid */
1244         dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1245
1246         if (plane_id == PLANE_PRIMARY)
1247                 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1248
1249  out:
1250         if (dirty) {
1251                 DRM_DEBUG_KMS("%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1252                               plane->base.name,
1253                               crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1254                               crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1255                               crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1256
1257                 if (plane_id == PLANE_PRIMARY)
1258                         DRM_DEBUG_KMS("FBC watermarks: SR=%d, HPLL=%d\n",
1259                                       crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1260                                       crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1261         }
1262
1263         return dirty;
1264 }
1265
1266 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1267                                       enum plane_id plane_id, int level)
1268 {
1269         const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1270
1271         return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1272 }
1273
1274 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1275                                      int level)
1276 {
1277         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1278
1279         if (level > dev_priv->wm.max_level)
1280                 return false;
1281
1282         return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1283                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1284                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1285 }
1286
1287 /* mark all levels starting from 'level' as invalid */
1288 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1289                                struct g4x_wm_state *wm_state, int level)
1290 {
1291         if (level <= G4X_WM_LEVEL_NORMAL) {
1292                 enum plane_id plane_id;
1293
1294                 for_each_plane_id_on_crtc(crtc, plane_id)
1295                         wm_state->wm.plane[plane_id] = USHRT_MAX;
1296         }
1297
1298         if (level <= G4X_WM_LEVEL_SR) {
1299                 wm_state->cxsr = false;
1300                 wm_state->sr.cursor = USHRT_MAX;
1301                 wm_state->sr.plane = USHRT_MAX;
1302                 wm_state->sr.fbc = USHRT_MAX;
1303         }
1304
1305         if (level <= G4X_WM_LEVEL_HPLL) {
1306                 wm_state->hpll_en = false;
1307                 wm_state->hpll.cursor = USHRT_MAX;
1308                 wm_state->hpll.plane = USHRT_MAX;
1309                 wm_state->hpll.fbc = USHRT_MAX;
1310         }
1311 }
1312
1313 static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1314 {
1315         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1316         struct intel_atomic_state *state =
1317                 to_intel_atomic_state(crtc_state->base.state);
1318         struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1319         int num_active_planes = hweight32(crtc_state->active_planes &
1320                                           ~BIT(PLANE_CURSOR));
1321         const struct g4x_pipe_wm *raw;
1322         const struct intel_plane_state *old_plane_state;
1323         const struct intel_plane_state *new_plane_state;
1324         struct intel_plane *plane;
1325         enum plane_id plane_id;
1326         int i, level;
1327         unsigned int dirty = 0;
1328
1329         for_each_oldnew_intel_plane_in_state(state, plane,
1330                                              old_plane_state,
1331                                              new_plane_state, i) {
1332                 if (new_plane_state->base.crtc != &crtc->base &&
1333                     old_plane_state->base.crtc != &crtc->base)
1334                         continue;
1335
1336                 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1337                         dirty |= BIT(plane->id);
1338         }
1339
1340         if (!dirty)
1341                 return 0;
1342
1343         level = G4X_WM_LEVEL_NORMAL;
1344         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1345                 goto out;
1346
1347         raw = &crtc_state->wm.g4x.raw[level];
1348         for_each_plane_id_on_crtc(crtc, plane_id)
1349                 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1350
1351         level = G4X_WM_LEVEL_SR;
1352
1353         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1354                 goto out;
1355
1356         raw = &crtc_state->wm.g4x.raw[level];
1357         wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1358         wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1359         wm_state->sr.fbc = raw->fbc;
1360
1361         wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1362
1363         level = G4X_WM_LEVEL_HPLL;
1364
1365         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1366                 goto out;
1367
1368         raw = &crtc_state->wm.g4x.raw[level];
1369         wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1370         wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1371         wm_state->hpll.fbc = raw->fbc;
1372
1373         wm_state->hpll_en = wm_state->cxsr;
1374
1375         level++;
1376
1377  out:
1378         if (level == G4X_WM_LEVEL_NORMAL)
1379                 return -EINVAL;
1380
1381         /* invalidate the higher levels */
1382         g4x_invalidate_wms(crtc, wm_state, level);
1383
1384         /*
1385          * Determine if the FBC watermark(s) can be used. IF
1386          * this isn't the case we prefer to disable the FBC
1387          ( watermark(s) rather than disable the SR/HPLL
1388          * level(s) entirely.
1389          */
1390         wm_state->fbc_en = level > G4X_WM_LEVEL_NORMAL;
1391
1392         if (level >= G4X_WM_LEVEL_SR &&
1393             wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1394                 wm_state->fbc_en = false;
1395         else if (level >= G4X_WM_LEVEL_HPLL &&
1396                  wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1397                 wm_state->fbc_en = false;
1398
1399         return 0;
1400 }
1401
1402 static int g4x_compute_intermediate_wm(struct drm_device *dev,
1403                                        struct intel_crtc *crtc,
1404                                        struct intel_crtc_state *new_crtc_state)
1405 {
1406         struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1407         const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1408         struct intel_atomic_state *intel_state =
1409                 to_intel_atomic_state(new_crtc_state->base.state);
1410         const struct intel_crtc_state *old_crtc_state =
1411                 intel_atomic_get_old_crtc_state(intel_state, crtc);
1412         const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1413         enum plane_id plane_id;
1414
1415         if (!new_crtc_state->base.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) {
1416                 *intermediate = *optimal;
1417
1418                 intermediate->cxsr = false;
1419                 intermediate->hpll_en = false;
1420                 goto out;
1421         }
1422
1423         intermediate->cxsr = optimal->cxsr && active->cxsr &&
1424                 !new_crtc_state->disable_cxsr;
1425         intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1426                 !new_crtc_state->disable_cxsr;
1427         intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1428
1429         for_each_plane_id_on_crtc(crtc, plane_id) {
1430                 intermediate->wm.plane[plane_id] =
1431                         max(optimal->wm.plane[plane_id],
1432                             active->wm.plane[plane_id]);
1433
1434                 WARN_ON(intermediate->wm.plane[plane_id] >
1435                         g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1436         }
1437
1438         intermediate->sr.plane = max(optimal->sr.plane,
1439                                      active->sr.plane);
1440         intermediate->sr.cursor = max(optimal->sr.cursor,
1441                                       active->sr.cursor);
1442         intermediate->sr.fbc = max(optimal->sr.fbc,
1443                                    active->sr.fbc);
1444
1445         intermediate->hpll.plane = max(optimal->hpll.plane,
1446                                        active->hpll.plane);
1447         intermediate->hpll.cursor = max(optimal->hpll.cursor,
1448                                         active->hpll.cursor);
1449         intermediate->hpll.fbc = max(optimal->hpll.fbc,
1450                                      active->hpll.fbc);
1451
1452         WARN_ON((intermediate->sr.plane >
1453                  g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1454                  intermediate->sr.cursor >
1455                  g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1456                 intermediate->cxsr);
1457         WARN_ON((intermediate->sr.plane >
1458                  g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1459                  intermediate->sr.cursor >
1460                  g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1461                 intermediate->hpll_en);
1462
1463         WARN_ON(intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1464                 intermediate->fbc_en && intermediate->cxsr);
1465         WARN_ON(intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1466                 intermediate->fbc_en && intermediate->hpll_en);
1467
1468 out:
1469         /*
1470          * If our intermediate WM are identical to the final WM, then we can
1471          * omit the post-vblank programming; only update if it's different.
1472          */
1473         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1474                 new_crtc_state->wm.need_postvbl_update = true;
1475
1476         return 0;
1477 }
1478
1479 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1480                          struct g4x_wm_values *wm)
1481 {
1482         struct intel_crtc *crtc;
1483         int num_active_crtcs = 0;
1484
1485         wm->cxsr = true;
1486         wm->hpll_en = true;
1487         wm->fbc_en = true;
1488
1489         for_each_intel_crtc(&dev_priv->drm, crtc) {
1490                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1491
1492                 if (!crtc->active)
1493                         continue;
1494
1495                 if (!wm_state->cxsr)
1496                         wm->cxsr = false;
1497                 if (!wm_state->hpll_en)
1498                         wm->hpll_en = false;
1499                 if (!wm_state->fbc_en)
1500                         wm->fbc_en = false;
1501
1502                 num_active_crtcs++;
1503         }
1504
1505         if (num_active_crtcs != 1) {
1506                 wm->cxsr = false;
1507                 wm->hpll_en = false;
1508                 wm->fbc_en = false;
1509         }
1510
1511         for_each_intel_crtc(&dev_priv->drm, crtc) {
1512                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1513                 enum pipe pipe = crtc->pipe;
1514
1515                 wm->pipe[pipe] = wm_state->wm;
1516                 if (crtc->active && wm->cxsr)
1517                         wm->sr = wm_state->sr;
1518                 if (crtc->active && wm->hpll_en)
1519                         wm->hpll = wm_state->hpll;
1520         }
1521 }
1522
1523 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1524 {
1525         struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1526         struct g4x_wm_values new_wm = {};
1527
1528         g4x_merge_wm(dev_priv, &new_wm);
1529
1530         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1531                 return;
1532
1533         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1534                 _intel_set_memory_cxsr(dev_priv, false);
1535
1536         g4x_write_wm_values(dev_priv, &new_wm);
1537
1538         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1539                 _intel_set_memory_cxsr(dev_priv, true);
1540
1541         *old_wm = new_wm;
1542 }
1543
1544 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1545                                    struct intel_crtc_state *crtc_state)
1546 {
1547         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1548         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1549
1550         mutex_lock(&dev_priv->wm.wm_mutex);
1551         crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1552         g4x_program_watermarks(dev_priv);
1553         mutex_unlock(&dev_priv->wm.wm_mutex);
1554 }
1555
1556 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1557                                     struct intel_crtc_state *crtc_state)
1558 {
1559         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1560         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
1561
1562         if (!crtc_state->wm.need_postvbl_update)
1563                 return;
1564
1565         mutex_lock(&dev_priv->wm.wm_mutex);
1566         intel_crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1567         g4x_program_watermarks(dev_priv);
1568         mutex_unlock(&dev_priv->wm.wm_mutex);
1569 }
1570
1571 /* latency must be in 0.1us units. */
1572 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1573                                    unsigned int htotal,
1574                                    unsigned int width,
1575                                    unsigned int cpp,
1576                                    unsigned int latency)
1577 {
1578         unsigned int ret;
1579
1580         ret = intel_wm_method2(pixel_rate, htotal,
1581                                width, cpp, latency);
1582         ret = DIV_ROUND_UP(ret, 64);
1583
1584         return ret;
1585 }
1586
1587 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1588 {
1589         /* all latencies in usec */
1590         dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1591
1592         dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1593
1594         if (IS_CHERRYVIEW(dev_priv)) {
1595                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1596                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1597
1598                 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1599         }
1600 }
1601
1602 static uint16_t vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1603                                      const struct intel_plane_state *plane_state,
1604                                      int level)
1605 {
1606         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1607         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1608         const struct drm_display_mode *adjusted_mode =
1609                 &crtc_state->base.adjusted_mode;
1610         unsigned int clock, htotal, cpp, width, wm;
1611
1612         if (dev_priv->wm.pri_latency[level] == 0)
1613                 return USHRT_MAX;
1614
1615         if (!intel_wm_plane_visible(crtc_state, plane_state))
1616                 return 0;
1617
1618         cpp = plane_state->base.fb->format->cpp[0];
1619         clock = adjusted_mode->crtc_clock;
1620         htotal = adjusted_mode->crtc_htotal;
1621         width = crtc_state->pipe_src_w;
1622
1623         if (plane->id == PLANE_CURSOR) {
1624                 /*
1625                  * FIXME the formula gives values that are
1626                  * too big for the cursor FIFO, and hence we
1627                  * would never be able to use cursors. For
1628                  * now just hardcode the watermark.
1629                  */
1630                 wm = 63;
1631         } else {
1632                 wm = vlv_wm_method2(clock, htotal, width, cpp,
1633                                     dev_priv->wm.pri_latency[level] * 10);
1634         }
1635
1636         return min_t(unsigned int, wm, USHRT_MAX);
1637 }
1638
1639 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1640 {
1641         return (active_planes & (BIT(PLANE_SPRITE0) |
1642                                  BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1643 }
1644
1645 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1646 {
1647         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1648         const struct g4x_pipe_wm *raw =
1649                 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1650         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1651         unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1652         int num_active_planes = hweight32(active_planes);
1653         const int fifo_size = 511;
1654         int fifo_extra, fifo_left = fifo_size;
1655         int sprite0_fifo_extra = 0;
1656         unsigned int total_rate;
1657         enum plane_id plane_id;
1658
1659         /*
1660          * When enabling sprite0 after sprite1 has already been enabled
1661          * we tend to get an underrun unless sprite0 already has some
1662          * FIFO space allcoated. Hence we always allocate at least one
1663          * cacheline for sprite0 whenever sprite1 is enabled.
1664          *
1665          * All other plane enable sequences appear immune to this problem.
1666          */
1667         if (vlv_need_sprite0_fifo_workaround(active_planes))
1668                 sprite0_fifo_extra = 1;
1669
1670         total_rate = raw->plane[PLANE_PRIMARY] +
1671                 raw->plane[PLANE_SPRITE0] +
1672                 raw->plane[PLANE_SPRITE1] +
1673                 sprite0_fifo_extra;
1674
1675         if (total_rate > fifo_size)
1676                 return -EINVAL;
1677
1678         if (total_rate == 0)
1679                 total_rate = 1;
1680
1681         for_each_plane_id_on_crtc(crtc, plane_id) {
1682                 unsigned int rate;
1683
1684                 if ((active_planes & BIT(plane_id)) == 0) {
1685                         fifo_state->plane[plane_id] = 0;
1686                         continue;
1687                 }
1688
1689                 rate = raw->plane[plane_id];
1690                 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1691                 fifo_left -= fifo_state->plane[plane_id];
1692         }
1693
1694         fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1695         fifo_left -= sprite0_fifo_extra;
1696
1697         fifo_state->plane[PLANE_CURSOR] = 63;
1698
1699         fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1700
1701         /* spread the remainder evenly */
1702         for_each_plane_id_on_crtc(crtc, plane_id) {
1703                 int plane_extra;
1704
1705                 if (fifo_left == 0)
1706                         break;
1707
1708                 if ((active_planes & BIT(plane_id)) == 0)
1709                         continue;
1710
1711                 plane_extra = min(fifo_extra, fifo_left);
1712                 fifo_state->plane[plane_id] += plane_extra;
1713                 fifo_left -= plane_extra;
1714         }
1715
1716         WARN_ON(active_planes != 0 && fifo_left != 0);
1717
1718         /* give it all to the first plane if none are active */
1719         if (active_planes == 0) {
1720                 WARN_ON(fifo_left != fifo_size);
1721                 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1722         }
1723
1724         return 0;
1725 }
1726
1727 /* mark all levels starting from 'level' as invalid */
1728 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1729                                struct vlv_wm_state *wm_state, int level)
1730 {
1731         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1732
1733         for (; level < intel_wm_num_levels(dev_priv); level++) {
1734                 enum plane_id plane_id;
1735
1736                 for_each_plane_id_on_crtc(crtc, plane_id)
1737                         wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1738
1739                 wm_state->sr[level].cursor = USHRT_MAX;
1740                 wm_state->sr[level].plane = USHRT_MAX;
1741         }
1742 }
1743
1744 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1745 {
1746         if (wm > fifo_size)
1747                 return USHRT_MAX;
1748         else
1749                 return fifo_size - wm;
1750 }
1751
1752 /*
1753  * Starting from 'level' set all higher
1754  * levels to 'value' in the "raw" watermarks.
1755  */
1756 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1757                                  int level, enum plane_id plane_id, u16 value)
1758 {
1759         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1760         int num_levels = intel_wm_num_levels(dev_priv);
1761         bool dirty = false;
1762
1763         for (; level < num_levels; level++) {
1764                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1765
1766                 dirty |= raw->plane[plane_id] != value;
1767                 raw->plane[plane_id] = value;
1768         }
1769
1770         return dirty;
1771 }
1772
1773 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1774                                      const struct intel_plane_state *plane_state)
1775 {
1776         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1777         enum plane_id plane_id = plane->id;
1778         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1779         int level;
1780         bool dirty = false;
1781
1782         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1783                 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1784                 goto out;
1785         }
1786
1787         for (level = 0; level < num_levels; level++) {
1788                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1789                 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1790                 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1791
1792                 if (wm > max_wm)
1793                         break;
1794
1795                 dirty |= raw->plane[plane_id] != wm;
1796                 raw->plane[plane_id] = wm;
1797         }
1798
1799         /* mark all higher levels as invalid */
1800         dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1801
1802 out:
1803         if (dirty)
1804                 DRM_DEBUG_KMS("%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1805                               plane->base.name,
1806                               crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1807                               crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1808                               crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1809
1810         return dirty;
1811 }
1812
1813 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1814                                       enum plane_id plane_id, int level)
1815 {
1816         const struct g4x_pipe_wm *raw =
1817                 &crtc_state->wm.vlv.raw[level];
1818         const struct vlv_fifo_state *fifo_state =
1819                 &crtc_state->wm.vlv.fifo_state;
1820
1821         return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1822 }
1823
1824 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1825 {
1826         return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1827                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1828                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1829                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1830 }
1831
1832 static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1833 {
1834         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1835         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1836         struct intel_atomic_state *state =
1837                 to_intel_atomic_state(crtc_state->base.state);
1838         struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1839         const struct vlv_fifo_state *fifo_state =
1840                 &crtc_state->wm.vlv.fifo_state;
1841         int num_active_planes = hweight32(crtc_state->active_planes &
1842                                           ~BIT(PLANE_CURSOR));
1843         bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->base);
1844         const struct intel_plane_state *old_plane_state;
1845         const struct intel_plane_state *new_plane_state;
1846         struct intel_plane *plane;
1847         enum plane_id plane_id;
1848         int level, ret, i;
1849         unsigned int dirty = 0;
1850
1851         for_each_oldnew_intel_plane_in_state(state, plane,
1852                                              old_plane_state,
1853                                              new_plane_state, i) {
1854                 if (new_plane_state->base.crtc != &crtc->base &&
1855                     old_plane_state->base.crtc != &crtc->base)
1856                         continue;
1857
1858                 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1859                         dirty |= BIT(plane->id);
1860         }
1861
1862         /*
1863          * DSPARB registers may have been reset due to the
1864          * power well being turned off. Make sure we restore
1865          * them to a consistent state even if no primary/sprite
1866          * planes are initially active.
1867          */
1868         if (needs_modeset)
1869                 crtc_state->fifo_changed = true;
1870
1871         if (!dirty)
1872                 return 0;
1873
1874         /* cursor changes don't warrant a FIFO recompute */
1875         if (dirty & ~BIT(PLANE_CURSOR)) {
1876                 const struct intel_crtc_state *old_crtc_state =
1877                         intel_atomic_get_old_crtc_state(state, crtc);
1878                 const struct vlv_fifo_state *old_fifo_state =
1879                         &old_crtc_state->wm.vlv.fifo_state;
1880
1881                 ret = vlv_compute_fifo(crtc_state);
1882                 if (ret)
1883                         return ret;
1884
1885                 if (needs_modeset ||
1886                     memcmp(old_fifo_state, fifo_state,
1887                            sizeof(*fifo_state)) != 0)
1888                         crtc_state->fifo_changed = true;
1889         }
1890
1891         /* initially allow all levels */
1892         wm_state->num_levels = intel_wm_num_levels(dev_priv);
1893         /*
1894          * Note that enabling cxsr with no primary/sprite planes
1895          * enabled can wedge the pipe. Hence we only allow cxsr
1896          * with exactly one enabled primary/sprite plane.
1897          */
1898         wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1899
1900         for (level = 0; level < wm_state->num_levels; level++) {
1901                 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1902                 const int sr_fifo_size = INTEL_INFO(dev_priv)->num_pipes * 512 - 1;
1903
1904                 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1905                         break;
1906
1907                 for_each_plane_id_on_crtc(crtc, plane_id) {
1908                         wm_state->wm[level].plane[plane_id] =
1909                                 vlv_invert_wm_value(raw->plane[plane_id],
1910                                                     fifo_state->plane[plane_id]);
1911                 }
1912
1913                 wm_state->sr[level].plane =
1914                         vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1915                                                  raw->plane[PLANE_SPRITE0],
1916                                                  raw->plane[PLANE_SPRITE1]),
1917                                             sr_fifo_size);
1918
1919                 wm_state->sr[level].cursor =
1920                         vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1921                                             63);
1922         }
1923
1924         if (level == 0)
1925                 return -EINVAL;
1926
1927         /* limit to only levels we can actually handle */
1928         wm_state->num_levels = level;
1929
1930         /* invalidate the higher levels */
1931         vlv_invalidate_wms(crtc, wm_state, level);
1932
1933         return 0;
1934 }
1935
1936 #define VLV_FIFO(plane, value) \
1937         (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1938
1939 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1940                                    struct intel_crtc_state *crtc_state)
1941 {
1942         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1943         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1944         const struct vlv_fifo_state *fifo_state =
1945                 &crtc_state->wm.vlv.fifo_state;
1946         int sprite0_start, sprite1_start, fifo_size;
1947
1948         if (!crtc_state->fifo_changed)
1949                 return;
1950
1951         sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1952         sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1953         fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1954
1955         WARN_ON(fifo_state->plane[PLANE_CURSOR] != 63);
1956         WARN_ON(fifo_size != 511);
1957
1958         trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1959
1960         /*
1961          * uncore.lock serves a double purpose here. It allows us to
1962          * use the less expensive I915_{READ,WRITE}_FW() functions, and
1963          * it protects the DSPARB registers from getting clobbered by
1964          * parallel updates from multiple pipes.
1965          *
1966          * intel_pipe_update_start() has already disabled interrupts
1967          * for us, so a plain spin_lock() is sufficient here.
1968          */
1969         spin_lock(&dev_priv->uncore.lock);
1970
1971         switch (crtc->pipe) {
1972                 uint32_t dsparb, dsparb2, dsparb3;
1973         case PIPE_A:
1974                 dsparb = I915_READ_FW(DSPARB);
1975                 dsparb2 = I915_READ_FW(DSPARB2);
1976
1977                 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
1978                             VLV_FIFO(SPRITEB, 0xff));
1979                 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
1980                            VLV_FIFO(SPRITEB, sprite1_start));
1981
1982                 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
1983                              VLV_FIFO(SPRITEB_HI, 0x1));
1984                 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
1985                            VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
1986
1987                 I915_WRITE_FW(DSPARB, dsparb);
1988                 I915_WRITE_FW(DSPARB2, dsparb2);
1989                 break;
1990         case PIPE_B:
1991                 dsparb = I915_READ_FW(DSPARB);
1992                 dsparb2 = I915_READ_FW(DSPARB2);
1993
1994                 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
1995                             VLV_FIFO(SPRITED, 0xff));
1996                 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
1997                            VLV_FIFO(SPRITED, sprite1_start));
1998
1999                 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2000                              VLV_FIFO(SPRITED_HI, 0xff));
2001                 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2002                            VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2003
2004                 I915_WRITE_FW(DSPARB, dsparb);
2005                 I915_WRITE_FW(DSPARB2, dsparb2);
2006                 break;
2007         case PIPE_C:
2008                 dsparb3 = I915_READ_FW(DSPARB3);
2009                 dsparb2 = I915_READ_FW(DSPARB2);
2010
2011                 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2012                              VLV_FIFO(SPRITEF, 0xff));
2013                 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2014                             VLV_FIFO(SPRITEF, sprite1_start));
2015
2016                 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2017                              VLV_FIFO(SPRITEF_HI, 0xff));
2018                 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2019                            VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2020
2021                 I915_WRITE_FW(DSPARB3, dsparb3);
2022                 I915_WRITE_FW(DSPARB2, dsparb2);
2023                 break;
2024         default:
2025                 break;
2026         }
2027
2028         POSTING_READ_FW(DSPARB);
2029
2030         spin_unlock(&dev_priv->uncore.lock);
2031 }
2032
2033 #undef VLV_FIFO
2034
2035 static int vlv_compute_intermediate_wm(struct drm_device *dev,
2036                                        struct intel_crtc *crtc,
2037                                        struct intel_crtc_state *new_crtc_state)
2038 {
2039         struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2040         const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2041         struct intel_atomic_state *intel_state =
2042                 to_intel_atomic_state(new_crtc_state->base.state);
2043         const struct intel_crtc_state *old_crtc_state =
2044                 intel_atomic_get_old_crtc_state(intel_state, crtc);
2045         const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2046         int level;
2047
2048         if (!new_crtc_state->base.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) {
2049                 *intermediate = *optimal;
2050
2051                 intermediate->cxsr = false;
2052                 goto out;
2053         }
2054
2055         intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2056         intermediate->cxsr = optimal->cxsr && active->cxsr &&
2057                 !new_crtc_state->disable_cxsr;
2058
2059         for (level = 0; level < intermediate->num_levels; level++) {
2060                 enum plane_id plane_id;
2061
2062                 for_each_plane_id_on_crtc(crtc, plane_id) {
2063                         intermediate->wm[level].plane[plane_id] =
2064                                 min(optimal->wm[level].plane[plane_id],
2065                                     active->wm[level].plane[plane_id]);
2066                 }
2067
2068                 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2069                                                     active->sr[level].plane);
2070                 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2071                                                      active->sr[level].cursor);
2072         }
2073
2074         vlv_invalidate_wms(crtc, intermediate, level);
2075
2076 out:
2077         /*
2078          * If our intermediate WM are identical to the final WM, then we can
2079          * omit the post-vblank programming; only update if it's different.
2080          */
2081         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2082                 new_crtc_state->wm.need_postvbl_update = true;
2083
2084         return 0;
2085 }
2086
2087 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2088                          struct vlv_wm_values *wm)
2089 {
2090         struct intel_crtc *crtc;
2091         int num_active_crtcs = 0;
2092
2093         wm->level = dev_priv->wm.max_level;
2094         wm->cxsr = true;
2095
2096         for_each_intel_crtc(&dev_priv->drm, crtc) {
2097                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2098
2099                 if (!crtc->active)
2100                         continue;
2101
2102                 if (!wm_state->cxsr)
2103                         wm->cxsr = false;
2104
2105                 num_active_crtcs++;
2106                 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2107         }
2108
2109         if (num_active_crtcs != 1)
2110                 wm->cxsr = false;
2111
2112         if (num_active_crtcs > 1)
2113                 wm->level = VLV_WM_LEVEL_PM2;
2114
2115         for_each_intel_crtc(&dev_priv->drm, crtc) {
2116                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2117                 enum pipe pipe = crtc->pipe;
2118
2119                 wm->pipe[pipe] = wm_state->wm[wm->level];
2120                 if (crtc->active && wm->cxsr)
2121                         wm->sr = wm_state->sr[wm->level];
2122
2123                 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2124                 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2125                 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2126                 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2127         }
2128 }
2129
2130 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2131 {
2132         struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2133         struct vlv_wm_values new_wm = {};
2134
2135         vlv_merge_wm(dev_priv, &new_wm);
2136
2137         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2138                 return;
2139
2140         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2141                 chv_set_memory_dvfs(dev_priv, false);
2142
2143         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2144                 chv_set_memory_pm5(dev_priv, false);
2145
2146         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2147                 _intel_set_memory_cxsr(dev_priv, false);
2148
2149         vlv_write_wm_values(dev_priv, &new_wm);
2150
2151         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2152                 _intel_set_memory_cxsr(dev_priv, true);
2153
2154         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2155                 chv_set_memory_pm5(dev_priv, true);
2156
2157         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2158                 chv_set_memory_dvfs(dev_priv, true);
2159
2160         *old_wm = new_wm;
2161 }
2162
2163 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2164                                    struct intel_crtc_state *crtc_state)
2165 {
2166         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
2167         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
2168
2169         mutex_lock(&dev_priv->wm.wm_mutex);
2170         crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2171         vlv_program_watermarks(dev_priv);
2172         mutex_unlock(&dev_priv->wm.wm_mutex);
2173 }
2174
2175 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2176                                     struct intel_crtc_state *crtc_state)
2177 {
2178         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
2179         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
2180
2181         if (!crtc_state->wm.need_postvbl_update)
2182                 return;
2183
2184         mutex_lock(&dev_priv->wm.wm_mutex);
2185         intel_crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2186         vlv_program_watermarks(dev_priv);
2187         mutex_unlock(&dev_priv->wm.wm_mutex);
2188 }
2189
2190 static void i965_update_wm(struct intel_crtc *unused_crtc)
2191 {
2192         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2193         struct intel_crtc *crtc;
2194         int srwm = 1;
2195         int cursor_sr = 16;
2196         bool cxsr_enabled;
2197
2198         /* Calc sr entries for one plane configs */
2199         crtc = single_enabled_crtc(dev_priv);
2200         if (crtc) {
2201                 /* self-refresh has much higher latency */
2202                 static const int sr_latency_ns = 12000;
2203                 const struct drm_display_mode *adjusted_mode =
2204                         &crtc->config->base.adjusted_mode;
2205                 const struct drm_framebuffer *fb =
2206                         crtc->base.primary->state->fb;
2207                 int clock = adjusted_mode->crtc_clock;
2208                 int htotal = adjusted_mode->crtc_htotal;
2209                 int hdisplay = crtc->config->pipe_src_w;
2210                 int cpp = fb->format->cpp[0];
2211                 int entries;
2212
2213                 entries = intel_wm_method2(clock, htotal,
2214                                            hdisplay, cpp, sr_latency_ns / 100);
2215                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2216                 srwm = I965_FIFO_SIZE - entries;
2217                 if (srwm < 0)
2218                         srwm = 1;
2219                 srwm &= 0x1ff;
2220                 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
2221                               entries, srwm);
2222
2223                 entries = intel_wm_method2(clock, htotal,
2224                                            crtc->base.cursor->state->crtc_w, 4,
2225                                            sr_latency_ns / 100);
2226                 entries = DIV_ROUND_UP(entries,
2227                                        i965_cursor_wm_info.cacheline_size) +
2228                         i965_cursor_wm_info.guard_size;
2229
2230                 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2231                 if (cursor_sr > i965_cursor_wm_info.max_wm)
2232                         cursor_sr = i965_cursor_wm_info.max_wm;
2233
2234                 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
2235                               "cursor %d\n", srwm, cursor_sr);
2236
2237                 cxsr_enabled = true;
2238         } else {
2239                 cxsr_enabled = false;
2240                 /* Turn off self refresh if both pipes are enabled */
2241                 intel_set_memory_cxsr(dev_priv, false);
2242         }
2243
2244         DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2245                       srwm);
2246
2247         /* 965 has limitations... */
2248         I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
2249                    FW_WM(8, CURSORB) |
2250                    FW_WM(8, PLANEB) |
2251                    FW_WM(8, PLANEA));
2252         I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
2253                    FW_WM(8, PLANEC_OLD));
2254         /* update cursor SR watermark */
2255         I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2256
2257         if (cxsr_enabled)
2258                 intel_set_memory_cxsr(dev_priv, true);
2259 }
2260
2261 #undef FW_WM
2262
2263 static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2264 {
2265         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2266         const struct intel_watermark_params *wm_info;
2267         uint32_t fwater_lo;
2268         uint32_t fwater_hi;
2269         int cwm, srwm = 1;
2270         int fifo_size;
2271         int planea_wm, planeb_wm;
2272         struct intel_crtc *crtc, *enabled = NULL;
2273
2274         if (IS_I945GM(dev_priv))
2275                 wm_info = &i945_wm_info;
2276         else if (!IS_GEN2(dev_priv))
2277                 wm_info = &i915_wm_info;
2278         else
2279                 wm_info = &i830_a_wm_info;
2280
2281         fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2282         crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2283         if (intel_crtc_active(crtc)) {
2284                 const struct drm_display_mode *adjusted_mode =
2285                         &crtc->config->base.adjusted_mode;
2286                 const struct drm_framebuffer *fb =
2287                         crtc->base.primary->state->fb;
2288                 int cpp;
2289
2290                 if (IS_GEN2(dev_priv))
2291                         cpp = 4;
2292                 else
2293                         cpp = fb->format->cpp[0];
2294
2295                 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2296                                                wm_info, fifo_size, cpp,
2297                                                pessimal_latency_ns);
2298                 enabled = crtc;
2299         } else {
2300                 planea_wm = fifo_size - wm_info->guard_size;
2301                 if (planea_wm > (long)wm_info->max_wm)
2302                         planea_wm = wm_info->max_wm;
2303         }
2304
2305         if (IS_GEN2(dev_priv))
2306                 wm_info = &i830_bc_wm_info;
2307
2308         fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2309         crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2310         if (intel_crtc_active(crtc)) {
2311                 const struct drm_display_mode *adjusted_mode =
2312                         &crtc->config->base.adjusted_mode;
2313                 const struct drm_framebuffer *fb =
2314                         crtc->base.primary->state->fb;
2315                 int cpp;
2316
2317                 if (IS_GEN2(dev_priv))
2318                         cpp = 4;
2319                 else
2320                         cpp = fb->format->cpp[0];
2321
2322                 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2323                                                wm_info, fifo_size, cpp,
2324                                                pessimal_latency_ns);
2325                 if (enabled == NULL)
2326                         enabled = crtc;
2327                 else
2328                         enabled = NULL;
2329         } else {
2330                 planeb_wm = fifo_size - wm_info->guard_size;
2331                 if (planeb_wm > (long)wm_info->max_wm)
2332                         planeb_wm = wm_info->max_wm;
2333         }
2334
2335         DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2336
2337         if (IS_I915GM(dev_priv) && enabled) {
2338                 struct drm_i915_gem_object *obj;
2339
2340                 obj = intel_fb_obj(enabled->base.primary->state->fb);
2341
2342                 /* self-refresh seems busted with untiled */
2343                 if (!i915_gem_object_is_tiled(obj))
2344                         enabled = NULL;
2345         }
2346
2347         /*
2348          * Overlay gets an aggressive default since video jitter is bad.
2349          */
2350         cwm = 2;
2351
2352         /* Play safe and disable self-refresh before adjusting watermarks. */
2353         intel_set_memory_cxsr(dev_priv, false);
2354
2355         /* Calc sr entries for one plane configs */
2356         if (HAS_FW_BLC(dev_priv) && enabled) {
2357                 /* self-refresh has much higher latency */
2358                 static const int sr_latency_ns = 6000;
2359                 const struct drm_display_mode *adjusted_mode =
2360                         &enabled->config->base.adjusted_mode;
2361                 const struct drm_framebuffer *fb =
2362                         enabled->base.primary->state->fb;
2363                 int clock = adjusted_mode->crtc_clock;
2364                 int htotal = adjusted_mode->crtc_htotal;
2365                 int hdisplay = enabled->config->pipe_src_w;
2366                 int cpp;
2367                 int entries;
2368
2369                 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2370                         cpp = 4;
2371                 else
2372                         cpp = fb->format->cpp[0];
2373
2374                 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2375                                            sr_latency_ns / 100);
2376                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2377                 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
2378                 srwm = wm_info->fifo_size - entries;
2379                 if (srwm < 0)
2380                         srwm = 1;
2381
2382                 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2383                         I915_WRITE(FW_BLC_SELF,
2384                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2385                 else
2386                         I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
2387         }
2388
2389         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2390                       planea_wm, planeb_wm, cwm, srwm);
2391
2392         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2393         fwater_hi = (cwm & 0x1f);
2394
2395         /* Set request length to 8 cachelines per fetch */
2396         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2397         fwater_hi = fwater_hi | (1 << 8);
2398
2399         I915_WRITE(FW_BLC, fwater_lo);
2400         I915_WRITE(FW_BLC2, fwater_hi);
2401
2402         if (enabled)
2403                 intel_set_memory_cxsr(dev_priv, true);
2404 }
2405
2406 static void i845_update_wm(struct intel_crtc *unused_crtc)
2407 {
2408         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2409         struct intel_crtc *crtc;
2410         const struct drm_display_mode *adjusted_mode;
2411         uint32_t fwater_lo;
2412         int planea_wm;
2413
2414         crtc = single_enabled_crtc(dev_priv);
2415         if (crtc == NULL)
2416                 return;
2417
2418         adjusted_mode = &crtc->config->base.adjusted_mode;
2419         planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2420                                        &i845_wm_info,
2421                                        dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2422                                        4, pessimal_latency_ns);
2423         fwater_lo = I915_READ(FW_BLC) & ~0xfff;
2424         fwater_lo |= (3<<8) | planea_wm;
2425
2426         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
2427
2428         I915_WRITE(FW_BLC, fwater_lo);
2429 }
2430
2431 /* latency must be in 0.1us units. */
2432 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2433                                    unsigned int cpp,
2434                                    unsigned int latency)
2435 {
2436         unsigned int ret;
2437
2438         ret = intel_wm_method1(pixel_rate, cpp, latency);
2439         ret = DIV_ROUND_UP(ret, 64) + 2;
2440
2441         return ret;
2442 }
2443
2444 /* latency must be in 0.1us units. */
2445 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2446                                    unsigned int htotal,
2447                                    unsigned int width,
2448                                    unsigned int cpp,
2449                                    unsigned int latency)
2450 {
2451         unsigned int ret;
2452
2453         ret = intel_wm_method2(pixel_rate, htotal,
2454                                width, cpp, latency);
2455         ret = DIV_ROUND_UP(ret, 64) + 2;
2456
2457         return ret;
2458 }
2459
2460 static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
2461                            uint8_t cpp)
2462 {
2463         /*
2464          * Neither of these should be possible since this function shouldn't be
2465          * called if the CRTC is off or the plane is invisible.  But let's be
2466          * extra paranoid to avoid a potential divide-by-zero if we screw up
2467          * elsewhere in the driver.
2468          */
2469         if (WARN_ON(!cpp))
2470                 return 0;
2471         if (WARN_ON(!horiz_pixels))
2472                 return 0;
2473
2474         return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2475 }
2476
2477 struct ilk_wm_maximums {
2478         uint16_t pri;
2479         uint16_t spr;
2480         uint16_t cur;
2481         uint16_t fbc;
2482 };
2483
2484 /*
2485  * For both WM_PIPE and WM_LP.
2486  * mem_value must be in 0.1us units.
2487  */
2488 static uint32_t ilk_compute_pri_wm(const struct intel_crtc_state *cstate,
2489                                    const struct intel_plane_state *pstate,
2490                                    uint32_t mem_value,
2491                                    bool is_lp)
2492 {
2493         uint32_t method1, method2;
2494         int cpp;
2495
2496         if (!intel_wm_plane_visible(cstate, pstate))
2497                 return 0;
2498
2499         cpp = pstate->base.fb->format->cpp[0];
2500
2501         method1 = ilk_wm_method1(cstate->pixel_rate, cpp, mem_value);
2502
2503         if (!is_lp)
2504                 return method1;
2505
2506         method2 = ilk_wm_method2(cstate->pixel_rate,
2507                                  cstate->base.adjusted_mode.crtc_htotal,
2508                                  drm_rect_width(&pstate->base.dst),
2509                                  cpp, mem_value);
2510
2511         return min(method1, method2);
2512 }
2513
2514 /*
2515  * For both WM_PIPE and WM_LP.
2516  * mem_value must be in 0.1us units.
2517  */
2518 static uint32_t ilk_compute_spr_wm(const struct intel_crtc_state *cstate,
2519                                    const struct intel_plane_state *pstate,
2520                                    uint32_t mem_value)
2521 {
2522         uint32_t method1, method2;
2523         int cpp;
2524
2525         if (!intel_wm_plane_visible(cstate, pstate))
2526                 return 0;
2527
2528         cpp = pstate->base.fb->format->cpp[0];
2529
2530         method1 = ilk_wm_method1(cstate->pixel_rate, cpp, mem_value);
2531         method2 = ilk_wm_method2(cstate->pixel_rate,
2532                                  cstate->base.adjusted_mode.crtc_htotal,
2533                                  drm_rect_width(&pstate->base.dst),
2534                                  cpp, mem_value);
2535         return min(method1, method2);
2536 }
2537
2538 /*
2539  * For both WM_PIPE and WM_LP.
2540  * mem_value must be in 0.1us units.
2541  */
2542 static uint32_t ilk_compute_cur_wm(const struct intel_crtc_state *cstate,
2543                                    const struct intel_plane_state *pstate,
2544                                    uint32_t mem_value)
2545 {
2546         int cpp;
2547
2548         if (!intel_wm_plane_visible(cstate, pstate))
2549                 return 0;
2550
2551         cpp = pstate->base.fb->format->cpp[0];
2552
2553         return ilk_wm_method2(cstate->pixel_rate,
2554                               cstate->base.adjusted_mode.crtc_htotal,
2555                               pstate->base.crtc_w, cpp, mem_value);
2556 }
2557
2558 /* Only for WM_LP. */
2559 static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
2560                                    const struct intel_plane_state *pstate,
2561                                    uint32_t pri_val)
2562 {
2563         int cpp;
2564
2565         if (!intel_wm_plane_visible(cstate, pstate))
2566                 return 0;
2567
2568         cpp = pstate->base.fb->format->cpp[0];
2569
2570         return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->base.dst), cpp);
2571 }
2572
2573 static unsigned int
2574 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2575 {
2576         if (INTEL_GEN(dev_priv) >= 8)
2577                 return 3072;
2578         else if (INTEL_GEN(dev_priv) >= 7)
2579                 return 768;
2580         else
2581                 return 512;
2582 }
2583
2584 static unsigned int
2585 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2586                      int level, bool is_sprite)
2587 {
2588         if (INTEL_GEN(dev_priv) >= 8)
2589                 /* BDW primary/sprite plane watermarks */
2590                 return level == 0 ? 255 : 2047;
2591         else if (INTEL_GEN(dev_priv) >= 7)
2592                 /* IVB/HSW primary/sprite plane watermarks */
2593                 return level == 0 ? 127 : 1023;
2594         else if (!is_sprite)
2595                 /* ILK/SNB primary plane watermarks */
2596                 return level == 0 ? 127 : 511;
2597         else
2598                 /* ILK/SNB sprite plane watermarks */
2599                 return level == 0 ? 63 : 255;
2600 }
2601
2602 static unsigned int
2603 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2604 {
2605         if (INTEL_GEN(dev_priv) >= 7)
2606                 return level == 0 ? 63 : 255;
2607         else
2608                 return level == 0 ? 31 : 63;
2609 }
2610
2611 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2612 {
2613         if (INTEL_GEN(dev_priv) >= 8)
2614                 return 31;
2615         else
2616                 return 15;
2617 }
2618
2619 /* Calculate the maximum primary/sprite plane watermark */
2620 static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
2621                                      int level,
2622                                      const struct intel_wm_config *config,
2623                                      enum intel_ddb_partitioning ddb_partitioning,
2624                                      bool is_sprite)
2625 {
2626         struct drm_i915_private *dev_priv = to_i915(dev);
2627         unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2628
2629         /* if sprites aren't enabled, sprites get nothing */
2630         if (is_sprite && !config->sprites_enabled)
2631                 return 0;
2632
2633         /* HSW allows LP1+ watermarks even with multiple pipes */
2634         if (level == 0 || config->num_pipes_active > 1) {
2635                 fifo_size /= INTEL_INFO(dev_priv)->num_pipes;
2636
2637                 /*
2638                  * For some reason the non self refresh
2639                  * FIFO size is only half of the self
2640                  * refresh FIFO size on ILK/SNB.
2641                  */
2642                 if (INTEL_GEN(dev_priv) <= 6)
2643                         fifo_size /= 2;
2644         }
2645
2646         if (config->sprites_enabled) {
2647                 /* level 0 is always calculated with 1:1 split */
2648                 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2649                         if (is_sprite)
2650                                 fifo_size *= 5;
2651                         fifo_size /= 6;
2652                 } else {
2653                         fifo_size /= 2;
2654                 }
2655         }
2656
2657         /* clamp to max that the registers can hold */
2658         return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2659 }
2660
2661 /* Calculate the maximum cursor plane watermark */
2662 static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
2663                                       int level,
2664                                       const struct intel_wm_config *config)
2665 {
2666         /* HSW LP1+ watermarks w/ multiple pipes */
2667         if (level > 0 && config->num_pipes_active > 1)
2668                 return 64;
2669
2670         /* otherwise just report max that registers can hold */
2671         return ilk_cursor_wm_reg_max(to_i915(dev), level);
2672 }
2673
2674 static void ilk_compute_wm_maximums(const struct drm_device *dev,
2675                                     int level,
2676                                     const struct intel_wm_config *config,
2677                                     enum intel_ddb_partitioning ddb_partitioning,
2678                                     struct ilk_wm_maximums *max)
2679 {
2680         max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
2681         max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
2682         max->cur = ilk_cursor_wm_max(dev, level, config);
2683         max->fbc = ilk_fbc_wm_reg_max(to_i915(dev));
2684 }
2685
2686 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2687                                         int level,
2688                                         struct ilk_wm_maximums *max)
2689 {
2690         max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2691         max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2692         max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2693         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2694 }
2695
2696 static bool ilk_validate_wm_level(int level,
2697                                   const struct ilk_wm_maximums *max,
2698                                   struct intel_wm_level *result)
2699 {
2700         bool ret;
2701
2702         /* already determined to be invalid? */
2703         if (!result->enable)
2704                 return false;
2705
2706         result->enable = result->pri_val <= max->pri &&
2707                          result->spr_val <= max->spr &&
2708                          result->cur_val <= max->cur;
2709
2710         ret = result->enable;
2711
2712         /*
2713          * HACK until we can pre-compute everything,
2714          * and thus fail gracefully if LP0 watermarks
2715          * are exceeded...
2716          */
2717         if (level == 0 && !result->enable) {
2718                 if (result->pri_val > max->pri)
2719                         DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2720                                       level, result->pri_val, max->pri);
2721                 if (result->spr_val > max->spr)
2722                         DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2723                                       level, result->spr_val, max->spr);
2724                 if (result->cur_val > max->cur)
2725                         DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2726                                       level, result->cur_val, max->cur);
2727
2728                 result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
2729                 result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
2730                 result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
2731                 result->enable = true;
2732         }
2733
2734         return ret;
2735 }
2736
2737 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2738                                  const struct intel_crtc *intel_crtc,
2739                                  int level,
2740                                  struct intel_crtc_state *cstate,
2741                                  const struct intel_plane_state *pristate,
2742                                  const struct intel_plane_state *sprstate,
2743                                  const struct intel_plane_state *curstate,
2744                                  struct intel_wm_level *result)
2745 {
2746         uint16_t pri_latency = dev_priv->wm.pri_latency[level];
2747         uint16_t spr_latency = dev_priv->wm.spr_latency[level];
2748         uint16_t cur_latency = dev_priv->wm.cur_latency[level];
2749
2750         /* WM1+ latency values stored in 0.5us units */
2751         if (level > 0) {
2752                 pri_latency *= 5;
2753                 spr_latency *= 5;
2754                 cur_latency *= 5;
2755         }
2756
2757         if (pristate) {
2758                 result->pri_val = ilk_compute_pri_wm(cstate, pristate,
2759                                                      pri_latency, level);
2760                 result->fbc_val = ilk_compute_fbc_wm(cstate, pristate, result->pri_val);
2761         }
2762
2763         if (sprstate)
2764                 result->spr_val = ilk_compute_spr_wm(cstate, sprstate, spr_latency);
2765
2766         if (curstate)
2767                 result->cur_val = ilk_compute_cur_wm(cstate, curstate, cur_latency);
2768
2769         result->enable = true;
2770 }
2771
2772 static uint32_t
2773 hsw_compute_linetime_wm(const struct intel_crtc_state *cstate)
2774 {
2775         const struct intel_atomic_state *intel_state =
2776                 to_intel_atomic_state(cstate->base.state);
2777         const struct drm_display_mode *adjusted_mode =
2778                 &cstate->base.adjusted_mode;
2779         u32 linetime, ips_linetime;
2780
2781         if (!cstate->base.active)
2782                 return 0;
2783         if (WARN_ON(adjusted_mode->crtc_clock == 0))
2784                 return 0;
2785         if (WARN_ON(intel_state->cdclk.logical.cdclk == 0))
2786                 return 0;
2787
2788         /* The WM are computed with base on how long it takes to fill a single
2789          * row at the given clock rate, multiplied by 8.
2790          * */
2791         linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2792                                      adjusted_mode->crtc_clock);
2793         ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2794                                          intel_state->cdclk.logical.cdclk);
2795
2796         return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
2797                PIPE_WM_LINETIME_TIME(linetime);
2798 }
2799
2800 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2801                                   uint16_t wm[8])
2802 {
2803         if (INTEL_GEN(dev_priv) >= 9) {
2804                 uint32_t val;
2805                 int ret, i;
2806                 int level, max_level = ilk_wm_max_level(dev_priv);
2807
2808                 /* read the first set of memory latencies[0:3] */
2809                 val = 0; /* data0 to be programmed to 0 for first set */
2810                 mutex_lock(&dev_priv->pcu_lock);
2811                 ret = sandybridge_pcode_read(dev_priv,
2812                                              GEN9_PCODE_READ_MEM_LATENCY,
2813                                              &val);
2814                 mutex_unlock(&dev_priv->pcu_lock);
2815
2816                 if (ret) {
2817                         DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2818                         return;
2819                 }
2820
2821                 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2822                 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2823                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2824                 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2825                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2826                 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2827                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2828
2829                 /* read the second set of memory latencies[4:7] */
2830                 val = 1; /* data0 to be programmed to 1 for second set */
2831                 mutex_lock(&dev_priv->pcu_lock);
2832                 ret = sandybridge_pcode_read(dev_priv,
2833                                              GEN9_PCODE_READ_MEM_LATENCY,
2834                                              &val);
2835                 mutex_unlock(&dev_priv->pcu_lock);
2836                 if (ret) {
2837                         DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2838                         return;
2839                 }
2840
2841                 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2842                 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2843                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2844                 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2845                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2846                 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2847                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2848
2849                 /*
2850                  * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2851                  * need to be disabled. We make sure to sanitize the values out
2852                  * of the punit to satisfy this requirement.
2853                  */
2854                 for (level = 1; level <= max_level; level++) {
2855                         if (wm[level] == 0) {
2856                                 for (i = level + 1; i <= max_level; i++)
2857                                         wm[i] = 0;
2858                                 break;
2859                         }
2860                 }
2861
2862                 /*
2863                  * WaWmMemoryReadLatency:skl+,glk
2864                  *
2865                  * punit doesn't take into account the read latency so we need
2866                  * to add 2us to the various latency levels we retrieve from the
2867                  * punit when level 0 response data us 0us.
2868                  */
2869                 if (wm[0] == 0) {
2870                         wm[0] += 2;
2871                         for (level = 1; level <= max_level; level++) {
2872                                 if (wm[level] == 0)
2873                                         break;
2874                                 wm[level] += 2;
2875                         }
2876                 }
2877
2878                 /*
2879                  * WA Level-0 adjustment for 16GB DIMMs: SKL+
2880                  * If we could not get dimm info enable this WA to prevent from
2881                  * any underrun. If not able to get Dimm info assume 16GB dimm
2882                  * to avoid any underrun.
2883                  */
2884                 if (!dev_priv->dram_info.valid_dimm ||
2885                     dev_priv->dram_info.is_16gb_dimm)
2886                         wm[0] += 1;
2887
2888         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2889                 uint64_t sskpd = I915_READ64(MCH_SSKPD);
2890
2891                 wm[0] = (sskpd >> 56) & 0xFF;
2892                 if (wm[0] == 0)
2893                         wm[0] = sskpd & 0xF;
2894                 wm[1] = (sskpd >> 4) & 0xFF;
2895                 wm[2] = (sskpd >> 12) & 0xFF;
2896                 wm[3] = (sskpd >> 20) & 0x1FF;
2897                 wm[4] = (sskpd >> 32) & 0x1FF;
2898         } else if (INTEL_GEN(dev_priv) >= 6) {
2899                 uint32_t sskpd = I915_READ(MCH_SSKPD);
2900
2901                 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2902                 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2903                 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2904                 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2905         } else if (INTEL_GEN(dev_priv) >= 5) {
2906                 uint32_t mltr = I915_READ(MLTR_ILK);
2907
2908                 /* ILK primary LP0 latency is 700 ns */
2909                 wm[0] = 7;
2910                 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2911                 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2912         } else {
2913                 MISSING_CASE(INTEL_DEVID(dev_priv));
2914         }
2915 }
2916
2917 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2918                                        uint16_t wm[5])
2919 {
2920         /* ILK sprite LP0 latency is 1300 ns */
2921         if (IS_GEN5(dev_priv))
2922                 wm[0] = 13;
2923 }
2924
2925 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2926                                        uint16_t wm[5])
2927 {
2928         /* ILK cursor LP0 latency is 1300 ns */
2929         if (IS_GEN5(dev_priv))
2930                 wm[0] = 13;
2931 }
2932
2933 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2934 {
2935         /* how many WM levels are we expecting */
2936         if (INTEL_GEN(dev_priv) >= 9)
2937                 return 7;
2938         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2939                 return 4;
2940         else if (INTEL_GEN(dev_priv) >= 6)
2941                 return 3;
2942         else
2943                 return 2;
2944 }
2945
2946 static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2947                                    const char *name,
2948                                    const uint16_t wm[8])
2949 {
2950         int level, max_level = ilk_wm_max_level(dev_priv);
2951
2952         for (level = 0; level <= max_level; level++) {
2953                 unsigned int latency = wm[level];
2954
2955                 if (latency == 0) {
2956                         DRM_DEBUG_KMS("%s WM%d latency not provided\n",
2957                                       name, level);
2958                         continue;
2959                 }
2960
2961                 /*
2962                  * - latencies are in us on gen9.
2963                  * - before then, WM1+ latency values are in 0.5us units
2964                  */
2965                 if (INTEL_GEN(dev_priv) >= 9)
2966                         latency *= 10;
2967                 else if (level > 0)
2968                         latency *= 5;
2969
2970                 DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
2971                               name, level, wm[level],
2972                               latency / 10, latency % 10);
2973         }
2974 }
2975
2976 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2977                                     uint16_t wm[5], uint16_t min)
2978 {
2979         int level, max_level = ilk_wm_max_level(dev_priv);
2980
2981         if (wm[0] >= min)
2982                 return false;
2983
2984         wm[0] = max(wm[0], min);
2985         for (level = 1; level <= max_level; level++)
2986                 wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
2987
2988         return true;
2989 }
2990
2991 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
2992 {
2993         bool changed;
2994
2995         /*
2996          * The BIOS provided WM memory latency values are often
2997          * inadequate for high resolution displays. Adjust them.
2998          */
2999         changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
3000                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
3001                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3002
3003         if (!changed)
3004                 return;
3005
3006         DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
3007         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3008         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3009         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3010 }
3011
3012 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3013 {
3014         intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3015
3016         memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3017                sizeof(dev_priv->wm.pri_latency));
3018         memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3019                sizeof(dev_priv->wm.pri_latency));
3020
3021         intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3022         intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3023
3024         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3025         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3026         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3027
3028         if (IS_GEN6(dev_priv))
3029                 snb_wm_latency_quirk(dev_priv);
3030 }
3031
3032 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3033 {
3034         intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3035         intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3036 }
3037
3038 static bool ilk_validate_pipe_wm(struct drm_device *dev,
3039                                  struct intel_pipe_wm *pipe_wm)
3040 {
3041         /* LP0 watermark maximums depend on this pipe alone */
3042         const struct intel_wm_config config = {
3043                 .num_pipes_active = 1,
3044                 .sprites_enabled = pipe_wm->sprites_enabled,
3045                 .sprites_scaled = pipe_wm->sprites_scaled,
3046         };
3047         struct ilk_wm_maximums max;
3048
3049         /* LP0 watermarks always use 1/2 DDB partitioning */
3050         ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
3051
3052         /* At least LP0 must be valid */
3053         if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3054                 DRM_DEBUG_KMS("LP0 watermark invalid\n");
3055                 return false;
3056         }
3057
3058         return true;
3059 }
3060
3061 /* Compute new watermarks for the pipe */
3062 static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
3063 {
3064         struct drm_atomic_state *state = cstate->base.state;
3065         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
3066         struct intel_pipe_wm *pipe_wm;
3067         struct drm_device *dev = state->dev;
3068         const struct drm_i915_private *dev_priv = to_i915(dev);
3069         struct drm_plane *plane;
3070         const struct drm_plane_state *plane_state;
3071         const struct intel_plane_state *pristate = NULL;
3072         const struct intel_plane_state *sprstate = NULL;
3073         const struct intel_plane_state *curstate = NULL;
3074         int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3075         struct ilk_wm_maximums max;
3076
3077         pipe_wm = &cstate->wm.ilk.optimal;
3078
3079         drm_atomic_crtc_state_for_each_plane_state(plane, plane_state, &cstate->base) {
3080                 const struct intel_plane_state *ps = to_intel_plane_state(plane_state);
3081
3082                 if (plane->type == DRM_PLANE_TYPE_PRIMARY)
3083                         pristate = ps;
3084                 else if (plane->type == DRM_PLANE_TYPE_OVERLAY)
3085                         sprstate = ps;
3086                 else if (plane->type == DRM_PLANE_TYPE_CURSOR)
3087                         curstate = ps;
3088         }
3089
3090         pipe_wm->pipe_enabled = cstate->base.active;
3091         if (sprstate) {
3092                 pipe_wm->sprites_enabled = sprstate->base.visible;
3093                 pipe_wm->sprites_scaled = sprstate->base.visible &&
3094                         (drm_rect_width(&sprstate->base.dst) != drm_rect_width(&sprstate->base.src) >> 16 ||
3095                          drm_rect_height(&sprstate->base.dst) != drm_rect_height(&sprstate->base.src) >> 16);
3096         }
3097
3098         usable_level = max_level;
3099
3100         /* ILK/SNB: LP2+ watermarks only w/o sprites */
3101         if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3102                 usable_level = 1;
3103
3104         /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3105         if (pipe_wm->sprites_scaled)
3106                 usable_level = 0;
3107
3108         memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3109         ilk_compute_wm_level(dev_priv, intel_crtc, 0, cstate,
3110                              pristate, sprstate, curstate, &pipe_wm->wm[0]);
3111
3112         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3113                 pipe_wm->linetime = hsw_compute_linetime_wm(cstate);
3114
3115         if (!ilk_validate_pipe_wm(dev, pipe_wm))
3116                 return -EINVAL;
3117
3118         ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3119
3120         for (level = 1; level <= usable_level; level++) {
3121                 struct intel_wm_level *wm = &pipe_wm->wm[level];
3122
3123                 ilk_compute_wm_level(dev_priv, intel_crtc, level, cstate,
3124                                      pristate, sprstate, curstate, wm);
3125
3126                 /*
3127                  * Disable any watermark level that exceeds the
3128                  * register maximums since such watermarks are
3129                  * always invalid.
3130                  */
3131                 if (!ilk_validate_wm_level(level, &max, wm)) {
3132                         memset(wm, 0, sizeof(*wm));
3133                         break;
3134                 }
3135         }
3136
3137         return 0;
3138 }
3139
3140 /*
3141  * Build a set of 'intermediate' watermark values that satisfy both the old
3142  * state and the new state.  These can be programmed to the hardware
3143  * immediately.
3144  */
3145 static int ilk_compute_intermediate_wm(struct drm_device *dev,
3146                                        struct intel_crtc *intel_crtc,
3147                                        struct intel_crtc_state *newstate)
3148 {
3149         struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3150         struct intel_atomic_state *intel_state =
3151                 to_intel_atomic_state(newstate->base.state);
3152         const struct intel_crtc_state *oldstate =
3153                 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3154         const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3155         int level, max_level = ilk_wm_max_level(to_i915(dev));
3156
3157         /*
3158          * Start with the final, target watermarks, then combine with the
3159          * currently active watermarks to get values that are safe both before
3160          * and after the vblank.
3161          */
3162         *a = newstate->wm.ilk.optimal;
3163         if (!newstate->base.active || drm_atomic_crtc_needs_modeset(&newstate->base))
3164                 return 0;
3165
3166         a->pipe_enabled |= b->pipe_enabled;
3167         a->sprites_enabled |= b->sprites_enabled;
3168         a->sprites_scaled |= b->sprites_scaled;
3169
3170         for (level = 0; level <= max_level; level++) {
3171                 struct intel_wm_level *a_wm = &a->wm[level];
3172                 const struct intel_wm_level *b_wm = &b->wm[level];
3173
3174                 a_wm->enable &= b_wm->enable;
3175                 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3176                 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3177                 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3178                 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3179         }
3180
3181         /*
3182          * We need to make sure that these merged watermark values are
3183          * actually a valid configuration themselves.  If they're not,
3184          * there's no safe way to transition from the old state to
3185          * the new state, so we need to fail the atomic transaction.
3186          */
3187         if (!ilk_validate_pipe_wm(dev, a))
3188                 return -EINVAL;
3189
3190         /*
3191          * If our intermediate WM are identical to the final WM, then we can
3192          * omit the post-vblank programming; only update if it's different.
3193          */
3194         if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3195                 newstate->wm.need_postvbl_update = true;
3196
3197         return 0;
3198 }
3199
3200 /*
3201  * Merge the watermarks from all active pipes for a specific level.
3202  */
3203 static void ilk_merge_wm_level(struct drm_device *dev,
3204                                int level,
3205                                struct intel_wm_level *ret_wm)
3206 {
3207         const struct intel_crtc *intel_crtc;
3208
3209         ret_wm->enable = true;
3210
3211         for_each_intel_crtc(dev, intel_crtc) {
3212                 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3213                 const struct intel_wm_level *wm = &active->wm[level];
3214
3215                 if (!active->pipe_enabled)
3216                         continue;
3217
3218                 /*
3219                  * The watermark values may have been used in the past,
3220                  * so we must maintain them in the registers for some
3221                  * time even if the level is now disabled.
3222                  */
3223                 if (!wm->enable)
3224                         ret_wm->enable = false;
3225
3226                 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3227                 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3228                 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3229                 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3230         }
3231 }
3232
3233 /*
3234  * Merge all low power watermarks for all active pipes.
3235  */
3236 static void ilk_wm_merge(struct drm_device *dev,
3237                          const struct intel_wm_config *config,
3238                          const struct ilk_wm_maximums *max,
3239                          struct intel_pipe_wm *merged)
3240 {
3241         struct drm_i915_private *dev_priv = to_i915(dev);
3242         int level, max_level = ilk_wm_max_level(dev_priv);
3243         int last_enabled_level = max_level;
3244
3245         /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3246         if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3247             config->num_pipes_active > 1)
3248                 last_enabled_level = 0;
3249
3250         /* ILK: FBC WM must be disabled always */
3251         merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
3252
3253         /* merge each WM1+ level */
3254         for (level = 1; level <= max_level; level++) {
3255                 struct intel_wm_level *wm = &merged->wm[level];
3256
3257                 ilk_merge_wm_level(dev, level, wm);
3258
3259                 if (level > last_enabled_level)
3260                         wm->enable = false;
3261                 else if (!ilk_validate_wm_level(level, max, wm))
3262                         /* make sure all following levels get disabled */
3263                         last_enabled_level = level - 1;
3264
3265                 /*
3266                  * The spec says it is preferred to disable
3267                  * FBC WMs instead of disabling a WM level.
3268                  */
3269                 if (wm->fbc_val > max->fbc) {
3270                         if (wm->enable)
3271                                 merged->fbc_wm_enabled = false;
3272                         wm->fbc_val = 0;
3273                 }
3274         }
3275
3276         /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3277         /*
3278          * FIXME this is racy. FBC might get enabled later.
3279          * What we should check here is whether FBC can be
3280          * enabled sometime later.
3281          */
3282         if (IS_GEN5(dev_priv) && !merged->fbc_wm_enabled &&
3283             intel_fbc_is_active(dev_priv)) {
3284                 for (level = 2; level <= max_level; level++) {
3285                         struct intel_wm_level *wm = &merged->wm[level];
3286
3287                         wm->enable = false;
3288                 }
3289         }
3290 }
3291
3292 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3293 {
3294         /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3295         return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3296 }
3297
3298 /* The value we need to program into the WM_LPx latency field */
3299 static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
3300 {
3301         struct drm_i915_private *dev_priv = to_i915(dev);
3302
3303         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3304                 return 2 * level;
3305         else
3306                 return dev_priv->wm.pri_latency[level];
3307 }
3308
3309 static void ilk_compute_wm_results(struct drm_device *dev,
3310                                    const struct intel_pipe_wm *merged,
3311                                    enum intel_ddb_partitioning partitioning,
3312                                    struct ilk_wm_values *results)
3313 {
3314         struct drm_i915_private *dev_priv = to_i915(dev);
3315         struct intel_crtc *intel_crtc;
3316         int level, wm_lp;
3317
3318         results->enable_fbc_wm = merged->fbc_wm_enabled;
3319         results->partitioning = partitioning;
3320
3321         /* LP1+ register values */
3322         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3323                 const struct intel_wm_level *r;
3324
3325                 level = ilk_wm_lp_to_level(wm_lp, merged);
3326
3327                 r = &merged->wm[level];
3328
3329                 /*
3330                  * Maintain the watermark values even if the level is
3331                  * disabled. Doing otherwise could cause underruns.
3332                  */
3333                 results->wm_lp[wm_lp - 1] =
3334                         (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) |
3335                         (r->pri_val << WM1_LP_SR_SHIFT) |
3336                         r->cur_val;
3337
3338                 if (r->enable)
3339                         results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3340
3341                 if (INTEL_GEN(dev_priv) >= 8)
3342                         results->wm_lp[wm_lp - 1] |=
3343                                 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3344                 else
3345                         results->wm_lp[wm_lp - 1] |=
3346                                 r->fbc_val << WM1_LP_FBC_SHIFT;
3347
3348                 /*
3349                  * Always set WM1S_LP_EN when spr_val != 0, even if the
3350                  * level is disabled. Doing otherwise could cause underruns.
3351                  */
3352                 if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
3353                         WARN_ON(wm_lp != 1);
3354                         results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3355                 } else
3356                         results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3357         }
3358
3359         /* LP0 register values */
3360         for_each_intel_crtc(dev, intel_crtc) {
3361                 enum pipe pipe = intel_crtc->pipe;
3362                 const struct intel_wm_level *r =
3363                         &intel_crtc->wm.active.ilk.wm[0];
3364
3365                 if (WARN_ON(!r->enable))
3366                         continue;
3367
3368                 results->wm_linetime[pipe] = intel_crtc->wm.active.ilk.linetime;
3369
3370                 results->wm_pipe[pipe] =
3371                         (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3372                         (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3373                         r->cur_val;
3374         }
3375 }
3376
3377 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3378  * case both are at the same level. Prefer r1 in case they're the same. */
3379 static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev,
3380                                                   struct intel_pipe_wm *r1,
3381                                                   struct intel_pipe_wm *r2)
3382 {
3383         int level, max_level = ilk_wm_max_level(to_i915(dev));
3384         int level1 = 0, level2 = 0;
3385
3386         for (level = 1; level <= max_level; level++) {
3387                 if (r1->wm[level].enable)
3388                         level1 = level;
3389                 if (r2->wm[level].enable)
3390                         level2 = level;
3391         }
3392
3393         if (level1 == level2) {
3394                 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3395                         return r2;
3396                 else
3397                         return r1;
3398         } else if (level1 > level2) {
3399                 return r1;
3400         } else {
3401                 return r2;
3402         }
3403 }
3404
3405 /* dirty bits used to track which watermarks need changes */
3406 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3407 #define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
3408 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3409 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3410 #define WM_DIRTY_FBC (1 << 24)
3411 #define WM_DIRTY_DDB (1 << 25)
3412
3413 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3414                                          const struct ilk_wm_values *old,
3415                                          const struct ilk_wm_values *new)
3416 {
3417         unsigned int dirty = 0;
3418         enum pipe pipe;
3419         int wm_lp;
3420
3421         for_each_pipe(dev_priv, pipe) {
3422                 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
3423                         dirty |= WM_DIRTY_LINETIME(pipe);
3424                         /* Must disable LP1+ watermarks too */
3425                         dirty |= WM_DIRTY_LP_ALL;
3426                 }
3427
3428                 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3429                         dirty |= WM_DIRTY_PIPE(pipe);
3430                         /* Must disable LP1+ watermarks too */
3431                         dirty |= WM_DIRTY_LP_ALL;
3432                 }
3433         }
3434
3435         if (old->enable_fbc_wm != new->enable_fbc_wm) {
3436                 dirty |= WM_DIRTY_FBC;
3437                 /* Must disable LP1+ watermarks too */
3438                 dirty |= WM_DIRTY_LP_ALL;
3439         }
3440
3441         if (old->partitioning != new->partitioning) {
3442                 dirty |= WM_DIRTY_DDB;
3443                 /* Must disable LP1+ watermarks too */
3444                 dirty |= WM_DIRTY_LP_ALL;
3445         }
3446
3447         /* LP1+ watermarks already deemed dirty, no need to continue */
3448         if (dirty & WM_DIRTY_LP_ALL)
3449                 return dirty;
3450
3451         /* Find the lowest numbered LP1+ watermark in need of an update... */
3452         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3453                 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3454                     old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3455                         break;
3456         }
3457
3458         /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3459         for (; wm_lp <= 3; wm_lp++)
3460                 dirty |= WM_DIRTY_LP(wm_lp);
3461
3462         return dirty;
3463 }
3464
3465 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3466                                unsigned int dirty)
3467 {
3468         struct ilk_wm_values *previous = &dev_priv->wm.hw;
3469         bool changed = false;
3470
3471         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3472                 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3473                 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
3474                 changed = true;
3475         }
3476         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3477                 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3478                 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
3479                 changed = true;
3480         }
3481         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3482                 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3483                 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
3484                 changed = true;
3485         }
3486
3487         /*
3488          * Don't touch WM1S_LP_EN here.
3489          * Doing so could cause underruns.
3490          */
3491
3492         return changed;
3493 }
3494
3495 /*
3496  * The spec says we shouldn't write when we don't need, because every write
3497  * causes WMs to be re-evaluated, expending some power.
3498  */
3499 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3500                                 struct ilk_wm_values *results)
3501 {
3502         struct ilk_wm_values *previous = &dev_priv->wm.hw;
3503         unsigned int dirty;
3504         uint32_t val;
3505
3506         dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3507         if (!dirty)
3508                 return;
3509
3510         _ilk_disable_lp_wm(dev_priv, dirty);
3511
3512         if (dirty & WM_DIRTY_PIPE(PIPE_A))
3513                 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
3514         if (dirty & WM_DIRTY_PIPE(PIPE_B))
3515                 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
3516         if (dirty & WM_DIRTY_PIPE(PIPE_C))
3517                 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
3518
3519         if (dirty & WM_DIRTY_LINETIME(PIPE_A))
3520                 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
3521         if (dirty & WM_DIRTY_LINETIME(PIPE_B))
3522                 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
3523         if (dirty & WM_DIRTY_LINETIME(PIPE_C))
3524                 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
3525
3526         if (dirty & WM_DIRTY_DDB) {
3527                 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3528                         val = I915_READ(WM_MISC);
3529                         if (results->partitioning == INTEL_DDB_PART_1_2)
3530                                 val &= ~WM_MISC_DATA_PARTITION_5_6;
3531                         else
3532                                 val |= WM_MISC_DATA_PARTITION_5_6;
3533                         I915_WRITE(WM_MISC, val);
3534                 } else {
3535                         val = I915_READ(DISP_ARB_CTL2);
3536                         if (results->partitioning == INTEL_DDB_PART_1_2)
3537                                 val &= ~DISP_DATA_PARTITION_5_6;
3538                         else
3539                                 val |= DISP_DATA_PARTITION_5_6;
3540                         I915_WRITE(DISP_ARB_CTL2, val);
3541                 }
3542         }
3543
3544         if (dirty & WM_DIRTY_FBC) {
3545                 val = I915_READ(DISP_ARB_CTL);
3546                 if (results->enable_fbc_wm)
3547                         val &= ~DISP_FBC_WM_DIS;
3548                 else
3549                         val |= DISP_FBC_WM_DIS;
3550                 I915_WRITE(DISP_ARB_CTL, val);
3551         }
3552
3553         if (dirty & WM_DIRTY_LP(1) &&
3554             previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3555                 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
3556
3557         if (INTEL_GEN(dev_priv) >= 7) {
3558                 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3559                         I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
3560                 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3561                         I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
3562         }
3563
3564         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3565                 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
3566         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3567                 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
3568         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3569                 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
3570
3571         dev_priv->wm.hw = *results;
3572 }
3573
3574 bool ilk_disable_lp_wm(struct drm_device *dev)
3575 {
3576         struct drm_i915_private *dev_priv = to_i915(dev);
3577
3578         return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3579 }
3580
3581 static u8 intel_enabled_dbuf_slices_num(struct drm_i915_private *dev_priv)
3582 {
3583         u8 enabled_slices;
3584
3585         /* Slice 1 will always be enabled */
3586         enabled_slices = 1;
3587
3588         /* Gen prior to GEN11 have only one DBuf slice */
3589         if (INTEL_GEN(dev_priv) < 11)
3590                 return enabled_slices;
3591
3592         if (I915_READ(DBUF_CTL_S2) & DBUF_POWER_STATE)
3593                 enabled_slices++;
3594
3595         return enabled_slices;
3596 }
3597
3598 /*
3599  * FIXME: We still don't have the proper code detect if we need to apply the WA,
3600  * so assume we'll always need it in order to avoid underruns.
3601  */
3602 static bool skl_needs_memory_bw_wa(struct intel_atomic_state *state)
3603 {
3604         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3605
3606         if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv))
3607                 return true;
3608
3609         return false;
3610 }
3611
3612 static bool
3613 intel_has_sagv(struct drm_i915_private *dev_priv)
3614 {
3615         if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv) ||
3616             IS_CANNONLAKE(dev_priv))
3617                 return true;
3618
3619         if (IS_SKYLAKE(dev_priv) &&
3620             dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED)
3621                 return true;
3622
3623         return false;
3624 }
3625
3626 /*
3627  * SAGV dynamically adjusts the system agent voltage and clock frequencies
3628  * depending on power and performance requirements. The display engine access
3629  * to system memory is blocked during the adjustment time. Because of the
3630  * blocking time, having this enabled can cause full system hangs and/or pipe
3631  * underruns if we don't meet all of the following requirements:
3632  *
3633  *  - <= 1 pipe enabled
3634  *  - All planes can enable watermarks for latencies >= SAGV engine block time
3635  *  - We're not using an interlaced display configuration
3636  */
3637 int
3638 intel_enable_sagv(struct drm_i915_private *dev_priv)
3639 {
3640         int ret;
3641
3642         if (!intel_has_sagv(dev_priv))
3643                 return 0;
3644
3645         if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3646                 return 0;
3647
3648         DRM_DEBUG_KMS("Enabling the SAGV\n");
3649         mutex_lock(&dev_priv->pcu_lock);
3650
3651         ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3652                                       GEN9_SAGV_ENABLE);
3653
3654         /* We don't need to wait for the SAGV when enabling */
3655         mutex_unlock(&dev_priv->pcu_lock);
3656
3657         /*
3658          * Some skl systems, pre-release machines in particular,
3659          * don't actually have an SAGV.
3660          */
3661         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3662                 DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
3663                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3664                 return 0;
3665         } else if (ret < 0) {
3666                 DRM_ERROR("Failed to enable the SAGV\n");
3667                 return ret;
3668         }
3669
3670         dev_priv->sagv_status = I915_SAGV_ENABLED;
3671         return 0;
3672 }
3673
3674 int
3675 intel_disable_sagv(struct drm_i915_private *dev_priv)
3676 {
3677         int ret;
3678
3679         if (!intel_has_sagv(dev_priv))
3680                 return 0;
3681
3682         if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3683                 return 0;
3684
3685         DRM_DEBUG_KMS("Disabling the SAGV\n");
3686         mutex_lock(&dev_priv->pcu_lock);
3687
3688         /* bspec says to keep retrying for at least 1 ms */
3689         ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3690                                 GEN9_SAGV_DISABLE,
3691                                 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3692                                 1);
3693         mutex_unlock(&dev_priv->pcu_lock);
3694
3695         /*
3696          * Some skl systems, pre-release machines in particular,
3697          * don't actually have an SAGV.
3698          */
3699         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3700                 DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
3701                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3702                 return 0;
3703         } else if (ret < 0) {
3704                 DRM_ERROR("Failed to disable the SAGV (%d)\n", ret);
3705                 return ret;
3706         }
3707
3708         dev_priv->sagv_status = I915_SAGV_DISABLED;
3709         return 0;
3710 }
3711
3712 bool intel_can_enable_sagv(struct drm_atomic_state *state)
3713 {
3714         struct drm_device *dev = state->dev;
3715         struct drm_i915_private *dev_priv = to_i915(dev);
3716         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
3717         struct intel_crtc *crtc;
3718         struct intel_plane *plane;
3719         struct intel_crtc_state *cstate;
3720         enum pipe pipe;
3721         int level, latency;
3722         int sagv_block_time_us;
3723
3724         if (!intel_has_sagv(dev_priv))
3725                 return false;
3726
3727         if (IS_GEN9(dev_priv))
3728                 sagv_block_time_us = 30;
3729         else if (IS_GEN10(dev_priv))
3730                 sagv_block_time_us = 20;
3731         else
3732                 sagv_block_time_us = 10;
3733
3734         /*
3735          * SKL+ workaround: bspec recommends we disable the SAGV when we have
3736          * more then one pipe enabled
3737          *
3738          * If there are no active CRTCs, no additional checks need be performed
3739          */
3740         if (hweight32(intel_state->active_crtcs) == 0)
3741                 return true;
3742         else if (hweight32(intel_state->active_crtcs) > 1)
3743                 return false;
3744
3745         /* Since we're now guaranteed to only have one active CRTC... */
3746         pipe = ffs(intel_state->active_crtcs) - 1;
3747         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
3748         cstate = to_intel_crtc_state(crtc->base.state);
3749
3750         if (crtc->base.state->adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
3751                 return false;
3752
3753         for_each_intel_plane_on_crtc(dev, crtc, plane) {
3754                 struct skl_plane_wm *wm =
3755                         &cstate->wm.skl.optimal.planes[plane->id];
3756
3757                 /* Skip this plane if it's not enabled */
3758                 if (!wm->wm[0].plane_en)
3759                         continue;
3760
3761                 /* Find the highest enabled wm level for this plane */
3762                 for (level = ilk_wm_max_level(dev_priv);
3763                      !wm->wm[level].plane_en; --level)
3764                      { }
3765
3766                 latency = dev_priv->wm.skl_latency[level];
3767
3768                 if (skl_needs_memory_bw_wa(intel_state) &&
3769                     plane->base.state->fb->modifier ==
3770                     I915_FORMAT_MOD_X_TILED)
3771                         latency += 15;
3772
3773                 /*
3774                  * If any of the planes on this pipe don't enable wm levels that
3775                  * incur memory latencies higher than sagv_block_time_us we
3776                  * can't enable the SAGV.
3777                  */
3778                 if (latency < sagv_block_time_us)
3779                         return false;
3780         }
3781
3782         return true;
3783 }
3784
3785 static u16 intel_get_ddb_size(struct drm_i915_private *dev_priv,
3786                               const struct intel_crtc_state *cstate,
3787                               const unsigned int total_data_rate,
3788                               const int num_active,
3789                               struct skl_ddb_allocation *ddb)
3790 {
3791         const struct drm_display_mode *adjusted_mode;
3792         u64 total_data_bw;
3793         u16 ddb_size = INTEL_INFO(dev_priv)->ddb_size;
3794
3795         WARN_ON(ddb_size == 0);
3796
3797         if (INTEL_GEN(dev_priv) < 11)
3798                 return ddb_size - 4; /* 4 blocks for bypass path allocation */
3799
3800         adjusted_mode = &cstate->base.adjusted_mode;
3801         total_data_bw = (u64)total_data_rate * drm_mode_vrefresh(adjusted_mode);
3802
3803         /*
3804          * 12GB/s is maximum BW supported by single DBuf slice.
3805          */
3806         if (total_data_bw >= GBps(12) || num_active > 1) {
3807                 ddb->enabled_slices = 2;
3808         } else {
3809                 ddb->enabled_slices = 1;
3810                 ddb_size /= 2;
3811         }
3812
3813         return ddb_size;
3814 }
3815
3816 static void
3817 skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
3818                                    const struct intel_crtc_state *cstate,
3819                                    const unsigned int total_data_rate,
3820                                    struct skl_ddb_allocation *ddb,
3821                                    struct skl_ddb_entry *alloc, /* out */
3822                                    int *num_active /* out */)
3823 {
3824         struct drm_atomic_state *state = cstate->base.state;
3825         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
3826         struct drm_i915_private *dev_priv = to_i915(dev);
3827         struct drm_crtc *for_crtc = cstate->base.crtc;
3828         const struct drm_crtc_state *crtc_state;
3829         const struct drm_crtc *crtc;
3830         u32 pipe_width = 0, total_width = 0, width_before_pipe = 0;
3831         enum pipe for_pipe = to_intel_crtc(for_crtc)->pipe;
3832         u16 ddb_size;
3833         u32 i;
3834
3835         if (WARN_ON(!state) || !cstate->base.active) {
3836                 alloc->start = 0;
3837                 alloc->end = 0;
3838                 *num_active = hweight32(dev_priv->active_crtcs);
3839                 return;
3840         }
3841
3842         if (intel_state->active_pipe_changes)
3843                 *num_active = hweight32(intel_state->active_crtcs);
3844         else
3845                 *num_active = hweight32(dev_priv->active_crtcs);
3846
3847         ddb_size = intel_get_ddb_size(dev_priv, cstate, total_data_rate,
3848                                       *num_active, ddb);
3849
3850         /*
3851          * If the state doesn't change the active CRTC's or there is no
3852          * modeset request, then there's no need to recalculate;
3853          * the existing pipe allocation limits should remain unchanged.
3854          * Note that we're safe from racing commits since any racing commit
3855          * that changes the active CRTC list or do modeset would need to
3856          * grab _all_ crtc locks, including the one we currently hold.
3857          */
3858         if (!intel_state->active_pipe_changes && !intel_state->modeset) {
3859                 /*
3860                  * alloc may be cleared by clear_intel_crtc_state,
3861                  * copy from old state to be sure
3862                  */
3863                 *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb;
3864                 return;
3865         }
3866
3867         /*
3868          * Watermark/ddb requirement highly depends upon width of the
3869          * framebuffer, So instead of allocating DDB equally among pipes
3870          * distribute DDB based on resolution/width of the display.
3871          */
3872         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
3873                 const struct drm_display_mode *adjusted_mode;
3874                 int hdisplay, vdisplay;
3875                 enum pipe pipe;
3876
3877                 if (!crtc_state->enable)
3878                         continue;
3879
3880                 pipe = to_intel_crtc(crtc)->pipe;
3881                 adjusted_mode = &crtc_state->adjusted_mode;
3882                 drm_mode_get_hv_timing(adjusted_mode, &hdisplay, &vdisplay);
3883                 total_width += hdisplay;
3884
3885                 if (pipe < for_pipe)
3886                         width_before_pipe += hdisplay;
3887                 else if (pipe == for_pipe)
3888                         pipe_width = hdisplay;
3889         }
3890
3891         alloc->start = ddb_size * width_before_pipe / total_width;
3892         alloc->end = ddb_size * (width_before_pipe + pipe_width) / total_width;
3893 }
3894
3895 static unsigned int skl_cursor_allocation(int num_active)
3896 {
3897         if (num_active == 1)
3898                 return 32;
3899
3900         return 8;
3901 }
3902
3903 static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
3904                                        struct skl_ddb_entry *entry, u32 reg)
3905 {
3906         u16 mask;
3907
3908         if (INTEL_GEN(dev_priv) >= 11)
3909                 mask = ICL_DDB_ENTRY_MASK;
3910         else
3911                 mask = SKL_DDB_ENTRY_MASK;
3912         entry->start = reg & mask;
3913         entry->end = (reg >> DDB_ENTRY_END_SHIFT) & mask;
3914
3915         if (entry->end)
3916                 entry->end += 1;
3917 }
3918
3919 static void
3920 skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
3921                            const enum pipe pipe,
3922                            const enum plane_id plane_id,
3923                            struct skl_ddb_allocation *ddb /* out */)
3924 {
3925         u32 val, val2 = 0;
3926         int fourcc, pixel_format;
3927
3928         /* Cursor doesn't support NV12/planar, so no extra calculation needed */
3929         if (plane_id == PLANE_CURSOR) {
3930                 val = I915_READ(CUR_BUF_CFG(pipe));
3931                 skl_ddb_entry_init_from_hw(dev_priv,
3932                                            &ddb->plane[pipe][plane_id], val);
3933                 return;
3934         }
3935
3936         val = I915_READ(PLANE_CTL(pipe, plane_id));
3937
3938         /* No DDB allocated for disabled planes */
3939         if (!(val & PLANE_CTL_ENABLE))
3940                 return;
3941
3942         pixel_format = val & PLANE_CTL_FORMAT_MASK;
3943         fourcc = skl_format_to_fourcc(pixel_format,
3944                                       val & PLANE_CTL_ORDER_RGBX,
3945                                       val & PLANE_CTL_ALPHA_MASK);
3946
3947         val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
3948         /*
3949          * FIXME: add proper NV12 support for ICL. Avoid reading unclaimed
3950          * registers for now.
3951          */
3952         if (INTEL_GEN(dev_priv) < 11)
3953                 val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id));
3954
3955         if (fourcc == DRM_FORMAT_NV12) {
3956                 skl_ddb_entry_init_from_hw(dev_priv,
3957                                            &ddb->plane[pipe][plane_id], val2);
3958                 skl_ddb_entry_init_from_hw(dev_priv,
3959                                            &ddb->uv_plane[pipe][plane_id], val);
3960         } else {
3961                 skl_ddb_entry_init_from_hw(dev_priv,
3962                                            &ddb->plane[pipe][plane_id], val);
3963         }
3964 }
3965
3966 void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
3967                           struct skl_ddb_allocation *ddb /* out */)
3968 {
3969         struct intel_crtc *crtc;
3970
3971         memset(ddb, 0, sizeof(*ddb));
3972
3973         ddb->enabled_slices = intel_enabled_dbuf_slices_num(dev_priv);
3974
3975         for_each_intel_crtc(&dev_priv->drm, crtc) {
3976                 enum intel_display_power_domain power_domain;
3977                 enum plane_id plane_id;
3978                 enum pipe pipe = crtc->pipe;
3979
3980                 power_domain = POWER_DOMAIN_PIPE(pipe);
3981                 if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
3982                         continue;
3983
3984                 for_each_plane_id_on_crtc(crtc, plane_id)
3985                         skl_ddb_get_hw_plane_state(dev_priv, pipe,
3986                                                    plane_id, ddb);
3987
3988                 intel_display_power_put(dev_priv, power_domain);
3989         }
3990 }
3991
3992 /*
3993  * Determines the downscale amount of a plane for the purposes of watermark calculations.
3994  * The bspec defines downscale amount as:
3995  *
3996  * """
3997  * Horizontal down scale amount = maximum[1, Horizontal source size /
3998  *                                           Horizontal destination size]
3999  * Vertical down scale amount = maximum[1, Vertical source size /
4000  *                                         Vertical destination size]
4001  * Total down scale amount = Horizontal down scale amount *
4002  *                           Vertical down scale amount
4003  * """
4004  *
4005  * Return value is provided in 16.16 fixed point form to retain fractional part.
4006  * Caller should take care of dividing & rounding off the value.
4007  */
4008 static uint_fixed_16_16_t
4009 skl_plane_downscale_amount(const struct intel_crtc_state *cstate,
4010                            const struct intel_plane_state *pstate)
4011 {
4012         struct intel_plane *plane = to_intel_plane(pstate->base.plane);
4013         uint32_t src_w, src_h, dst_w, dst_h;
4014         uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4015         uint_fixed_16_16_t downscale_h, downscale_w;
4016
4017         if (WARN_ON(!intel_wm_plane_visible(cstate, pstate)))
4018                 return u32_to_fixed16(0);
4019
4020         /* n.b., src is 16.16 fixed point, dst is whole integer */
4021         if (plane->id == PLANE_CURSOR) {
4022                 /*
4023                  * Cursors only support 0/180 degree rotation,
4024                  * hence no need to account for rotation here.
4025                  */
4026                 src_w = pstate->base.src_w >> 16;
4027                 src_h = pstate->base.src_h >> 16;
4028                 dst_w = pstate->base.crtc_w;
4029                 dst_h = pstate->base.crtc_h;
4030         } else {
4031                 /*
4032                  * Src coordinates are already rotated by 270 degrees for
4033                  * the 90/270 degree plane rotation cases (to match the
4034                  * GTT mapping), hence no need to account for rotation here.
4035                  */
4036                 src_w = drm_rect_width(&pstate->base.src) >> 16;
4037                 src_h = drm_rect_height(&pstate->base.src) >> 16;
4038                 dst_w = drm_rect_width(&pstate->base.dst);
4039                 dst_h = drm_rect_height(&pstate->base.dst);
4040         }
4041
4042         fp_w_ratio = div_fixed16(src_w, dst_w);
4043         fp_h_ratio = div_fixed16(src_h, dst_h);
4044         downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4045         downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4046
4047         return mul_fixed16(downscale_w, downscale_h);
4048 }
4049
4050 static uint_fixed_16_16_t
4051 skl_pipe_downscale_amount(const struct intel_crtc_state *crtc_state)
4052 {
4053         uint_fixed_16_16_t pipe_downscale = u32_to_fixed16(1);
4054
4055         if (!crtc_state->base.enable)
4056                 return pipe_downscale;
4057
4058         if (crtc_state->pch_pfit.enabled) {
4059                 uint32_t src_w, src_h, dst_w, dst_h;
4060                 uint32_t pfit_size = crtc_state->pch_pfit.size;
4061                 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4062                 uint_fixed_16_16_t downscale_h, downscale_w;
4063
4064                 src_w = crtc_state->pipe_src_w;
4065                 src_h = crtc_state->pipe_src_h;
4066                 dst_w = pfit_size >> 16;
4067                 dst_h = pfit_size & 0xffff;
4068
4069                 if (!dst_w || !dst_h)
4070                         return pipe_downscale;
4071
4072                 fp_w_ratio = div_fixed16(src_w, dst_w);
4073                 fp_h_ratio = div_fixed16(src_h, dst_h);
4074                 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4075                 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
4076
4077                 pipe_downscale = mul_fixed16(downscale_w, downscale_h);
4078         }
4079
4080         return pipe_downscale;
4081 }
4082
4083 int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
4084                                   struct intel_crtc_state *cstate)
4085 {
4086         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
4087         struct drm_crtc_state *crtc_state = &cstate->base;
4088         struct drm_atomic_state *state = crtc_state->state;
4089         struct drm_plane *plane;
4090         const struct drm_plane_state *pstate;
4091         struct intel_plane_state *intel_pstate;
4092         int crtc_clock, dotclk;
4093         uint32_t pipe_max_pixel_rate;
4094         uint_fixed_16_16_t pipe_downscale;
4095         uint_fixed_16_16_t max_downscale = u32_to_fixed16(1);
4096
4097         if (!cstate->base.enable)
4098                 return 0;
4099
4100         drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
4101                 uint_fixed_16_16_t plane_downscale;
4102                 uint_fixed_16_16_t fp_9_div_8 = div_fixed16(9, 8);
4103                 int bpp;
4104
4105                 if (!intel_wm_plane_visible(cstate,
4106                                             to_intel_plane_state(pstate)))
4107                         continue;
4108
4109                 if (WARN_ON(!pstate->fb))
4110                         return -EINVAL;
4111
4112                 intel_pstate = to_intel_plane_state(pstate);
4113                 plane_downscale = skl_plane_downscale_amount(cstate,
4114                                                              intel_pstate);
4115                 bpp = pstate->fb->format->cpp[0] * 8;
4116                 if (bpp == 64)
4117                         plane_downscale = mul_fixed16(plane_downscale,
4118                                                       fp_9_div_8);
4119
4120                 max_downscale = max_fixed16(plane_downscale, max_downscale);
4121         }
4122         pipe_downscale = skl_pipe_downscale_amount(cstate);
4123
4124         pipe_downscale = mul_fixed16(pipe_downscale, max_downscale);
4125
4126         crtc_clock = crtc_state->adjusted_mode.crtc_clock;
4127         dotclk = to_intel_atomic_state(state)->cdclk.logical.cdclk;
4128
4129         if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10)
4130                 dotclk *= 2;
4131
4132         pipe_max_pixel_rate = div_round_up_u32_fixed16(dotclk, pipe_downscale);
4133
4134         if (pipe_max_pixel_rate < crtc_clock) {
4135                 DRM_DEBUG_KMS("Max supported pixel clock with scaling exceeded\n");
4136                 return -EINVAL;
4137         }
4138
4139         return 0;
4140 }
4141
4142 static unsigned int
4143 skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
4144                              const struct drm_plane_state *pstate,
4145                              const int plane)
4146 {
4147         struct intel_plane *intel_plane = to_intel_plane(pstate->plane);
4148         struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
4149         uint32_t data_rate;
4150         uint32_t width = 0, height = 0;
4151         struct drm_framebuffer *fb;
4152         u32 format;
4153         uint_fixed_16_16_t down_scale_amount;
4154
4155         if (!intel_pstate->base.visible)
4156                 return 0;
4157
4158         fb = pstate->fb;
4159         format = fb->format->format;
4160
4161         if (intel_plane->id == PLANE_CURSOR)
4162                 return 0;
4163         if (plane == 1 && format != DRM_FORMAT_NV12)
4164                 return 0;
4165
4166         /*
4167          * Src coordinates are already rotated by 270 degrees for
4168          * the 90/270 degree plane rotation cases (to match the
4169          * GTT mapping), hence no need to account for rotation here.
4170          */
4171         width = drm_rect_width(&intel_pstate->base.src) >> 16;
4172         height = drm_rect_height(&intel_pstate->base.src) >> 16;
4173
4174         /* UV plane does 1/2 pixel sub-sampling */
4175         if (plane == 1 && format == DRM_FORMAT_NV12) {
4176                 width /= 2;
4177                 height /= 2;
4178         }
4179
4180         data_rate = width * height * fb->format->cpp[plane];
4181
4182         down_scale_amount = skl_plane_downscale_amount(cstate, intel_pstate);
4183
4184         return mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4185 }
4186
4187 /*
4188  * We don't overflow 32 bits. Worst case is 3 planes enabled, each fetching
4189  * a 8192x4096@32bpp framebuffer:
4190  *   3 * 4096 * 8192  * 4 < 2^32
4191  */
4192 static unsigned int
4193 skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate,
4194                                  unsigned int *plane_data_rate,
4195                                  unsigned int *uv_plane_data_rate)
4196 {
4197         struct drm_crtc_state *cstate = &intel_cstate->base;
4198         struct drm_atomic_state *state = cstate->state;
4199         struct drm_plane *plane;
4200         const struct drm_plane_state *pstate;
4201         unsigned int total_data_rate = 0;
4202
4203         if (WARN_ON(!state))
4204                 return 0;
4205
4206         /* Calculate and cache data rate for each plane */
4207         drm_atomic_crtc_state_for_each_plane_state(plane, pstate, cstate) {
4208                 enum plane_id plane_id = to_intel_plane(plane)->id;
4209                 unsigned int rate;
4210
4211                 /* packed/y */
4212                 rate = skl_plane_relative_data_rate(intel_cstate,
4213                                                     pstate, 0);
4214                 plane_data_rate[plane_id] = rate;
4215
4216                 total_data_rate += rate;
4217
4218                 /* uv-plane */
4219                 rate = skl_plane_relative_data_rate(intel_cstate,
4220                                                     pstate, 1);
4221                 uv_plane_data_rate[plane_id] = rate;
4222
4223                 total_data_rate += rate;
4224         }
4225
4226         return total_data_rate;
4227 }
4228
4229 static uint16_t
4230 skl_ddb_min_alloc(const struct drm_plane_state *pstate, const int plane)
4231 {
4232         struct drm_framebuffer *fb = pstate->fb;
4233         struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
4234         uint32_t src_w, src_h;
4235         uint32_t min_scanlines = 8;
4236         uint8_t plane_bpp;
4237
4238         if (WARN_ON(!fb))
4239                 return 0;
4240
4241         /* For packed formats, and uv-plane, return 0 */
4242         if (plane == 1 && fb->format->format != DRM_FORMAT_NV12)
4243                 return 0;
4244
4245         /* For Non Y-tile return 8-blocks */
4246         if (fb->modifier != I915_FORMAT_MOD_Y_TILED &&
4247             fb->modifier != I915_FORMAT_MOD_Yf_TILED &&
4248             fb->modifier != I915_FORMAT_MOD_Y_TILED_CCS &&
4249             fb->modifier != I915_FORMAT_MOD_Yf_TILED_CCS)
4250                 return 8;
4251
4252         /*
4253          * Src coordinates are already rotated by 270 degrees for
4254          * the 90/270 degree plane rotation cases (to match the
4255          * GTT mapping), hence no need to account for rotation here.
4256          */
4257         src_w = drm_rect_width(&intel_pstate->base.src) >> 16;
4258         src_h = drm_rect_height(&intel_pstate->base.src) >> 16;
4259
4260         /* Halve UV plane width and height for NV12 */
4261         if (plane == 1) {
4262                 src_w /= 2;
4263                 src_h /= 2;
4264         }
4265
4266         plane_bpp = fb->format->cpp[plane];
4267
4268         if (drm_rotation_90_or_270(pstate->rotation)) {
4269                 switch (plane_bpp) {
4270                 case 1:
4271                         min_scanlines = 32;
4272                         break;
4273                 case 2:
4274                         min_scanlines = 16;
4275                         break;
4276                 case 4:
4277                         min_scanlines = 8;
4278                         break;
4279                 case 8:
4280                         min_scanlines = 4;
4281                         break;
4282                 default:
4283                         WARN(1, "Unsupported pixel depth %u for rotation",
4284                              plane_bpp);
4285                         min_scanlines = 32;
4286                 }
4287         }
4288
4289         return DIV_ROUND_UP((4 * src_w * plane_bpp), 512) * min_scanlines/4 + 3;
4290 }
4291
4292 static void
4293 skl_ddb_calc_min(const struct intel_crtc_state *cstate, int num_active,
4294                  uint16_t *minimum, uint16_t *uv_minimum)
4295 {
4296         const struct drm_plane_state *pstate;
4297         struct drm_plane *plane;
4298
4299         drm_atomic_crtc_state_for_each_plane_state(plane, pstate, &cstate->base) {
4300                 enum plane_id plane_id = to_intel_plane(plane)->id;
4301
4302                 if (plane_id == PLANE_CURSOR)
4303                         continue;
4304
4305                 if (!pstate->visible)
4306                         continue;
4307
4308                 minimum[plane_id] = skl_ddb_min_alloc(pstate, 0);
4309                 uv_minimum[plane_id] = skl_ddb_min_alloc(pstate, 1);
4310         }
4311
4312         minimum[PLANE_CURSOR] = skl_cursor_allocation(num_active);
4313 }
4314
4315 static int
4316 skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
4317                       struct skl_ddb_allocation *ddb /* out */)
4318 {
4319         struct drm_atomic_state *state = cstate->base.state;
4320         struct drm_crtc *crtc = cstate->base.crtc;
4321         struct drm_device *dev = crtc->dev;
4322         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4323         enum pipe pipe = intel_crtc->pipe;
4324         struct skl_ddb_entry *alloc = &cstate->wm.skl.ddb;
4325         uint16_t alloc_size, start;
4326         uint16_t minimum[I915_MAX_PLANES] = {};
4327         uint16_t uv_minimum[I915_MAX_PLANES] = {};
4328         unsigned int total_data_rate;
4329         enum plane_id plane_id;
4330         int num_active;
4331         unsigned int plane_data_rate[I915_MAX_PLANES] = {};
4332         unsigned int uv_plane_data_rate[I915_MAX_PLANES] = {};
4333         uint16_t total_min_blocks = 0;
4334
4335         /* Clear the partitioning for disabled planes. */
4336         memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
4337         memset(ddb->uv_plane[pipe], 0, sizeof(ddb->uv_plane[pipe]));
4338
4339         if (WARN_ON(!state))
4340                 return 0;
4341
4342         if (!cstate->base.active) {
4343                 alloc->start = alloc->end = 0;
4344                 return 0;
4345         }
4346
4347         total_data_rate = skl_get_total_relative_data_rate(cstate,
4348                                                            plane_data_rate,
4349                                                            uv_plane_data_rate);
4350         skl_ddb_get_pipe_allocation_limits(dev, cstate, total_data_rate, ddb,
4351                                            alloc, &num_active);
4352         alloc_size = skl_ddb_entry_size(alloc);
4353         if (alloc_size == 0)
4354                 return 0;
4355
4356         skl_ddb_calc_min(cstate, num_active, minimum, uv_minimum);
4357
4358         /*
4359          * 1. Allocate the mininum required blocks for each active plane
4360          * and allocate the cursor, it doesn't require extra allocation
4361          * proportional to the data rate.
4362          */
4363
4364         for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4365                 total_min_blocks += minimum[plane_id];
4366                 total_min_blocks += uv_minimum[plane_id];
4367         }
4368
4369         if (total_min_blocks > alloc_size) {
4370                 DRM_DEBUG_KMS("Requested display configuration exceeds system DDB limitations");
4371                 DRM_DEBUG_KMS("minimum required %d/%d\n", total_min_blocks,
4372                                                         alloc_size);
4373                 return -EINVAL;
4374         }
4375
4376         alloc_size -= total_min_blocks;
4377         ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - minimum[PLANE_CURSOR];
4378         ddb->plane[pipe][PLANE_CURSOR].end = alloc->end;
4379
4380         /*
4381          * 2. Distribute the remaining space in proportion to the amount of
4382          * data each plane needs to fetch from memory.
4383          *
4384          * FIXME: we may not allocate every single block here.
4385          */
4386         if (total_data_rate == 0)
4387                 return 0;
4388
4389         start = alloc->start;
4390         for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4391                 unsigned int data_rate, uv_data_rate;
4392                 uint16_t plane_blocks, uv_plane_blocks;
4393
4394                 if (plane_id == PLANE_CURSOR)
4395                         continue;
4396
4397                 data_rate = plane_data_rate[plane_id];
4398
4399                 /*
4400                  * allocation for (packed formats) or (uv-plane part of planar format):
4401                  * promote the expression to 64 bits to avoid overflowing, the
4402                  * result is < available as data_rate / total_data_rate < 1
4403                  */
4404                 plane_blocks = minimum[plane_id];
4405                 plane_blocks += div_u64((uint64_t)alloc_size * data_rate,
4406                                         total_data_rate);
4407
4408                 /* Leave disabled planes at (0,0) */
4409                 if (data_rate) {
4410                         ddb->plane[pipe][plane_id].start = start;
4411                         ddb->plane[pipe][plane_id].end = start + plane_blocks;
4412                 }
4413
4414                 start += plane_blocks;
4415
4416                 /* Allocate DDB for UV plane for planar format/NV12 */
4417                 uv_data_rate = uv_plane_data_rate[plane_id];
4418
4419                 uv_plane_blocks = uv_minimum[plane_id];
4420                 uv_plane_blocks += div_u64((uint64_t)alloc_size * uv_data_rate,
4421                                            total_data_rate);
4422
4423                 if (uv_data_rate) {
4424                         ddb->uv_plane[pipe][plane_id].start = start;
4425                         ddb->uv_plane[pipe][plane_id].end =
4426                                 start + uv_plane_blocks;
4427                 }
4428
4429                 start += uv_plane_blocks;
4430         }
4431
4432         return 0;
4433 }
4434
4435 /*
4436  * The max latency should be 257 (max the punit can code is 255 and we add 2us
4437  * for the read latency) and cpp should always be <= 8, so that
4438  * should allow pixel_rate up to ~2 GHz which seems sufficient since max
4439  * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
4440 */
4441 static uint_fixed_16_16_t
4442 skl_wm_method1(const struct drm_i915_private *dev_priv, uint32_t pixel_rate,
4443                uint8_t cpp, uint32_t latency, uint32_t dbuf_block_size)
4444 {
4445         uint32_t wm_intermediate_val;
4446         uint_fixed_16_16_t ret;
4447
4448         if (latency == 0)
4449                 return FP_16_16_MAX;
4450
4451         wm_intermediate_val = latency * pixel_rate * cpp;
4452         ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
4453
4454         if (INTEL_GEN(dev_priv) >= 10)
4455                 ret = add_fixed16_u32(ret, 1);
4456
4457         return ret;
4458 }
4459
4460 static uint_fixed_16_16_t skl_wm_method2(uint32_t pixel_rate,
4461                         uint32_t pipe_htotal,
4462                         uint32_t latency,
4463                         uint_fixed_16_16_t plane_blocks_per_line)
4464 {
4465         uint32_t wm_intermediate_val;
4466         uint_fixed_16_16_t ret;
4467
4468         if (latency == 0)
4469                 return FP_16_16_MAX;
4470
4471         wm_intermediate_val = latency * pixel_rate;
4472         wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
4473                                            pipe_htotal * 1000);
4474         ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
4475         return ret;
4476 }
4477
4478 static uint_fixed_16_16_t
4479 intel_get_linetime_us(struct intel_crtc_state *cstate)
4480 {
4481         uint32_t pixel_rate;
4482         uint32_t crtc_htotal;
4483         uint_fixed_16_16_t linetime_us;
4484
4485         if (!cstate->base.active)
4486                 return u32_to_fixed16(0);
4487
4488         pixel_rate = cstate->pixel_rate;
4489
4490         if (WARN_ON(pixel_rate == 0))
4491                 return u32_to_fixed16(0);
4492
4493         crtc_htotal = cstate->base.adjusted_mode.crtc_htotal;
4494         linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
4495
4496         return linetime_us;
4497 }
4498
4499 static uint32_t
4500 skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *cstate,
4501                               const struct intel_plane_state *pstate)
4502 {
4503         uint64_t adjusted_pixel_rate;
4504         uint_fixed_16_16_t downscale_amount;
4505
4506         /* Shouldn't reach here on disabled planes... */
4507         if (WARN_ON(!intel_wm_plane_visible(cstate, pstate)))
4508                 return 0;
4509
4510         /*
4511          * Adjusted plane pixel rate is just the pipe's adjusted pixel rate
4512          * with additional adjustments for plane-specific scaling.
4513          */
4514         adjusted_pixel_rate = cstate->pixel_rate;
4515         downscale_amount = skl_plane_downscale_amount(cstate, pstate);
4516
4517         return mul_round_up_u32_fixed16(adjusted_pixel_rate,
4518                                             downscale_amount);
4519 }
4520
4521 static int
4522 skl_compute_plane_wm_params(const struct drm_i915_private *dev_priv,
4523                             struct intel_crtc_state *cstate,
4524                             const struct intel_plane_state *intel_pstate,
4525                             struct skl_wm_params *wp, int plane_id)
4526 {
4527         struct intel_plane *plane = to_intel_plane(intel_pstate->base.plane);
4528         const struct drm_plane_state *pstate = &intel_pstate->base;
4529         const struct drm_framebuffer *fb = pstate->fb;
4530         uint32_t interm_pbpl;
4531         struct intel_atomic_state *state =
4532                 to_intel_atomic_state(cstate->base.state);
4533         bool apply_memory_bw_wa = skl_needs_memory_bw_wa(state);
4534
4535         if (!intel_wm_plane_visible(cstate, intel_pstate))
4536                 return 0;
4537
4538         /* only NV12 format has two planes */
4539         if (plane_id == 1 && fb->format->format != DRM_FORMAT_NV12) {
4540                 DRM_DEBUG_KMS("Non NV12 format have single plane\n");
4541                 return -EINVAL;
4542         }
4543
4544         wp->y_tiled = fb->modifier == I915_FORMAT_MOD_Y_TILED ||
4545                       fb->modifier == I915_FORMAT_MOD_Yf_TILED ||
4546                       fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
4547                       fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
4548         wp->x_tiled = fb->modifier == I915_FORMAT_MOD_X_TILED;
4549         wp->rc_surface = fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
4550                          fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
4551         wp->is_planar = fb->format->format == DRM_FORMAT_NV12;
4552
4553         if (plane->id == PLANE_CURSOR) {
4554                 wp->width = intel_pstate->base.crtc_w;
4555         } else {
4556                 /*
4557                  * Src coordinates are already rotated by 270 degrees for
4558                  * the 90/270 degree plane rotation cases (to match the
4559                  * GTT mapping), hence no need to account for rotation here.
4560                  */
4561                 wp->width = drm_rect_width(&intel_pstate->base.src) >> 16;
4562         }
4563
4564         if (plane_id == 1 && wp->is_planar)
4565                 wp->width /= 2;
4566
4567         wp->cpp = fb->format->cpp[plane_id];
4568         wp->plane_pixel_rate = skl_adjusted_plane_pixel_rate(cstate,
4569                                                              intel_pstate);
4570
4571         if (INTEL_GEN(dev_priv) >= 11 &&
4572             fb->modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 8)
4573                 wp->dbuf_block_size = 256;
4574         else
4575                 wp->dbuf_block_size = 512;
4576
4577         if (drm_rotation_90_or_270(pstate->rotation)) {
4578
4579                 switch (wp->cpp) {
4580                 case 1:
4581                         wp->y_min_scanlines = 16;
4582                         break;
4583                 case 2:
4584                         wp->y_min_scanlines = 8;
4585                         break;
4586                 case 4:
4587                         wp->y_min_scanlines = 4;
4588                         break;
4589                 default:
4590                         MISSING_CASE(wp->cpp);
4591                         return -EINVAL;
4592                 }
4593         } else {
4594                 wp->y_min_scanlines = 4;
4595         }
4596
4597         if (apply_memory_bw_wa)
4598                 wp->y_min_scanlines *= 2;
4599
4600         wp->plane_bytes_per_line = wp->width * wp->cpp;
4601         if (wp->y_tiled) {
4602                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
4603                                            wp->y_min_scanlines,
4604                                            wp->dbuf_block_size);
4605
4606                 if (INTEL_GEN(dev_priv) >= 10)
4607                         interm_pbpl++;
4608
4609                 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
4610                                                         wp->y_min_scanlines);
4611         } else if (wp->x_tiled && IS_GEN9(dev_priv)) {
4612                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
4613                                            wp->dbuf_block_size);
4614                 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
4615         } else {
4616                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
4617                                            wp->dbuf_block_size) + 1;
4618                 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
4619         }
4620
4621         wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
4622                                              wp->plane_blocks_per_line);
4623         wp->linetime_us = fixed16_to_u32_round_up(
4624                                         intel_get_linetime_us(cstate));
4625
4626         return 0;
4627 }
4628
4629 static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
4630                                 struct intel_crtc_state *cstate,
4631                                 const struct intel_plane_state *intel_pstate,
4632                                 uint16_t ddb_allocation,
4633                                 int level,
4634                                 const struct skl_wm_params *wp,
4635                                 const struct skl_wm_level *result_prev,
4636                                 struct skl_wm_level *result /* out */)
4637 {
4638         const struct drm_plane_state *pstate = &intel_pstate->base;
4639         uint32_t latency = dev_priv->wm.skl_latency[level];
4640         uint_fixed_16_16_t method1, method2;
4641         uint_fixed_16_16_t selected_result;
4642         uint32_t res_blocks, res_lines;
4643         struct intel_atomic_state *state =
4644                 to_intel_atomic_state(cstate->base.state);
4645         bool apply_memory_bw_wa = skl_needs_memory_bw_wa(state);
4646         uint32_t min_disp_buf_needed;
4647
4648         if (latency == 0 ||
4649             !intel_wm_plane_visible(cstate, intel_pstate)) {
4650                 result->plane_en = false;
4651                 return 0;
4652         }
4653
4654         /* Display WA #1141: kbl,cfl */
4655         if ((IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv) ||
4656             IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_B0)) &&
4657             dev_priv->ipc_enabled)
4658                 latency += 4;
4659
4660         if (apply_memory_bw_wa && wp->x_tiled)
4661                 latency += 15;
4662
4663         method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
4664                                  wp->cpp, latency, wp->dbuf_block_size);
4665         method2 = skl_wm_method2(wp->plane_pixel_rate,
4666                                  cstate->base.adjusted_mode.crtc_htotal,
4667                                  latency,
4668                                  wp->plane_blocks_per_line);
4669
4670         if (wp->y_tiled) {
4671                 selected_result = max_fixed16(method2, wp->y_tile_minimum);
4672         } else {
4673                 if ((wp->cpp * cstate->base.adjusted_mode.crtc_htotal /
4674                      wp->dbuf_block_size < 1) &&
4675                      (wp->plane_bytes_per_line / wp->dbuf_block_size < 1))
4676                         selected_result = method2;
4677                 else if (ddb_allocation >=
4678                          fixed16_to_u32_round_up(wp->plane_blocks_per_line))
4679                         selected_result = min_fixed16(method1, method2);
4680                 else if (latency >= wp->linetime_us)
4681                         selected_result = min_fixed16(method1, method2);
4682                 else
4683                         selected_result = method1;
4684         }
4685
4686         res_blocks = fixed16_to_u32_round_up(selected_result) + 1;
4687         res_lines = div_round_up_fixed16(selected_result,
4688                                          wp->plane_blocks_per_line);
4689
4690         /* Display WA #1125: skl,bxt,kbl,glk */
4691         if (level == 0 && wp->rc_surface)
4692                 res_blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
4693
4694         /* Display WA #1126: skl,bxt,kbl,glk */
4695         if (level >= 1 && level <= 7) {
4696                 if (wp->y_tiled) {
4697                         res_blocks += fixed16_to_u32_round_up(
4698                                                         wp->y_tile_minimum);
4699                         res_lines += wp->y_min_scanlines;
4700                 } else {
4701                         res_blocks++;
4702                 }
4703
4704                 /*
4705                  * Make sure result blocks for higher latency levels are atleast
4706                  * as high as level below the current level.
4707                  * Assumption in DDB algorithm optimization for special cases.
4708                  * Also covers Display WA #1125 for RC.
4709                  */
4710                 if (result_prev->plane_res_b > res_blocks)
4711                         res_blocks = result_prev->plane_res_b;
4712         }
4713
4714         if (INTEL_GEN(dev_priv) >= 11) {
4715                 if (wp->y_tiled) {
4716                         uint32_t extra_lines;
4717                         uint_fixed_16_16_t fp_min_disp_buf_needed;
4718
4719                         if (res_lines % wp->y_min_scanlines == 0)
4720                                 extra_lines = wp->y_min_scanlines;
4721                         else
4722                                 extra_lines = wp->y_min_scanlines * 2 -
4723                                               res_lines % wp->y_min_scanlines;
4724
4725                         fp_min_disp_buf_needed = mul_u32_fixed16(res_lines +
4726                                                 extra_lines,
4727                                                 wp->plane_blocks_per_line);
4728                         min_disp_buf_needed = fixed16_to_u32_round_up(
4729                                                 fp_min_disp_buf_needed);
4730                 } else {
4731                         min_disp_buf_needed = DIV_ROUND_UP(res_blocks * 11, 10);
4732                 }
4733         } else {
4734                 min_disp_buf_needed = res_blocks;
4735         }
4736
4737         if ((level > 0 && res_lines > 31) ||
4738             res_blocks >= ddb_allocation ||
4739             min_disp_buf_needed >= ddb_allocation) {
4740                 result->plane_en = false;
4741
4742                 /*
4743                  * If there are no valid level 0 watermarks, then we can't
4744                  * support this display configuration.
4745                  */
4746                 if (level) {
4747                         return 0;
4748                 } else {
4749                         struct drm_plane *plane = pstate->plane;
4750
4751                         DRM_DEBUG_KMS("Requested display configuration exceeds system watermark limitations\n");
4752                         DRM_DEBUG_KMS("[PLANE:%d:%s] blocks required = %u/%u, lines required = %u/31\n",
4753                                       plane->base.id, plane->name,
4754                                       res_blocks, ddb_allocation, res_lines);
4755                         return -EINVAL;
4756                 }
4757         }
4758
4759         /*
4760          * Display WA #826 (SKL:ALL, BXT:ALL) & #1059 (CNL:A)
4761          * disable wm level 1-7 on NV12 planes
4762          */
4763         if (wp->is_planar && level >= 1 &&
4764             (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv) ||
4765              IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_A0))) {
4766                 result->plane_en = false;
4767                 return 0;
4768         }
4769
4770         /* The number of lines are ignored for the level 0 watermark. */
4771         result->plane_res_b = res_blocks;
4772         result->plane_res_l = res_lines;
4773         result->plane_en = true;
4774
4775         return 0;
4776 }
4777
4778 static int
4779 skl_compute_wm_levels(const struct drm_i915_private *dev_priv,
4780                       struct skl_ddb_allocation *ddb,
4781                       struct intel_crtc_state *cstate,
4782                       const struct intel_plane_state *intel_pstate,
4783                       const struct skl_wm_params *wm_params,
4784                       struct skl_plane_wm *wm,
4785                       int plane_id)
4786 {
4787         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
4788         struct drm_plane *plane = intel_pstate->base.plane;
4789         struct intel_plane *intel_plane = to_intel_plane(plane);
4790         uint16_t ddb_blocks;
4791         enum pipe pipe = intel_crtc->pipe;
4792         int level, max_level = ilk_wm_max_level(dev_priv);
4793         enum plane_id intel_plane_id = intel_plane->id;
4794         int ret;
4795
4796         if (WARN_ON(!intel_pstate->base.fb))
4797                 return -EINVAL;
4798
4799         ddb_blocks = plane_id ?
4800                      skl_ddb_entry_size(&ddb->uv_plane[pipe][intel_plane_id]) :
4801                      skl_ddb_entry_size(&ddb->plane[pipe][intel_plane_id]);
4802
4803         for (level = 0; level <= max_level; level++) {
4804                 struct skl_wm_level *result = plane_id ? &wm->uv_wm[level] :
4805                                                           &wm->wm[level];
4806                 struct skl_wm_level *result_prev;
4807
4808                 if (level)
4809                         result_prev = plane_id ? &wm->uv_wm[level - 1] :
4810                                                   &wm->wm[level - 1];
4811                 else
4812                         result_prev = plane_id ? &wm->uv_wm[0] : &wm->wm[0];
4813
4814                 ret = skl_compute_plane_wm(dev_priv,
4815                                            cstate,
4816                                            intel_pstate,
4817                                            ddb_blocks,
4818                                            level,
4819                                            wm_params,
4820                                            result_prev,
4821                                            result);
4822                 if (ret)
4823                         return ret;
4824         }
4825
4826         if (intel_pstate->base.fb->format->format == DRM_FORMAT_NV12)
4827                 wm->is_planar = true;
4828
4829         return 0;
4830 }
4831
4832 static uint32_t
4833 skl_compute_linetime_wm(struct intel_crtc_state *cstate)
4834 {
4835         struct drm_atomic_state *state = cstate->base.state;
4836         struct drm_i915_private *dev_priv = to_i915(state->dev);
4837         uint_fixed_16_16_t linetime_us;
4838         uint32_t linetime_wm;
4839
4840         linetime_us = intel_get_linetime_us(cstate);
4841
4842         if (is_fixed16_zero(linetime_us))
4843                 return 0;
4844
4845         linetime_wm = fixed16_to_u32_round_up(mul_u32_fixed16(8, linetime_us));
4846
4847         /* Display WA #1135: bxt:ALL GLK:ALL */
4848         if ((IS_BROXTON(dev_priv) || IS_GEMINILAKE(dev_priv)) &&
4849             dev_priv->ipc_enabled)
4850                 linetime_wm /= 2;
4851
4852         return linetime_wm;
4853 }
4854
4855 static void skl_compute_transition_wm(struct intel_crtc_state *cstate,
4856                                       struct skl_wm_params *wp,
4857                                       struct skl_wm_level *wm_l0,
4858                                       uint16_t ddb_allocation,
4859                                       struct skl_wm_level *trans_wm /* out */)
4860 {
4861         struct drm_device *dev = cstate->base.crtc->dev;
4862         const struct drm_i915_private *dev_priv = to_i915(dev);
4863         uint16_t trans_min, trans_y_tile_min;
4864         const uint16_t trans_amount = 10; /* This is configurable amount */
4865         uint16_t trans_offset_b, res_blocks;
4866
4867         if (!cstate->base.active)
4868                 goto exit;
4869
4870         /* Transition WM are not recommended by HW team for GEN9 */
4871         if (INTEL_GEN(dev_priv) <= 9)
4872                 goto exit;
4873
4874         /* Transition WM don't make any sense if ipc is disabled */
4875         if (!dev_priv->ipc_enabled)
4876                 goto exit;
4877
4878         trans_min = 0;
4879         if (INTEL_GEN(dev_priv) >= 10)
4880                 trans_min = 4;
4881
4882         trans_offset_b = trans_min + trans_amount;
4883
4884         if (wp->y_tiled) {
4885                 trans_y_tile_min = (uint16_t) mul_round_up_u32_fixed16(2,
4886                                                         wp->y_tile_minimum);
4887                 res_blocks = max(wm_l0->plane_res_b, trans_y_tile_min) +
4888                                 trans_offset_b;
4889         } else {
4890                 res_blocks = wm_l0->plane_res_b + trans_offset_b;
4891
4892                 /* WA BUG:1938466 add one block for non y-tile planes */
4893                 if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_A0))
4894                         res_blocks += 1;
4895
4896         }
4897
4898         res_blocks += 1;
4899
4900         if (res_blocks < ddb_allocation) {
4901                 trans_wm->plane_res_b = res_blocks;
4902                 trans_wm->plane_en = true;
4903                 return;
4904         }
4905
4906 exit:
4907         trans_wm->plane_en = false;
4908 }
4909
4910 static int skl_build_pipe_wm(struct intel_crtc_state *cstate,
4911                              struct skl_ddb_allocation *ddb,
4912                              struct skl_pipe_wm *pipe_wm)
4913 {
4914         struct drm_device *dev = cstate->base.crtc->dev;
4915         struct drm_crtc_state *crtc_state = &cstate->base;
4916         const struct drm_i915_private *dev_priv = to_i915(dev);
4917         struct drm_plane *plane;
4918         const struct drm_plane_state *pstate;
4919         struct skl_plane_wm *wm;
4920         int ret;
4921
4922         /*
4923          * We'll only calculate watermarks for planes that are actually
4924          * enabled, so make sure all other planes are set as disabled.
4925          */
4926         memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));
4927
4928         drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
4929                 const struct intel_plane_state *intel_pstate =
4930                                                 to_intel_plane_state(pstate);
4931                 enum plane_id plane_id = to_intel_plane(plane)->id;
4932                 struct skl_wm_params wm_params;
4933                 enum pipe pipe = to_intel_crtc(cstate->base.crtc)->pipe;
4934                 uint16_t ddb_blocks;
4935
4936                 wm = &pipe_wm->planes[plane_id];
4937                 ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][plane_id]);
4938
4939                 ret = skl_compute_plane_wm_params(dev_priv, cstate,
4940                                                   intel_pstate, &wm_params, 0);
4941                 if (ret)
4942                         return ret;
4943
4944                 ret = skl_compute_wm_levels(dev_priv, ddb, cstate,
4945                                             intel_pstate, &wm_params, wm, 0);
4946                 if (ret)
4947                         return ret;
4948
4949                 skl_compute_transition_wm(cstate, &wm_params, &wm->wm[0],
4950                                           ddb_blocks, &wm->trans_wm);
4951
4952                 /* uv plane watermarks must also be validated for NV12/Planar */
4953                 if (wm_params.is_planar) {
4954                         memset(&wm_params, 0, sizeof(struct skl_wm_params));
4955                         wm->is_planar = true;
4956
4957                         ret = skl_compute_plane_wm_params(dev_priv, cstate,
4958                                                           intel_pstate,
4959                                                           &wm_params, 1);
4960                         if (ret)
4961                                 return ret;
4962
4963                         ret = skl_compute_wm_levels(dev_priv, ddb, cstate,
4964                                                     intel_pstate, &wm_params,
4965                                                     wm, 1);
4966                         if (ret)
4967                                 return ret;
4968                 }
4969         }
4970
4971         pipe_wm->linetime = skl_compute_linetime_wm(cstate);
4972
4973         return 0;
4974 }
4975
4976 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
4977                                 i915_reg_t reg,
4978                                 const struct skl_ddb_entry *entry)
4979 {
4980         if (entry->end)
4981                 I915_WRITE(reg, (entry->end - 1) << 16 | entry->start);
4982         else
4983                 I915_WRITE(reg, 0);
4984 }
4985
4986 static void skl_write_wm_level(struct drm_i915_private *dev_priv,
4987                                i915_reg_t reg,
4988                                const struct skl_wm_level *level)
4989 {
4990         uint32_t val = 0;
4991
4992         if (level->plane_en) {
4993                 val |= PLANE_WM_EN;
4994                 val |= level->plane_res_b;
4995                 val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
4996         }
4997
4998         I915_WRITE(reg, val);
4999 }
5000
5001 static void skl_write_plane_wm(struct intel_crtc *intel_crtc,
5002                                const struct skl_plane_wm *wm,
5003                                const struct skl_ddb_allocation *ddb,
5004                                enum plane_id plane_id)
5005 {
5006         struct drm_crtc *crtc = &intel_crtc->base;
5007         struct drm_device *dev = crtc->dev;
5008         struct drm_i915_private *dev_priv = to_i915(dev);
5009         int level, max_level = ilk_wm_max_level(dev_priv);
5010         enum pipe pipe = intel_crtc->pipe;
5011
5012         for (level = 0; level <= max_level; level++) {
5013                 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5014                                    &wm->wm[level]);
5015         }
5016         skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5017                            &wm->trans_wm);
5018
5019         skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id),
5020                             &ddb->plane[pipe][plane_id]);
5021         /* FIXME: add proper NV12 support for ICL. */
5022         if (INTEL_GEN(dev_priv) >= 11)
5023                 return skl_ddb_entry_write(dev_priv,
5024                                            PLANE_BUF_CFG(pipe, plane_id),
5025                                            &ddb->plane[pipe][plane_id]);
5026         if (wm->is_planar) {
5027                 skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id),
5028                                     &ddb->uv_plane[pipe][plane_id]);
5029                 skl_ddb_entry_write(dev_priv,
5030                                     PLANE_NV12_BUF_CFG(pipe, plane_id),
5031                                     &ddb->plane[pipe][plane_id]);
5032         } else {
5033                 skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id),
5034                                     &ddb->plane[pipe][plane_id]);
5035                 I915_WRITE(PLANE_NV12_BUF_CFG(pipe, plane_id), 0x0);
5036         }
5037 }
5038
5039 static void skl_write_cursor_wm(struct intel_crtc *intel_crtc,
5040                                 const struct skl_plane_wm *wm,
5041                                 const struct skl_ddb_allocation *ddb)
5042 {
5043         struct drm_crtc *crtc = &intel_crtc->base;
5044         struct drm_device *dev = crtc->dev;
5045         struct drm_i915_private *dev_priv = to_i915(dev);
5046         int level, max_level = ilk_wm_max_level(dev_priv);
5047         enum pipe pipe = intel_crtc->pipe;
5048
5049         for (level = 0; level <= max_level; level++) {
5050                 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5051                                    &wm->wm[level]);
5052         }
5053         skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm);
5054
5055         skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
5056                             &ddb->plane[pipe][PLANE_CURSOR]);
5057 }
5058
5059 bool skl_wm_level_equals(const struct skl_wm_level *l1,
5060                          const struct skl_wm_level *l2)
5061 {
5062         if (l1->plane_en != l2->plane_en)
5063                 return false;
5064
5065         /* If both planes aren't enabled, the rest shouldn't matter */
5066         if (!l1->plane_en)
5067                 return true;
5068
5069         return (l1->plane_res_l == l2->plane_res_l &&
5070                 l1->plane_res_b == l2->plane_res_b);
5071 }
5072
5073 static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5074                                            const struct skl_ddb_entry *b)
5075 {
5076         return a->start < b->end && b->start < a->end;
5077 }
5078
5079 bool skl_ddb_allocation_overlaps(struct drm_i915_private *dev_priv,
5080                                  const struct skl_ddb_entry **entries,
5081                                  const struct skl_ddb_entry *ddb,
5082                                  int ignore)
5083 {
5084         enum pipe pipe;
5085
5086         for_each_pipe(dev_priv, pipe) {
5087                 if (pipe != ignore && entries[pipe] &&
5088                     skl_ddb_entries_overlap(ddb, entries[pipe]))
5089                         return true;
5090         }
5091
5092         return false;
5093 }
5094
5095 static int skl_update_pipe_wm(struct drm_crtc_state *cstate,
5096                               const struct skl_pipe_wm *old_pipe_wm,
5097                               struct skl_pipe_wm *pipe_wm, /* out */
5098                               struct skl_ddb_allocation *ddb, /* out */
5099                               bool *changed /* out */)
5100 {
5101         struct intel_crtc_state *intel_cstate = to_intel_crtc_state(cstate);
5102         int ret;
5103
5104         ret = skl_build_pipe_wm(intel_cstate, ddb, pipe_wm);
5105         if (ret)
5106                 return ret;
5107
5108         if (!memcmp(old_pipe_wm, pipe_wm, sizeof(*pipe_wm)))
5109                 *changed = false;
5110         else
5111                 *changed = true;
5112
5113         return 0;
5114 }
5115
5116 static uint32_t
5117 pipes_modified(struct drm_atomic_state *state)
5118 {
5119         struct drm_crtc *crtc;
5120         struct drm_crtc_state *cstate;
5121         uint32_t i, ret = 0;
5122
5123         for_each_new_crtc_in_state(state, crtc, cstate, i)
5124                 ret |= drm_crtc_mask(crtc);
5125
5126         return ret;
5127 }
5128
5129 static int
5130 skl_ddb_add_affected_planes(struct intel_crtc_state *cstate)
5131 {
5132         struct drm_atomic_state *state = cstate->base.state;
5133         struct drm_device *dev = state->dev;
5134         struct drm_crtc *crtc = cstate->base.crtc;
5135         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5136         struct drm_i915_private *dev_priv = to_i915(dev);
5137         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
5138         struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb;
5139         struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
5140         struct drm_plane_state *plane_state;
5141         struct drm_plane *plane;
5142         enum pipe pipe = intel_crtc->pipe;
5143
5144         drm_for_each_plane_mask(plane, dev, cstate->base.plane_mask) {
5145                 enum plane_id plane_id = to_intel_plane(plane)->id;
5146
5147                 if (skl_ddb_entry_equal(&cur_ddb->plane[pipe][plane_id],
5148                                         &new_ddb->plane[pipe][plane_id]) &&
5149                     skl_ddb_entry_equal(&cur_ddb->uv_plane[pipe][plane_id],
5150                                         &new_ddb->uv_plane[pipe][plane_id]))
5151                         continue;
5152
5153                 plane_state = drm_atomic_get_plane_state(state, plane);
5154                 if (IS_ERR(plane_state))
5155                         return PTR_ERR(plane_state);
5156         }
5157
5158         return 0;
5159 }
5160
5161 static int
5162 skl_compute_ddb(struct drm_atomic_state *state)
5163 {
5164         const struct drm_i915_private *dev_priv = to_i915(state->dev);
5165         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
5166         struct skl_ddb_allocation *ddb = &intel_state->wm_results.ddb;
5167         struct intel_crtc *crtc;
5168         struct intel_crtc_state *cstate;
5169         int ret, i;
5170
5171         memcpy(ddb, &dev_priv->wm.skl_hw.ddb, sizeof(*ddb));
5172
5173         for_each_new_intel_crtc_in_state(intel_state, crtc, cstate, i) {
5174                 ret = skl_allocate_pipe_ddb(cstate, ddb);
5175                 if (ret)
5176                         return ret;
5177
5178                 ret = skl_ddb_add_affected_planes(cstate);
5179                 if (ret)
5180                         return ret;
5181         }
5182
5183         return 0;
5184 }
5185
5186 static void
5187 skl_print_wm_changes(const struct drm_atomic_state *state)
5188 {
5189         const struct drm_device *dev = state->dev;
5190         const struct drm_i915_private *dev_priv = to_i915(dev);
5191         const struct intel_atomic_state *intel_state =
5192                 to_intel_atomic_state(state);
5193         const struct drm_crtc *crtc;
5194         const struct drm_crtc_state *cstate;
5195         const struct intel_plane *intel_plane;
5196         const struct skl_ddb_allocation *old_ddb = &dev_priv->wm.skl_hw.ddb;
5197         const struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb;
5198         int i;
5199
5200         for_each_new_crtc_in_state(state, crtc, cstate, i) {
5201                 const struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5202                 enum pipe pipe = intel_crtc->pipe;
5203
5204                 for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
5205                         enum plane_id plane_id = intel_plane->id;
5206                         const struct skl_ddb_entry *old, *new;
5207
5208                         old = &old_ddb->plane[pipe][plane_id];
5209                         new = &new_ddb->plane[pipe][plane_id];
5210
5211                         if (skl_ddb_entry_equal(old, new))
5212                                 continue;
5213
5214                         DRM_DEBUG_ATOMIC("[PLANE:%d:%s] ddb (%d - %d) -> (%d - %d)\n",
5215                                          intel_plane->base.base.id,
5216                                          intel_plane->base.name,
5217                                          old->start, old->end,
5218                                          new->start, new->end);
5219                 }
5220         }
5221 }
5222
5223 static int
5224 skl_ddb_add_affected_pipes(struct drm_atomic_state *state, bool *changed)
5225 {
5226         struct drm_device *dev = state->dev;
5227         const struct drm_i915_private *dev_priv = to_i915(dev);
5228         const struct drm_crtc *crtc;
5229         const struct drm_crtc_state *cstate;
5230         struct intel_crtc *intel_crtc;
5231         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
5232         uint32_t realloc_pipes = pipes_modified(state);
5233         int ret, i;
5234
5235         /*
5236          * When we distrust bios wm we always need to recompute to set the
5237          * expected DDB allocations for each CRTC.
5238          */
5239         if (dev_priv->wm.distrust_bios_wm)
5240                 (*changed) = true;
5241
5242         /*
5243          * If this transaction isn't actually touching any CRTC's, don't
5244          * bother with watermark calculation.  Note that if we pass this
5245          * test, we're guaranteed to hold at least one CRTC state mutex,
5246          * which means we can safely use values like dev_priv->active_crtcs
5247          * since any racing commits that want to update them would need to
5248          * hold _all_ CRTC state mutexes.
5249          */
5250         for_each_new_crtc_in_state(state, crtc, cstate, i)
5251                 (*changed) = true;
5252
5253         if (!*changed)
5254                 return 0;
5255
5256         /*
5257          * If this is our first atomic update following hardware readout,
5258          * we can't trust the DDB that the BIOS programmed for us.  Let's
5259          * pretend that all pipes switched active status so that we'll
5260          * ensure a full DDB recompute.
5261          */
5262         if (dev_priv->wm.distrust_bios_wm) {
5263                 ret = drm_modeset_lock(&dev->mode_config.connection_mutex,
5264                                        state->acquire_ctx);
5265                 if (ret)
5266                         return ret;
5267
5268                 intel_state->active_pipe_changes = ~0;
5269
5270                 /*
5271                  * We usually only initialize intel_state->active_crtcs if we
5272                  * we're doing a modeset; make sure this field is always
5273                  * initialized during the sanitization process that happens
5274                  * on the first commit too.
5275                  */
5276                 if (!intel_state->modeset)
5277                         intel_state->active_crtcs = dev_priv->active_crtcs;
5278         }
5279
5280         /*
5281          * If the modeset changes which CRTC's are active, we need to
5282          * recompute the DDB allocation for *all* active pipes, even
5283          * those that weren't otherwise being modified in any way by this
5284          * atomic commit.  Due to the shrinking of the per-pipe allocations
5285          * when new active CRTC's are added, it's possible for a pipe that
5286          * we were already using and aren't changing at all here to suddenly
5287          * become invalid if its DDB needs exceeds its new allocation.
5288          *
5289          * Note that if we wind up doing a full DDB recompute, we can't let
5290          * any other display updates race with this transaction, so we need
5291          * to grab the lock on *all* CRTC's.
5292          */
5293         if (intel_state->active_pipe_changes || intel_state->modeset) {
5294                 realloc_pipes = ~0;
5295                 intel_state->wm_results.dirty_pipes = ~0;
5296         }
5297
5298         /*
5299          * We're not recomputing for the pipes not included in the commit, so
5300          * make sure we start with the current state.
5301          */
5302         for_each_intel_crtc_mask(dev, intel_crtc, realloc_pipes) {
5303                 struct intel_crtc_state *cstate;
5304
5305                 cstate = intel_atomic_get_crtc_state(state, intel_crtc);
5306                 if (IS_ERR(cstate))
5307                         return PTR_ERR(cstate);
5308         }
5309
5310         return 0;
5311 }
5312
5313 static int
5314 skl_compute_wm(struct drm_atomic_state *state)
5315 {
5316         struct drm_crtc *crtc;
5317         struct drm_crtc_state *cstate;
5318         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
5319         struct skl_ddb_values *results = &intel_state->wm_results;
5320         struct skl_pipe_wm *pipe_wm;
5321         bool changed = false;
5322         int ret, i;
5323
5324         /* Clear all dirty flags */
5325         results->dirty_pipes = 0;
5326
5327         ret = skl_ddb_add_affected_pipes(state, &changed);
5328         if (ret || !changed)
5329                 return ret;
5330
5331         ret = skl_compute_ddb(state);
5332         if (ret)
5333                 return ret;
5334
5335         /*
5336          * Calculate WM's for all pipes that are part of this transaction.
5337          * Note that the DDB allocation above may have added more CRTC's that
5338          * weren't otherwise being modified (and set bits in dirty_pipes) if
5339          * pipe allocations had to change.
5340          *
5341          * FIXME:  Now that we're doing this in the atomic check phase, we
5342          * should allow skl_update_pipe_wm() to return failure in cases where
5343          * no suitable watermark values can be found.
5344          */
5345         for_each_new_crtc_in_state(state, crtc, cstate, i) {
5346                 struct intel_crtc_state *intel_cstate =
5347                         to_intel_crtc_state(cstate);
5348                 const struct skl_pipe_wm *old_pipe_wm =
5349                         &to_intel_crtc_state(crtc->state)->wm.skl.optimal;
5350
5351                 pipe_wm = &intel_cstate->wm.skl.optimal;
5352                 ret = skl_update_pipe_wm(cstate, old_pipe_wm, pipe_wm,
5353                                          &results->ddb, &changed);
5354                 if (ret)
5355                         return ret;
5356
5357                 if (changed)
5358                         results->dirty_pipes |= drm_crtc_mask(crtc);
5359
5360                 if ((results->dirty_pipes & drm_crtc_mask(crtc)) == 0)
5361                         /* This pipe's WM's did not change */
5362                         continue;
5363
5364                 intel_cstate->update_wm_pre = true;
5365         }
5366
5367         skl_print_wm_changes(state);
5368
5369         return 0;
5370 }
5371
5372 static void skl_atomic_update_crtc_wm(struct intel_atomic_state *state,
5373                                       struct intel_crtc_state *cstate)
5374 {
5375         struct intel_crtc *crtc = to_intel_crtc(cstate->base.crtc);
5376         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5377         struct skl_pipe_wm *pipe_wm = &cstate->wm.skl.optimal;
5378         const struct skl_ddb_allocation *ddb = &state->wm_results.ddb;
5379         enum pipe pipe = crtc->pipe;
5380         enum plane_id plane_id;
5381
5382         if (!(state->wm_results.dirty_pipes & drm_crtc_mask(&crtc->base)))
5383                 return;
5384
5385         I915_WRITE(PIPE_WM_LINETIME(pipe), pipe_wm->linetime);
5386
5387         for_each_plane_id_on_crtc(crtc, plane_id) {
5388                 if (plane_id != PLANE_CURSOR)
5389                         skl_write_plane_wm(crtc, &pipe_wm->planes[plane_id],
5390                                            ddb, plane_id);
5391                 else
5392                         skl_write_cursor_wm(crtc, &pipe_wm->planes[plane_id],
5393                                             ddb);
5394         }
5395 }
5396
5397 static void skl_initial_wm(struct intel_atomic_state *state,
5398                            struct intel_crtc_state *cstate)
5399 {
5400         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
5401         struct drm_device *dev = intel_crtc->base.dev;
5402         struct drm_i915_private *dev_priv = to_i915(dev);
5403         struct skl_ddb_values *results = &state->wm_results;
5404         struct skl_ddb_values *hw_vals = &dev_priv->wm.skl_hw;
5405         enum pipe pipe = intel_crtc->pipe;
5406
5407         if ((results->dirty_pipes & drm_crtc_mask(&intel_crtc->base)) == 0)
5408                 return;
5409
5410         mutex_lock(&dev_priv->wm.wm_mutex);
5411
5412         if (cstate->base.active_changed)
5413                 skl_atomic_update_crtc_wm(state, cstate);
5414
5415         memcpy(hw_vals->ddb.uv_plane[pipe], results->ddb.uv_plane[pipe],
5416                sizeof(hw_vals->ddb.uv_plane[pipe]));
5417         memcpy(hw_vals->ddb.plane[pipe], results->ddb.plane[pipe],
5418                sizeof(hw_vals->ddb.plane[pipe]));
5419
5420         mutex_unlock(&dev_priv->wm.wm_mutex);
5421 }
5422
5423 static void ilk_compute_wm_config(struct drm_device *dev,
5424                                   struct intel_wm_config *config)
5425 {
5426         struct intel_crtc *crtc;
5427
5428         /* Compute the currently _active_ config */
5429         for_each_intel_crtc(dev, crtc) {
5430                 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
5431
5432                 if (!wm->pipe_enabled)
5433                         continue;
5434
5435                 config->sprites_enabled |= wm->sprites_enabled;
5436                 config->sprites_scaled |= wm->sprites_scaled;
5437                 config->num_pipes_active++;
5438         }
5439 }
5440
5441 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
5442 {
5443         struct drm_device *dev = &dev_priv->drm;
5444         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
5445         struct ilk_wm_maximums max;
5446         struct intel_wm_config config = {};
5447         struct ilk_wm_values results = {};
5448         enum intel_ddb_partitioning partitioning;
5449
5450         ilk_compute_wm_config(dev, &config);
5451
5452         ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
5453         ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
5454
5455         /* 5/6 split only in single pipe config on IVB+ */
5456         if (INTEL_GEN(dev_priv) >= 7 &&
5457             config.num_pipes_active == 1 && config.sprites_enabled) {
5458                 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
5459                 ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
5460
5461                 best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
5462         } else {
5463                 best_lp_wm = &lp_wm_1_2;
5464         }
5465
5466         partitioning = (best_lp_wm == &lp_wm_1_2) ?
5467                        INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
5468
5469         ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
5470
5471         ilk_write_wm_values(dev_priv, &results);
5472 }
5473
5474 static void ilk_initial_watermarks(struct intel_atomic_state *state,
5475                                    struct intel_crtc_state *cstate)
5476 {
5477         struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
5478         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
5479
5480         mutex_lock(&dev_priv->wm.wm_mutex);
5481         intel_crtc->wm.active.ilk = cstate->wm.ilk.intermediate;
5482         ilk_program_watermarks(dev_priv);
5483         mutex_unlock(&dev_priv->wm.wm_mutex);
5484 }
5485
5486 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
5487                                     struct intel_crtc_state *cstate)
5488 {
5489         struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
5490         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
5491
5492         mutex_lock(&dev_priv->wm.wm_mutex);
5493         if (cstate->wm.need_postvbl_update) {
5494                 intel_crtc->wm.active.ilk = cstate->wm.ilk.optimal;
5495                 ilk_program_watermarks(dev_priv);
5496         }
5497         mutex_unlock(&dev_priv->wm.wm_mutex);
5498 }
5499
5500 static inline void skl_wm_level_from_reg_val(uint32_t val,
5501                                              struct skl_wm_level *level)
5502 {
5503         level->plane_en = val & PLANE_WM_EN;
5504         level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
5505         level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
5506                 PLANE_WM_LINES_MASK;
5507 }
5508
5509 void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc,
5510                               struct skl_pipe_wm *out)
5511 {
5512         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
5513         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5514         enum pipe pipe = intel_crtc->pipe;
5515         int level, max_level;
5516         enum plane_id plane_id;
5517         uint32_t val;
5518
5519         max_level = ilk_wm_max_level(dev_priv);
5520
5521         for_each_plane_id_on_crtc(intel_crtc, plane_id) {
5522                 struct skl_plane_wm *wm = &out->planes[plane_id];
5523
5524                 for (level = 0; level <= max_level; level++) {
5525                         if (plane_id != PLANE_CURSOR)
5526                                 val = I915_READ(PLANE_WM(pipe, plane_id, level));
5527                         else
5528                                 val = I915_READ(CUR_WM(pipe, level));
5529
5530                         skl_wm_level_from_reg_val(val, &wm->wm[level]);
5531                 }
5532
5533                 if (plane_id != PLANE_CURSOR)
5534                         val = I915_READ(PLANE_WM_TRANS(pipe, plane_id));
5535                 else
5536                         val = I915_READ(CUR_WM_TRANS(pipe));
5537
5538                 skl_wm_level_from_reg_val(val, &wm->trans_wm);
5539         }
5540
5541         if (!intel_crtc->active)
5542                 return;
5543
5544         out->linetime = I915_READ(PIPE_WM_LINETIME(pipe));
5545 }
5546
5547 void skl_wm_get_hw_state(struct drm_device *dev)
5548 {
5549         struct drm_i915_private *dev_priv = to_i915(dev);
5550         struct skl_ddb_values *hw = &dev_priv->wm.skl_hw;
5551         struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
5552         struct drm_crtc *crtc;
5553         struct intel_crtc *intel_crtc;
5554         struct intel_crtc_state *cstate;
5555
5556         skl_ddb_get_hw_state(dev_priv, ddb);
5557         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
5558                 intel_crtc = to_intel_crtc(crtc);
5559                 cstate = to_intel_crtc_state(crtc->state);
5560
5561                 skl_pipe_wm_get_hw_state(crtc, &cstate->wm.skl.optimal);
5562
5563                 if (intel_crtc->active)
5564                         hw->dirty_pipes |= drm_crtc_mask(crtc);
5565         }
5566
5567         if (dev_priv->active_crtcs) {
5568                 /* Fully recompute DDB on first atomic commit */
5569                 dev_priv->wm.distrust_bios_wm = true;
5570         } else {
5571                 /*
5572                  * Easy/common case; just sanitize DDB now if everything off
5573                  * Keep dbuf slice info intact
5574                  */
5575                 memset(ddb->plane, 0, sizeof(ddb->plane));
5576                 memset(ddb->uv_plane, 0, sizeof(ddb->uv_plane));
5577         }
5578 }
5579
5580 static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
5581 {
5582         struct drm_device *dev = crtc->dev;
5583         struct drm_i915_private *dev_priv = to_i915(dev);
5584         struct ilk_wm_values *hw = &dev_priv->wm.hw;
5585         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5586         struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
5587         struct intel_pipe_wm *active = &cstate->wm.ilk.optimal;
5588         enum pipe pipe = intel_crtc->pipe;
5589         static const i915_reg_t wm0_pipe_reg[] = {
5590                 [PIPE_A] = WM0_PIPEA_ILK,
5591                 [PIPE_B] = WM0_PIPEB_ILK,
5592                 [PIPE_C] = WM0_PIPEC_IVB,
5593         };
5594
5595         hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
5596         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
5597                 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
5598
5599         memset(active, 0, sizeof(*active));
5600
5601         active->pipe_enabled = intel_crtc->active;
5602
5603         if (active->pipe_enabled) {
5604                 u32 tmp = hw->wm_pipe[pipe];
5605
5606                 /*
5607                  * For active pipes LP0 watermark is marked as
5608                  * enabled, and LP1+ watermaks as disabled since
5609                  * we can't really reverse compute them in case
5610                  * multiple pipes are active.
5611                  */
5612                 active->wm[0].enable = true;
5613                 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
5614                 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
5615                 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
5616                 active->linetime = hw->wm_linetime[pipe];
5617         } else {
5618                 int level, max_level = ilk_wm_max_level(dev_priv);
5619
5620                 /*
5621                  * For inactive pipes, all watermark levels
5622                  * should be marked as enabled but zeroed,
5623                  * which is what we'd compute them to.
5624                  */
5625                 for (level = 0; level <= max_level; level++)
5626                         active->wm[level].enable = true;
5627         }
5628
5629         intel_crtc->wm.active.ilk = *active;
5630 }
5631
5632 #define _FW_WM(value, plane) \
5633         (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
5634 #define _FW_WM_VLV(value, plane) \
5635         (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
5636
5637 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
5638                                struct g4x_wm_values *wm)
5639 {
5640         uint32_t tmp;
5641
5642         tmp = I915_READ(DSPFW1);
5643         wm->sr.plane = _FW_WM(tmp, SR);
5644         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
5645         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
5646         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
5647
5648         tmp = I915_READ(DSPFW2);
5649         wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
5650         wm->sr.fbc = _FW_WM(tmp, FBC_SR);
5651         wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
5652         wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
5653         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
5654         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
5655
5656         tmp = I915_READ(DSPFW3);
5657         wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
5658         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
5659         wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
5660         wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
5661 }
5662
5663 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
5664                                struct vlv_wm_values *wm)
5665 {
5666         enum pipe pipe;
5667         uint32_t tmp;
5668
5669         for_each_pipe(dev_priv, pipe) {
5670                 tmp = I915_READ(VLV_DDL(pipe));
5671
5672                 wm->ddl[pipe].plane[PLANE_PRIMARY] =
5673                         (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5674                 wm->ddl[pipe].plane[PLANE_CURSOR] =
5675                         (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5676                 wm->ddl[pipe].plane[PLANE_SPRITE0] =
5677                         (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5678                 wm->ddl[pipe].plane[PLANE_SPRITE1] =
5679                         (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5680         }
5681
5682         tmp = I915_READ(DSPFW1);
5683         wm->sr.plane = _FW_WM(tmp, SR);
5684         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
5685         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
5686         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
5687
5688         tmp = I915_READ(DSPFW2);
5689         wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
5690         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
5691         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
5692
5693         tmp = I915_READ(DSPFW3);
5694         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
5695
5696         if (IS_CHERRYVIEW(dev_priv)) {
5697                 tmp = I915_READ(DSPFW7_CHV);
5698                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
5699                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
5700
5701                 tmp = I915_READ(DSPFW8_CHV);
5702                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
5703                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
5704
5705                 tmp = I915_READ(DSPFW9_CHV);
5706                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
5707                 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
5708
5709                 tmp = I915_READ(DSPHOWM);
5710                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
5711                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
5712                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
5713                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
5714                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
5715                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
5716                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
5717                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
5718                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
5719                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
5720         } else {
5721                 tmp = I915_READ(DSPFW7);
5722                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
5723                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
5724
5725                 tmp = I915_READ(DSPHOWM);
5726                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
5727                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
5728                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
5729                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
5730                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
5731                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
5732                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
5733         }
5734 }
5735
5736 #undef _FW_WM
5737 #undef _FW_WM_VLV
5738
5739 void g4x_wm_get_hw_state(struct drm_device *dev)
5740 {
5741         struct drm_i915_private *dev_priv = to_i915(dev);
5742         struct g4x_wm_values *wm = &dev_priv->wm.g4x;
5743         struct intel_crtc *crtc;
5744
5745         g4x_read_wm_values(dev_priv, wm);
5746
5747         wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
5748
5749         for_each_intel_crtc(dev, crtc) {
5750                 struct intel_crtc_state *crtc_state =
5751                         to_intel_crtc_state(crtc->base.state);
5752                 struct g4x_wm_state *active = &crtc->wm.active.g4x;
5753                 struct g4x_pipe_wm *raw;
5754                 enum pipe pipe = crtc->pipe;
5755                 enum plane_id plane_id;
5756                 int level, max_level;
5757
5758                 active->cxsr = wm->cxsr;
5759                 active->hpll_en = wm->hpll_en;
5760                 active->fbc_en = wm->fbc_en;
5761
5762                 active->sr = wm->sr;
5763                 active->hpll = wm->hpll;
5764
5765                 for_each_plane_id_on_crtc(crtc, plane_id) {
5766                         active->wm.plane[plane_id] =
5767                                 wm->pipe[pipe].plane[plane_id];
5768                 }
5769
5770                 if (wm->cxsr && wm->hpll_en)
5771                         max_level = G4X_WM_LEVEL_HPLL;
5772                 else if (wm->cxsr)
5773                         max_level = G4X_WM_LEVEL_SR;
5774                 else
5775                         max_level = G4X_WM_LEVEL_NORMAL;
5776
5777                 level = G4X_WM_LEVEL_NORMAL;
5778                 raw = &crtc_state->wm.g4x.raw[level];
5779                 for_each_plane_id_on_crtc(crtc, plane_id)
5780                         raw->plane[plane_id] = active->wm.plane[plane_id];
5781
5782                 if (++level > max_level)
5783                         goto out;
5784
5785                 raw = &crtc_state->wm.g4x.raw[level];
5786                 raw->plane[PLANE_PRIMARY] = active->sr.plane;
5787                 raw->plane[PLANE_CURSOR] = active->sr.cursor;
5788                 raw->plane[PLANE_SPRITE0] = 0;
5789                 raw->fbc = active->sr.fbc;
5790
5791                 if (++level > max_level)
5792                         goto out;
5793
5794                 raw = &crtc_state->wm.g4x.raw[level];
5795                 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
5796                 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
5797                 raw->plane[PLANE_SPRITE0] = 0;
5798                 raw->fbc = active->hpll.fbc;
5799
5800         out:
5801                 for_each_plane_id_on_crtc(crtc, plane_id)
5802                         g4x_raw_plane_wm_set(crtc_state, level,
5803                                              plane_id, USHRT_MAX);
5804                 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
5805
5806                 crtc_state->wm.g4x.optimal = *active;
5807                 crtc_state->wm.g4x.intermediate = *active;
5808
5809                 DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
5810                               pipe_name(pipe),
5811                               wm->pipe[pipe].plane[PLANE_PRIMARY],
5812                               wm->pipe[pipe].plane[PLANE_CURSOR],
5813                               wm->pipe[pipe].plane[PLANE_SPRITE0]);
5814         }
5815
5816         DRM_DEBUG_KMS("Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
5817                       wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
5818         DRM_DEBUG_KMS("Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
5819                       wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
5820         DRM_DEBUG_KMS("Initial SR=%s HPLL=%s FBC=%s\n",
5821                       yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
5822 }
5823
5824 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
5825 {
5826         struct intel_plane *plane;
5827         struct intel_crtc *crtc;
5828
5829         mutex_lock(&dev_priv->wm.wm_mutex);
5830
5831         for_each_intel_plane(&dev_priv->drm, plane) {
5832                 struct intel_crtc *crtc =
5833                         intel_get_crtc_for_pipe(dev_priv, plane->pipe);
5834                 struct intel_crtc_state *crtc_state =
5835                         to_intel_crtc_state(crtc->base.state);
5836                 struct intel_plane_state *plane_state =
5837                         to_intel_plane_state(plane->base.state);
5838                 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
5839                 enum plane_id plane_id = plane->id;
5840                 int level;
5841
5842                 if (plane_state->base.visible)
5843                         continue;
5844
5845                 for (level = 0; level < 3; level++) {
5846                         struct g4x_pipe_wm *raw =
5847                                 &crtc_state->wm.g4x.raw[level];
5848
5849                         raw->plane[plane_id] = 0;
5850                         wm_state->wm.plane[plane_id] = 0;
5851                 }
5852
5853                 if (plane_id == PLANE_PRIMARY) {
5854                         for (level = 0; level < 3; level++) {
5855                                 struct g4x_pipe_wm *raw =
5856                                         &crtc_state->wm.g4x.raw[level];
5857                                 raw->fbc = 0;
5858                         }
5859
5860                         wm_state->sr.fbc = 0;
5861                         wm_state->hpll.fbc = 0;
5862                         wm_state->fbc_en = false;
5863                 }
5864         }
5865
5866         for_each_intel_crtc(&dev_priv->drm, crtc) {
5867                 struct intel_crtc_state *crtc_state =
5868                         to_intel_crtc_state(crtc->base.state);
5869
5870                 crtc_state->wm.g4x.intermediate =
5871                         crtc_state->wm.g4x.optimal;
5872                 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
5873         }
5874
5875         g4x_program_watermarks(dev_priv);
5876
5877         mutex_unlock(&dev_priv->wm.wm_mutex);
5878 }
5879
5880 void vlv_wm_get_hw_state(struct drm_device *dev)
5881 {
5882         struct drm_i915_private *dev_priv = to_i915(dev);
5883         struct vlv_wm_values *wm = &dev_priv->wm.vlv;
5884         struct intel_crtc *crtc;
5885         u32 val;
5886
5887         vlv_read_wm_values(dev_priv, wm);
5888
5889         wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
5890         wm->level = VLV_WM_LEVEL_PM2;
5891
5892         if (IS_CHERRYVIEW(dev_priv)) {
5893                 mutex_lock(&dev_priv->pcu_lock);
5894
5895                 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
5896                 if (val & DSP_MAXFIFO_PM5_ENABLE)
5897                         wm->level = VLV_WM_LEVEL_PM5;
5898
5899                 /*
5900                  * If DDR DVFS is disabled in the BIOS, Punit
5901                  * will never ack the request. So if that happens
5902                  * assume we don't have to enable/disable DDR DVFS
5903                  * dynamically. To test that just set the REQ_ACK
5904                  * bit to poke the Punit, but don't change the
5905                  * HIGH/LOW bits so that we don't actually change
5906                  * the current state.
5907                  */
5908                 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
5909                 val |= FORCE_DDR_FREQ_REQ_ACK;
5910                 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
5911
5912                 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
5913                               FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
5914                         DRM_DEBUG_KMS("Punit not acking DDR DVFS request, "
5915                                       "assuming DDR DVFS is disabled\n");
5916                         dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
5917                 } else {
5918                         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
5919                         if ((val & FORCE_DDR_HIGH_FREQ) == 0)
5920                                 wm->level = VLV_WM_LEVEL_DDR_DVFS;
5921                 }
5922
5923                 mutex_unlock(&dev_priv->pcu_lock);
5924         }
5925
5926         for_each_intel_crtc(dev, crtc) {
5927                 struct intel_crtc_state *crtc_state =
5928                         to_intel_crtc_state(crtc->base.state);
5929                 struct vlv_wm_state *active = &crtc->wm.active.vlv;
5930                 const struct vlv_fifo_state *fifo_state =
5931                         &crtc_state->wm.vlv.fifo_state;
5932                 enum pipe pipe = crtc->pipe;
5933                 enum plane_id plane_id;
5934                 int level;
5935
5936                 vlv_get_fifo_size(crtc_state);
5937
5938                 active->num_levels = wm->level + 1;
5939                 active->cxsr = wm->cxsr;
5940
5941                 for (level = 0; level < active->num_levels; level++) {
5942                         struct g4x_pipe_wm *raw =
5943                                 &crtc_state->wm.vlv.raw[level];
5944
5945                         active->sr[level].plane = wm->sr.plane;
5946                         active->sr[level].cursor = wm->sr.cursor;
5947
5948                         for_each_plane_id_on_crtc(crtc, plane_id) {
5949                                 active->wm[level].plane[plane_id] =
5950                                         wm->pipe[pipe].plane[plane_id];
5951
5952                                 raw->plane[plane_id] =
5953                                         vlv_invert_wm_value(active->wm[level].plane[plane_id],
5954                                                             fifo_state->plane[plane_id]);
5955                         }
5956                 }
5957
5958                 for_each_plane_id_on_crtc(crtc, plane_id)
5959                         vlv_raw_plane_wm_set(crtc_state, level,
5960                                              plane_id, USHRT_MAX);
5961                 vlv_invalidate_wms(crtc, active, level);
5962
5963                 crtc_state->wm.vlv.optimal = *active;
5964                 crtc_state->wm.vlv.intermediate = *active;
5965
5966                 DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
5967                               pipe_name(pipe),
5968                               wm->pipe[pipe].plane[PLANE_PRIMARY],
5969                               wm->pipe[pipe].plane[PLANE_CURSOR],
5970                               wm->pipe[pipe].plane[PLANE_SPRITE0],
5971                               wm->pipe[pipe].plane[PLANE_SPRITE1]);
5972         }
5973
5974         DRM_DEBUG_KMS("Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
5975                       wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
5976 }
5977
5978 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
5979 {
5980         struct intel_plane *plane;
5981         struct intel_crtc *crtc;
5982
5983         mutex_lock(&dev_priv->wm.wm_mutex);
5984
5985         for_each_intel_plane(&dev_priv->drm, plane) {
5986                 struct intel_crtc *crtc =
5987                         intel_get_crtc_for_pipe(dev_priv, plane->pipe);
5988                 struct intel_crtc_state *crtc_state =
5989                         to_intel_crtc_state(crtc->base.state);
5990                 struct intel_plane_state *plane_state =
5991                         to_intel_plane_state(plane->base.state);
5992                 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
5993                 const struct vlv_fifo_state *fifo_state =
5994                         &crtc_state->wm.vlv.fifo_state;
5995                 enum plane_id plane_id = plane->id;
5996                 int level;
5997
5998                 if (plane_state->base.visible)
5999                         continue;
6000
6001                 for (level = 0; level < wm_state->num_levels; level++) {
6002                         struct g4x_pipe_wm *raw =
6003                                 &crtc_state->wm.vlv.raw[level];
6004
6005                         raw->plane[plane_id] = 0;
6006
6007                         wm_state->wm[level].plane[plane_id] =
6008                                 vlv_invert_wm_value(raw->plane[plane_id],
6009                                                     fifo_state->plane[plane_id]);
6010                 }
6011         }
6012
6013         for_each_intel_crtc(&dev_priv->drm, crtc) {
6014                 struct intel_crtc_state *crtc_state =
6015                         to_intel_crtc_state(crtc->base.state);
6016
6017                 crtc_state->wm.vlv.intermediate =
6018                         crtc_state->wm.vlv.optimal;
6019                 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
6020         }
6021
6022         vlv_program_watermarks(dev_priv);
6023
6024         mutex_unlock(&dev_priv->wm.wm_mutex);
6025 }
6026
6027 /*
6028  * FIXME should probably kill this and improve
6029  * the real watermark readout/sanitation instead
6030  */
6031 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
6032 {
6033         I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
6034         I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
6035         I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
6036
6037         /*
6038          * Don't touch WM1S_LP_EN here.
6039          * Doing so could cause underruns.
6040          */
6041 }
6042
6043 void ilk_wm_get_hw_state(struct drm_device *dev)
6044 {
6045         struct drm_i915_private *dev_priv = to_i915(dev);
6046         struct ilk_wm_values *hw = &dev_priv->wm.hw;
6047         struct drm_crtc *crtc;
6048
6049         ilk_init_lp_watermarks(dev_priv);
6050
6051         for_each_crtc(dev, crtc)
6052                 ilk_pipe_wm_get_hw_state(crtc);
6053
6054         hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
6055         hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
6056         hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
6057
6058         hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
6059         if (INTEL_GEN(dev_priv) >= 7) {
6060                 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
6061                 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
6062         }
6063
6064         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6065                 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
6066                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6067         else if (IS_IVYBRIDGE(dev_priv))
6068                 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
6069                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
6070
6071         hw->enable_fbc_wm =
6072                 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
6073 }
6074
6075 /**
6076  * intel_update_watermarks - update FIFO watermark values based on current modes
6077  * @crtc: the #intel_crtc on which to compute the WM
6078  *
6079  * Calculate watermark values for the various WM regs based on current mode
6080  * and plane configuration.
6081  *
6082  * There are several cases to deal with here:
6083  *   - normal (i.e. non-self-refresh)
6084  *   - self-refresh (SR) mode
6085  *   - lines are large relative to FIFO size (buffer can hold up to 2)
6086  *   - lines are small relative to FIFO size (buffer can hold more than 2
6087  *     lines), so need to account for TLB latency
6088  *
6089  *   The normal calculation is:
6090  *     watermark = dotclock * bytes per pixel * latency
6091  *   where latency is platform & configuration dependent (we assume pessimal
6092  *   values here).
6093  *
6094  *   The SR calculation is:
6095  *     watermark = (trunc(latency/line time)+1) * surface width *
6096  *       bytes per pixel
6097  *   where
6098  *     line time = htotal / dotclock
6099  *     surface width = hdisplay for normal plane and 64 for cursor
6100  *   and latency is assumed to be high, as above.
6101  *
6102  * The final value programmed to the register should always be rounded up,
6103  * and include an extra 2 entries to account for clock crossings.
6104  *
6105  * We don't use the sprite, so we can ignore that.  And on Crestline we have
6106  * to set the non-SR watermarks to 8.
6107  */
6108 void intel_update_watermarks(struct intel_crtc *crtc)
6109 {
6110         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6111
6112         if (dev_priv->display.update_wm)
6113                 dev_priv->display.update_wm(crtc);
6114 }
6115
6116 void intel_enable_ipc(struct drm_i915_private *dev_priv)
6117 {
6118         u32 val;
6119
6120         /* Display WA #0477 WaDisableIPC: skl */
6121         if (IS_SKYLAKE(dev_priv))
6122                 dev_priv->ipc_enabled = false;
6123
6124         /* Display WA #1141: SKL:all KBL:all CFL */
6125         if ((IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) &&
6126             !dev_priv->dram_info.symmetric_memory)
6127                 dev_priv->ipc_enabled = false;
6128
6129         val = I915_READ(DISP_ARB_CTL2);
6130
6131         if (dev_priv->ipc_enabled)
6132                 val |= DISP_IPC_ENABLE;
6133         else
6134                 val &= ~DISP_IPC_ENABLE;
6135
6136         I915_WRITE(DISP_ARB_CTL2, val);
6137 }
6138
6139 void intel_init_ipc(struct drm_i915_private *dev_priv)
6140 {
6141         dev_priv->ipc_enabled = false;
6142         if (!HAS_IPC(dev_priv))
6143                 return;
6144
6145         dev_priv->ipc_enabled = true;
6146         intel_enable_ipc(dev_priv);
6147 }
6148
6149 /*
6150  * Lock protecting IPS related data structures
6151  */
6152 DEFINE_SPINLOCK(mchdev_lock);
6153
6154 /* Global for IPS driver to get at the current i915 device. Protected by
6155  * mchdev_lock. */
6156 static struct drm_i915_private *i915_mch_dev;
6157
6158 bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val)
6159 {
6160         u16 rgvswctl;
6161
6162         lockdep_assert_held(&mchdev_lock);
6163
6164         rgvswctl = I915_READ16(MEMSWCTL);
6165         if (rgvswctl & MEMCTL_CMD_STS) {
6166                 DRM_DEBUG("gpu busy, RCS change rejected\n");
6167                 return false; /* still busy with another command */
6168         }
6169
6170         rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
6171                 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
6172         I915_WRITE16(MEMSWCTL, rgvswctl);
6173         POSTING_READ16(MEMSWCTL);
6174
6175         rgvswctl |= MEMCTL_CMD_STS;
6176         I915_WRITE16(MEMSWCTL, rgvswctl);
6177
6178         return true;
6179 }
6180
6181 static void ironlake_enable_drps(struct drm_i915_private *dev_priv)
6182 {
6183         u32 rgvmodectl;
6184         u8 fmax, fmin, fstart, vstart;
6185
6186         spin_lock_irq(&mchdev_lock);
6187
6188         rgvmodectl = I915_READ(MEMMODECTL);
6189
6190         /* Enable temp reporting */
6191         I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
6192         I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
6193
6194         /* 100ms RC evaluation intervals */
6195         I915_WRITE(RCUPEI, 100000);
6196         I915_WRITE(RCDNEI, 100000);
6197
6198         /* Set max/min thresholds to 90ms and 80ms respectively */
6199         I915_WRITE(RCBMAXAVG, 90000);
6200         I915_WRITE(RCBMINAVG, 80000);
6201
6202         I915_WRITE(MEMIHYST, 1);
6203
6204         /* Set up min, max, and cur for interrupt handling */
6205         fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
6206         fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
6207         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
6208                 MEMMODE_FSTART_SHIFT;
6209
6210         vstart = (I915_READ(PXVFREQ(fstart)) & PXVFREQ_PX_MASK) >>
6211                 PXVFREQ_PX_SHIFT;
6212
6213         dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
6214         dev_priv->ips.fstart = fstart;
6215
6216         dev_priv->ips.max_delay = fstart;
6217         dev_priv->ips.min_delay = fmin;
6218         dev_priv->ips.cur_delay = fstart;
6219
6220         DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
6221                          fmax, fmin, fstart);
6222
6223         I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
6224
6225         /*
6226          * Interrupts will be enabled in ironlake_irq_postinstall
6227          */
6228
6229         I915_WRITE(VIDSTART, vstart);
6230         POSTING_READ(VIDSTART);
6231
6232         rgvmodectl |= MEMMODE_SWMODE_EN;
6233         I915_WRITE(MEMMODECTL, rgvmodectl);
6234
6235         if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
6236                 DRM_ERROR("stuck trying to change perf mode\n");
6237         mdelay(1);
6238
6239         ironlake_set_drps(dev_priv, fstart);
6240
6241         dev_priv->ips.last_count1 = I915_READ(DMIEC) +
6242                 I915_READ(DDREC) + I915_READ(CSIEC);
6243         dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
6244         dev_priv->ips.last_count2 = I915_READ(GFXEC);
6245         dev_priv->ips.last_time2 = ktime_get_raw_ns();
6246
6247         spin_unlock_irq(&mchdev_lock);
6248 }
6249
6250 static void ironlake_disable_drps(struct drm_i915_private *dev_priv)
6251 {
6252         u16 rgvswctl;
6253
6254         spin_lock_irq(&mchdev_lock);
6255
6256         rgvswctl = I915_READ16(MEMSWCTL);
6257
6258         /* Ack interrupts, disable EFC interrupt */
6259         I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
6260         I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
6261         I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
6262         I915_WRITE(DEIIR, DE_PCU_EVENT);
6263         I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
6264
6265         /* Go back to the starting frequency */
6266         ironlake_set_drps(dev_priv, dev_priv->ips.fstart);
6267         mdelay(1);
6268         rgvswctl |= MEMCTL_CMD_STS;
6269         I915_WRITE(MEMSWCTL, rgvswctl);
6270         mdelay(1);
6271
6272         spin_unlock_irq(&mchdev_lock);
6273 }
6274
6275 /* There's a funny hw issue where the hw returns all 0 when reading from
6276  * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
6277  * ourselves, instead of doing a rmw cycle (which might result in us clearing
6278  * all limits and the gpu stuck at whatever frequency it is at atm).
6279  */
6280 static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val)
6281 {
6282         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6283         u32 limits;
6284
6285         /* Only set the down limit when we've reached the lowest level to avoid
6286          * getting more interrupts, otherwise leave this clear. This prevents a
6287          * race in the hw when coming out of rc6: There's a tiny window where
6288          * the hw runs at the minimal clock before selecting the desired
6289          * frequency, if the down threshold expires in that window we will not
6290          * receive a down interrupt. */
6291         if (INTEL_GEN(dev_priv) >= 9) {
6292                 limits = (rps->max_freq_softlimit) << 23;
6293                 if (val <= rps->min_freq_softlimit)
6294                         limits |= (rps->min_freq_softlimit) << 14;
6295         } else {
6296                 limits = rps->max_freq_softlimit << 24;
6297                 if (val <= rps->min_freq_softlimit)
6298                         limits |= rps->min_freq_softlimit << 16;
6299         }
6300
6301         return limits;
6302 }
6303
6304 static void rps_set_power(struct drm_i915_private *dev_priv, int new_power)
6305 {
6306         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6307         u32 threshold_up = 0, threshold_down = 0; /* in % */
6308         u32 ei_up = 0, ei_down = 0;
6309
6310         lockdep_assert_held(&rps->power.mutex);
6311
6312         if (new_power == rps->power.mode)
6313                 return;
6314
6315         /* Note the units here are not exactly 1us, but 1280ns. */
6316         switch (new_power) {
6317         case LOW_POWER:
6318                 /* Upclock if more than 95% busy over 16ms */
6319                 ei_up = 16000;
6320                 threshold_up = 95;
6321
6322                 /* Downclock if less than 85% busy over 32ms */
6323                 ei_down = 32000;
6324                 threshold_down = 85;
6325                 break;
6326
6327         case BETWEEN:
6328                 /* Upclock if more than 90% busy over 13ms */
6329                 ei_up = 13000;
6330                 threshold_up = 90;
6331
6332                 /* Downclock if less than 75% busy over 32ms */
6333                 ei_down = 32000;
6334                 threshold_down = 75;
6335                 break;
6336
6337         case HIGH_POWER:
6338                 /* Upclock if more than 85% busy over 10ms */
6339                 ei_up = 10000;
6340                 threshold_up = 85;
6341
6342                 /* Downclock if less than 60% busy over 32ms */
6343                 ei_down = 32000;
6344                 threshold_down = 60;
6345                 break;
6346         }
6347
6348         /* When byt can survive without system hang with dynamic
6349          * sw freq adjustments, this restriction can be lifted.
6350          */
6351         if (IS_VALLEYVIEW(dev_priv))
6352                 goto skip_hw_write;
6353
6354         I915_WRITE(GEN6_RP_UP_EI,
6355                    GT_INTERVAL_FROM_US(dev_priv, ei_up));
6356         I915_WRITE(GEN6_RP_UP_THRESHOLD,
6357                    GT_INTERVAL_FROM_US(dev_priv,
6358                                        ei_up * threshold_up / 100));
6359
6360         I915_WRITE(GEN6_RP_DOWN_EI,
6361                    GT_INTERVAL_FROM_US(dev_priv, ei_down));
6362         I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
6363                    GT_INTERVAL_FROM_US(dev_priv,
6364                                        ei_down * threshold_down / 100));
6365
6366         I915_WRITE(GEN6_RP_CONTROL,
6367                    GEN6_RP_MEDIA_TURBO |
6368                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
6369                    GEN6_RP_MEDIA_IS_GFX |
6370                    GEN6_RP_ENABLE |
6371                    GEN6_RP_UP_BUSY_AVG |
6372                    GEN6_RP_DOWN_IDLE_AVG);
6373
6374 skip_hw_write:
6375         rps->power.mode = new_power;
6376         rps->power.up_threshold = threshold_up;
6377         rps->power.down_threshold = threshold_down;
6378 }
6379
6380 static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
6381 {
6382         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6383         int new_power;
6384
6385         new_power = rps->power.mode;
6386         switch (rps->power.mode) {
6387         case LOW_POWER:
6388                 if (val > rps->efficient_freq + 1 &&
6389                     val > rps->cur_freq)
6390                         new_power = BETWEEN;
6391                 break;
6392
6393         case BETWEEN:
6394                 if (val <= rps->efficient_freq &&
6395                     val < rps->cur_freq)
6396                         new_power = LOW_POWER;
6397                 else if (val >= rps->rp0_freq &&
6398                          val > rps->cur_freq)
6399                         new_power = HIGH_POWER;
6400                 break;
6401
6402         case HIGH_POWER:
6403                 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
6404                     val < rps->cur_freq)
6405                         new_power = BETWEEN;
6406                 break;
6407         }
6408         /* Max/min bins are special */
6409         if (val <= rps->min_freq_softlimit)
6410                 new_power = LOW_POWER;
6411         if (val >= rps->max_freq_softlimit)
6412                 new_power = HIGH_POWER;
6413
6414         mutex_lock(&rps->power.mutex);
6415         if (rps->power.interactive)
6416                 new_power = HIGH_POWER;
6417         rps_set_power(dev_priv, new_power);
6418         mutex_unlock(&rps->power.mutex);
6419 }
6420
6421 void intel_rps_mark_interactive(struct drm_i915_private *i915, bool interactive)
6422 {
6423         struct intel_rps *rps = &i915->gt_pm.rps;
6424
6425         if (INTEL_GEN(i915) < 6)
6426                 return;
6427
6428         mutex_lock(&rps->power.mutex);
6429         if (interactive) {
6430                 if (!rps->power.interactive++ && READ_ONCE(i915->gt.awake))
6431                         rps_set_power(i915, HIGH_POWER);
6432         } else {
6433                 GEM_BUG_ON(!rps->power.interactive);
6434                 rps->power.interactive--;
6435         }
6436         mutex_unlock(&rps->power.mutex);
6437 }
6438
6439 static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
6440 {
6441         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6442         u32 mask = 0;
6443
6444         /* We use UP_EI_EXPIRED interupts for both up/down in manual mode */
6445         if (val > rps->min_freq_softlimit)
6446                 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT;
6447         if (val < rps->max_freq_softlimit)
6448                 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
6449
6450         mask &= dev_priv->pm_rps_events;
6451
6452         return gen6_sanitize_rps_pm_mask(dev_priv, ~mask);
6453 }
6454
6455 /* gen6_set_rps is called to update the frequency request, but should also be
6456  * called when the range (min_delay and max_delay) is modified so that we can
6457  * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
6458 static int gen6_set_rps(struct drm_i915_private *dev_priv, u8 val)
6459 {
6460         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6461
6462         /* min/max delay may still have been modified so be sure to
6463          * write the limits value.
6464          */
6465         if (val != rps->cur_freq) {
6466                 gen6_set_rps_thresholds(dev_priv, val);
6467
6468                 if (INTEL_GEN(dev_priv) >= 9)
6469                         I915_WRITE(GEN6_RPNSWREQ,
6470                                    GEN9_FREQUENCY(val));
6471                 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6472                         I915_WRITE(GEN6_RPNSWREQ,
6473                                    HSW_FREQUENCY(val));
6474                 else
6475                         I915_WRITE(GEN6_RPNSWREQ,
6476                                    GEN6_FREQUENCY(val) |
6477                                    GEN6_OFFSET(0) |
6478                                    GEN6_AGGRESSIVE_TURBO);
6479         }
6480
6481         /* Make sure we continue to get interrupts
6482          * until we hit the minimum or maximum frequencies.
6483          */
6484         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val));
6485         I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
6486
6487         rps->cur_freq = val;
6488         trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
6489
6490         return 0;
6491 }
6492
6493 static int valleyview_set_rps(struct drm_i915_private *dev_priv, u8 val)
6494 {
6495         int err;
6496
6497         if (WARN_ONCE(IS_CHERRYVIEW(dev_priv) && (val & 1),
6498                       "Odd GPU freq value\n"))
6499                 val &= ~1;
6500
6501         I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
6502
6503         if (val != dev_priv->gt_pm.rps.cur_freq) {
6504                 err = vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
6505                 if (err)
6506                         return err;
6507
6508                 gen6_set_rps_thresholds(dev_priv, val);
6509         }
6510
6511         dev_priv->gt_pm.rps.cur_freq = val;
6512         trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
6513
6514         return 0;
6515 }
6516
6517 /* vlv_set_rps_idle: Set the frequency to idle, if Gfx clocks are down
6518  *
6519  * * If Gfx is Idle, then
6520  * 1. Forcewake Media well.
6521  * 2. Request idle freq.
6522  * 3. Release Forcewake of Media well.
6523 */
6524 static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
6525 {
6526         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6527         u32 val = rps->idle_freq;
6528         int err;
6529
6530         if (rps->cur_freq <= val)
6531                 return;
6532
6533         /* The punit delays the write of the frequency and voltage until it
6534          * determines the GPU is awake. During normal usage we don't want to
6535          * waste power changing the frequency if the GPU is sleeping (rc6).
6536          * However, the GPU and driver is now idle and we do not want to delay
6537          * switching to minimum voltage (reducing power whilst idle) as we do
6538          * not expect to be woken in the near future and so must flush the
6539          * change by waking the device.
6540          *
6541          * We choose to take the media powerwell (either would do to trick the
6542          * punit into committing the voltage change) as that takes a lot less
6543          * power than the render powerwell.
6544          */
6545         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_MEDIA);
6546         err = valleyview_set_rps(dev_priv, val);
6547         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_MEDIA);
6548
6549         if (err)
6550                 DRM_ERROR("Failed to set RPS for idle\n");
6551 }
6552
6553 void gen6_rps_busy(struct drm_i915_private *dev_priv)
6554 {
6555         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6556
6557         mutex_lock(&dev_priv->pcu_lock);
6558         if (rps->enabled) {
6559                 u8 freq;
6560
6561                 if (dev_priv->pm_rps_events & GEN6_PM_RP_UP_EI_EXPIRED)
6562                         gen6_rps_reset_ei(dev_priv);
6563                 I915_WRITE(GEN6_PMINTRMSK,
6564                            gen6_rps_pm_mask(dev_priv, rps->cur_freq));
6565
6566                 gen6_enable_rps_interrupts(dev_priv);
6567
6568                 /* Use the user's desired frequency as a guide, but for better
6569                  * performance, jump directly to RPe as our starting frequency.
6570                  */
6571                 freq = max(rps->cur_freq,
6572                            rps->efficient_freq);
6573
6574                 if (intel_set_rps(dev_priv,
6575                                   clamp(freq,
6576                                         rps->min_freq_softlimit,
6577                                         rps->max_freq_softlimit)))
6578                         DRM_DEBUG_DRIVER("Failed to set idle frequency\n");
6579         }
6580         mutex_unlock(&dev_priv->pcu_lock);
6581 }
6582
6583 void gen6_rps_idle(struct drm_i915_private *dev_priv)
6584 {
6585         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6586
6587         /* Flush our bottom-half so that it does not race with us
6588          * setting the idle frequency and so that it is bounded by
6589          * our rpm wakeref. And then disable the interrupts to stop any
6590          * futher RPS reclocking whilst we are asleep.
6591          */
6592         gen6_disable_rps_interrupts(dev_priv);
6593
6594         mutex_lock(&dev_priv->pcu_lock);
6595         if (rps->enabled) {
6596                 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
6597                         vlv_set_rps_idle(dev_priv);
6598                 else
6599                         gen6_set_rps(dev_priv, rps->idle_freq);
6600                 rps->last_adj = 0;
6601                 I915_WRITE(GEN6_PMINTRMSK,
6602                            gen6_sanitize_rps_pm_mask(dev_priv, ~0));
6603         }
6604         mutex_unlock(&dev_priv->pcu_lock);
6605 }
6606
6607 void gen6_rps_boost(struct i915_request *rq,
6608                     struct intel_rps_client *rps_client)
6609 {
6610         struct intel_rps *rps = &rq->i915->gt_pm.rps;
6611         unsigned long flags;
6612         bool boost;
6613
6614         /* This is intentionally racy! We peek at the state here, then
6615          * validate inside the RPS worker.
6616          */
6617         if (!rps->enabled)
6618                 return;
6619
6620         if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags))
6621                 return;
6622
6623         /* Serializes with i915_request_retire() */
6624         boost = false;
6625         spin_lock_irqsave(&rq->lock, flags);
6626         if (!rq->waitboost && !dma_fence_is_signaled_locked(&rq->fence)) {
6627                 boost = !atomic_fetch_inc(&rps->num_waiters);
6628                 rq->waitboost = true;
6629         }
6630         spin_unlock_irqrestore(&rq->lock, flags);
6631         if (!boost)
6632                 return;
6633
6634         if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
6635                 schedule_work(&rps->work);
6636
6637         atomic_inc(rps_client ? &rps_client->boosts : &rps->boosts);
6638 }
6639
6640 int intel_set_rps(struct drm_i915_private *dev_priv, u8 val)
6641 {
6642         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6643         int err;
6644
6645         lockdep_assert_held(&dev_priv->pcu_lock);
6646         GEM_BUG_ON(val > rps->max_freq);
6647         GEM_BUG_ON(val < rps->min_freq);
6648
6649         if (!rps->enabled) {
6650                 rps->cur_freq = val;
6651                 return 0;
6652         }
6653
6654         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
6655                 err = valleyview_set_rps(dev_priv, val);
6656         else
6657                 err = gen6_set_rps(dev_priv, val);
6658
6659         return err;
6660 }
6661
6662 static void gen9_disable_rc6(struct drm_i915_private *dev_priv)
6663 {
6664         I915_WRITE(GEN6_RC_CONTROL, 0);
6665         I915_WRITE(GEN9_PG_ENABLE, 0);
6666 }
6667
6668 static void gen9_disable_rps(struct drm_i915_private *dev_priv)
6669 {
6670         I915_WRITE(GEN6_RP_CONTROL, 0);
6671 }
6672
6673 static void gen6_disable_rc6(struct drm_i915_private *dev_priv)
6674 {
6675         I915_WRITE(GEN6_RC_CONTROL, 0);
6676 }
6677
6678 static void gen6_disable_rps(struct drm_i915_private *dev_priv)
6679 {
6680         I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
6681         I915_WRITE(GEN6_RP_CONTROL, 0);
6682 }
6683
6684 static void cherryview_disable_rc6(struct drm_i915_private *dev_priv)
6685 {
6686         I915_WRITE(GEN6_RC_CONTROL, 0);
6687 }
6688
6689 static void cherryview_disable_rps(struct drm_i915_private *dev_priv)
6690 {
6691         I915_WRITE(GEN6_RP_CONTROL, 0);
6692 }
6693
6694 static void valleyview_disable_rc6(struct drm_i915_private *dev_priv)
6695 {
6696         /* We're doing forcewake before Disabling RC6,
6697          * This what the BIOS expects when going into suspend */
6698         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6699
6700         I915_WRITE(GEN6_RC_CONTROL, 0);
6701
6702         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6703 }
6704
6705 static void valleyview_disable_rps(struct drm_i915_private *dev_priv)
6706 {
6707         I915_WRITE(GEN6_RP_CONTROL, 0);
6708 }
6709
6710 static bool bxt_check_bios_rc6_setup(struct drm_i915_private *dev_priv)
6711 {
6712         bool enable_rc6 = true;
6713         unsigned long rc6_ctx_base;
6714         u32 rc_ctl;
6715         int rc_sw_target;
6716
6717         rc_ctl = I915_READ(GEN6_RC_CONTROL);
6718         rc_sw_target = (I915_READ(GEN6_RC_STATE) & RC_SW_TARGET_STATE_MASK) >>
6719                        RC_SW_TARGET_STATE_SHIFT;
6720         DRM_DEBUG_DRIVER("BIOS enabled RC states: "
6721                          "HW_CTRL %s HW_RC6 %s SW_TARGET_STATE %x\n",
6722                          onoff(rc_ctl & GEN6_RC_CTL_HW_ENABLE),
6723                          onoff(rc_ctl & GEN6_RC_CTL_RC6_ENABLE),
6724                          rc_sw_target);
6725
6726         if (!(I915_READ(RC6_LOCATION) & RC6_CTX_IN_DRAM)) {
6727                 DRM_DEBUG_DRIVER("RC6 Base location not set properly.\n");
6728                 enable_rc6 = false;
6729         }
6730
6731         /*
6732          * The exact context size is not known for BXT, so assume a page size
6733          * for this check.
6734          */
6735         rc6_ctx_base = I915_READ(RC6_CTX_BASE) & RC6_CTX_BASE_MASK;
6736         if (!((rc6_ctx_base >= dev_priv->dsm_reserved.start) &&
6737               (rc6_ctx_base + PAGE_SIZE < dev_priv->dsm_reserved.end))) {
6738                 DRM_DEBUG_DRIVER("RC6 Base address not as expected.\n");
6739                 enable_rc6 = false;
6740         }
6741
6742         if (!(((I915_READ(PWRCTX_MAXCNT_RCSUNIT) & IDLE_TIME_MASK) > 1) &&
6743               ((I915_READ(PWRCTX_MAXCNT_VCSUNIT0) & IDLE_TIME_MASK) > 1) &&
6744               ((I915_READ(PWRCTX_MAXCNT_BCSUNIT) & IDLE_TIME_MASK) > 1) &&
6745               ((I915_READ(PWRCTX_MAXCNT_VECSUNIT) & IDLE_TIME_MASK) > 1))) {
6746                 DRM_DEBUG_DRIVER("Engine Idle wait time not set properly.\n");
6747                 enable_rc6 = false;
6748         }
6749
6750         if (!I915_READ(GEN8_PUSHBUS_CONTROL) ||
6751             !I915_READ(GEN8_PUSHBUS_ENABLE) ||
6752             !I915_READ(GEN8_PUSHBUS_SHIFT)) {
6753                 DRM_DEBUG_DRIVER("Pushbus not setup properly.\n");
6754                 enable_rc6 = false;
6755         }
6756
6757         if (!I915_READ(GEN6_GFXPAUSE)) {
6758                 DRM_DEBUG_DRIVER("GFX pause not setup properly.\n");
6759                 enable_rc6 = false;
6760         }
6761
6762         if (!I915_READ(GEN8_MISC_CTRL0)) {
6763                 DRM_DEBUG_DRIVER("GPM control not setup properly.\n");
6764                 enable_rc6 = false;
6765         }
6766
6767         return enable_rc6;
6768 }
6769
6770 static bool sanitize_rc6(struct drm_i915_private *i915)
6771 {
6772         struct intel_device_info *info = mkwrite_device_info(i915);
6773
6774         /* Powersaving is controlled by the host when inside a VM */
6775         if (intel_vgpu_active(i915))
6776                 info->has_rc6 = 0;
6777
6778         if (info->has_rc6 &&
6779             IS_GEN9_LP(i915) && !bxt_check_bios_rc6_setup(i915)) {
6780                 DRM_INFO("RC6 disabled by BIOS\n");
6781                 info->has_rc6 = 0;
6782         }
6783
6784         /*
6785          * We assume that we do not have any deep rc6 levels if we don't have
6786          * have the previous rc6 level supported, i.e. we use HAS_RC6()
6787          * as the initial coarse check for rc6 in general, moving on to
6788          * progressively finer/deeper levels.
6789          */
6790         if (!info->has_rc6 && info->has_rc6p)
6791                 info->has_rc6p = 0;
6792
6793         return info->has_rc6;
6794 }
6795
6796 static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv)
6797 {
6798         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6799
6800         /* All of these values are in units of 50MHz */
6801
6802         /* static values from HW: RP0 > RP1 > RPn (min_freq) */
6803         if (IS_GEN9_LP(dev_priv)) {
6804                 u32 rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
6805                 rps->rp0_freq = (rp_state_cap >> 16) & 0xff;
6806                 rps->rp1_freq = (rp_state_cap >>  8) & 0xff;
6807                 rps->min_freq = (rp_state_cap >>  0) & 0xff;
6808         } else {
6809                 u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
6810                 rps->rp0_freq = (rp_state_cap >>  0) & 0xff;
6811                 rps->rp1_freq = (rp_state_cap >>  8) & 0xff;
6812                 rps->min_freq = (rp_state_cap >> 16) & 0xff;
6813         }
6814         /* hw_max = RP0 until we check for overclocking */
6815         rps->max_freq = rps->rp0_freq;
6816
6817         rps->efficient_freq = rps->rp1_freq;
6818         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) ||
6819             IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
6820                 u32 ddcc_status = 0;
6821
6822                 if (sandybridge_pcode_read(dev_priv,
6823                                            HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
6824                                            &ddcc_status) == 0)
6825                         rps->efficient_freq =
6826                                 clamp_t(u8,
6827                                         ((ddcc_status >> 8) & 0xff),
6828                                         rps->min_freq,
6829                                         rps->max_freq);
6830         }
6831
6832         if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
6833                 /* Store the frequency values in 16.66 MHZ units, which is
6834                  * the natural hardware unit for SKL
6835                  */
6836                 rps->rp0_freq *= GEN9_FREQ_SCALER;
6837                 rps->rp1_freq *= GEN9_FREQ_SCALER;
6838                 rps->min_freq *= GEN9_FREQ_SCALER;
6839                 rps->max_freq *= GEN9_FREQ_SCALER;
6840                 rps->efficient_freq *= GEN9_FREQ_SCALER;
6841         }
6842 }
6843
6844 static void reset_rps(struct drm_i915_private *dev_priv,
6845                       int (*set)(struct drm_i915_private *, u8))
6846 {
6847         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6848         u8 freq = rps->cur_freq;
6849
6850         /* force a reset */
6851         rps->power.mode = -1;
6852         rps->cur_freq = -1;
6853
6854         if (set(dev_priv, freq))
6855                 DRM_ERROR("Failed to reset RPS to initial values\n");
6856 }
6857
6858 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
6859 static void gen9_enable_rps(struct drm_i915_private *dev_priv)
6860 {
6861         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6862
6863         /* Program defaults and thresholds for RPS */
6864         if (IS_GEN9(dev_priv))
6865                 I915_WRITE(GEN6_RC_VIDEO_FREQ,
6866                         GEN9_FREQUENCY(dev_priv->gt_pm.rps.rp1_freq));
6867
6868         /* 1 second timeout*/
6869         I915_WRITE(GEN6_RP_DOWN_TIMEOUT,
6870                 GT_INTERVAL_FROM_US(dev_priv, 1000000));
6871
6872         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa);
6873
6874         /* Leaning on the below call to gen6_set_rps to program/setup the
6875          * Up/Down EI & threshold registers, as well as the RP_CONTROL,
6876          * RP_INTERRUPT_LIMITS & RPNSWREQ registers */
6877         reset_rps(dev_priv, gen6_set_rps);
6878
6879         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6880 }
6881
6882 static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
6883 {
6884         struct intel_engine_cs *engine;
6885         enum intel_engine_id id;
6886         u32 rc6_mode;
6887
6888         /* 1a: Software RC state - RC0 */
6889         I915_WRITE(GEN6_RC_STATE, 0);
6890
6891         /* 1b: Get forcewake during program sequence. Although the driver
6892          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6893         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6894
6895         /* 2a: Disable RC states. */
6896         I915_WRITE(GEN6_RC_CONTROL, 0);
6897
6898         /* 2b: Program RC6 thresholds.*/
6899         if (INTEL_GEN(dev_priv) >= 10) {
6900                 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85);
6901                 I915_WRITE(GEN10_MEDIA_WAKE_RATE_LIMIT, 150);
6902         } else if (IS_SKYLAKE(dev_priv)) {
6903                 /*
6904                  * WaRsDoubleRc6WrlWithCoarsePowerGating:skl Doubling WRL only
6905                  * when CPG is enabled
6906                  */
6907                 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
6908         } else {
6909                 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
6910         }
6911
6912         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
6913         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
6914         for_each_engine(engine, dev_priv, id)
6915                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
6916
6917         if (HAS_GUC(dev_priv))
6918                 I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
6919
6920         I915_WRITE(GEN6_RC_SLEEP, 0);
6921
6922         /*
6923          * 2c: Program Coarse Power Gating Policies.
6924          *
6925          * Bspec's guidance is to use 25us (really 25 * 1280ns) here. What we
6926          * use instead is a more conservative estimate for the maximum time
6927          * it takes us to service a CS interrupt and submit a new ELSP - that
6928          * is the time which the GPU is idle waiting for the CPU to select the
6929          * next request to execute. If the idle hysteresis is less than that
6930          * interrupt service latency, the hardware will automatically gate
6931          * the power well and we will then incur the wake up cost on top of
6932          * the service latency. A similar guide from intel_pstate is that we
6933          * do not want the enable hysteresis to less than the wakeup latency.
6934          *
6935          * igt/gem_exec_nop/sequential provides a rough estimate for the
6936          * service latency, and puts it around 10us for Broadwell (and other
6937          * big core) and around 40us for Broxton (and other low power cores).
6938          * [Note that for legacy ringbuffer submission, this is less than 1us!]
6939          * However, the wakeup latency on Broxton is closer to 100us. To be
6940          * conservative, we have to factor in a context switch on top (due
6941          * to ksoftirqd).
6942          */
6943         I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 250);
6944         I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 250);
6945
6946         /* 3a: Enable RC6 */
6947         I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
6948
6949         /* WaRsUseTimeoutMode:cnl (pre-prod) */
6950         if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_C0))
6951                 rc6_mode = GEN7_RC_CTL_TO_MODE;
6952         else
6953                 rc6_mode = GEN6_RC_CTL_EI_MODE(1);
6954
6955         I915_WRITE(GEN6_RC_CONTROL,
6956                    GEN6_RC_CTL_HW_ENABLE |
6957                    GEN6_RC_CTL_RC6_ENABLE |
6958                    rc6_mode);
6959
6960         /*
6961          * 3b: Enable Coarse Power Gating only when RC6 is enabled.
6962          * WaRsDisableCoarsePowerGating:skl,cnl - Render/Media PG need to be disabled with RC6.
6963          */
6964         if (NEEDS_WaRsDisableCoarsePowerGating(dev_priv))
6965                 I915_WRITE(GEN9_PG_ENABLE, 0);
6966         else
6967                 I915_WRITE(GEN9_PG_ENABLE,
6968                            GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE);
6969
6970         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6971 }
6972
6973 static void gen8_enable_rc6(struct drm_i915_private *dev_priv)
6974 {
6975         struct intel_engine_cs *engine;
6976         enum intel_engine_id id;
6977
6978         /* 1a: Software RC state - RC0 */
6979         I915_WRITE(GEN6_RC_STATE, 0);
6980
6981         /* 1b: Get forcewake during program sequence. Although the driver
6982          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6983         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6984
6985         /* 2a: Disable RC states. */
6986         I915_WRITE(GEN6_RC_CONTROL, 0);
6987
6988         /* 2b: Program RC6 thresholds.*/
6989         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
6990         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
6991         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
6992         for_each_engine(engine, dev_priv, id)
6993                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
6994         I915_WRITE(GEN6_RC_SLEEP, 0);
6995         I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
6996
6997         /* 3: Enable RC6 */
6998
6999         I915_WRITE(GEN6_RC_CONTROL,
7000                    GEN6_RC_CTL_HW_ENABLE |
7001                    GEN7_RC_CTL_TO_MODE |
7002                    GEN6_RC_CTL_RC6_ENABLE);
7003
7004         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7005 }
7006
7007 static void gen8_enable_rps(struct drm_i915_private *dev_priv)
7008 {
7009         struct intel_rps *rps = &dev_priv->gt_pm.rps;
7010
7011         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7012
7013         /* 1 Program defaults and thresholds for RPS*/
7014         I915_WRITE(GEN6_RPNSWREQ,
7015                    HSW_FREQUENCY(rps->rp1_freq));
7016         I915_WRITE(GEN6_RC_VIDEO_FREQ,
7017                    HSW_FREQUENCY(rps->rp1_freq));
7018         /* NB: Docs say 1s, and 1000000 - which aren't equivalent */
7019         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
7020
7021         /* Docs recommend 900MHz, and 300 MHz respectively */
7022         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
7023                    rps->max_freq_softlimit << 24 |
7024                    rps->min_freq_softlimit << 16);
7025
7026         I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
7027         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
7028         I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
7029         I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
7030
7031         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7032
7033         /* 2: Enable RPS */
7034         I915_WRITE(GEN6_RP_CONTROL,
7035                    GEN6_RP_MEDIA_TURBO |
7036                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
7037                    GEN6_RP_MEDIA_IS_GFX |
7038                    GEN6_RP_ENABLE |
7039                    GEN6_RP_UP_BUSY_AVG |
7040                    GEN6_RP_DOWN_IDLE_AVG);
7041
7042         reset_rps(dev_priv, gen6_set_rps);
7043
7044         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7045 }
7046
7047 static void gen6_enable_rc6(struct drm_i915_private *dev_priv)
7048 {
7049         struct intel_engine_cs *engine;
7050         enum intel_engine_id id;
7051         u32 rc6vids, rc6_mask;
7052         u32 gtfifodbg;
7053         int ret;
7054
7055         I915_WRITE(GEN6_RC_STATE, 0);
7056
7057         /* Clear the DBG now so we don't confuse earlier errors */
7058         gtfifodbg = I915_READ(GTFIFODBG);
7059         if (gtfifodbg) {
7060                 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
7061                 I915_WRITE(GTFIFODBG, gtfifodbg);
7062         }
7063
7064         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7065
7066         /* disable the counters and set deterministic thresholds */
7067         I915_WRITE(GEN6_RC_CONTROL, 0);
7068
7069         I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
7070         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
7071         I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
7072         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7073         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7074
7075         for_each_engine(engine, dev_priv, id)
7076                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7077
7078         I915_WRITE(GEN6_RC_SLEEP, 0);
7079         I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
7080         if (IS_IVYBRIDGE(dev_priv))
7081                 I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
7082         else
7083                 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
7084         I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
7085         I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
7086
7087         /* We don't use those on Haswell */
7088         rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
7089         if (HAS_RC6p(dev_priv))
7090                 rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
7091         if (HAS_RC6pp(dev_priv))
7092                 rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
7093         I915_WRITE(GEN6_RC_CONTROL,
7094                    rc6_mask |
7095                    GEN6_RC_CTL_EI_MODE(1) |
7096                    GEN6_RC_CTL_HW_ENABLE);
7097
7098         rc6vids = 0;
7099         ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
7100         if (IS_GEN6(dev_priv) && ret) {
7101                 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
7102         } else if (IS_GEN6(dev_priv) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
7103                 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
7104                           GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
7105                 rc6vids &= 0xffff00;
7106                 rc6vids |= GEN6_ENCODE_RC6_VID(450);
7107                 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
7108                 if (ret)
7109                         DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
7110         }
7111
7112         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7113 }
7114
7115 static void gen6_enable_rps(struct drm_i915_private *dev_priv)
7116 {
7117         /* Here begins a magic sequence of register writes to enable
7118          * auto-downclocking.
7119          *
7120          * Perhaps there might be some value in exposing these to
7121          * userspace...
7122          */
7123         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7124
7125         /* Power down if completely idle for over 50ms */
7126         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
7127         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7128
7129         reset_rps(dev_priv, gen6_set_rps);
7130
7131         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7132 }
7133
7134 static void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
7135 {
7136         struct intel_rps *rps = &dev_priv->gt_pm.rps;
7137         const int min_freq = 15;
7138         const int scaling_factor = 180;
7139         unsigned int gpu_freq;
7140         unsigned int max_ia_freq, min_ring_freq;
7141         unsigned int max_gpu_freq, min_gpu_freq;
7142         struct cpufreq_policy *policy;
7143
7144         WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
7145
7146         if (rps->max_freq <= rps->min_freq)
7147                 return;
7148
7149         policy = cpufreq_cpu_get(0);
7150         if (policy) {
7151                 max_ia_freq = policy->cpuinfo.max_freq;
7152                 cpufreq_cpu_put(policy);
7153         } else {
7154                 /*
7155                  * Default to measured freq if none found, PCU will ensure we
7156                  * don't go over
7157                  */
7158                 max_ia_freq = tsc_khz;
7159         }
7160
7161         /* Convert from kHz to MHz */
7162         max_ia_freq /= 1000;
7163
7164         min_ring_freq = I915_READ(DCLK) & 0xf;
7165         /* convert DDR frequency from units of 266.6MHz to bandwidth */
7166         min_ring_freq = mult_frac(min_ring_freq, 8, 3);
7167
7168         min_gpu_freq = rps->min_freq;
7169         max_gpu_freq = rps->max_freq;
7170         if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
7171                 /* Convert GT frequency to 50 HZ units */
7172                 min_gpu_freq /= GEN9_FREQ_SCALER;
7173                 max_gpu_freq /= GEN9_FREQ_SCALER;
7174         }
7175
7176         /*
7177          * For each potential GPU frequency, load a ring frequency we'd like
7178          * to use for memory access.  We do this by specifying the IA frequency
7179          * the PCU should use as a reference to determine the ring frequency.
7180          */
7181         for (gpu_freq = max_gpu_freq; gpu_freq >= min_gpu_freq; gpu_freq--) {
7182                 const int diff = max_gpu_freq - gpu_freq;
7183                 unsigned int ia_freq = 0, ring_freq = 0;
7184
7185                 if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
7186                         /*
7187                          * ring_freq = 2 * GT. ring_freq is in 100MHz units
7188                          * No floor required for ring frequency on SKL.
7189                          */
7190                         ring_freq = gpu_freq;
7191                 } else if (INTEL_GEN(dev_priv) >= 8) {
7192                         /* max(2 * GT, DDR). NB: GT is 50MHz units */
7193                         ring_freq = max(min_ring_freq, gpu_freq);
7194                 } else if (IS_HASWELL(dev_priv)) {
7195                         ring_freq = mult_frac(gpu_freq, 5, 4);
7196                         ring_freq = max(min_ring_freq, ring_freq);
7197                         /* leave ia_freq as the default, chosen by cpufreq */
7198                 } else {
7199                         /* On older processors, there is no separate ring
7200                          * clock domain, so in order to boost the bandwidth
7201                          * of the ring, we need to upclock the CPU (ia_freq).
7202                          *
7203                          * For GPU frequencies less than 750MHz,
7204                          * just use the lowest ring freq.
7205                          */
7206                         if (gpu_freq < min_freq)
7207                                 ia_freq = 800;
7208                         else
7209                                 ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
7210                         ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
7211                 }
7212
7213                 sandybridge_pcode_write(dev_priv,
7214                                         GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
7215                                         ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT |
7216                                         ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT |
7217                                         gpu_freq);
7218         }
7219 }
7220
7221 static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
7222 {
7223         u32 val, rp0;
7224
7225         val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
7226
7227         switch (INTEL_INFO(dev_priv)->sseu.eu_total) {
7228         case 8:
7229                 /* (2 * 4) config */
7230                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
7231                 break;
7232         case 12:
7233                 /* (2 * 6) config */
7234                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
7235                 break;
7236         case 16:
7237                 /* (2 * 8) config */
7238         default:
7239                 /* Setting (2 * 8) Min RP0 for any other combination */
7240                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
7241                 break;
7242         }
7243
7244         rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
7245
7246         return rp0;
7247 }
7248
7249 static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
7250 {
7251         u32 val, rpe;
7252
7253         val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
7254         rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
7255
7256         return rpe;
7257 }
7258
7259 static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
7260 {
7261         u32 val, rp1;
7262
7263         val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
7264         rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
7265
7266         return rp1;
7267 }
7268
7269 static u32 cherryview_rps_min_freq(struct drm_i915_private *dev_priv)
7270 {
7271         u32 val, rpn;
7272
7273         val = vlv_punit_read(dev_priv, FB_GFX_FMIN_AT_VMIN_FUSE);
7274         rpn = ((val >> FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT) &
7275                        FB_GFX_FREQ_FUSE_MASK);
7276
7277         return rpn;
7278 }
7279
7280 static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
7281 {
7282         u32 val, rp1;
7283
7284         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
7285
7286         rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
7287
7288         return rp1;
7289 }
7290
7291 static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
7292 {
7293         u32 val, rp0;
7294
7295         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
7296
7297         rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
7298         /* Clamp to max */
7299         rp0 = min_t(u32, rp0, 0xea);
7300
7301         return rp0;
7302 }
7303
7304 static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
7305 {
7306         u32 val, rpe;
7307
7308         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
7309         rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
7310         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
7311         rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
7312
7313         return rpe;
7314 }
7315
7316 static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
7317 {
7318         u32 val;
7319
7320         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
7321         /*
7322          * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
7323          * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
7324          * a BYT-M B0 the above register contains 0xbf. Moreover when setting
7325          * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
7326          * to make sure it matches what Punit accepts.
7327          */
7328         return max_t(u32, val, 0xc0);
7329 }
7330
7331 /* Check that the pctx buffer wasn't move under us. */
7332 static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
7333 {
7334         unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
7335
7336         WARN_ON(pctx_addr != dev_priv->dsm.start +
7337                              dev_priv->vlv_pctx->stolen->start);
7338 }
7339
7340
7341 /* Check that the pcbr address is not empty. */
7342 static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
7343 {
7344         unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
7345
7346         WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
7347 }
7348
7349 static void cherryview_setup_pctx(struct drm_i915_private *dev_priv)
7350 {
7351         resource_size_t pctx_paddr, paddr;
7352         resource_size_t pctx_size = 32*1024;
7353         u32 pcbr;
7354
7355         pcbr = I915_READ(VLV_PCBR);
7356         if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
7357                 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
7358                 paddr = dev_priv->dsm.end + 1 - pctx_size;
7359                 GEM_BUG_ON(paddr > U32_MAX);
7360
7361                 pctx_paddr = (paddr & (~4095));
7362                 I915_WRITE(VLV_PCBR, pctx_paddr);
7363         }
7364
7365         DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
7366 }
7367
7368 static void valleyview_setup_pctx(struct drm_i915_private *dev_priv)
7369 {
7370         struct drm_i915_gem_object *pctx;
7371         resource_size_t pctx_paddr;
7372         resource_size_t pctx_size = 24*1024;
7373         u32 pcbr;
7374
7375         pcbr = I915_READ(VLV_PCBR);
7376         if (pcbr) {
7377                 /* BIOS set it up already, grab the pre-alloc'd space */
7378                 resource_size_t pcbr_offset;
7379
7380                 pcbr_offset = (pcbr & (~4095)) - dev_priv->dsm.start;
7381                 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv,
7382                                                                       pcbr_offset,
7383                                                                       I915_GTT_OFFSET_NONE,
7384                                                                       pctx_size);
7385                 goto out;
7386         }
7387
7388         DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
7389
7390         /*
7391          * From the Gunit register HAS:
7392          * The Gfx driver is expected to program this register and ensure
7393          * proper allocation within Gfx stolen memory.  For example, this
7394          * register should be programmed such than the PCBR range does not
7395          * overlap with other ranges, such as the frame buffer, protected
7396          * memory, or any other relevant ranges.
7397          */
7398         pctx = i915_gem_object_create_stolen(dev_priv, pctx_size);
7399         if (!pctx) {
7400                 DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
7401                 goto out;
7402         }
7403
7404         GEM_BUG_ON(range_overflows_t(u64,
7405                                      dev_priv->dsm.start,
7406                                      pctx->stolen->start,
7407                                      U32_MAX));
7408         pctx_paddr = dev_priv->dsm.start + pctx->stolen->start;
7409         I915_WRITE(VLV_PCBR, pctx_paddr);
7410
7411 out:
7412         DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
7413         dev_priv->vlv_pctx = pctx;
7414 }
7415
7416 static void valleyview_cleanup_pctx(struct drm_i915_private *dev_priv)
7417 {
7418         struct drm_i915_gem_object *pctx;
7419
7420         pctx = fetch_and_zero(&dev_priv->vlv_pctx);
7421         if (pctx)
7422                 i915_gem_object_put(pctx);
7423 }
7424
7425 static void vlv_init_gpll_ref_freq(struct drm_i915_private *dev_priv)
7426 {
7427         dev_priv->gt_pm.rps.gpll_ref_freq =
7428                 vlv_get_cck_clock(dev_priv, "GPLL ref",
7429                                   CCK_GPLL_CLOCK_CONTROL,
7430                                   dev_priv->czclk_freq);
7431
7432         DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n",
7433                          dev_priv->gt_pm.rps.gpll_ref_freq);
7434 }
7435
7436 static void valleyview_init_gt_powersave(struct drm_i915_private *dev_priv)
7437 {
7438         struct intel_rps *rps = &dev_priv->gt_pm.rps;
7439         u32 val;
7440
7441         valleyview_setup_pctx(dev_priv);
7442
7443         vlv_init_gpll_ref_freq(dev_priv);
7444
7445         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
7446         switch ((val >> 6) & 3) {
7447         case 0:
7448         case 1:
7449                 dev_priv->mem_freq = 800;
7450                 break;
7451         case 2:
7452                 dev_priv->mem_freq = 1066;
7453                 break;
7454         case 3:
7455                 dev_priv->mem_freq = 1333;
7456                 break;
7457         }
7458         DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
7459
7460         rps->max_freq = valleyview_rps_max_freq(dev_priv);
7461         rps->rp0_freq = rps->max_freq;
7462         DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
7463                          intel_gpu_freq(dev_priv, rps->max_freq),
7464                          rps->max_freq);
7465
7466         rps->efficient_freq = valleyview_rps_rpe_freq(dev_priv);
7467         DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
7468                          intel_gpu_freq(dev_priv, rps->efficient_freq),
7469                          rps->efficient_freq);
7470
7471         rps->rp1_freq = valleyview_rps_guar_freq(dev_priv);
7472         DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
7473                          intel_gpu_freq(dev_priv, rps->rp1_freq),
7474                          rps->rp1_freq);
7475
7476         rps->min_freq = valleyview_rps_min_freq(dev_priv);
7477         DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
7478                          intel_gpu_freq(dev_priv, rps->min_freq),
7479                          rps->min_freq);
7480 }
7481
7482 static void cherryview_init_gt_powersave(struct drm_i915_private *dev_priv)
7483 {
7484         struct intel_rps *rps = &dev_priv->gt_pm.rps;
7485         u32 val;
7486
7487         cherryview_setup_pctx(dev_priv);
7488
7489         vlv_init_gpll_ref_freq(dev_priv);
7490
7491         mutex_lock(&dev_priv->sb_lock);
7492         val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
7493         mutex_unlock(&dev_priv->sb_lock);
7494
7495         switch ((val >> 2) & 0x7) {
7496         case 3:
7497                 dev_priv->mem_freq = 2000;
7498                 break;
7499         default:
7500                 dev_priv->mem_freq = 1600;
7501                 break;
7502         }
7503         DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
7504
7505         rps->max_freq = cherryview_rps_max_freq(dev_priv);
7506         rps->rp0_freq = rps->max_freq;
7507         DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
7508                          intel_gpu_freq(dev_priv, rps->max_freq),
7509                          rps->max_freq);
7510
7511         rps->efficient_freq = cherryview_rps_rpe_freq(dev_priv);
7512         DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
7513                          intel_gpu_freq(dev_priv, rps->efficient_freq),
7514                          rps->efficient_freq);
7515
7516         rps->rp1_freq = cherryview_rps_guar_freq(dev_priv);
7517         DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
7518                          intel_gpu_freq(dev_priv, rps->rp1_freq),
7519                          rps->rp1_freq);
7520
7521         rps->min_freq = cherryview_rps_min_freq(dev_priv);
7522         DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
7523                          intel_gpu_freq(dev_priv, rps->min_freq),
7524                          rps->min_freq);
7525
7526         WARN_ONCE((rps->max_freq | rps->efficient_freq | rps->rp1_freq |
7527                    rps->min_freq) & 1,
7528                   "Odd GPU freq values\n");
7529 }
7530
7531 static void valleyview_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
7532 {
7533         valleyview_cleanup_pctx(dev_priv);
7534 }
7535
7536 static void cherryview_enable_rc6(struct drm_i915_private *dev_priv)
7537 {
7538         struct intel_engine_cs *engine;
7539         enum intel_engine_id id;
7540         u32 gtfifodbg, rc6_mode, pcbr;
7541
7542         gtfifodbg = I915_READ(GTFIFODBG) & ~(GT_FIFO_SBDEDICATE_FREE_ENTRY_CHV |
7543                                              GT_FIFO_FREE_ENTRIES_CHV);
7544         if (gtfifodbg) {
7545                 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
7546                                  gtfifodbg);
7547                 I915_WRITE(GTFIFODBG, gtfifodbg);
7548         }
7549
7550         cherryview_check_pctx(dev_priv);
7551
7552         /* 1a & 1b: Get forcewake during program sequence. Although the driver
7553          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
7554         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7555
7556         /*  Disable RC states. */
7557         I915_WRITE(GEN6_RC_CONTROL, 0);
7558
7559         /* 2a: Program RC6 thresholds.*/
7560         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
7561         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
7562         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
7563
7564         for_each_engine(engine, dev_priv, id)
7565                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7566         I915_WRITE(GEN6_RC_SLEEP, 0);
7567
7568         /* TO threshold set to 500 us ( 0x186 * 1.28 us) */
7569         I915_WRITE(GEN6_RC6_THRESHOLD, 0x186);
7570
7571         /* Allows RC6 residency counter to work */
7572         I915_WRITE(VLV_COUNTER_CONTROL,
7573                    _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
7574                                       VLV_MEDIA_RC6_COUNT_EN |
7575                                       VLV_RENDER_RC6_COUNT_EN));
7576
7577         /* For now we assume BIOS is allocating and populating the PCBR  */
7578         pcbr = I915_READ(VLV_PCBR);
7579
7580         /* 3: Enable RC6 */
7581         rc6_mode = 0;
7582         if (pcbr >> VLV_PCBR_ADDR_SHIFT)
7583                 rc6_mode = GEN7_RC_CTL_TO_MODE;
7584         I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
7585
7586         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7587 }
7588
7589 static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
7590 {
7591         u32 val;
7592
7593         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7594
7595         /* 1: Program defaults and thresholds for RPS*/
7596         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
7597         I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
7598         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
7599         I915_WRITE(GEN6_RP_UP_EI, 66000);
7600         I915_WRITE(GEN6_RP_DOWN_EI, 350000);
7601
7602         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7603
7604         /* 2: Enable RPS */
7605         I915_WRITE(GEN6_RP_CONTROL,
7606                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
7607                    GEN6_RP_MEDIA_IS_GFX |
7608                    GEN6_RP_ENABLE |
7609                    GEN6_RP_UP_BUSY_AVG |
7610                    GEN6_RP_DOWN_IDLE_AVG);
7611
7612         /* Setting Fixed Bias */
7613         val = VLV_OVERRIDE_EN |
7614                   VLV_SOC_TDP_EN |
7615                   CHV_BIAS_CPU_50_SOC_50;
7616         vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
7617
7618         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
7619
7620         /* RPS code assumes GPLL is used */
7621         WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
7622
7623         DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
7624         DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
7625
7626         reset_rps(dev_priv, valleyview_set_rps);
7627
7628         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7629 }
7630
7631 static void valleyview_enable_rc6(struct drm_i915_private *dev_priv)
7632 {
7633         struct intel_engine_cs *engine;
7634         enum intel_engine_id id;
7635         u32 gtfifodbg;
7636
7637         valleyview_check_pctx(dev_priv);
7638
7639         gtfifodbg = I915_READ(GTFIFODBG);
7640         if (gtfifodbg) {
7641                 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
7642                                  gtfifodbg);
7643                 I915_WRITE(GTFIFODBG, gtfifodbg);
7644         }
7645
7646         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7647
7648         /*  Disable RC states. */
7649         I915_WRITE(GEN6_RC_CONTROL, 0);
7650
7651         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
7652         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7653         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7654
7655         for_each_engine(engine, dev_priv, id)
7656                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7657
7658         I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
7659
7660         /* Allows RC6 residency counter to work */
7661         I915_WRITE(VLV_COUNTER_CONTROL,
7662                    _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
7663                                       VLV_MEDIA_RC0_COUNT_EN |
7664                                       VLV_RENDER_RC0_COUNT_EN |
7665                                       VLV_MEDIA_RC6_COUNT_EN |
7666                                       VLV_RENDER_RC6_COUNT_EN));
7667
7668         I915_WRITE(GEN6_RC_CONTROL,
7669                    GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL);
7670
7671         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7672 }
7673
7674 static void valleyview_enable_rps(struct drm_i915_private *dev_priv)
7675 {
7676         u32 val;
7677
7678         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7679
7680         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
7681         I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
7682         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
7683         I915_WRITE(GEN6_RP_UP_EI, 66000);
7684         I915_WRITE(GEN6_RP_DOWN_EI, 350000);
7685
7686         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7687
7688         I915_WRITE(GEN6_RP_CONTROL,
7689                    GEN6_RP_MEDIA_TURBO |
7690                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
7691                    GEN6_RP_MEDIA_IS_GFX |
7692                    GEN6_RP_ENABLE |
7693                    GEN6_RP_UP_BUSY_AVG |
7694                    GEN6_RP_DOWN_IDLE_CONT);
7695
7696         /* Setting Fixed Bias */
7697         val = VLV_OVERRIDE_EN |
7698                   VLV_SOC_TDP_EN |
7699                   VLV_BIAS_CPU_125_SOC_875;
7700         vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
7701
7702         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
7703
7704         /* RPS code assumes GPLL is used */
7705         WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
7706
7707         DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
7708         DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
7709
7710         reset_rps(dev_priv, valleyview_set_rps);
7711
7712         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7713 }
7714
7715 static unsigned long intel_pxfreq(u32 vidfreq)
7716 {
7717         unsigned long freq;
7718         int div = (vidfreq & 0x3f0000) >> 16;
7719         int post = (vidfreq & 0x3000) >> 12;
7720         int pre = (vidfreq & 0x7);
7721
7722         if (!pre)
7723                 return 0;
7724
7725         freq = ((div * 133333) / ((1<<post) * pre));
7726
7727         return freq;
7728 }
7729
7730 static const struct cparams {
7731         u16 i;
7732         u16 t;
7733         u16 m;
7734         u16 c;
7735 } cparams[] = {
7736         { 1, 1333, 301, 28664 },
7737         { 1, 1066, 294, 24460 },
7738         { 1, 800, 294, 25192 },
7739         { 0, 1333, 276, 27605 },
7740         { 0, 1066, 276, 27605 },
7741         { 0, 800, 231, 23784 },
7742 };
7743
7744 static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
7745 {
7746         u64 total_count, diff, ret;
7747         u32 count1, count2, count3, m = 0, c = 0;
7748         unsigned long now = jiffies_to_msecs(jiffies), diff1;
7749         int i;
7750
7751         lockdep_assert_held(&mchdev_lock);
7752
7753         diff1 = now - dev_priv->ips.last_time1;
7754
7755         /* Prevent division-by-zero if we are asking too fast.
7756          * Also, we don't get interesting results if we are polling
7757          * faster than once in 10ms, so just return the saved value
7758          * in such cases.
7759          */
7760         if (diff1 <= 10)
7761                 return dev_priv->ips.chipset_power;
7762
7763         count1 = I915_READ(DMIEC);
7764         count2 = I915_READ(DDREC);
7765         count3 = I915_READ(CSIEC);
7766
7767         total_count = count1 + count2 + count3;
7768
7769         /* FIXME: handle per-counter overflow */
7770         if (total_count < dev_priv->ips.last_count1) {
7771                 diff = ~0UL - dev_priv->ips.last_count1;
7772                 diff += total_count;
7773         } else {
7774                 diff = total_count - dev_priv->ips.last_count1;
7775         }
7776
7777         for (i = 0; i < ARRAY_SIZE(cparams); i++) {
7778                 if (cparams[i].i == dev_priv->ips.c_m &&
7779                     cparams[i].t == dev_priv->ips.r_t) {
7780                         m = cparams[i].m;
7781                         c = cparams[i].c;
7782                         break;
7783                 }
7784         }
7785
7786         diff = div_u64(diff, diff1);
7787         ret = ((m * diff) + c);
7788         ret = div_u64(ret, 10);
7789
7790         dev_priv->ips.last_count1 = total_count;
7791         dev_priv->ips.last_time1 = now;
7792
7793         dev_priv->ips.chipset_power = ret;
7794
7795         return ret;
7796 }
7797
7798 unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
7799 {
7800         unsigned long val;
7801
7802         if (!IS_GEN5(dev_priv))
7803                 return 0;
7804
7805         spin_lock_irq(&mchdev_lock);
7806
7807         val = __i915_chipset_val(dev_priv);
7808
7809         spin_unlock_irq(&mchdev_lock);
7810
7811         return val;
7812 }
7813
7814 unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
7815 {
7816         unsigned long m, x, b;
7817         u32 tsfs;
7818
7819         tsfs = I915_READ(TSFS);
7820
7821         m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
7822         x = I915_READ8(TR1);
7823
7824         b = tsfs & TSFS_INTR_MASK;
7825
7826         return ((m * x) / 127) - b;
7827 }
7828
7829 static int _pxvid_to_vd(u8 pxvid)
7830 {
7831         if (pxvid == 0)
7832                 return 0;
7833
7834         if (pxvid >= 8 && pxvid < 31)
7835                 pxvid = 31;
7836
7837         return (pxvid + 2) * 125;
7838 }
7839
7840 static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
7841 {
7842         const int vd = _pxvid_to_vd(pxvid);
7843         const int vm = vd - 1125;
7844
7845         if (INTEL_INFO(dev_priv)->is_mobile)
7846                 return vm > 0 ? vm : 0;
7847
7848         return vd;
7849 }
7850
7851 static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
7852 {
7853         u64 now, diff, diffms;
7854         u32 count;
7855
7856         lockdep_assert_held(&mchdev_lock);
7857
7858         now = ktime_get_raw_ns();
7859         diffms = now - dev_priv->ips.last_time2;
7860         do_div(diffms, NSEC_PER_MSEC);
7861
7862         /* Don't divide by 0 */
7863         if (!diffms)
7864                 return;
7865
7866         count = I915_READ(GFXEC);
7867
7868         if (count < dev_priv->ips.last_count2) {
7869                 diff = ~0UL - dev_priv->ips.last_count2;
7870                 diff += count;
7871         } else {
7872                 diff = count - dev_priv->ips.last_count2;
7873         }
7874
7875         dev_priv->ips.last_count2 = count;
7876         dev_priv->ips.last_time2 = now;
7877
7878         /* More magic constants... */
7879         diff = diff * 1181;
7880         diff = div_u64(diff, diffms * 10);
7881         dev_priv->ips.gfx_power = diff;
7882 }
7883
7884 void i915_update_gfx_val(struct drm_i915_private *dev_priv)
7885 {
7886         if (!IS_GEN5(dev_priv))
7887                 return;
7888
7889         spin_lock_irq(&mchdev_lock);
7890
7891         __i915_update_gfx_val(dev_priv);
7892
7893         spin_unlock_irq(&mchdev_lock);
7894 }
7895
7896 static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
7897 {
7898         unsigned long t, corr, state1, corr2, state2;
7899         u32 pxvid, ext_v;
7900
7901         lockdep_assert_held(&mchdev_lock);
7902
7903         pxvid = I915_READ(PXVFREQ(dev_priv->gt_pm.rps.cur_freq));
7904         pxvid = (pxvid >> 24) & 0x7f;
7905         ext_v = pvid_to_extvid(dev_priv, pxvid);
7906
7907         state1 = ext_v;
7908
7909         t = i915_mch_val(dev_priv);
7910
7911         /* Revel in the empirically derived constants */
7912
7913         /* Correction factor in 1/100000 units */
7914         if (t > 80)
7915                 corr = ((t * 2349) + 135940);
7916         else if (t >= 50)
7917                 corr = ((t * 964) + 29317);
7918         else /* < 50 */
7919                 corr = ((t * 301) + 1004);
7920
7921         corr = corr * ((150142 * state1) / 10000 - 78642);
7922         corr /= 100000;
7923         corr2 = (corr * dev_priv->ips.corr);
7924
7925         state2 = (corr2 * state1) / 10000;
7926         state2 /= 100; /* convert to mW */
7927
7928         __i915_update_gfx_val(dev_priv);
7929
7930         return dev_priv->ips.gfx_power + state2;
7931 }
7932
7933 unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
7934 {
7935         unsigned long val;
7936
7937         if (!IS_GEN5(dev_priv))
7938                 return 0;
7939
7940         spin_lock_irq(&mchdev_lock);
7941
7942         val = __i915_gfx_val(dev_priv);
7943
7944         spin_unlock_irq(&mchdev_lock);
7945
7946         return val;
7947 }
7948
7949 /**
7950  * i915_read_mch_val - return value for IPS use
7951  *
7952  * Calculate and return a value for the IPS driver to use when deciding whether
7953  * we have thermal and power headroom to increase CPU or GPU power budget.
7954  */
7955 unsigned long i915_read_mch_val(void)
7956 {
7957         struct drm_i915_private *dev_priv;
7958         unsigned long chipset_val, graphics_val, ret = 0;
7959
7960         spin_lock_irq(&mchdev_lock);
7961         if (!i915_mch_dev)
7962                 goto out_unlock;
7963         dev_priv = i915_mch_dev;
7964
7965         chipset_val = __i915_chipset_val(dev_priv);
7966         graphics_val = __i915_gfx_val(dev_priv);
7967
7968         ret = chipset_val + graphics_val;
7969
7970 out_unlock:
7971         spin_unlock_irq(&mchdev_lock);
7972
7973         return ret;
7974 }
7975 EXPORT_SYMBOL_GPL(i915_read_mch_val);
7976
7977 /**
7978  * i915_gpu_raise - raise GPU frequency limit
7979  *
7980  * Raise the limit; IPS indicates we have thermal headroom.
7981  */
7982 bool i915_gpu_raise(void)
7983 {
7984         struct drm_i915_private *dev_priv;
7985         bool ret = true;
7986
7987         spin_lock_irq(&mchdev_lock);
7988         if (!i915_mch_dev) {
7989                 ret = false;
7990                 goto out_unlock;
7991         }
7992         dev_priv = i915_mch_dev;
7993
7994         if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
7995                 dev_priv->ips.max_delay--;
7996
7997 out_unlock:
7998         spin_unlock_irq(&mchdev_lock);
7999
8000         return ret;
8001 }
8002 EXPORT_SYMBOL_GPL(i915_gpu_raise);
8003
8004 /**
8005  * i915_gpu_lower - lower GPU frequency limit
8006  *
8007  * IPS indicates we're close to a thermal limit, so throttle back the GPU
8008  * frequency maximum.
8009  */
8010 bool i915_gpu_lower(void)
8011 {
8012         struct drm_i915_private *dev_priv;
8013         bool ret = true;
8014
8015         spin_lock_irq(&mchdev_lock);
8016         if (!i915_mch_dev) {
8017                 ret = false;
8018                 goto out_unlock;
8019         }
8020         dev_priv = i915_mch_dev;
8021
8022         if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
8023                 dev_priv->ips.max_delay++;
8024
8025 out_unlock:
8026         spin_unlock_irq(&mchdev_lock);
8027
8028         return ret;
8029 }
8030 EXPORT_SYMBOL_GPL(i915_gpu_lower);
8031
8032 /**
8033  * i915_gpu_busy - indicate GPU business to IPS
8034  *
8035  * Tell the IPS driver whether or not the GPU is busy.
8036  */
8037 bool i915_gpu_busy(void)
8038 {
8039         bool ret = false;
8040
8041         spin_lock_irq(&mchdev_lock);
8042         if (i915_mch_dev)
8043                 ret = i915_mch_dev->gt.awake;
8044         spin_unlock_irq(&mchdev_lock);
8045
8046         return ret;
8047 }
8048 EXPORT_SYMBOL_GPL(i915_gpu_busy);
8049
8050 /**
8051  * i915_gpu_turbo_disable - disable graphics turbo
8052  *
8053  * Disable graphics turbo by resetting the max frequency and setting the
8054  * current frequency to the default.
8055  */
8056 bool i915_gpu_turbo_disable(void)
8057 {
8058         struct drm_i915_private *dev_priv;
8059         bool ret = true;
8060
8061         spin_lock_irq(&mchdev_lock);
8062         if (!i915_mch_dev) {
8063                 ret = false;
8064                 goto out_unlock;
8065         }
8066         dev_priv = i915_mch_dev;
8067
8068         dev_priv->ips.max_delay = dev_priv->ips.fstart;
8069
8070         if (!ironlake_set_drps(dev_priv, dev_priv->ips.fstart))
8071                 ret = false;
8072
8073 out_unlock:
8074         spin_unlock_irq(&mchdev_lock);
8075
8076         return ret;
8077 }
8078 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
8079
8080 /**
8081  * Tells the intel_ips driver that the i915 driver is now loaded, if
8082  * IPS got loaded first.
8083  *
8084  * This awkward dance is so that neither module has to depend on the
8085  * other in order for IPS to do the appropriate communication of
8086  * GPU turbo limits to i915.
8087  */
8088 static void
8089 ips_ping_for_i915_load(void)
8090 {
8091         void (*link)(void);
8092
8093         link = symbol_get(ips_link_to_i915_driver);
8094         if (link) {
8095                 link();
8096                 symbol_put(ips_link_to_i915_driver);
8097         }
8098 }
8099
8100 void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
8101 {
8102         /* We only register the i915 ips part with intel-ips once everything is
8103          * set up, to avoid intel-ips sneaking in and reading bogus values. */
8104         spin_lock_irq(&mchdev_lock);
8105         i915_mch_dev = dev_priv;
8106         spin_unlock_irq(&mchdev_lock);
8107
8108         ips_ping_for_i915_load();
8109 }
8110
8111 void intel_gpu_ips_teardown(void)
8112 {
8113         spin_lock_irq(&mchdev_lock);
8114         i915_mch_dev = NULL;
8115         spin_unlock_irq(&mchdev_lock);
8116 }
8117
8118 static void intel_init_emon(struct drm_i915_private *dev_priv)
8119 {
8120         u32 lcfuse;
8121         u8 pxw[16];
8122         int i;
8123
8124         /* Disable to program */
8125         I915_WRITE(ECR, 0);
8126         POSTING_READ(ECR);
8127
8128         /* Program energy weights for various events */
8129         I915_WRITE(SDEW, 0x15040d00);
8130         I915_WRITE(CSIEW0, 0x007f0000);
8131         I915_WRITE(CSIEW1, 0x1e220004);
8132         I915_WRITE(CSIEW2, 0x04000004);
8133
8134         for (i = 0; i < 5; i++)
8135                 I915_WRITE(PEW(i), 0);
8136         for (i = 0; i < 3; i++)
8137                 I915_WRITE(DEW(i), 0);
8138
8139         /* Program P-state weights to account for frequency power adjustment */
8140         for (i = 0; i < 16; i++) {
8141                 u32 pxvidfreq = I915_READ(PXVFREQ(i));
8142                 unsigned long freq = intel_pxfreq(pxvidfreq);
8143                 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
8144                         PXVFREQ_PX_SHIFT;
8145                 unsigned long val;
8146
8147                 val = vid * vid;
8148                 val *= (freq / 1000);
8149                 val *= 255;
8150                 val /= (127*127*900);
8151                 if (val > 0xff)
8152                         DRM_ERROR("bad pxval: %ld\n", val);
8153                 pxw[i] = val;
8154         }
8155         /* Render standby states get 0 weight */
8156         pxw[14] = 0;
8157         pxw[15] = 0;
8158
8159         for (i = 0; i < 4; i++) {
8160                 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
8161                         (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
8162                 I915_WRITE(PXW(i), val);
8163         }
8164
8165         /* Adjust magic regs to magic values (more experimental results) */
8166         I915_WRITE(OGW0, 0);
8167         I915_WRITE(OGW1, 0);
8168         I915_WRITE(EG0, 0x00007f00);
8169         I915_WRITE(EG1, 0x0000000e);
8170         I915_WRITE(EG2, 0x000e0000);
8171         I915_WRITE(EG3, 0x68000300);
8172         I915_WRITE(EG4, 0x42000000);
8173         I915_WRITE(EG5, 0x00140031);
8174         I915_WRITE(EG6, 0);
8175         I915_WRITE(EG7, 0);
8176
8177         for (i = 0; i < 8; i++)
8178                 I915_WRITE(PXWL(i), 0);
8179
8180         /* Enable PMON + select events */
8181         I915_WRITE(ECR, 0x80000019);
8182
8183         lcfuse = I915_READ(LCFUSE02);
8184
8185         dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
8186 }
8187
8188 void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
8189 {
8190         struct intel_rps *rps = &dev_priv->gt_pm.rps;
8191
8192         /*
8193          * RPM depends on RC6 to save restore the GT HW context, so make RC6 a
8194          * requirement.
8195          */
8196         if (!sanitize_rc6(dev_priv)) {
8197                 DRM_INFO("RC6 disabled, disabling runtime PM support\n");
8198                 pm_runtime_get(&dev_priv->drm.pdev->dev);
8199         }
8200
8201         mutex_lock(&dev_priv->pcu_lock);
8202
8203         /* Initialize RPS limits (for userspace) */
8204         if (IS_CHERRYVIEW(dev_priv))
8205                 cherryview_init_gt_powersave(dev_priv);
8206         else if (IS_VALLEYVIEW(dev_priv))
8207                 valleyview_init_gt_powersave(dev_priv);
8208         else if (INTEL_GEN(dev_priv) >= 6)
8209                 gen6_init_rps_frequencies(dev_priv);
8210
8211         /* Derive initial user preferences/limits from the hardware limits */
8212         rps->idle_freq = rps->min_freq;
8213         rps->cur_freq = rps->idle_freq;
8214
8215         rps->max_freq_softlimit = rps->max_freq;
8216         rps->min_freq_softlimit = rps->min_freq;
8217
8218         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
8219                 rps->min_freq_softlimit =
8220                         max_t(int,
8221                               rps->efficient_freq,
8222                               intel_freq_opcode(dev_priv, 450));
8223
8224         /* After setting max-softlimit, find the overclock max freq */
8225         if (IS_GEN6(dev_priv) ||
8226             IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) {
8227                 u32 params = 0;
8228
8229                 sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &params);
8230                 if (params & BIT(31)) { /* OC supported */
8231                         DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n",
8232                                          (rps->max_freq & 0xff) * 50,
8233                                          (params & 0xff) * 50);
8234                         rps->max_freq = params & 0xff;
8235                 }
8236         }
8237
8238         /* Finally allow us to boost to max by default */
8239         rps->boost_freq = rps->max_freq;
8240
8241         mutex_unlock(&dev_priv->pcu_lock);
8242 }
8243
8244 void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
8245 {
8246         if (IS_VALLEYVIEW(dev_priv))
8247                 valleyview_cleanup_gt_powersave(dev_priv);
8248
8249         if (!HAS_RC6(dev_priv))
8250                 pm_runtime_put(&dev_priv->drm.pdev->dev);
8251 }
8252
8253 /**
8254  * intel_suspend_gt_powersave - suspend PM work and helper threads
8255  * @dev_priv: i915 device
8256  *
8257  * We don't want to disable RC6 or other features here, we just want
8258  * to make sure any work we've queued has finished and won't bother
8259  * us while we're suspended.
8260  */
8261 void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv)
8262 {
8263         if (INTEL_GEN(dev_priv) < 6)
8264                 return;
8265
8266         /* gen6_rps_idle() will be called later to disable interrupts */
8267 }
8268
8269 void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv)
8270 {
8271         dev_priv->gt_pm.rps.enabled = true; /* force RPS disabling */
8272         dev_priv->gt_pm.rc6.enabled = true; /* force RC6 disabling */
8273         intel_disable_gt_powersave(dev_priv);
8274
8275         if (INTEL_GEN(dev_priv) >= 11)
8276                 gen11_reset_rps_interrupts(dev_priv);
8277         else if (INTEL_GEN(dev_priv) >= 6)
8278                 gen6_reset_rps_interrupts(dev_priv);
8279 }
8280
8281 static inline void intel_disable_llc_pstate(struct drm_i915_private *i915)
8282 {
8283         lockdep_assert_held(&i915->pcu_lock);
8284
8285         if (!i915->gt_pm.llc_pstate.enabled)
8286                 return;
8287
8288         /* Currently there is no HW configuration to be done to disable. */
8289
8290         i915->gt_pm.llc_pstate.enabled = false;
8291 }
8292
8293 static void intel_disable_rc6(struct drm_i915_private *dev_priv)
8294 {
8295         lockdep_assert_held(&dev_priv->pcu_lock);
8296
8297         if (!dev_priv->gt_pm.rc6.enabled)
8298                 return;
8299
8300         if (INTEL_GEN(dev_priv) >= 9)
8301                 gen9_disable_rc6(dev_priv);
8302         else if (IS_CHERRYVIEW(dev_priv))
8303                 cherryview_disable_rc6(dev_priv);
8304         else if (IS_VALLEYVIEW(dev_priv))
8305                 valleyview_disable_rc6(dev_priv);
8306         else if (INTEL_GEN(dev_priv) >= 6)
8307                 gen6_disable_rc6(dev_priv);
8308
8309         dev_priv->gt_pm.rc6.enabled = false;
8310 }
8311
8312 static void intel_disable_rps(struct drm_i915_private *dev_priv)
8313 {
8314         lockdep_assert_held(&dev_priv->pcu_lock);
8315
8316         if (!dev_priv->gt_pm.rps.enabled)
8317                 return;
8318
8319         if (INTEL_GEN(dev_priv) >= 9)
8320                 gen9_disable_rps(dev_priv);
8321         else if (IS_CHERRYVIEW(dev_priv))
8322                 cherryview_disable_rps(dev_priv);
8323         else if (IS_VALLEYVIEW(dev_priv))
8324                 valleyview_disable_rps(dev_priv);
8325         else if (INTEL_GEN(dev_priv) >= 6)
8326                 gen6_disable_rps(dev_priv);
8327         else if (IS_IRONLAKE_M(dev_priv))
8328                 ironlake_disable_drps(dev_priv);
8329
8330         dev_priv->gt_pm.rps.enabled = false;
8331 }
8332
8333 void intel_disable_gt_powersave(struct drm_i915_private *dev_priv)
8334 {
8335         mutex_lock(&dev_priv->pcu_lock);
8336
8337         intel_disable_rc6(dev_priv);
8338         intel_disable_rps(dev_priv);
8339         if (HAS_LLC(dev_priv))
8340                 intel_disable_llc_pstate(dev_priv);
8341
8342         mutex_unlock(&dev_priv->pcu_lock);
8343 }
8344
8345 static inline void intel_enable_llc_pstate(struct drm_i915_private *i915)
8346 {
8347         lockdep_assert_held(&i915->pcu_lock);
8348
8349         if (i915->gt_pm.llc_pstate.enabled)
8350                 return;
8351
8352         gen6_update_ring_freq(i915);
8353
8354         i915->gt_pm.llc_pstate.enabled = true;
8355 }
8356
8357 static void intel_enable_rc6(struct drm_i915_private *dev_priv)
8358 {
8359         lockdep_assert_held(&dev_priv->pcu_lock);
8360
8361         if (dev_priv->gt_pm.rc6.enabled)
8362                 return;
8363
8364         if (IS_CHERRYVIEW(dev_priv))
8365                 cherryview_enable_rc6(dev_priv);
8366         else if (IS_VALLEYVIEW(dev_priv))
8367                 valleyview_enable_rc6(dev_priv);
8368         else if (INTEL_GEN(dev_priv) >= 9)
8369                 gen9_enable_rc6(dev_priv);
8370         else if (IS_BROADWELL(dev_priv))
8371                 gen8_enable_rc6(dev_priv);
8372         else if (INTEL_GEN(dev_priv) >= 6)
8373                 gen6_enable_rc6(dev_priv);
8374
8375         dev_priv->gt_pm.rc6.enabled = true;
8376 }
8377
8378 static void intel_enable_rps(struct drm_i915_private *dev_priv)
8379 {
8380         struct intel_rps *rps = &dev_priv->gt_pm.rps;
8381
8382         lockdep_assert_held(&dev_priv->pcu_lock);
8383
8384         if (rps->enabled)
8385                 return;
8386
8387         if (IS_CHERRYVIEW(dev_priv)) {
8388                 cherryview_enable_rps(dev_priv);
8389         } else if (IS_VALLEYVIEW(dev_priv)) {
8390                 valleyview_enable_rps(dev_priv);
8391         } else if (INTEL_GEN(dev_priv) >= 9) {
8392                 gen9_enable_rps(dev_priv);
8393         } else if (IS_BROADWELL(dev_priv)) {
8394                 gen8_enable_rps(dev_priv);
8395         } else if (INTEL_GEN(dev_priv) >= 6) {
8396                 gen6_enable_rps(dev_priv);
8397         } else if (IS_IRONLAKE_M(dev_priv)) {
8398                 ironlake_enable_drps(dev_priv);
8399                 intel_init_emon(dev_priv);
8400         }
8401
8402         WARN_ON(rps->max_freq < rps->min_freq);
8403         WARN_ON(rps->idle_freq > rps->max_freq);
8404
8405         WARN_ON(rps->efficient_freq < rps->min_freq);
8406         WARN_ON(rps->efficient_freq > rps->max_freq);
8407
8408         rps->enabled = true;
8409 }
8410
8411 void intel_enable_gt_powersave(struct drm_i915_private *dev_priv)
8412 {
8413         /* Powersaving is controlled by the host when inside a VM */
8414         if (intel_vgpu_active(dev_priv))
8415                 return;
8416
8417         mutex_lock(&dev_priv->pcu_lock);
8418
8419         if (HAS_RC6(dev_priv))
8420                 intel_enable_rc6(dev_priv);
8421         intel_enable_rps(dev_priv);
8422         if (HAS_LLC(dev_priv))
8423                 intel_enable_llc_pstate(dev_priv);
8424
8425         mutex_unlock(&dev_priv->pcu_lock);
8426 }
8427
8428 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
8429 {
8430         /*
8431          * On Ibex Peak and Cougar Point, we need to disable clock
8432          * gating for the panel power sequencer or it will fail to
8433          * start up when no ports are active.
8434          */
8435         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
8436 }
8437
8438 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
8439 {
8440         enum pipe pipe;
8441
8442         for_each_pipe(dev_priv, pipe) {
8443                 I915_WRITE(DSPCNTR(pipe),
8444                            I915_READ(DSPCNTR(pipe)) |
8445                            DISPPLANE_TRICKLE_FEED_DISABLE);
8446
8447                 I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
8448                 POSTING_READ(DSPSURF(pipe));
8449         }
8450 }
8451
8452 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
8453 {
8454         uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
8455
8456         /*
8457          * Required for FBC
8458          * WaFbcDisableDpfcClockGating:ilk
8459          */
8460         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
8461                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
8462                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
8463
8464         I915_WRITE(PCH_3DCGDIS0,
8465                    MARIUNIT_CLOCK_GATE_DISABLE |
8466                    SVSMUNIT_CLOCK_GATE_DISABLE);
8467         I915_WRITE(PCH_3DCGDIS1,
8468                    VFMUNIT_CLOCK_GATE_DISABLE);
8469
8470         /*
8471          * According to the spec the following bits should be set in
8472          * order to enable memory self-refresh
8473          * The bit 22/21 of 0x42004
8474          * The bit 5 of 0x42020
8475          * The bit 15 of 0x45000
8476          */
8477         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8478                    (I915_READ(ILK_DISPLAY_CHICKEN2) |
8479                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
8480         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
8481         I915_WRITE(DISP_ARB_CTL,
8482                    (I915_READ(DISP_ARB_CTL) |
8483                     DISP_FBC_WM_DIS));
8484
8485         /*
8486          * Based on the document from hardware guys the following bits
8487          * should be set unconditionally in order to enable FBC.
8488          * The bit 22 of 0x42000
8489          * The bit 22 of 0x42004
8490          * The bit 7,8,9 of 0x42020.
8491          */
8492         if (IS_IRONLAKE_M(dev_priv)) {
8493                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
8494                 I915_WRITE(ILK_DISPLAY_CHICKEN1,
8495                            I915_READ(ILK_DISPLAY_CHICKEN1) |
8496                            ILK_FBCQ_DIS);
8497                 I915_WRITE(ILK_DISPLAY_CHICKEN2,
8498                            I915_READ(ILK_DISPLAY_CHICKEN2) |
8499                            ILK_DPARB_GATE);
8500         }
8501
8502         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
8503
8504         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8505                    I915_READ(ILK_DISPLAY_CHICKEN2) |
8506                    ILK_ELPIN_409_SELECT);
8507         I915_WRITE(_3D_CHICKEN2,
8508                    _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
8509                    _3D_CHICKEN2_WM_READ_PIPELINED);
8510
8511         /* WaDisableRenderCachePipelinedFlush:ilk */
8512         I915_WRITE(CACHE_MODE_0,
8513                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
8514
8515         /* WaDisable_RenderCache_OperationalFlush:ilk */
8516         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8517
8518         g4x_disable_trickle_feed(dev_priv);
8519
8520         ibx_init_clock_gating(dev_priv);
8521 }
8522
8523 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
8524 {
8525         int pipe;
8526         uint32_t val;
8527
8528         /*
8529          * On Ibex Peak and Cougar Point, we need to disable clock
8530          * gating for the panel power sequencer or it will fail to
8531          * start up when no ports are active.
8532          */
8533         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
8534                    PCH_DPLUNIT_CLOCK_GATE_DISABLE |
8535                    PCH_CPUNIT_CLOCK_GATE_DISABLE);
8536         I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
8537                    DPLS_EDP_PPS_FIX_DIS);
8538         /* The below fixes the weird display corruption, a few pixels shifted
8539          * downward, on (only) LVDS of some HP laptops with IVY.
8540          */
8541         for_each_pipe(dev_priv, pipe) {
8542                 val = I915_READ(TRANS_CHICKEN2(pipe));
8543                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
8544                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
8545                 if (dev_priv->vbt.fdi_rx_polarity_inverted)
8546                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
8547                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
8548                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
8549                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
8550                 I915_WRITE(TRANS_CHICKEN2(pipe), val);
8551         }
8552         /* WADP0ClockGatingDisable */
8553         for_each_pipe(dev_priv, pipe) {
8554                 I915_WRITE(TRANS_CHICKEN1(pipe),
8555                            TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
8556         }
8557 }
8558
8559 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
8560 {
8561         uint32_t tmp;
8562
8563         tmp = I915_READ(MCH_SSKPD);
8564         if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
8565                 DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
8566                               tmp);
8567 }
8568
8569 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
8570 {
8571         uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
8572
8573         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
8574
8575         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8576                    I915_READ(ILK_DISPLAY_CHICKEN2) |
8577                    ILK_ELPIN_409_SELECT);
8578
8579         /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
8580         I915_WRITE(_3D_CHICKEN,
8581                    _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
8582
8583         /* WaDisable_RenderCache_OperationalFlush:snb */
8584         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8585
8586         /*
8587          * BSpec recoomends 8x4 when MSAA is used,
8588          * however in practice 16x4 seems fastest.
8589          *
8590          * Note that PS/WM thread counts depend on the WIZ hashing
8591          * disable bit, which we don't touch here, but it's good
8592          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
8593          */
8594         I915_WRITE(GEN6_GT_MODE,
8595                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
8596
8597         I915_WRITE(CACHE_MODE_0,
8598                    _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
8599
8600         I915_WRITE(GEN6_UCGCTL1,
8601                    I915_READ(GEN6_UCGCTL1) |
8602                    GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
8603                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
8604
8605         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
8606          * gating disable must be set.  Failure to set it results in
8607          * flickering pixels due to Z write ordering failures after
8608          * some amount of runtime in the Mesa "fire" demo, and Unigine
8609          * Sanctuary and Tropics, and apparently anything else with
8610          * alpha test or pixel discard.
8611          *
8612          * According to the spec, bit 11 (RCCUNIT) must also be set,
8613          * but we didn't debug actual testcases to find it out.
8614          *
8615          * WaDisableRCCUnitClockGating:snb
8616          * WaDisableRCPBUnitClockGating:snb
8617          */
8618         I915_WRITE(GEN6_UCGCTL2,
8619                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
8620                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
8621
8622         /* WaStripsFansDisableFastClipPerformanceFix:snb */
8623         I915_WRITE(_3D_CHICKEN3,
8624                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
8625
8626         /*
8627          * Bspec says:
8628          * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
8629          * 3DSTATE_SF number of SF output attributes is more than 16."
8630          */
8631         I915_WRITE(_3D_CHICKEN3,
8632                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
8633
8634         /*
8635          * According to the spec the following bits should be
8636          * set in order to enable memory self-refresh and fbc:
8637          * The bit21 and bit22 of 0x42000
8638          * The bit21 and bit22 of 0x42004
8639          * The bit5 and bit7 of 0x42020
8640          * The bit14 of 0x70180
8641          * The bit14 of 0x71180
8642          *
8643          * WaFbcAsynchFlipDisableFbcQueue:snb
8644          */
8645         I915_WRITE(ILK_DISPLAY_CHICKEN1,
8646                    I915_READ(ILK_DISPLAY_CHICKEN1) |
8647                    ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
8648         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8649                    I915_READ(ILK_DISPLAY_CHICKEN2) |
8650                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
8651         I915_WRITE(ILK_DSPCLK_GATE_D,
8652                    I915_READ(ILK_DSPCLK_GATE_D) |
8653                    ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
8654                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
8655
8656         g4x_disable_trickle_feed(dev_priv);
8657
8658         cpt_init_clock_gating(dev_priv);
8659
8660         gen6_check_mch_setup(dev_priv);
8661 }
8662
8663 static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
8664 {
8665         uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
8666
8667         /*
8668          * WaVSThreadDispatchOverride:ivb,vlv
8669          *
8670          * This actually overrides the dispatch
8671          * mode for all thread types.
8672          */
8673         reg &= ~GEN7_FF_SCHED_MASK;
8674         reg |= GEN7_FF_TS_SCHED_HW;
8675         reg |= GEN7_FF_VS_SCHED_HW;
8676         reg |= GEN7_FF_DS_SCHED_HW;
8677
8678         I915_WRITE(GEN7_FF_THREAD_MODE, reg);
8679 }
8680
8681 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
8682 {
8683         /*
8684          * TODO: this bit should only be enabled when really needed, then
8685          * disabled when not needed anymore in order to save power.
8686          */
8687         if (HAS_PCH_LPT_LP(dev_priv))
8688                 I915_WRITE(SOUTH_DSPCLK_GATE_D,
8689                            I915_READ(SOUTH_DSPCLK_GATE_D) |
8690                            PCH_LP_PARTITION_LEVEL_DISABLE);
8691
8692         /* WADPOClockGatingDisable:hsw */
8693         I915_WRITE(TRANS_CHICKEN1(PIPE_A),
8694                    I915_READ(TRANS_CHICKEN1(PIPE_A)) |
8695                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
8696 }
8697
8698 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
8699 {
8700         if (HAS_PCH_LPT_LP(dev_priv)) {
8701                 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
8702
8703                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
8704                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
8705         }
8706 }
8707
8708 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
8709                                    int general_prio_credits,
8710                                    int high_prio_credits)
8711 {
8712         u32 misccpctl;
8713         u32 val;
8714
8715         /* WaTempDisableDOPClkGating:bdw */
8716         misccpctl = I915_READ(GEN7_MISCCPCTL);
8717         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
8718
8719         val = I915_READ(GEN8_L3SQCREG1);
8720         val &= ~L3_PRIO_CREDITS_MASK;
8721         val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
8722         val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
8723         I915_WRITE(GEN8_L3SQCREG1, val);
8724
8725         /*
8726          * Wait at least 100 clocks before re-enabling clock gating.
8727          * See the definition of L3SQCREG1 in BSpec.
8728          */
8729         POSTING_READ(GEN8_L3SQCREG1);
8730         udelay(1);
8731         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
8732 }
8733
8734 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
8735 {
8736         /* This is not an Wa. Enable to reduce Sampler power */
8737         I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN,
8738                    I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
8739 }
8740
8741 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
8742 {
8743         if (!HAS_PCH_CNP(dev_priv))
8744                 return;
8745
8746         /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
8747         I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) |
8748                    CNP_PWM_CGE_GATING_DISABLE);
8749 }
8750
8751 static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
8752 {
8753         u32 val;
8754         cnp_init_clock_gating(dev_priv);
8755
8756         /* This is not an Wa. Enable for better image quality */
8757         I915_WRITE(_3D_CHICKEN3,
8758                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
8759
8760         /* WaEnableChickenDCPR:cnl */
8761         I915_WRITE(GEN8_CHICKEN_DCPR_1,
8762                    I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
8763
8764         /* WaFbcWakeMemOn:cnl */
8765         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
8766                    DISP_FBC_MEMORY_WAKE);
8767
8768         val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE);
8769         /* ReadHitWriteOnlyDisable:cnl */
8770         val |= RCCUNIT_CLKGATE_DIS;
8771         /* WaSarbUnitClockGatingDisable:cnl (pre-prod) */
8772         if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_B0))
8773                 val |= SARBUNIT_CLKGATE_DIS;
8774         I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val);
8775
8776         /* Wa_2201832410:cnl */
8777         val = I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE);
8778         val |= GWUNIT_CLKGATE_DIS;
8779         I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE, val);
8780
8781         /* WaDisableVFclkgate:cnl */
8782         /* WaVFUnitClockGatingDisable:cnl */
8783         val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE);
8784         val |= VFUNIT_CLKGATE_DIS;
8785         I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val);
8786 }
8787
8788 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
8789 {
8790         cnp_init_clock_gating(dev_priv);
8791         gen9_init_clock_gating(dev_priv);
8792
8793         /* WaFbcNukeOnHostModify:cfl */
8794         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
8795                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
8796 }
8797
8798 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
8799 {
8800         gen9_init_clock_gating(dev_priv);
8801
8802         /* WaDisableSDEUnitClockGating:kbl */
8803         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
8804                 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
8805                            GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
8806
8807         /* WaDisableGamClockGating:kbl */
8808         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
8809                 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
8810                            GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
8811
8812         /* WaFbcNukeOnHostModify:kbl */
8813         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
8814                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
8815 }
8816
8817 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
8818 {
8819         gen9_init_clock_gating(dev_priv);
8820
8821         /* WAC6entrylatency:skl */
8822         I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
8823                    FBC_LLC_FULLY_OPEN);
8824
8825         /* WaFbcNukeOnHostModify:skl */
8826         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
8827                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
8828 }
8829
8830 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
8831 {
8832         /* The GTT cache must be disabled if the system is using 2M pages. */
8833         bool can_use_gtt_cache = !HAS_PAGE_SIZES(dev_priv,
8834                                                  I915_GTT_PAGE_SIZE_2M);
8835         enum pipe pipe;
8836
8837         /* WaSwitchSolVfFArbitrationPriority:bdw */
8838         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
8839
8840         /* WaPsrDPAMaskVBlankInSRD:bdw */
8841         I915_WRITE(CHICKEN_PAR1_1,
8842                    I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
8843
8844         /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
8845         for_each_pipe(dev_priv, pipe) {
8846                 I915_WRITE(CHICKEN_PIPESL_1(pipe),
8847                            I915_READ(CHICKEN_PIPESL_1(pipe)) |
8848                            BDW_DPRS_MASK_VBLANK_SRD);
8849         }
8850
8851         /* WaVSRefCountFullforceMissDisable:bdw */
8852         /* WaDSRefCountFullforceMissDisable:bdw */
8853         I915_WRITE(GEN7_FF_THREAD_MODE,
8854                    I915_READ(GEN7_FF_THREAD_MODE) &
8855                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
8856
8857         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
8858                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
8859
8860         /* WaDisableSDEUnitClockGating:bdw */
8861         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
8862                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
8863
8864         /* WaProgramL3SqcReg1Default:bdw */
8865         gen8_set_l3sqc_credits(dev_priv, 30, 2);
8866
8867         /* WaGttCachingOffByDefault:bdw */
8868         I915_WRITE(HSW_GTT_CACHE_EN, can_use_gtt_cache ? GTT_CACHE_EN_ALL : 0);
8869
8870         /* WaKVMNotificationOnConfigChange:bdw */
8871         I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
8872                    | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
8873
8874         lpt_init_clock_gating(dev_priv);
8875
8876         /* WaDisableDopClockGating:bdw
8877          *
8878          * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
8879          * clock gating.
8880          */
8881         I915_WRITE(GEN6_UCGCTL1,
8882                    I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
8883 }
8884
8885 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
8886 {
8887         /* L3 caching of data atomics doesn't work -- disable it. */
8888         I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
8889         I915_WRITE(HSW_ROW_CHICKEN3,
8890                    _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
8891
8892         /* This is required by WaCatErrorRejectionIssue:hsw */
8893         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
8894                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
8895                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
8896
8897         /* WaVSRefCountFullforceMissDisable:hsw */
8898         I915_WRITE(GEN7_FF_THREAD_MODE,
8899                    I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
8900
8901         /* WaDisable_RenderCache_OperationalFlush:hsw */
8902         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8903
8904         /* enable HiZ Raw Stall Optimization */
8905         I915_WRITE(CACHE_MODE_0_GEN7,
8906                    _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
8907
8908         /* WaDisable4x2SubspanOptimization:hsw */
8909         I915_WRITE(CACHE_MODE_1,
8910                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
8911
8912         /*
8913          * BSpec recommends 8x4 when MSAA is used,
8914          * however in practice 16x4 seems fastest.
8915          *
8916          * Note that PS/WM thread counts depend on the WIZ hashing
8917          * disable bit, which we don't touch here, but it's good
8918          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
8919          */
8920         I915_WRITE(GEN7_GT_MODE,
8921                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
8922
8923         /* WaSampleCChickenBitEnable:hsw */
8924         I915_WRITE(HALF_SLICE_CHICKEN3,
8925                    _MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
8926
8927         /* WaSwitchSolVfFArbitrationPriority:hsw */
8928         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
8929
8930         lpt_init_clock_gating(dev_priv);
8931 }
8932
8933 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
8934 {
8935         uint32_t snpcr;
8936
8937         I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
8938
8939         /* WaDisableEarlyCull:ivb */
8940         I915_WRITE(_3D_CHICKEN3,
8941                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
8942
8943         /* WaDisableBackToBackFlipFix:ivb */
8944         I915_WRITE(IVB_CHICKEN3,
8945                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
8946                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
8947
8948         /* WaDisablePSDDualDispatchEnable:ivb */
8949         if (IS_IVB_GT1(dev_priv))
8950                 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
8951                            _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
8952
8953         /* WaDisable_RenderCache_OperationalFlush:ivb */
8954         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8955
8956         /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
8957         I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
8958                    GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
8959
8960         /* WaApplyL3ControlAndL3ChickenMode:ivb */
8961         I915_WRITE(GEN7_L3CNTLREG1,
8962                         GEN7_WA_FOR_GEN7_L3_CONTROL);
8963         I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
8964                    GEN7_WA_L3_CHICKEN_MODE);
8965         if (IS_IVB_GT1(dev_priv))
8966                 I915_WRITE(GEN7_ROW_CHICKEN2,
8967                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
8968         else {
8969                 /* must write both registers */
8970                 I915_WRITE(GEN7_ROW_CHICKEN2,
8971                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
8972                 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
8973                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
8974         }
8975
8976         /* WaForceL3Serialization:ivb */
8977         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
8978                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
8979
8980         /*
8981          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
8982          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
8983          */
8984         I915_WRITE(GEN6_UCGCTL2,
8985                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
8986
8987         /* This is required by WaCatErrorRejectionIssue:ivb */
8988         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
8989                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
8990                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
8991
8992         g4x_disable_trickle_feed(dev_priv);
8993
8994         gen7_setup_fixed_func_scheduler(dev_priv);
8995
8996         if (0) { /* causes HiZ corruption on ivb:gt1 */
8997                 /* enable HiZ Raw Stall Optimization */
8998                 I915_WRITE(CACHE_MODE_0_GEN7,
8999                            _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
9000         }
9001
9002         /* WaDisable4x2SubspanOptimization:ivb */
9003         I915_WRITE(CACHE_MODE_1,
9004                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
9005
9006         /*
9007          * BSpec recommends 8x4 when MSAA is used,
9008          * however in practice 16x4 seems fastest.
9009          *
9010          * Note that PS/WM thread counts depend on the WIZ hashing
9011          * disable bit, which we don't touch here, but it's good
9012          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
9013          */
9014         I915_WRITE(GEN7_GT_MODE,
9015                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
9016
9017         snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
9018         snpcr &= ~GEN6_MBC_SNPCR_MASK;
9019         snpcr |= GEN6_MBC_SNPCR_MED;
9020         I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
9021
9022         if (!HAS_PCH_NOP(dev_priv))
9023                 cpt_init_clock_gating(dev_priv);
9024
9025         gen6_check_mch_setup(dev_priv);
9026 }
9027
9028 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
9029 {
9030         /* WaDisableEarlyCull:vlv */
9031         I915_WRITE(_3D_CHICKEN3,
9032                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
9033
9034         /* WaDisableBackToBackFlipFix:vlv */
9035         I915_WRITE(IVB_CHICKEN3,
9036                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
9037                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
9038
9039         /* WaPsdDispatchEnable:vlv */
9040         /* WaDisablePSDDualDispatchEnable:vlv */
9041         I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
9042                    _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
9043                                       GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
9044
9045         /* WaDisable_RenderCache_OperationalFlush:vlv */
9046         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9047
9048         /* WaForceL3Serialization:vlv */
9049         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
9050                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
9051
9052         /* WaDisableDopClockGating:vlv */
9053         I915_WRITE(GEN7_ROW_CHICKEN2,
9054                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
9055
9056         /* This is required by WaCatErrorRejectionIssue:vlv */
9057         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
9058                    I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
9059                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
9060
9061         gen7_setup_fixed_func_scheduler(dev_priv);
9062
9063         /*
9064          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
9065          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
9066          */
9067         I915_WRITE(GEN6_UCGCTL2,
9068                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
9069
9070         /* WaDisableL3Bank2xClockGate:vlv
9071          * Disabling L3 clock gating- MMIO 940c[25] = 1
9072          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
9073         I915_WRITE(GEN7_UCGCTL4,
9074                    I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
9075
9076         /*
9077          * BSpec says this must be set, even though
9078          * WaDisable4x2SubspanOptimization isn't listed for VLV.
9079          */
9080         I915_WRITE(CACHE_MODE_1,
9081                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
9082
9083         /*
9084          * BSpec recommends 8x4 when MSAA is used,
9085          * however in practice 16x4 seems fastest.
9086          *
9087          * Note that PS/WM thread counts depend on the WIZ hashing
9088          * disable bit, which we don't touch here, but it's good
9089          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
9090          */
9091         I915_WRITE(GEN7_GT_MODE,
9092                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
9093
9094         /*
9095          * WaIncreaseL3CreditsForVLVB0:vlv
9096          * This is the hardware default actually.
9097          */
9098         I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
9099
9100         /*
9101          * WaDisableVLVClockGating_VBIIssue:vlv
9102          * Disable clock gating on th GCFG unit to prevent a delay
9103          * in the reporting of vblank events.
9104          */
9105         I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
9106 }
9107
9108 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
9109 {
9110         /* WaVSRefCountFullforceMissDisable:chv */
9111         /* WaDSRefCountFullforceMissDisable:chv */
9112         I915_WRITE(GEN7_FF_THREAD_MODE,
9113                    I915_READ(GEN7_FF_THREAD_MODE) &
9114                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
9115
9116         /* WaDisableSemaphoreAndSyncFlipWait:chv */
9117         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
9118                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
9119
9120         /* WaDisableCSUnitClockGating:chv */
9121         I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
9122                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
9123
9124         /* WaDisableSDEUnitClockGating:chv */
9125         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
9126                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
9127
9128         /*
9129          * WaProgramL3SqcReg1Default:chv
9130          * See gfxspecs/Related Documents/Performance Guide/
9131          * LSQC Setting Recommendations.
9132          */
9133         gen8_set_l3sqc_credits(dev_priv, 38, 2);
9134
9135         /*
9136          * GTT cache may not work with big pages, so if those
9137          * are ever enabled GTT cache may need to be disabled.
9138          */
9139         I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
9140 }
9141
9142 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
9143 {
9144         uint32_t dspclk_gate;
9145
9146         I915_WRITE(RENCLK_GATE_D1, 0);
9147         I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
9148                    GS_UNIT_CLOCK_GATE_DISABLE |
9149                    CL_UNIT_CLOCK_GATE_DISABLE);
9150         I915_WRITE(RAMCLK_GATE_D, 0);
9151         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
9152                 OVRUNIT_CLOCK_GATE_DISABLE |
9153                 OVCUNIT_CLOCK_GATE_DISABLE;
9154         if (IS_GM45(dev_priv))
9155                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
9156         I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
9157
9158         /* WaDisableRenderCachePipelinedFlush */
9159         I915_WRITE(CACHE_MODE_0,
9160                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
9161
9162         /* WaDisable_RenderCache_OperationalFlush:g4x */
9163         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9164
9165         g4x_disable_trickle_feed(dev_priv);
9166 }
9167
9168 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
9169 {
9170         I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
9171         I915_WRITE(RENCLK_GATE_D2, 0);
9172         I915_WRITE(DSPCLK_GATE_D, 0);
9173         I915_WRITE(RAMCLK_GATE_D, 0);
9174         I915_WRITE16(DEUC, 0);
9175         I915_WRITE(MI_ARB_STATE,
9176                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
9177
9178         /* WaDisable_RenderCache_OperationalFlush:gen4 */
9179         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9180 }
9181
9182 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
9183 {
9184         I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
9185                    I965_RCC_CLOCK_GATE_DISABLE |
9186                    I965_RCPB_CLOCK_GATE_DISABLE |
9187                    I965_ISC_CLOCK_GATE_DISABLE |
9188                    I965_FBC_CLOCK_GATE_DISABLE);
9189         I915_WRITE(RENCLK_GATE_D2, 0);
9190         I915_WRITE(MI_ARB_STATE,
9191                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
9192
9193         /* WaDisable_RenderCache_OperationalFlush:gen4 */
9194         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
9195 }
9196
9197 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
9198 {
9199         u32 dstate = I915_READ(D_STATE);
9200
9201         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
9202                 DSTATE_DOT_CLOCK_GATING;
9203         I915_WRITE(D_STATE, dstate);
9204
9205         if (IS_PINEVIEW(dev_priv))
9206                 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
9207
9208         /* IIR "flip pending" means done if this bit is set */
9209         I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
9210
9211         /* interrupts should cause a wake up from C3 */
9212         I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
9213
9214         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
9215         I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
9216
9217         I915_WRITE(MI_ARB_STATE,
9218                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
9219 }
9220
9221 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
9222 {
9223         I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
9224
9225         /* interrupts should cause a wake up from C3 */
9226         I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
9227                    _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
9228
9229         I915_WRITE(MEM_MODE,
9230                    _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
9231 }
9232
9233 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
9234 {
9235         I915_WRITE(MEM_MODE,
9236                    _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
9237                    _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
9238 }
9239
9240 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
9241 {
9242         dev_priv->display.init_clock_gating(dev_priv);
9243 }
9244
9245 void intel_suspend_hw(struct drm_i915_private *dev_priv)
9246 {
9247         if (HAS_PCH_LPT(dev_priv))
9248                 lpt_suspend_hw(dev_priv);
9249 }
9250
9251 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
9252 {
9253         DRM_DEBUG_KMS("No clock gating settings or workarounds applied.\n");
9254 }
9255
9256 /**
9257  * intel_init_clock_gating_hooks - setup the clock gating hooks
9258  * @dev_priv: device private
9259  *
9260  * Setup the hooks that configure which clocks of a given platform can be
9261  * gated and also apply various GT and display specific workarounds for these
9262  * platforms. Note that some GT specific workarounds are applied separately
9263  * when GPU contexts or batchbuffers start their execution.
9264  */
9265 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
9266 {
9267         if (IS_ICELAKE(dev_priv))
9268                 dev_priv->display.init_clock_gating = icl_init_clock_gating;
9269         else if (IS_CANNONLAKE(dev_priv))
9270                 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
9271         else if (IS_COFFEELAKE(dev_priv))
9272                 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
9273         else if (IS_SKYLAKE(dev_priv))
9274                 dev_priv->display.init_clock_gating = skl_init_clock_gating;
9275         else if (IS_KABYLAKE(dev_priv))
9276                 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
9277         else if (IS_BROXTON(dev_priv))
9278                 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
9279         else if (IS_GEMINILAKE(dev_priv))
9280                 dev_priv->display.init_clock_gating = glk_init_clock_gating;
9281         else if (IS_BROADWELL(dev_priv))
9282                 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
9283         else if (IS_CHERRYVIEW(dev_priv))
9284                 dev_priv->display.init_clock_gating = chv_init_clock_gating;
9285         else if (IS_HASWELL(dev_priv))
9286                 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
9287         else if (IS_IVYBRIDGE(dev_priv))
9288                 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
9289         else if (IS_VALLEYVIEW(dev_priv))
9290                 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
9291         else if (IS_GEN6(dev_priv))
9292                 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
9293         else if (IS_GEN5(dev_priv))
9294                 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
9295         else if (IS_G4X(dev_priv))
9296                 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
9297         else if (IS_I965GM(dev_priv))
9298                 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
9299         else if (IS_I965G(dev_priv))
9300                 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
9301         else if (IS_GEN3(dev_priv))
9302                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
9303         else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
9304                 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
9305         else if (IS_GEN2(dev_priv))
9306                 dev_priv->display.init_clock_gating = i830_init_clock_gating;
9307         else {
9308                 MISSING_CASE(INTEL_DEVID(dev_priv));
9309                 dev_priv->display.init_clock_gating = nop_init_clock_gating;
9310         }
9311 }
9312
9313 /* Set up chip specific power management-related functions */
9314 void intel_init_pm(struct drm_i915_private *dev_priv)
9315 {
9316         intel_fbc_init(dev_priv);
9317
9318         /* For cxsr */
9319         if (IS_PINEVIEW(dev_priv))
9320                 i915_pineview_get_mem_freq(dev_priv);
9321         else if (IS_GEN5(dev_priv))
9322                 i915_ironlake_get_mem_freq(dev_priv);
9323
9324         /* For FIFO watermark updates */
9325         if (INTEL_GEN(dev_priv) >= 9) {
9326                 skl_setup_wm_latency(dev_priv);
9327                 dev_priv->display.initial_watermarks = skl_initial_wm;
9328                 dev_priv->display.atomic_update_watermarks = skl_atomic_update_crtc_wm;
9329                 dev_priv->display.compute_global_watermarks = skl_compute_wm;
9330         } else if (HAS_PCH_SPLIT(dev_priv)) {
9331                 ilk_setup_wm_latency(dev_priv);
9332
9333                 if ((IS_GEN5(dev_priv) && dev_priv->wm.pri_latency[1] &&
9334                      dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
9335                     (!IS_GEN5(dev_priv) && dev_priv->wm.pri_latency[0] &&
9336                      dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
9337                         dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
9338                         dev_priv->display.compute_intermediate_wm =
9339                                 ilk_compute_intermediate_wm;
9340                         dev_priv->display.initial_watermarks =
9341                                 ilk_initial_watermarks;
9342                         dev_priv->display.optimize_watermarks =
9343                                 ilk_optimize_watermarks;
9344                 } else {
9345                         DRM_DEBUG_KMS("Failed to read display plane latency. "
9346                                       "Disable CxSR\n");
9347                 }
9348         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
9349                 vlv_setup_wm_latency(dev_priv);
9350                 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
9351                 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
9352                 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
9353                 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
9354                 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
9355         } else if (IS_G4X(dev_priv)) {
9356                 g4x_setup_wm_latency(dev_priv);
9357                 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
9358                 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
9359                 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
9360                 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
9361         } else if (IS_PINEVIEW(dev_priv)) {
9362                 if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv),
9363                                             dev_priv->is_ddr3,
9364                                             dev_priv->fsb_freq,
9365                                             dev_priv->mem_freq)) {
9366                         DRM_INFO("failed to find known CxSR latency "
9367                                  "(found ddr%s fsb freq %d, mem freq %d), "
9368                                  "disabling CxSR\n",
9369                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
9370                                  dev_priv->fsb_freq, dev_priv->mem_freq);
9371                         /* Disable CxSR and never update its watermark again */
9372                         intel_set_memory_cxsr(dev_priv, false);
9373                         dev_priv->display.update_wm = NULL;
9374                 } else
9375                         dev_priv->display.update_wm = pineview_update_wm;
9376         } else if (IS_GEN4(dev_priv)) {
9377                 dev_priv->display.update_wm = i965_update_wm;
9378         } else if (IS_GEN3(dev_priv)) {
9379                 dev_priv->display.update_wm = i9xx_update_wm;
9380                 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
9381         } else if (IS_GEN2(dev_priv)) {
9382                 if (INTEL_INFO(dev_priv)->num_pipes == 1) {
9383                         dev_priv->display.update_wm = i845_update_wm;
9384                         dev_priv->display.get_fifo_size = i845_get_fifo_size;
9385                 } else {
9386                         dev_priv->display.update_wm = i9xx_update_wm;
9387                         dev_priv->display.get_fifo_size = i830_get_fifo_size;
9388                 }
9389         } else {
9390                 DRM_ERROR("unexpected fall-through in intel_init_pm\n");
9391         }
9392 }
9393
9394 static inline int gen6_check_mailbox_status(struct drm_i915_private *dev_priv)
9395 {
9396         uint32_t flags =
9397                 I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_ERROR_MASK;
9398
9399         switch (flags) {
9400         case GEN6_PCODE_SUCCESS:
9401                 return 0;
9402         case GEN6_PCODE_UNIMPLEMENTED_CMD:
9403                 return -ENODEV;
9404         case GEN6_PCODE_ILLEGAL_CMD:
9405                 return -ENXIO;
9406         case GEN6_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
9407         case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
9408                 return -EOVERFLOW;
9409         case GEN6_PCODE_TIMEOUT:
9410                 return -ETIMEDOUT;
9411         default:
9412                 MISSING_CASE(flags);
9413                 return 0;
9414         }
9415 }
9416
9417 static inline int gen7_check_mailbox_status(struct drm_i915_private *dev_priv)
9418 {
9419         uint32_t flags =
9420                 I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_ERROR_MASK;
9421
9422         switch (flags) {
9423         case GEN6_PCODE_SUCCESS:
9424                 return 0;
9425         case GEN6_PCODE_ILLEGAL_CMD:
9426                 return -ENXIO;
9427         case GEN7_PCODE_TIMEOUT:
9428                 return -ETIMEDOUT;
9429         case GEN7_PCODE_ILLEGAL_DATA:
9430                 return -EINVAL;
9431         case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
9432                 return -EOVERFLOW;
9433         default:
9434                 MISSING_CASE(flags);
9435                 return 0;
9436         }
9437 }
9438
9439 int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val)
9440 {
9441         int status;
9442
9443         WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
9444
9445         /* GEN6_PCODE_* are outside of the forcewake domain, we can
9446          * use te fw I915_READ variants to reduce the amount of work
9447          * required when reading/writing.
9448          */
9449
9450         if (I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
9451                 DRM_DEBUG_DRIVER("warning: pcode (read from mbox %x) mailbox access failed for %ps\n",
9452                                  mbox, __builtin_return_address(0));
9453                 return -EAGAIN;
9454         }
9455
9456         I915_WRITE_FW(GEN6_PCODE_DATA, *val);
9457         I915_WRITE_FW(GEN6_PCODE_DATA1, 0);
9458         I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
9459
9460         if (__intel_wait_for_register_fw(dev_priv,
9461                                          GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
9462                                          500, 0, NULL)) {
9463                 DRM_ERROR("timeout waiting for pcode read (from mbox %x) to finish for %ps\n",
9464                           mbox, __builtin_return_address(0));
9465                 return -ETIMEDOUT;
9466         }
9467
9468         *val = I915_READ_FW(GEN6_PCODE_DATA);
9469         I915_WRITE_FW(GEN6_PCODE_DATA, 0);
9470
9471         if (INTEL_GEN(dev_priv) > 6)
9472                 status = gen7_check_mailbox_status(dev_priv);
9473         else
9474                 status = gen6_check_mailbox_status(dev_priv);
9475
9476         if (status) {
9477                 DRM_DEBUG_DRIVER("warning: pcode (read from mbox %x) mailbox access failed for %ps: %d\n",
9478                                  mbox, __builtin_return_address(0), status);
9479                 return status;
9480         }
9481
9482         return 0;
9483 }
9484
9485 int sandybridge_pcode_write_timeout(struct drm_i915_private *dev_priv,
9486                                     u32 mbox, u32 val,
9487                                     int fast_timeout_us, int slow_timeout_ms)
9488 {
9489         int status;
9490
9491         WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
9492
9493         /* GEN6_PCODE_* are outside of the forcewake domain, we can
9494          * use te fw I915_READ variants to reduce the amount of work
9495          * required when reading/writing.
9496          */
9497
9498         if (I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
9499                 DRM_DEBUG_DRIVER("warning: pcode (write of 0x%08x to mbox %x) mailbox access failed for %ps\n",
9500                                  val, mbox, __builtin_return_address(0));
9501                 return -EAGAIN;
9502         }
9503
9504         I915_WRITE_FW(GEN6_PCODE_DATA, val);
9505         I915_WRITE_FW(GEN6_PCODE_DATA1, 0);
9506         I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
9507
9508         if (__intel_wait_for_register_fw(dev_priv,
9509                                          GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
9510                                          fast_timeout_us, slow_timeout_ms,
9511                                          NULL)) {
9512                 DRM_ERROR("timeout waiting for pcode write of 0x%08x to mbox %x to finish for %ps\n",
9513                           val, mbox, __builtin_return_address(0));
9514                 return -ETIMEDOUT;
9515         }
9516
9517         I915_WRITE_FW(GEN6_PCODE_DATA, 0);
9518
9519         if (INTEL_GEN(dev_priv) > 6)
9520                 status = gen7_check_mailbox_status(dev_priv);
9521         else
9522                 status = gen6_check_mailbox_status(dev_priv);
9523
9524         if (status) {
9525                 DRM_DEBUG_DRIVER("warning: pcode (write of 0x%08x to mbox %x) mailbox access failed for %ps: %d\n",
9526                                  val, mbox, __builtin_return_address(0), status);
9527                 return status;
9528         }
9529
9530         return 0;
9531 }
9532
9533 static bool skl_pcode_try_request(struct drm_i915_private *dev_priv, u32 mbox,
9534                                   u32 request, u32 reply_mask, u32 reply,
9535                                   u32 *status)
9536 {
9537         u32 val = request;
9538
9539         *status = sandybridge_pcode_read(dev_priv, mbox, &val);
9540
9541         return *status || ((val & reply_mask) == reply);
9542 }
9543
9544 /**
9545  * skl_pcode_request - send PCODE request until acknowledgment
9546  * @dev_priv: device private
9547  * @mbox: PCODE mailbox ID the request is targeted for
9548  * @request: request ID
9549  * @reply_mask: mask used to check for request acknowledgment
9550  * @reply: value used to check for request acknowledgment
9551  * @timeout_base_ms: timeout for polling with preemption enabled
9552  *
9553  * Keep resending the @request to @mbox until PCODE acknowledges it, PCODE
9554  * reports an error or an overall timeout of @timeout_base_ms+50 ms expires.
9555  * The request is acknowledged once the PCODE reply dword equals @reply after
9556  * applying @reply_mask. Polling is first attempted with preemption enabled
9557  * for @timeout_base_ms and if this times out for another 50 ms with
9558  * preemption disabled.
9559  *
9560  * Returns 0 on success, %-ETIMEDOUT in case of a timeout, <0 in case of some
9561  * other error as reported by PCODE.
9562  */
9563 int skl_pcode_request(struct drm_i915_private *dev_priv, u32 mbox, u32 request,
9564                       u32 reply_mask, u32 reply, int timeout_base_ms)
9565 {
9566         u32 status;
9567         int ret;
9568
9569         WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
9570
9571 #define COND skl_pcode_try_request(dev_priv, mbox, request, reply_mask, reply, \
9572                                    &status)
9573
9574         /*
9575          * Prime the PCODE by doing a request first. Normally it guarantees
9576          * that a subsequent request, at most @timeout_base_ms later, succeeds.
9577          * _wait_for() doesn't guarantee when its passed condition is evaluated
9578          * first, so send the first request explicitly.
9579          */
9580         if (COND) {
9581                 ret = 0;
9582                 goto out;
9583         }
9584         ret = _wait_for(COND, timeout_base_ms * 1000, 10, 10);
9585         if (!ret)
9586                 goto out;
9587
9588         /*
9589          * The above can time out if the number of requests was low (2 in the
9590          * worst case) _and_ PCODE was busy for some reason even after a
9591          * (queued) request and @timeout_base_ms delay. As a workaround retry
9592          * the poll with preemption disabled to maximize the number of
9593          * requests. Increase the timeout from @timeout_base_ms to 50ms to
9594          * account for interrupts that could reduce the number of these
9595          * requests, and for any quirks of the PCODE firmware that delays
9596          * the request completion.
9597          */
9598         DRM_DEBUG_KMS("PCODE timeout, retrying with preemption disabled\n");
9599         WARN_ON_ONCE(timeout_base_ms > 3);
9600         preempt_disable();
9601         ret = wait_for_atomic(COND, 50);
9602         preempt_enable();
9603
9604 out:
9605         return ret ? ret : status;
9606 #undef COND
9607 }
9608
9609 static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
9610 {
9611         struct intel_rps *rps = &dev_priv->gt_pm.rps;
9612
9613         /*
9614          * N = val - 0xb7
9615          * Slow = Fast = GPLL ref * N
9616          */
9617         return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
9618 }
9619
9620 static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
9621 {
9622         struct intel_rps *rps = &dev_priv->gt_pm.rps;
9623
9624         return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
9625 }
9626
9627 static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
9628 {
9629         struct intel_rps *rps = &dev_priv->gt_pm.rps;
9630
9631         /*
9632          * N = val / 2
9633          * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
9634          */
9635         return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
9636 }
9637
9638 static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
9639 {
9640         struct intel_rps *rps = &dev_priv->gt_pm.rps;
9641
9642         /* CHV needs even values */
9643         return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
9644 }
9645
9646 int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
9647 {
9648         if (INTEL_GEN(dev_priv) >= 9)
9649                 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
9650                                          GEN9_FREQ_SCALER);
9651         else if (IS_CHERRYVIEW(dev_priv))
9652                 return chv_gpu_freq(dev_priv, val);
9653         else if (IS_VALLEYVIEW(dev_priv))
9654                 return byt_gpu_freq(dev_priv, val);
9655         else
9656                 return val * GT_FREQUENCY_MULTIPLIER;
9657 }
9658
9659 int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
9660 {
9661         if (INTEL_GEN(dev_priv) >= 9)
9662                 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
9663                                          GT_FREQUENCY_MULTIPLIER);
9664         else if (IS_CHERRYVIEW(dev_priv))
9665                 return chv_freq_opcode(dev_priv, val);
9666         else if (IS_VALLEYVIEW(dev_priv))
9667                 return byt_freq_opcode(dev_priv, val);
9668         else
9669                 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
9670 }
9671
9672 void intel_pm_setup(struct drm_i915_private *dev_priv)
9673 {
9674         mutex_init(&dev_priv->pcu_lock);
9675         mutex_init(&dev_priv->gt_pm.rps.power.mutex);
9676
9677         atomic_set(&dev_priv->gt_pm.rps.num_waiters, 0);
9678
9679         dev_priv->runtime_pm.suspended = false;
9680         atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
9681 }
9682
9683 static u64 vlv_residency_raw(struct drm_i915_private *dev_priv,
9684                              const i915_reg_t reg)
9685 {
9686         u32 lower, upper, tmp;
9687         int loop = 2;
9688
9689         /*
9690          * The register accessed do not need forcewake. We borrow
9691          * uncore lock to prevent concurrent access to range reg.
9692          */
9693         lockdep_assert_held(&dev_priv->uncore.lock);
9694
9695         /*
9696          * vlv and chv residency counters are 40 bits in width.
9697          * With a control bit, we can choose between upper or lower
9698          * 32bit window into this counter.
9699          *
9700          * Although we always use the counter in high-range mode elsewhere,
9701          * userspace may attempt to read the value before rc6 is initialised,
9702          * before we have set the default VLV_COUNTER_CONTROL value. So always
9703          * set the high bit to be safe.
9704          */
9705         I915_WRITE_FW(VLV_COUNTER_CONTROL,
9706                       _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
9707         upper = I915_READ_FW(reg);
9708         do {
9709                 tmp = upper;
9710
9711                 I915_WRITE_FW(VLV_COUNTER_CONTROL,
9712                               _MASKED_BIT_DISABLE(VLV_COUNT_RANGE_HIGH));
9713                 lower = I915_READ_FW(reg);
9714
9715                 I915_WRITE_FW(VLV_COUNTER_CONTROL,
9716                               _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
9717                 upper = I915_READ_FW(reg);
9718         } while (upper != tmp && --loop);
9719
9720         /*
9721          * Everywhere else we always use VLV_COUNTER_CONTROL with the
9722          * VLV_COUNT_RANGE_HIGH bit set - so it is safe to leave it set
9723          * now.
9724          */
9725
9726         return lower | (u64)upper << 8;
9727 }
9728
9729 u64 intel_rc6_residency_ns(struct drm_i915_private *dev_priv,
9730                            const i915_reg_t reg)
9731 {
9732         u64 time_hw, prev_hw, overflow_hw;
9733         unsigned int fw_domains;
9734         unsigned long flags;
9735         unsigned int i;
9736         u32 mul, div;
9737
9738         if (!HAS_RC6(dev_priv))
9739                 return 0;
9740
9741         /*
9742          * Store previous hw counter values for counter wrap-around handling.
9743          *
9744          * There are only four interesting registers and they live next to each
9745          * other so we can use the relative address, compared to the smallest
9746          * one as the index into driver storage.
9747          */
9748         i = (i915_mmio_reg_offset(reg) -
9749              i915_mmio_reg_offset(GEN6_GT_GFX_RC6_LOCKED)) / sizeof(u32);
9750         if (WARN_ON_ONCE(i >= ARRAY_SIZE(dev_priv->gt_pm.rc6.cur_residency)))
9751                 return 0;
9752
9753         fw_domains = intel_uncore_forcewake_for_reg(dev_priv, reg, FW_REG_READ);
9754
9755         spin_lock_irqsave(&dev_priv->uncore.lock, flags);
9756         intel_uncore_forcewake_get__locked(dev_priv, fw_domains);
9757
9758         /* On VLV and CHV, residency time is in CZ units rather than 1.28us */
9759         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
9760                 mul = 1000000;
9761                 div = dev_priv->czclk_freq;
9762                 overflow_hw = BIT_ULL(40);
9763                 time_hw = vlv_residency_raw(dev_priv, reg);
9764         } else {
9765                 /* 833.33ns units on Gen9LP, 1.28us elsewhere. */
9766                 if (IS_GEN9_LP(dev_priv)) {
9767                         mul = 10000;
9768                         div = 12;
9769                 } else {
9770                         mul = 1280;
9771                         div = 1;
9772                 }
9773
9774                 overflow_hw = BIT_ULL(32);
9775                 time_hw = I915_READ_FW(reg);
9776         }
9777
9778         /*
9779          * Counter wrap handling.
9780          *
9781          * But relying on a sufficient frequency of queries otherwise counters
9782          * can still wrap.
9783          */
9784         prev_hw = dev_priv->gt_pm.rc6.prev_hw_residency[i];
9785         dev_priv->gt_pm.rc6.prev_hw_residency[i] = time_hw;
9786
9787         /* RC6 delta from last sample. */
9788         if (time_hw >= prev_hw)
9789                 time_hw -= prev_hw;
9790         else
9791                 time_hw += overflow_hw - prev_hw;
9792
9793         /* Add delta to RC6 extended raw driver copy. */
9794         time_hw += dev_priv->gt_pm.rc6.cur_residency[i];
9795         dev_priv->gt_pm.rc6.cur_residency[i] = time_hw;
9796
9797         intel_uncore_forcewake_put__locked(dev_priv, fw_domains);
9798         spin_unlock_irqrestore(&dev_priv->uncore.lock, flags);
9799
9800         return mul_u64_u32_div(time_hw, mul, div);
9801 }
9802
9803 u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat)
9804 {
9805         u32 cagf;
9806
9807         if (INTEL_GEN(dev_priv) >= 9)
9808                 cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
9809         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
9810                 cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
9811         else
9812                 cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
9813
9814         return  cagf;
9815 }