drivers/gpu/drm/i915/display/intel_cdclk.c

   1 /*
   2  * Copyright © 2006-2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include <linux/time.h>
  25
  26 #include "intel_atomic.h"
  27 #include "intel_bw.h"
  28 #include "intel_cdclk.h"
  29 #include "intel_de.h"
  30 #include "intel_display_types.h"
  31 #include "intel_sideband.h"
  32
  33 /**
  34  * DOC: CDCLK / RAWCLK
  35  *
  36  * The display engine uses several different clocks to do its work. There
  37  * are two main clocks involved that aren't directly related to the actual
  38  * pixel clock or any symbol/bit clock of the actual output port. These
  39  * are the core display clock (CDCLK) and RAWCLK.
  40  *
  41  * CDCLK clocks most of the display pipe logic, and thus its frequency
  42  * must be high enough to support the rate at which pixels are flowing
  43  * through the pipes. Downscaling must also be accounted as that increases
  44  * the effective pixel rate.
  45  *
  46  * On several platforms the CDCLK frequency can be changed dynamically
  47  * to minimize power consumption for a given display configuration.
  48  * Typically changes to the CDCLK frequency require all the display pipes
  49  * to be shut down while the frequency is being changed.
  50  *
  51  * On SKL+ the DMC will toggle the CDCLK off/on during DC5/6 entry/exit.
  52  * DMC will not change the active CDCLK frequency however, so that part
  53  * will still be performed by the driver directly.
  54  *
  55  * RAWCLK is a fixed frequency clock, often used by various auxiliary
  56  * blocks such as AUX CH or backlight PWM. Hence the only thing we
  57  * really need to know about RAWCLK is its frequency so that various
  58  * dividers can be programmed correctly.
  59  */
  60
  61 static void fixed_133mhz_get_cdclk(struct drm_i915_private *dev_priv,
  62                                    struct intel_cdclk_config *cdclk_config)
  63 {
  64         cdclk_config->cdclk = 133333;
  65 }
  66
  67 static void fixed_200mhz_get_cdclk(struct drm_i915_private *dev_priv,
  68                                    struct intel_cdclk_config *cdclk_config)
  69 {
  70         cdclk_config->cdclk = 200000;
  71 }
  72
  73 static void fixed_266mhz_get_cdclk(struct drm_i915_private *dev_priv,
  74                                    struct intel_cdclk_config *cdclk_config)
  75 {
  76         cdclk_config->cdclk = 266667;
  77 }
  78
  79 static void fixed_333mhz_get_cdclk(struct drm_i915_private *dev_priv,
  80                                    struct intel_cdclk_config *cdclk_config)
  81 {
  82         cdclk_config->cdclk = 333333;
  83 }
  84
  85 static void fixed_400mhz_get_cdclk(struct drm_i915_private *dev_priv,
  86                                    struct intel_cdclk_config *cdclk_config)
  87 {
  88         cdclk_config->cdclk = 400000;
  89 }
  90
  91 static void fixed_450mhz_get_cdclk(struct drm_i915_private *dev_priv,
  92                                    struct intel_cdclk_config *cdclk_config)
  93 {
  94         cdclk_config->cdclk = 450000;
  95 }
  96
  97 static void i85x_get_cdclk(struct drm_i915_private *dev_priv,
  98                            struct intel_cdclk_config *cdclk_config)
  99 {
 100         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 101         u16 hpllcc = 0;
 102
 103         /*
 104          * 852GM/852GMV only supports 133 MHz and the HPLLCC
 105          * encoding is different :(
 106          * FIXME is this the right way to detect 852GM/852GMV?
 107          */
 108         if (pdev->revision == 0x1) {
 109                 cdclk_config->cdclk = 133333;
 110                 return;
 111         }
 112
 113         pci_bus_read_config_word(pdev->bus,
 114                                  PCI_DEVFN(0, 3), HPLLCC, &hpllcc);
 115
 116         /* Assume that the hardware is in the high speed state.  This
 117          * should be the default.
 118          */
 119         switch (hpllcc & GC_CLOCK_CONTROL_MASK) {
 120         case GC_CLOCK_133_200:
 121         case GC_CLOCK_133_200_2:
 122         case GC_CLOCK_100_200:
 123                 cdclk_config->cdclk = 200000;
 124                 break;
 125         case GC_CLOCK_166_250:
 126                 cdclk_config->cdclk = 250000;
 127                 break;
 128         case GC_CLOCK_100_133:
 129                 cdclk_config->cdclk = 133333;
 130                 break;
 131         case GC_CLOCK_133_266:
 132         case GC_CLOCK_133_266_2:
 133         case GC_CLOCK_166_266:
 134                 cdclk_config->cdclk = 266667;
 135                 break;
 136         }
 137 }
 138
 139 static void i915gm_get_cdclk(struct drm_i915_private *dev_priv,
 140                              struct intel_cdclk_config *cdclk_config)
 141 {
 142         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 143         u16 gcfgc = 0;
 144
 145         pci_read_config_word(pdev, GCFGC, &gcfgc);
 146
 147         if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
 148                 cdclk_config->cdclk = 133333;
 149                 return;
 150         }
 151
 152         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 153         case GC_DISPLAY_CLOCK_333_320_MHZ:
 154                 cdclk_config->cdclk = 333333;
 155                 break;
 156         default:
 157         case GC_DISPLAY_CLOCK_190_200_MHZ:
 158                 cdclk_config->cdclk = 190000;
 159                 break;
 160         }
 161 }
 162
 163 static void i945gm_get_cdclk(struct drm_i915_private *dev_priv,
 164                              struct intel_cdclk_config *cdclk_config)
 165 {
 166         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 167         u16 gcfgc = 0;
 168
 169         pci_read_config_word(pdev, GCFGC, &gcfgc);
 170
 171         if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
 172                 cdclk_config->cdclk = 133333;
 173                 return;
 174         }
 175
 176         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 177         case GC_DISPLAY_CLOCK_333_320_MHZ:
 178                 cdclk_config->cdclk = 320000;
 179                 break;
 180         default:
 181         case GC_DISPLAY_CLOCK_190_200_MHZ:
 182                 cdclk_config->cdclk = 200000;
 183                 break;
 184         }
 185 }
 186
 187 static unsigned int intel_hpll_vco(struct drm_i915_private *dev_priv)
 188 {
 189         static const unsigned int blb_vco[8] = {
 190                 [0] = 3200000,
 191                 [1] = 4000000,
 192                 [2] = 5333333,
 193                 [3] = 4800000,
 194                 [4] = 6400000,
 195         };
 196         static const unsigned int pnv_vco[8] = {
 197                 [0] = 3200000,
 198                 [1] = 4000000,
 199                 [2] = 5333333,
 200                 [3] = 4800000,
 201                 [4] = 2666667,
 202         };
 203         static const unsigned int cl_vco[8] = {
 204                 [0] = 3200000,
 205                 [1] = 4000000,
 206                 [2] = 5333333,
 207                 [3] = 6400000,
 208                 [4] = 3333333,
 209                 [5] = 3566667,
 210                 [6] = 4266667,
 211         };
 212         static const unsigned int elk_vco[8] = {
 213                 [0] = 3200000,
 214                 [1] = 4000000,
 215                 [2] = 5333333,
 216                 [3] = 4800000,
 217         };
 218         static const unsigned int ctg_vco[8] = {
 219                 [0] = 3200000,
 220                 [1] = 4000000,
 221                 [2] = 5333333,
 222                 [3] = 6400000,
 223                 [4] = 2666667,
 224                 [5] = 4266667,
 225         };
 226         const unsigned int *vco_table;
 227         unsigned int vco;
 228         u8 tmp = 0;
 229
 230         /* FIXME other chipsets? */
 231         if (IS_GM45(dev_priv))
 232                 vco_table = ctg_vco;
 233         else if (IS_G45(dev_priv))
 234                 vco_table = elk_vco;
 235         else if (IS_I965GM(dev_priv))
 236                 vco_table = cl_vco;
 237         else if (IS_PINEVIEW(dev_priv))
 238                 vco_table = pnv_vco;
 239         else if (IS_G33(dev_priv))
 240                 vco_table = blb_vco;
 241         else
 242                 return 0;
 243
 244         tmp = intel_de_read(dev_priv,
 245                             IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv) ? HPLLVCO_MOBILE : HPLLVCO);
 246
 247         vco = vco_table[tmp & 0x7];
 248         if (vco == 0)
 249                 drm_err(&dev_priv->drm, "Bad HPLL VCO (HPLLVCO=0x%02x)\n",
 250                         tmp);
 251         else
 252                 drm_dbg_kms(&dev_priv->drm, "HPLL VCO %u kHz\n", vco);
 253
 254         return vco;
 255 }
 256
 257 static void g33_get_cdclk(struct drm_i915_private *dev_priv,
 258                           struct intel_cdclk_config *cdclk_config)
 259 {
 260         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 261         static const u8 div_3200[] = { 12, 10,  8,  7, 5, 16 };
 262         static const u8 div_4000[] = { 14, 12, 10,  8, 6, 20 };
 263         static const u8 div_4800[] = { 20, 14, 12, 10, 8, 24 };
 264         static const u8 div_5333[] = { 20, 16, 12, 12, 8, 28 };
 265         const u8 *div_table;
 266         unsigned int cdclk_sel;
 267         u16 tmp = 0;
 268
 269         cdclk_config->vco = intel_hpll_vco(dev_priv);
 270
 271         pci_read_config_word(pdev, GCFGC, &tmp);
 272
 273         cdclk_sel = (tmp >> 4) & 0x7;
 274
 275         if (cdclk_sel >= ARRAY_SIZE(div_3200))
 276                 goto fail;
 277
 278         switch (cdclk_config->vco) {
 279         case 3200000:
 280                 div_table = div_3200;
 281                 break;
 282         case 4000000:
 283                 div_table = div_4000;
 284                 break;
 285         case 4800000:
 286                 div_table = div_4800;
 287                 break;
 288         case 5333333:
 289                 div_table = div_5333;
 290                 break;
 291         default:
 292                 goto fail;
 293         }
 294
 295         cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
 296                                                 div_table[cdclk_sel]);
 297         return;
 298
 299 fail:
 300         drm_err(&dev_priv->drm,
 301                 "Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%08x\n",
 302                 cdclk_config->vco, tmp);
 303         cdclk_config->cdclk = 190476;
 304 }
 305
 306 static void pnv_get_cdclk(struct drm_i915_private *dev_priv,
 307                           struct intel_cdclk_config *cdclk_config)
 308 {
 309         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 310         u16 gcfgc = 0;
 311
 312         pci_read_config_word(pdev, GCFGC, &gcfgc);
 313
 314         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 315         case GC_DISPLAY_CLOCK_267_MHZ_PNV:
 316                 cdclk_config->cdclk = 266667;
 317                 break;
 318         case GC_DISPLAY_CLOCK_333_MHZ_PNV:
 319                 cdclk_config->cdclk = 333333;
 320                 break;
 321         case GC_DISPLAY_CLOCK_444_MHZ_PNV:
 322                 cdclk_config->cdclk = 444444;
 323                 break;
 324         case GC_DISPLAY_CLOCK_200_MHZ_PNV:
 325                 cdclk_config->cdclk = 200000;
 326                 break;
 327         default:
 328                 drm_err(&dev_priv->drm,
 329                         "Unknown pnv display core clock 0x%04x\n", gcfgc);
 330                 fallthrough;
 331         case GC_DISPLAY_CLOCK_133_MHZ_PNV:
 332                 cdclk_config->cdclk = 133333;
 333                 break;
 334         case GC_DISPLAY_CLOCK_167_MHZ_PNV:
 335                 cdclk_config->cdclk = 166667;
 336                 break;
 337         }
 338 }
 339
 340 static void i965gm_get_cdclk(struct drm_i915_private *dev_priv,
 341                              struct intel_cdclk_config *cdclk_config)
 342 {
 343         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 344         static const u8 div_3200[] = { 16, 10,  8 };
 345         static const u8 div_4000[] = { 20, 12, 10 };
 346         static const u8 div_5333[] = { 24, 16, 14 };
 347         const u8 *div_table;
 348         unsigned int cdclk_sel;
 349         u16 tmp = 0;
 350
 351         cdclk_config->vco = intel_hpll_vco(dev_priv);
 352
 353         pci_read_config_word(pdev, GCFGC, &tmp);
 354
 355         cdclk_sel = ((tmp >> 8) & 0x1f) - 1;
 356
 357         if (cdclk_sel >= ARRAY_SIZE(div_3200))
 358                 goto fail;
 359
 360         switch (cdclk_config->vco) {
 361         case 3200000:
 362                 div_table = div_3200;
 363                 break;
 364         case 4000000:
 365                 div_table = div_4000;
 366                 break;
 367         case 5333333:
 368                 div_table = div_5333;
 369                 break;
 370         default:
 371                 goto fail;
 372         }
 373
 374         cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
 375                                                 div_table[cdclk_sel]);
 376         return;
 377
 378 fail:
 379         drm_err(&dev_priv->drm,
 380                 "Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%04x\n",
 381                 cdclk_config->vco, tmp);
 382         cdclk_config->cdclk = 200000;
 383 }
 384
 385 static void gm45_get_cdclk(struct drm_i915_private *dev_priv,
 386                            struct intel_cdclk_config *cdclk_config)
 387 {
 388         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 389         unsigned int cdclk_sel;
 390         u16 tmp = 0;
 391
 392         cdclk_config->vco = intel_hpll_vco(dev_priv);
 393
 394         pci_read_config_word(pdev, GCFGC, &tmp);
 395
 396         cdclk_sel = (tmp >> 12) & 0x1;
 397
 398         switch (cdclk_config->vco) {
 399         case 2666667:
 400         case 4000000:
 401         case 5333333:
 402                 cdclk_config->cdclk = cdclk_sel ? 333333 : 222222;
 403                 break;
 404         case 3200000:
 405                 cdclk_config->cdclk = cdclk_sel ? 320000 : 228571;
 406                 break;
 407         default:
 408                 drm_err(&dev_priv->drm,
 409                         "Unable to determine CDCLK. HPLL VCO=%u, CFGC=0x%04x\n",
 410                         cdclk_config->vco, tmp);
 411                 cdclk_config->cdclk = 222222;
 412                 break;
 413         }
 414 }
 415
 416 static void hsw_get_cdclk(struct drm_i915_private *dev_priv,
 417                           struct intel_cdclk_config *cdclk_config)
 418 {
 419         u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
 420         u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
 421
 422         if (lcpll & LCPLL_CD_SOURCE_FCLK)
 423                 cdclk_config->cdclk = 800000;
 424         else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
 425                 cdclk_config->cdclk = 450000;
 426         else if (freq == LCPLL_CLK_FREQ_450)
 427                 cdclk_config->cdclk = 450000;
 428         else if (IS_HSW_ULT(dev_priv))
 429                 cdclk_config->cdclk = 337500;
 430         else
 431                 cdclk_config->cdclk = 540000;
 432 }
 433
 434 static int vlv_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
 435 {
 436         int freq_320 = (dev_priv->hpll_freq <<  1) % 320000 != 0 ?
 437                 333333 : 320000;
 438
 439         /*
 440          * We seem to get an unstable or solid color picture at 200MHz.
 441          * Not sure what's wrong. For now use 200MHz only when all pipes
 442          * are off.
 443          */
 444         if (IS_VALLEYVIEW(dev_priv) && min_cdclk > freq_320)
 445                 return 400000;
 446         else if (min_cdclk > 266667)
 447                 return freq_320;
 448         else if (min_cdclk > 0)
 449                 return 266667;
 450         else
 451                 return 200000;
 452 }
 453
 454 static u8 vlv_calc_voltage_level(struct drm_i915_private *dev_priv, int cdclk)
 455 {
 456         if (IS_VALLEYVIEW(dev_priv)) {
 457                 if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */
 458                         return 2;
 459                 else if (cdclk >= 266667)
 460                         return 1;
 461                 else
 462                         return 0;
 463         } else {
 464                 /*
 465                  * Specs are full of misinformation, but testing on actual
 466                  * hardware has shown that we just need to write the desired
 467                  * CCK divider into the Punit register.
 468                  */
 469                 return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
 470         }
 471 }
 472
 473 static void vlv_get_cdclk(struct drm_i915_private *dev_priv,
 474                           struct intel_cdclk_config *cdclk_config)
 475 {
 476         u32 val;
 477
 478         vlv_iosf_sb_get(dev_priv,
 479                         BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
 480
 481         cdclk_config->vco = vlv_get_hpll_vco(dev_priv);
 482         cdclk_config->cdclk = vlv_get_cck_clock(dev_priv, "cdclk",
 483                                                 CCK_DISPLAY_CLOCK_CONTROL,
 484                                                 cdclk_config->vco);
 485
 486         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 487
 488         vlv_iosf_sb_put(dev_priv,
 489                         BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
 490
 491         if (IS_VALLEYVIEW(dev_priv))
 492                 cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK) >>
 493                         DSPFREQGUAR_SHIFT;
 494         else
 495                 cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK_CHV) >>
 496                         DSPFREQGUAR_SHIFT_CHV;
 497 }
 498
 499 static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
 500 {
 501         unsigned int credits, default_credits;
 502
 503         if (IS_CHERRYVIEW(dev_priv))
 504                 default_credits = PFI_CREDIT(12);
 505         else
 506                 default_credits = PFI_CREDIT(8);
 507
 508         if (dev_priv->cdclk.hw.cdclk >= dev_priv->czclk_freq) {
 509                 /* CHV suggested value is 31 or 63 */
 510                 if (IS_CHERRYVIEW(dev_priv))
 511                         credits = PFI_CREDIT_63;
 512                 else
 513                         credits = PFI_CREDIT(15);
 514         } else {
 515                 credits = default_credits;
 516         }
 517
 518         /*
 519          * WA - write default credits before re-programming
 520          * FIXME: should we also set the resend bit here?
 521          */
 522         intel_de_write(dev_priv, GCI_CONTROL,
 523                        VGA_FAST_MODE_DISABLE | default_credits);
 524
 525         intel_de_write(dev_priv, GCI_CONTROL,
 526                        VGA_FAST_MODE_DISABLE | credits | PFI_CREDIT_RESEND);
 527
 528         /*
 529          * FIXME is this guaranteed to clear
 530          * immediately or should we poll for it?
 531          */
 532         drm_WARN_ON(&dev_priv->drm,
 533                     intel_de_read(dev_priv, GCI_CONTROL) & PFI_CREDIT_RESEND);
 534 }
 535
 536 static void vlv_set_cdclk(struct drm_i915_private *dev_priv,
 537                           const struct intel_cdclk_config *cdclk_config,
 538                           enum pipe pipe)
 539 {
 540         int cdclk = cdclk_config->cdclk;
 541         u32 val, cmd = cdclk_config->voltage_level;
 542         intel_wakeref_t wakeref;
 543
 544         switch (cdclk) {
 545         case 400000:
 546         case 333333:
 547         case 320000:
 548         case 266667:
 549         case 200000:
 550                 break;
 551         default:
 552                 MISSING_CASE(cdclk);
 553                 return;
 554         }
 555
 556         /* There are cases where we can end up here with power domains
 557          * off and a CDCLK frequency other than the minimum, like when
 558          * issuing a modeset without actually changing any display after
 559          * a system suspend.  So grab the display core domain, which covers
 560          * the HW blocks needed for the following programming.
 561          */
 562         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
 563
 564         vlv_iosf_sb_get(dev_priv,
 565                         BIT(VLV_IOSF_SB_CCK) |
 566                         BIT(VLV_IOSF_SB_BUNIT) |
 567                         BIT(VLV_IOSF_SB_PUNIT));
 568
 569         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 570         val &= ~DSPFREQGUAR_MASK;
 571         val |= (cmd << DSPFREQGUAR_SHIFT);
 572         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 573         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
 574                       DSPFREQSTAT_MASK) == (cmd << DSPFREQSTAT_SHIFT),
 575                      50)) {
 576                 drm_err(&dev_priv->drm,
 577                         "timed out waiting for CDclk change\n");
 578         }
 579
 580         if (cdclk == 400000) {
 581                 u32 divider;
 582
 583                 divider = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1,
 584                                             cdclk) - 1;
 585
 586                 /* adjust cdclk divider */
 587                 val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL);
 588                 val &= ~CCK_FREQUENCY_VALUES;
 589                 val |= divider;
 590                 vlv_cck_write(dev_priv, CCK_DISPLAY_CLOCK_CONTROL, val);
 591
 592                 if (wait_for((vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL) &
 593                               CCK_FREQUENCY_STATUS) == (divider << CCK_FREQUENCY_STATUS_SHIFT),
 594                              50))
 595                         drm_err(&dev_priv->drm,
 596                                 "timed out waiting for CDclk change\n");
 597         }
 598
 599         /* adjust self-refresh exit latency value */
 600         val = vlv_bunit_read(dev_priv, BUNIT_REG_BISOC);
 601         val &= ~0x7f;
 602
 603         /*
 604          * For high bandwidth configs, we set a higher latency in the bunit
 605          * so that the core display fetch happens in time to avoid underruns.
 606          */
 607         if (cdclk == 400000)
 608                 val |= 4500 / 250; /* 4.5 usec */
 609         else
 610                 val |= 3000 / 250; /* 3.0 usec */
 611         vlv_bunit_write(dev_priv, BUNIT_REG_BISOC, val);
 612
 613         vlv_iosf_sb_put(dev_priv,
 614                         BIT(VLV_IOSF_SB_CCK) |
 615                         BIT(VLV_IOSF_SB_BUNIT) |
 616                         BIT(VLV_IOSF_SB_PUNIT));
 617
 618         intel_update_cdclk(dev_priv);
 619
 620         vlv_program_pfi_credits(dev_priv);
 621
 622         intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
 623 }
 624
 625 static void chv_set_cdclk(struct drm_i915_private *dev_priv,
 626                           const struct intel_cdclk_config *cdclk_config,
 627                           enum pipe pipe)
 628 {
 629         int cdclk = cdclk_config->cdclk;
 630         u32 val, cmd = cdclk_config->voltage_level;
 631         intel_wakeref_t wakeref;
 632
 633         switch (cdclk) {
 634         case 333333:
 635         case 320000:
 636         case 266667:
 637         case 200000:
 638                 break;
 639         default:
 640                 MISSING_CASE(cdclk);
 641                 return;
 642         }
 643
 644         /* There are cases where we can end up here with power domains
 645          * off and a CDCLK frequency other than the minimum, like when
 646          * issuing a modeset without actually changing any display after
 647          * a system suspend.  So grab the display core domain, which covers
 648          * the HW blocks needed for the following programming.
 649          */
 650         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
 651
 652         vlv_punit_get(dev_priv);
 653         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 654         val &= ~DSPFREQGUAR_MASK_CHV;
 655         val |= (cmd << DSPFREQGUAR_SHIFT_CHV);
 656         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 657         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
 658                       DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV),
 659                      50)) {
 660                 drm_err(&dev_priv->drm,
 661                         "timed out waiting for CDclk change\n");
 662         }
 663
 664         vlv_punit_put(dev_priv);
 665
 666         intel_update_cdclk(dev_priv);
 667
 668         vlv_program_pfi_credits(dev_priv);
 669
 670         intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
 671 }
 672
 673 static int bdw_calc_cdclk(int min_cdclk)
 674 {
 675         if (min_cdclk > 540000)
 676                 return 675000;
 677         else if (min_cdclk > 450000)
 678                 return 540000;
 679         else if (min_cdclk > 337500)
 680                 return 450000;
 681         else
 682                 return 337500;
 683 }
 684
 685 static u8 bdw_calc_voltage_level(int cdclk)
 686 {
 687         switch (cdclk) {
 688         default:
 689         case 337500:
 690                 return 2;
 691         case 450000:
 692                 return 0;
 693         case 540000:
 694                 return 1;
 695         case 675000:
 696                 return 3;
 697         }
 698 }
 699
 700 static void bdw_get_cdclk(struct drm_i915_private *dev_priv,
 701                           struct intel_cdclk_config *cdclk_config)
 702 {
 703         u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
 704         u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
 705
 706         if (lcpll & LCPLL_CD_SOURCE_FCLK)
 707                 cdclk_config->cdclk = 800000;
 708         else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
 709                 cdclk_config->cdclk = 450000;
 710         else if (freq == LCPLL_CLK_FREQ_450)
 711                 cdclk_config->cdclk = 450000;
 712         else if (freq == LCPLL_CLK_FREQ_54O_BDW)
 713                 cdclk_config->cdclk = 540000;
 714         else if (freq == LCPLL_CLK_FREQ_337_5_BDW)
 715                 cdclk_config->cdclk = 337500;
 716         else
 717                 cdclk_config->cdclk = 675000;
 718
 719         /*
 720          * Can't read this out :( Let's assume it's
 721          * at least what the CDCLK frequency requires.
 722          */
 723         cdclk_config->voltage_level =
 724                 bdw_calc_voltage_level(cdclk_config->cdclk);
 725 }
 726
 727 static u32 bdw_cdclk_freq_sel(int cdclk)
 728 {
 729         switch (cdclk) {
 730         default:
 731                 MISSING_CASE(cdclk);
 732                 fallthrough;
 733         case 337500:
 734                 return LCPLL_CLK_FREQ_337_5_BDW;
 735         case 450000:
 736                 return LCPLL_CLK_FREQ_450;
 737         case 540000:
 738                 return LCPLL_CLK_FREQ_54O_BDW;
 739         case 675000:
 740                 return LCPLL_CLK_FREQ_675_BDW;
 741         }
 742 }
 743
 744 static void bdw_set_cdclk(struct drm_i915_private *dev_priv,
 745                           const struct intel_cdclk_config *cdclk_config,
 746                           enum pipe pipe)
 747 {
 748         int cdclk = cdclk_config->cdclk;
 749         int ret;
 750
 751         if (drm_WARN(&dev_priv->drm,
 752                      (intel_de_read(dev_priv, LCPLL_CTL) &
 753                       (LCPLL_PLL_DISABLE | LCPLL_PLL_LOCK |
 754                        LCPLL_CD_CLOCK_DISABLE | LCPLL_ROOT_CD_CLOCK_DISABLE |
 755                        LCPLL_CD2X_CLOCK_DISABLE | LCPLL_POWER_DOWN_ALLOW |
 756                        LCPLL_CD_SOURCE_FCLK)) != LCPLL_PLL_LOCK,
 757                      "trying to change cdclk frequency with cdclk not enabled\n"))
 758                 return;
 759
 760         ret = sandybridge_pcode_write(dev_priv,
 761                                       BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0);
 762         if (ret) {
 763                 drm_err(&dev_priv->drm,
 764                         "failed to inform pcode about cdclk change\n");
 765                 return;
 766         }
 767
 768         intel_de_rmw(dev_priv, LCPLL_CTL,
 769                      0, LCPLL_CD_SOURCE_FCLK);
 770
 771         /*
 772          * According to the spec, it should be enough to poll for this 1 us.
 773          * However, extensive testing shows that this can take longer.
 774          */
 775         if (wait_for_us(intel_de_read(dev_priv, LCPLL_CTL) &
 776                         LCPLL_CD_SOURCE_FCLK_DONE, 100))
 777                 drm_err(&dev_priv->drm, "Switching to FCLK failed\n");
 778
 779         intel_de_rmw(dev_priv, LCPLL_CTL,
 780                      LCPLL_CLK_FREQ_MASK, bdw_cdclk_freq_sel(cdclk));
 781
 782         intel_de_rmw(dev_priv, LCPLL_CTL,
 783                      LCPLL_CD_SOURCE_FCLK, 0);
 784
 785         if (wait_for_us((intel_de_read(dev_priv, LCPLL_CTL) &
 786                          LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
 787                 drm_err(&dev_priv->drm, "Switching back to LCPLL failed\n");
 788
 789         sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ,
 790                                 cdclk_config->voltage_level);
 791
 792         intel_de_write(dev_priv, CDCLK_FREQ,
 793                        DIV_ROUND_CLOSEST(cdclk, 1000) - 1);
 794
 795         intel_update_cdclk(dev_priv);
 796 }
 797
 798 static int skl_calc_cdclk(int min_cdclk, int vco)
 799 {
 800         if (vco == 8640000) {
 801                 if (min_cdclk > 540000)
 802                         return 617143;
 803                 else if (min_cdclk > 432000)
 804                         return 540000;
 805                 else if (min_cdclk > 308571)
 806                         return 432000;
 807                 else
 808                         return 308571;
 809         } else {
 810                 if (min_cdclk > 540000)
 811                         return 675000;
 812                 else if (min_cdclk > 450000)
 813                         return 540000;
 814                 else if (min_cdclk > 337500)
 815                         return 450000;
 816                 else
 817                         return 337500;
 818         }
 819 }
 820
 821 static u8 skl_calc_voltage_level(int cdclk)
 822 {
 823         if (cdclk > 540000)
 824                 return 3;
 825         else if (cdclk > 450000)
 826                 return 2;
 827         else if (cdclk > 337500)
 828                 return 1;
 829         else
 830                 return 0;
 831 }
 832
 833 static void skl_dpll0_update(struct drm_i915_private *dev_priv,
 834                              struct intel_cdclk_config *cdclk_config)
 835 {
 836         u32 val;
 837
 838         cdclk_config->ref = 24000;
 839         cdclk_config->vco = 0;
 840
 841         val = intel_de_read(dev_priv, LCPLL1_CTL);
 842         if ((val & LCPLL_PLL_ENABLE) == 0)
 843                 return;
 844
 845         if (drm_WARN_ON(&dev_priv->drm, (val & LCPLL_PLL_LOCK) == 0))
 846                 return;
 847
 848         val = intel_de_read(dev_priv, DPLL_CTRL1);
 849
 850         if (drm_WARN_ON(&dev_priv->drm,
 851                         (val & (DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
 852                                 DPLL_CTRL1_SSC(SKL_DPLL0) |
 853                                 DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) !=
 854                         DPLL_CTRL1_OVERRIDE(SKL_DPLL0)))
 855                 return;
 856
 857         switch (val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) {
 858         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0):
 859         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1350, SKL_DPLL0):
 860         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1620, SKL_DPLL0):
 861         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2700, SKL_DPLL0):
 862                 cdclk_config->vco = 8100000;
 863                 break;
 864         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0):
 865         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2160, SKL_DPLL0):
 866                 cdclk_config->vco = 8640000;
 867                 break;
 868         default:
 869                 MISSING_CASE(val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0));
 870                 break;
 871         }
 872 }
 873
 874 static void skl_get_cdclk(struct drm_i915_private *dev_priv,
 875                           struct intel_cdclk_config *cdclk_config)
 876 {
 877         u32 cdctl;
 878
 879         skl_dpll0_update(dev_priv, cdclk_config);
 880
 881         cdclk_config->cdclk = cdclk_config->bypass = cdclk_config->ref;
 882
 883         if (cdclk_config->vco == 0)
 884                 goto out;
 885
 886         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
 887
 888         if (cdclk_config->vco == 8640000) {
 889                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
 890                 case CDCLK_FREQ_450_432:
 891                         cdclk_config->cdclk = 432000;
 892                         break;
 893                 case CDCLK_FREQ_337_308:
 894                         cdclk_config->cdclk = 308571;
 895                         break;
 896                 case CDCLK_FREQ_540:
 897                         cdclk_config->cdclk = 540000;
 898                         break;
 899                 case CDCLK_FREQ_675_617:
 900                         cdclk_config->cdclk = 617143;
 901                         break;
 902                 default:
 903                         MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
 904                         break;
 905                 }
 906         } else {
 907                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
 908                 case CDCLK_FREQ_450_432:
 909                         cdclk_config->cdclk = 450000;
 910                         break;
 911                 case CDCLK_FREQ_337_308:
 912                         cdclk_config->cdclk = 337500;
 913                         break;
 914                 case CDCLK_FREQ_540:
 915                         cdclk_config->cdclk = 540000;
 916                         break;
 917                 case CDCLK_FREQ_675_617:
 918                         cdclk_config->cdclk = 675000;
 919                         break;
 920                 default:
 921                         MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
 922                         break;
 923                 }
 924         }
 925
 926  out:
 927         /*
 928          * Can't read this out :( Let's assume it's
 929          * at least what the CDCLK frequency requires.
 930          */
 931         cdclk_config->voltage_level =
 932                 skl_calc_voltage_level(cdclk_config->cdclk);
 933 }
 934
 935 /* convert from kHz to .1 fixpoint MHz with -1MHz offset */
 936 static int skl_cdclk_decimal(int cdclk)
 937 {
 938         return DIV_ROUND_CLOSEST(cdclk - 1000, 500);
 939 }
 940
 941 static void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv,
 942                                         int vco)
 943 {
 944         bool changed = dev_priv->skl_preferred_vco_freq != vco;
 945
 946         dev_priv->skl_preferred_vco_freq = vco;
 947
 948         if (changed)
 949                 intel_update_max_cdclk(dev_priv);
 950 }
 951
 952 static u32 skl_dpll0_link_rate(struct drm_i915_private *dev_priv, int vco)
 953 {
 954         drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
 955
 956         /*
 957          * We always enable DPLL0 with the lowest link rate possible, but still
 958          * taking into account the VCO required to operate the eDP panel at the
 959          * desired frequency. The usual DP link rates operate with a VCO of
 960          * 8100 while the eDP 1.4 alternate link rates need a VCO of 8640.
 961          * The modeset code is responsible for the selection of the exact link
 962          * rate later on, with the constraint of choosing a frequency that
 963          * works with vco.
 964          */
 965         if (vco == 8640000)
 966                 return DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0);
 967         else
 968                 return DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0);
 969 }
 970
 971 static void skl_dpll0_enable(struct drm_i915_private *dev_priv, int vco)
 972 {
 973         intel_de_rmw(dev_priv, DPLL_CTRL1,
 974                      DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
 975                      DPLL_CTRL1_SSC(SKL_DPLL0) |
 976                      DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0),
 977                      DPLL_CTRL1_OVERRIDE(SKL_DPLL0) |
 978                      skl_dpll0_link_rate(dev_priv, vco));
 979         intel_de_posting_read(dev_priv, DPLL_CTRL1);
 980
 981         intel_de_rmw(dev_priv, LCPLL1_CTL,
 982                      0, LCPLL_PLL_ENABLE);
 983
 984         if (intel_de_wait_for_set(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 5))
 985                 drm_err(&dev_priv->drm, "DPLL0 not locked\n");
 986
 987         dev_priv->cdclk.hw.vco = vco;
 988
 989         /* We'll want to keep using the current vco from now on. */
 990         skl_set_preferred_cdclk_vco(dev_priv, vco);
 991 }
 992
 993 static void skl_dpll0_disable(struct drm_i915_private *dev_priv)
 994 {
 995         intel_de_rmw(dev_priv, LCPLL1_CTL,
 996                      LCPLL_PLL_ENABLE, 0);
 997
 998         if (intel_de_wait_for_clear(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 1))
 999                 drm_err(&dev_priv->drm, "Couldn't disable DPLL0\n");
1000
1001         dev_priv->cdclk.hw.vco = 0;
1002 }
1003
1004 static u32 skl_cdclk_freq_sel(struct drm_i915_private *dev_priv,
1005                               int cdclk, int vco)
1006 {
1007         switch (cdclk) {
1008         default:
1009                 drm_WARN_ON(&dev_priv->drm,
1010                             cdclk != dev_priv->cdclk.hw.bypass);
1011                 drm_WARN_ON(&dev_priv->drm, vco != 0);
1012                 fallthrough;
1013         case 308571:
1014         case 337500:
1015                 return CDCLK_FREQ_337_308;
1016         case 450000:
1017         case 432000:
1018                 return CDCLK_FREQ_450_432;
1019         case 540000:
1020                 return CDCLK_FREQ_540;
1021         case 617143:
1022         case 675000:
1023                 return CDCLK_FREQ_675_617;
1024         }
1025 }
1026
1027 static void skl_set_cdclk(struct drm_i915_private *dev_priv,
1028                           const struct intel_cdclk_config *cdclk_config,
1029                           enum pipe pipe)
1030 {
1031         int cdclk = cdclk_config->cdclk;
1032         int vco = cdclk_config->vco;
1033         u32 freq_select, cdclk_ctl;
1034         int ret;
1035
1036         /*
1037          * Based on WA#1183 CDCLK rates 308 and 617MHz CDCLK rates are
1038          * unsupported on SKL. In theory this should never happen since only
1039          * the eDP1.4 2.16 and 4.32Gbps rates require it, but eDP1.4 is not
1040          * supported on SKL either, see the above WA. WARN whenever trying to
1041          * use the corresponding VCO freq as that always leads to using the
1042          * minimum 308MHz CDCLK.
1043          */
1044         drm_WARN_ON_ONCE(&dev_priv->drm,
1045                          IS_SKYLAKE(dev_priv) && vco == 8640000);
1046
1047         ret = skl_pcode_request(dev_priv, SKL_PCODE_CDCLK_CONTROL,
1048                                 SKL_CDCLK_PREPARE_FOR_CHANGE,
1049                                 SKL_CDCLK_READY_FOR_CHANGE,
1050                                 SKL_CDCLK_READY_FOR_CHANGE, 3);
1051         if (ret) {
1052                 drm_err(&dev_priv->drm,
1053                         "Failed to inform PCU about cdclk change (%d)\n", ret);
1054                 return;
1055         }
1056
1057         freq_select = skl_cdclk_freq_sel(dev_priv, cdclk, vco);
1058
1059         if (dev_priv->cdclk.hw.vco != 0 &&
1060             dev_priv->cdclk.hw.vco != vco)
1061                 skl_dpll0_disable(dev_priv);
1062
1063         cdclk_ctl = intel_de_read(dev_priv, CDCLK_CTL);
1064
1065         if (dev_priv->cdclk.hw.vco != vco) {
1066                 /* Wa Display #1183: skl,kbl,cfl */
1067                 cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1068                 cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1069                 intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1070         }
1071
1072         /* Wa Display #1183: skl,kbl,cfl */
1073         cdclk_ctl |= CDCLK_DIVMUX_CD_OVERRIDE;
1074         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1075         intel_de_posting_read(dev_priv, CDCLK_CTL);
1076
1077         if (dev_priv->cdclk.hw.vco != vco)
1078                 skl_dpll0_enable(dev_priv, vco);
1079
1080         /* Wa Display #1183: skl,kbl,cfl */
1081         cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1082         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1083
1084         cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1085         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1086
1087         /* Wa Display #1183: skl,kbl,cfl */
1088         cdclk_ctl &= ~CDCLK_DIVMUX_CD_OVERRIDE;
1089         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1090         intel_de_posting_read(dev_priv, CDCLK_CTL);
1091
1092         /* inform PCU of the change */
1093         sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL,
1094                                 cdclk_config->voltage_level);
1095
1096         intel_update_cdclk(dev_priv);
1097 }
1098
1099 static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
1100 {
1101         u32 cdctl, expected;
1102
1103         /*
1104          * check if the pre-os initialized the display
1105          * There is SWF18 scratchpad register defined which is set by the
1106          * pre-os which can be used by the OS drivers to check the status
1107          */
1108         if ((intel_de_read(dev_priv, SWF_ILK(0x18)) & 0x00FFFFFF) == 0)
1109                 goto sanitize;
1110
1111         intel_update_cdclk(dev_priv);
1112         intel_dump_cdclk_config(&dev_priv->cdclk.hw, "Current CDCLK");
1113
1114         /* Is PLL enabled and locked ? */
1115         if (dev_priv->cdclk.hw.vco == 0 ||
1116             dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.bypass)
1117                 goto sanitize;
1118
1119         /* DPLL okay; verify the cdclock
1120          *
1121          * Noticed in some instances that the freq selection is correct but
1122          * decimal part is programmed wrong from BIOS where pre-os does not
1123          * enable display. Verify the same as well.
1124          */
1125         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
1126         expected = (cdctl & CDCLK_FREQ_SEL_MASK) |
1127                 skl_cdclk_decimal(dev_priv->cdclk.hw.cdclk);
1128         if (cdctl == expected)
1129                 /* All well; nothing to sanitize */
1130                 return;
1131
1132 sanitize:
1133         drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
1134
1135         /* force cdclk programming */
1136         dev_priv->cdclk.hw.cdclk = 0;
1137         /* force full PLL disable + enable */
1138         dev_priv->cdclk.hw.vco = -1;
1139 }
1140
1141 static void skl_cdclk_init_hw(struct drm_i915_private *dev_priv)
1142 {
1143         struct intel_cdclk_config cdclk_config;
1144
1145         skl_sanitize_cdclk(dev_priv);
1146
1147         if (dev_priv->cdclk.hw.cdclk != 0 &&
1148             dev_priv->cdclk.hw.vco != 0) {
1149                 /*
1150                  * Use the current vco as our initial
1151                  * guess as to what the preferred vco is.
1152                  */
1153                 if (dev_priv->skl_preferred_vco_freq == 0)
1154                         skl_set_preferred_cdclk_vco(dev_priv,
1155                                                     dev_priv->cdclk.hw.vco);
1156                 return;
1157         }
1158
1159         cdclk_config = dev_priv->cdclk.hw;
1160
1161         cdclk_config.vco = dev_priv->skl_preferred_vco_freq;
1162         if (cdclk_config.vco == 0)
1163                 cdclk_config.vco = 8100000;
1164         cdclk_config.cdclk = skl_calc_cdclk(0, cdclk_config.vco);
1165         cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1166
1167         skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1168 }
1169
1170 static void skl_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
1171 {
1172         struct intel_cdclk_config cdclk_config = dev_priv->cdclk.hw;
1173
1174         cdclk_config.cdclk = cdclk_config.bypass;
1175         cdclk_config.vco = 0;
1176         cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1177
1178         skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1179 }
1180
1181 static const struct intel_cdclk_vals bxt_cdclk_table[] = {
1182         { .refclk = 19200, .cdclk = 144000, .divider = 8, .ratio = 60 },
1183         { .refclk = 19200, .cdclk = 288000, .divider = 4, .ratio = 60 },
1184         { .refclk = 19200, .cdclk = 384000, .divider = 3, .ratio = 60 },
1185         { .refclk = 19200, .cdclk = 576000, .divider = 2, .ratio = 60 },
1186         { .refclk = 19200, .cdclk = 624000, .divider = 2, .ratio = 65 },
1187         {}
1188 };
1189
1190 static const struct intel_cdclk_vals glk_cdclk_table[] = {
1191         { .refclk = 19200, .cdclk =  79200, .divider = 8, .ratio = 33 },
1192         { .refclk = 19200, .cdclk = 158400, .divider = 4, .ratio = 33 },
1193         { .refclk = 19200, .cdclk = 316800, .divider = 2, .ratio = 33 },
1194         {}
1195 };
1196
1197 static const struct intel_cdclk_vals cnl_cdclk_table[] = {
1198         { .refclk = 19200, .cdclk = 168000, .divider = 4, .ratio = 35 },
1199         { .refclk = 19200, .cdclk = 336000, .divider = 2, .ratio = 35 },
1200         { .refclk = 19200, .cdclk = 528000, .divider = 2, .ratio = 55 },
1201
1202         { .refclk = 24000, .cdclk = 168000, .divider = 4, .ratio = 28 },
1203         { .refclk = 24000, .cdclk = 336000, .divider = 2, .ratio = 28 },
1204         { .refclk = 24000, .cdclk = 528000, .divider = 2, .ratio = 44 },
1205         {}
1206 };
1207
1208 static const struct intel_cdclk_vals icl_cdclk_table[] = {
1209         { .refclk = 19200, .cdclk = 172800, .divider = 2, .ratio = 18 },
1210         { .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 },
1211         { .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1212         { .refclk = 19200, .cdclk = 326400, .divider = 4, .ratio = 68 },
1213         { .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1214         { .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1215
1216         { .refclk = 24000, .cdclk = 180000, .divider = 2, .ratio = 15 },
1217         { .refclk = 24000, .cdclk = 192000, .divider = 2, .ratio = 16 },
1218         { .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1219         { .refclk = 24000, .cdclk = 324000, .divider = 4, .ratio = 54 },
1220         { .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1221         { .refclk = 24000, .cdclk = 648000, .divider = 2, .ratio = 54 },
1222
1223         { .refclk = 38400, .cdclk = 172800, .divider = 2, .ratio =  9 },
1224         { .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
1225         { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1226         { .refclk = 38400, .cdclk = 326400, .divider = 4, .ratio = 34 },
1227         { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1228         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1229         {}
1230 };
1231
1232 static const struct intel_cdclk_vals rkl_cdclk_table[] = {
1233         { .refclk = 19200, .cdclk = 172800, .divider = 4, .ratio =  36 },
1234         { .refclk = 19200, .cdclk = 192000, .divider = 4, .ratio =  40 },
1235         { .refclk = 19200, .cdclk = 307200, .divider = 4, .ratio =  64 },
1236         { .refclk = 19200, .cdclk = 326400, .divider = 8, .ratio = 136 },
1237         { .refclk = 19200, .cdclk = 556800, .divider = 4, .ratio = 116 },
1238         { .refclk = 19200, .cdclk = 652800, .divider = 4, .ratio = 136 },
1239
1240         { .refclk = 24000, .cdclk = 180000, .divider = 4, .ratio =  30 },
1241         { .refclk = 24000, .cdclk = 192000, .divider = 4, .ratio =  32 },
1242         { .refclk = 24000, .cdclk = 312000, .divider = 4, .ratio =  52 },
1243         { .refclk = 24000, .cdclk = 324000, .divider = 8, .ratio = 108 },
1244         { .refclk = 24000, .cdclk = 552000, .divider = 4, .ratio =  92 },
1245         { .refclk = 24000, .cdclk = 648000, .divider = 4, .ratio = 108 },
1246
1247         { .refclk = 38400, .cdclk = 172800, .divider = 4, .ratio = 18 },
1248         { .refclk = 38400, .cdclk = 192000, .divider = 4, .ratio = 20 },
1249         { .refclk = 38400, .cdclk = 307200, .divider = 4, .ratio = 32 },
1250         { .refclk = 38400, .cdclk = 326400, .divider = 8, .ratio = 68 },
1251         { .refclk = 38400, .cdclk = 556800, .divider = 4, .ratio = 58 },
1252         { .refclk = 38400, .cdclk = 652800, .divider = 4, .ratio = 68 },
1253         {}
1254 };
1255
1256 static const struct intel_cdclk_vals adlp_a_step_cdclk_table[] = {
1257         { .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1258         { .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1259         { .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1260
1261         { .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1262         { .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1263         { .refclk = 24400, .cdclk = 648000, .divider = 2, .ratio = 54 },
1264
1265         { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1266         { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1267         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1268         {}
1269 };
1270
1271 static const struct intel_cdclk_vals adlp_cdclk_table[] = {
1272         { .refclk = 19200, .cdclk = 172800, .divider = 3, .ratio = 27 },
1273         { .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 },
1274         { .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1275         { .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1276         { .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1277
1278         { .refclk = 24000, .cdclk = 176000, .divider = 3, .ratio = 22 },
1279         { .refclk = 24000, .cdclk = 192000, .divider = 2, .ratio = 16 },
1280         { .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1281         { .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1282         { .refclk = 24400, .cdclk = 648000, .divider = 2, .ratio = 54 },
1283
1284         { .refclk = 38400, .cdclk = 179200, .divider = 3, .ratio = 14 },
1285         { .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
1286         { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1287         { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1288         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1289         {}
1290 };
1291
1292 static int bxt_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
1293 {
1294         const struct intel_cdclk_vals *table = dev_priv->cdclk.table;
1295         int i;
1296
1297         for (i = 0; table[i].refclk; i++)
1298                 if (table[i].refclk == dev_priv->cdclk.hw.ref &&
1299                     table[i].cdclk >= min_cdclk)
1300                         return table[i].cdclk;
1301
1302         drm_WARN(&dev_priv->drm, 1,
1303                  "Cannot satisfy minimum cdclk %d with refclk %u\n",
1304                  min_cdclk, dev_priv->cdclk.hw.ref);
1305         return 0;
1306 }
1307
1308 static int bxt_calc_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
1309 {
1310         const struct intel_cdclk_vals *table = dev_priv->cdclk.table;
1311         int i;
1312
1313         if (cdclk == dev_priv->cdclk.hw.bypass)
1314                 return 0;
1315
1316         for (i = 0; table[i].refclk; i++)
1317                 if (table[i].refclk == dev_priv->cdclk.hw.ref &&
1318                     table[i].cdclk == cdclk)
1319                         return dev_priv->cdclk.hw.ref * table[i].ratio;
1320
1321         drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n",
1322                  cdclk, dev_priv->cdclk.hw.ref);
1323         return 0;
1324 }
1325
1326 static u8 bxt_calc_voltage_level(int cdclk)
1327 {
1328         return DIV_ROUND_UP(cdclk, 25000);
1329 }
1330
1331 static u8 cnl_calc_voltage_level(int cdclk)
1332 {
1333         if (cdclk > 336000)
1334                 return 2;
1335         else if (cdclk > 168000)
1336                 return 1;
1337         else
1338                 return 0;
1339 }
1340
1341 static u8 icl_calc_voltage_level(int cdclk)
1342 {
1343         if (cdclk > 556800)
1344                 return 2;
1345         else if (cdclk > 312000)
1346                 return 1;
1347         else
1348                 return 0;
1349 }
1350
1351 static u8 ehl_calc_voltage_level(int cdclk)
1352 {
1353         if (cdclk > 326400)
1354                 return 3;
1355         else if (cdclk > 312000)
1356                 return 2;
1357         else if (cdclk > 180000)
1358                 return 1;
1359         else
1360                 return 0;
1361 }
1362
1363 static u8 tgl_calc_voltage_level(int cdclk)
1364 {
1365         if (cdclk > 556800)
1366                 return 3;
1367         else if (cdclk > 326400)
1368                 return 2;
1369         else if (cdclk > 312000)
1370                 return 1;
1371         else
1372                 return 0;
1373 }
1374
1375 static void cnl_readout_refclk(struct drm_i915_private *dev_priv,
1376                                struct intel_cdclk_config *cdclk_config)
1377 {
1378         if (intel_de_read(dev_priv, SKL_DSSM) & CNL_DSSM_CDCLK_PLL_REFCLK_24MHz)
1379                 cdclk_config->ref = 24000;
1380         else
1381                 cdclk_config->ref = 19200;
1382 }
1383
1384 static void icl_readout_refclk(struct drm_i915_private *dev_priv,
1385                                struct intel_cdclk_config *cdclk_config)
1386 {
1387         u32 dssm = intel_de_read(dev_priv, SKL_DSSM) & ICL_DSSM_CDCLK_PLL_REFCLK_MASK;
1388
1389         switch (dssm) {
1390         default:
1391                 MISSING_CASE(dssm);
1392                 fallthrough;
1393         case ICL_DSSM_CDCLK_PLL_REFCLK_24MHz:
1394                 cdclk_config->ref = 24000;
1395                 break;
1396         case ICL_DSSM_CDCLK_PLL_REFCLK_19_2MHz:
1397                 cdclk_config->ref = 19200;
1398                 break;
1399         case ICL_DSSM_CDCLK_PLL_REFCLK_38_4MHz:
1400                 cdclk_config->ref = 38400;
1401                 break;
1402         }
1403 }
1404
1405 static void bxt_de_pll_readout(struct drm_i915_private *dev_priv,
1406                                struct intel_cdclk_config *cdclk_config)
1407 {
1408         u32 val, ratio;
1409
1410         if (DISPLAY_VER(dev_priv) >= 11)
1411                 icl_readout_refclk(dev_priv, cdclk_config);
1412         else if (IS_CANNONLAKE(dev_priv))
1413                 cnl_readout_refclk(dev_priv, cdclk_config);
1414         else
1415                 cdclk_config->ref = 19200;
1416
1417         val = intel_de_read(dev_priv, BXT_DE_PLL_ENABLE);
1418         if ((val & BXT_DE_PLL_PLL_ENABLE) == 0 ||
1419             (val & BXT_DE_PLL_LOCK) == 0) {
1420                 /*
1421                  * CDCLK PLL is disabled, the VCO/ratio doesn't matter, but
1422                  * setting it to zero is a way to signal that.
1423                  */
1424                 cdclk_config->vco = 0;
1425                 return;
1426         }
1427
1428         /*
1429          * CNL+ have the ratio directly in the PLL enable register, gen9lp had
1430          * it in a separate PLL control register.
1431          */
1432         if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv))
1433                 ratio = val & CNL_CDCLK_PLL_RATIO_MASK;
1434         else
1435                 ratio = intel_de_read(dev_priv, BXT_DE_PLL_CTL) & BXT_DE_PLL_RATIO_MASK;
1436
1437         cdclk_config->vco = ratio * cdclk_config->ref;
1438 }
1439
1440 static void bxt_get_cdclk(struct drm_i915_private *dev_priv,
1441                           struct intel_cdclk_config *cdclk_config)
1442 {
1443         u32 divider;
1444         int div;
1445
1446         bxt_de_pll_readout(dev_priv, cdclk_config);
1447
1448         if (DISPLAY_VER(dev_priv) >= 12)
1449                 cdclk_config->bypass = cdclk_config->ref / 2;
1450         else if (DISPLAY_VER(dev_priv) >= 11)
1451                 cdclk_config->bypass = 50000;
1452         else
1453                 cdclk_config->bypass = cdclk_config->ref;
1454
1455         if (cdclk_config->vco == 0) {
1456                 cdclk_config->cdclk = cdclk_config->bypass;
1457                 goto out;
1458         }
1459
1460         divider = intel_de_read(dev_priv, CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK;
1461
1462         switch (divider) {
1463         case BXT_CDCLK_CD2X_DIV_SEL_1:
1464                 div = 2;
1465                 break;
1466         case BXT_CDCLK_CD2X_DIV_SEL_1_5:
1467                 div = 3;
1468                 break;
1469         case BXT_CDCLK_CD2X_DIV_SEL_2:
1470                 div = 4;
1471                 break;
1472         case BXT_CDCLK_CD2X_DIV_SEL_4:
1473                 div = 8;
1474                 break;
1475         default:
1476                 MISSING_CASE(divider);
1477                 return;
1478         }
1479
1480         cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco, div);
1481
1482  out:
1483         /*
1484          * Can't read this out :( Let's assume it's
1485          * at least what the CDCLK frequency requires.
1486          */
1487         cdclk_config->voltage_level =
1488                 dev_priv->display.calc_voltage_level(cdclk_config->cdclk);
1489 }
1490
1491 static void bxt_de_pll_disable(struct drm_i915_private *dev_priv)
1492 {
1493         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, 0);
1494
1495         /* Timeout 200us */
1496         if (intel_de_wait_for_clear(dev_priv,
1497                                     BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1498                 drm_err(&dev_priv->drm, "timeout waiting for DE PLL unlock\n");
1499
1500         dev_priv->cdclk.hw.vco = 0;
1501 }
1502
1503 static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco)
1504 {
1505         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref);
1506
1507         intel_de_rmw(dev_priv, BXT_DE_PLL_CTL,
1508                      BXT_DE_PLL_RATIO_MASK, BXT_DE_PLL_RATIO(ratio));
1509
1510         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE);
1511
1512         /* Timeout 200us */
1513         if (intel_de_wait_for_set(dev_priv,
1514                                   BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1515                 drm_err(&dev_priv->drm, "timeout waiting for DE PLL lock\n");
1516
1517         dev_priv->cdclk.hw.vco = vco;
1518 }
1519
1520 static void cnl_cdclk_pll_disable(struct drm_i915_private *dev_priv)
1521 {
1522         intel_de_rmw(dev_priv, BXT_DE_PLL_ENABLE,
1523                      BXT_DE_PLL_PLL_ENABLE, 0);
1524
1525         /* Timeout 200us */
1526         if (intel_de_wait_for_clear(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1527                 drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL unlock\n");
1528
1529         dev_priv->cdclk.hw.vco = 0;
1530 }
1531
1532 static void cnl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco)
1533 {
1534         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref);
1535         u32 val;
1536
1537         val = CNL_CDCLK_PLL_RATIO(ratio);
1538         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1539
1540         val |= BXT_DE_PLL_PLL_ENABLE;
1541         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1542
1543         /* Timeout 200us */
1544         if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1545                 drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL lock\n");
1546
1547         dev_priv->cdclk.hw.vco = vco;
1548 }
1549
1550 static u32 bxt_cdclk_cd2x_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
1551 {
1552         if (DISPLAY_VER(dev_priv) >= 12) {
1553                 if (pipe == INVALID_PIPE)
1554                         return TGL_CDCLK_CD2X_PIPE_NONE;
1555                 else
1556                         return TGL_CDCLK_CD2X_PIPE(pipe);
1557         } else if (DISPLAY_VER(dev_priv) >= 11) {
1558                 if (pipe == INVALID_PIPE)
1559                         return ICL_CDCLK_CD2X_PIPE_NONE;
1560                 else
1561                         return ICL_CDCLK_CD2X_PIPE(pipe);
1562         } else {
1563                 if (pipe == INVALID_PIPE)
1564                         return BXT_CDCLK_CD2X_PIPE_NONE;
1565                 else
1566                         return BXT_CDCLK_CD2X_PIPE(pipe);
1567         }
1568 }
1569
1570 static u32 bxt_cdclk_cd2x_div_sel(struct drm_i915_private *dev_priv,
1571                                   int cdclk, int vco)
1572 {
1573         /* cdclk = vco / 2 / div{1,1.5,2,4} */
1574         switch (DIV_ROUND_CLOSEST(vco, cdclk)) {
1575         default:
1576                 drm_WARN_ON(&dev_priv->drm,
1577                             cdclk != dev_priv->cdclk.hw.bypass);
1578                 drm_WARN_ON(&dev_priv->drm, vco != 0);
1579                 fallthrough;
1580         case 2:
1581                 return BXT_CDCLK_CD2X_DIV_SEL_1;
1582         case 3:
1583                 return BXT_CDCLK_CD2X_DIV_SEL_1_5;
1584         case 4:
1585                 return BXT_CDCLK_CD2X_DIV_SEL_2;
1586         case 8:
1587                 return BXT_CDCLK_CD2X_DIV_SEL_4;
1588         }
1589 }
1590
1591 static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
1592                           const struct intel_cdclk_config *cdclk_config,
1593                           enum pipe pipe)
1594 {
1595         int cdclk = cdclk_config->cdclk;
1596         int vco = cdclk_config->vco;
1597         u32 val;
1598         int ret;
1599
1600         /* Inform power controller of upcoming frequency change. */
1601         if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv))
1602                 ret = skl_pcode_request(dev_priv, SKL_PCODE_CDCLK_CONTROL,
1603                                         SKL_CDCLK_PREPARE_FOR_CHANGE,
1604                                         SKL_CDCLK_READY_FOR_CHANGE,
1605                                         SKL_CDCLK_READY_FOR_CHANGE, 3);
1606         else
1607                 /*
1608                  * BSpec requires us to wait up to 150usec, but that leads to
1609                  * timeouts; the 2ms used here is based on experiment.
1610                  */
1611                 ret = sandybridge_pcode_write_timeout(dev_priv,
1612                                                       HSW_PCODE_DE_WRITE_FREQ_REQ,
1613                                                       0x80000000, 150, 2);
1614
1615         if (ret) {
1616                 drm_err(&dev_priv->drm,
1617                         "Failed to inform PCU about cdclk change (err %d, freq %d)\n",
1618                         ret, cdclk);
1619                 return;
1620         }
1621
1622         if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) {
1623                 if (dev_priv->cdclk.hw.vco != 0 &&
1624                     dev_priv->cdclk.hw.vco != vco)
1625                         cnl_cdclk_pll_disable(dev_priv);
1626
1627                 if (dev_priv->cdclk.hw.vco != vco)
1628                         cnl_cdclk_pll_enable(dev_priv, vco);
1629
1630         } else {
1631                 if (dev_priv->cdclk.hw.vco != 0 &&
1632                     dev_priv->cdclk.hw.vco != vco)
1633                         bxt_de_pll_disable(dev_priv);
1634
1635                 if (dev_priv->cdclk.hw.vco != vco)
1636                         bxt_de_pll_enable(dev_priv, vco);
1637         }
1638
1639         val = bxt_cdclk_cd2x_div_sel(dev_priv, cdclk, vco) |
1640                 bxt_cdclk_cd2x_pipe(dev_priv, pipe) |
1641                 skl_cdclk_decimal(cdclk);
1642
1643         /*
1644          * Disable SSA Precharge when CD clock frequency < 500 MHz,
1645          * enable otherwise.
1646          */
1647         if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
1648             cdclk >= 500000)
1649                 val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
1650         intel_de_write(dev_priv, CDCLK_CTL, val);
1651
1652         if (pipe != INVALID_PIPE)
1653                 intel_wait_for_vblank(dev_priv, pipe);
1654
1655         if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv)) {
1656                 ret = sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL,
1657                                               cdclk_config->voltage_level);
1658         } else {
1659                 /*
1660                  * The timeout isn't specified, the 2ms used here is based on
1661                  * experiment.
1662                  * FIXME: Waiting for the request completion could be delayed
1663                  * until the next PCODE request based on BSpec.
1664                  */
1665                 ret = sandybridge_pcode_write_timeout(dev_priv,
1666                                                       HSW_PCODE_DE_WRITE_FREQ_REQ,
1667                                                       cdclk_config->voltage_level,
1668                                                       150, 2);
1669         }
1670
1671         if (ret) {
1672                 drm_err(&dev_priv->drm,
1673                         "PCode CDCLK freq set failed, (err %d, freq %d)\n",
1674                         ret, cdclk);
1675                 return;
1676         }
1677
1678         intel_update_cdclk(dev_priv);
1679
1680         if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv))
1681                 /*
1682                  * Can't read out the voltage level :(
1683                  * Let's just assume everything is as expected.
1684                  */
1685                 dev_priv->cdclk.hw.voltage_level = cdclk_config->voltage_level;
1686 }
1687
1688 static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv)
1689 {
1690         u32 cdctl, expected;
1691         int cdclk, vco;
1692
1693         intel_update_cdclk(dev_priv);
1694         intel_dump_cdclk_config(&dev_priv->cdclk.hw, "Current CDCLK");
1695
1696         if (dev_priv->cdclk.hw.vco == 0 ||
1697             dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.bypass)
1698                 goto sanitize;
1699
1700         /* DPLL okay; verify the cdclock
1701          *
1702          * Some BIOS versions leave an incorrect decimal frequency value and
1703          * set reserved MBZ bits in CDCLK_CTL at least during exiting from S4,
1704          * so sanitize this register.
1705          */
1706         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
1707         /*
1708          * Let's ignore the pipe field, since BIOS could have configured the
1709          * dividers both synching to an active pipe, or asynchronously
1710          * (PIPE_NONE).
1711          */
1712         cdctl &= ~bxt_cdclk_cd2x_pipe(dev_priv, INVALID_PIPE);
1713
1714         /* Make sure this is a legal cdclk value for the platform */
1715         cdclk = bxt_calc_cdclk(dev_priv, dev_priv->cdclk.hw.cdclk);
1716         if (cdclk != dev_priv->cdclk.hw.cdclk)
1717                 goto sanitize;
1718
1719         /* Make sure the VCO is correct for the cdclk */
1720         vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
1721         if (vco != dev_priv->cdclk.hw.vco)
1722                 goto sanitize;
1723
1724         expected = skl_cdclk_decimal(cdclk);
1725
1726         /* Figure out what CD2X divider we should be using for this cdclk */
1727         expected |= bxt_cdclk_cd2x_div_sel(dev_priv,
1728                                            dev_priv->cdclk.hw.cdclk,
1729                                            dev_priv->cdclk.hw.vco);
1730
1731         /*
1732          * Disable SSA Precharge when CD clock frequency < 500 MHz,
1733          * enable otherwise.
1734          */
1735         if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
1736             dev_priv->cdclk.hw.cdclk >= 500000)
1737                 expected |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
1738
1739         if (cdctl == expected)
1740                 /* All well; nothing to sanitize */
1741                 return;
1742
1743 sanitize:
1744         drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
1745
1746         /* force cdclk programming */
1747         dev_priv->cdclk.hw.cdclk = 0;
1748
1749         /* force full PLL disable + enable */
1750         dev_priv->cdclk.hw.vco = -1;
1751 }
1752
1753 static void bxt_cdclk_init_hw(struct drm_i915_private *dev_priv)
1754 {
1755         struct intel_cdclk_config cdclk_config;
1756
1757         bxt_sanitize_cdclk(dev_priv);
1758
1759         if (dev_priv->cdclk.hw.cdclk != 0 &&
1760             dev_priv->cdclk.hw.vco != 0)
1761                 return;
1762
1763         cdclk_config = dev_priv->cdclk.hw;
1764
1765         /*
1766          * FIXME:
1767          * - The initial CDCLK needs to be read from VBT.
1768          *   Need to make this change after VBT has changes for BXT.
1769          */
1770         cdclk_config.cdclk = bxt_calc_cdclk(dev_priv, 0);
1771         cdclk_config.vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk_config.cdclk);
1772         cdclk_config.voltage_level =
1773                 dev_priv->display.calc_voltage_level(cdclk_config.cdclk);
1774
1775         bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1776 }
1777
1778 static void bxt_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
1779 {
1780         struct intel_cdclk_config cdclk_config = dev_priv->cdclk.hw;
1781
1782         cdclk_config.cdclk = cdclk_config.bypass;
1783         cdclk_config.vco = 0;
1784         cdclk_config.voltage_level =
1785                 dev_priv->display.calc_voltage_level(cdclk_config.cdclk);
1786
1787         bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1788 }
1789
1790 /**
1791  * intel_cdclk_init_hw - Initialize CDCLK hardware
1792  * @i915: i915 device
1793  *
1794  * Initialize CDCLK. This consists mainly of initializing dev_priv->cdclk.hw and
1795  * sanitizing the state of the hardware if needed. This is generally done only
1796  * during the display core initialization sequence, after which the DMC will
1797  * take care of turning CDCLK off/on as needed.
1798  */
1799 void intel_cdclk_init_hw(struct drm_i915_private *i915)
1800 {
1801         if (DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915))
1802                 bxt_cdclk_init_hw(i915);
1803         else if (DISPLAY_VER(i915) == 9)
1804                 skl_cdclk_init_hw(i915);
1805 }
1806
1807 /**
1808  * intel_cdclk_uninit_hw - Uninitialize CDCLK hardware
1809  * @i915: i915 device
1810  *
1811  * Uninitialize CDCLK. This is done only during the display core
1812  * uninitialization sequence.
1813  */
1814 void intel_cdclk_uninit_hw(struct drm_i915_private *i915)
1815 {
1816         if (DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915))
1817                 bxt_cdclk_uninit_hw(i915);
1818         else if (DISPLAY_VER(i915) == 9)
1819                 skl_cdclk_uninit_hw(i915);
1820 }
1821
1822 /**
1823  * intel_cdclk_needs_modeset - Determine if changong between the CDCLK
1824  *                             configurations requires a modeset on all pipes
1825  * @a: first CDCLK configuration
1826  * @b: second CDCLK configuration
1827  *
1828  * Returns:
1829  * True if changing between the two CDCLK configurations
1830  * requires all pipes to be off, false if not.
1831  */
1832 bool intel_cdclk_needs_modeset(const struct intel_cdclk_config *a,
1833                                const struct intel_cdclk_config *b)
1834 {
1835         return a->cdclk != b->cdclk ||
1836                 a->vco != b->vco ||
1837                 a->ref != b->ref;
1838 }
1839
1840 /**
1841  * intel_cdclk_can_cd2x_update - Determine if changing between the two CDCLK
1842  *                               configurations requires only a cd2x divider update
1843  * @dev_priv: i915 device
1844  * @a: first CDCLK configuration
1845  * @b: second CDCLK configuration
1846  *
1847  * Returns:
1848  * True if changing between the two CDCLK configurations
1849  * can be done with just a cd2x divider update, false if not.
1850  */
1851 static bool intel_cdclk_can_cd2x_update(struct drm_i915_private *dev_priv,
1852                                         const struct intel_cdclk_config *a,
1853                                         const struct intel_cdclk_config *b)
1854 {
1855         /* Older hw doesn't have the capability */
1856         if (DISPLAY_VER(dev_priv) < 10 && !IS_BROXTON(dev_priv))
1857                 return false;
1858
1859         return a->cdclk != b->cdclk &&
1860                 a->vco == b->vco &&
1861                 a->ref == b->ref;
1862 }
1863
1864 /**
1865  * intel_cdclk_changed - Determine if two CDCLK configurations are different
1866  * @a: first CDCLK configuration
1867  * @b: second CDCLK configuration
1868  *
1869  * Returns:
1870  * True if the CDCLK configurations don't match, false if they do.
1871  */
1872 static bool intel_cdclk_changed(const struct intel_cdclk_config *a,
1873                                 const struct intel_cdclk_config *b)
1874 {
1875         return intel_cdclk_needs_modeset(a, b) ||
1876                 a->voltage_level != b->voltage_level;
1877 }
1878
1879 void intel_dump_cdclk_config(const struct intel_cdclk_config *cdclk_config,
1880                              const char *context)
1881 {
1882         DRM_DEBUG_DRIVER("%s %d kHz, VCO %d kHz, ref %d kHz, bypass %d kHz, voltage level %d\n",
1883                          context, cdclk_config->cdclk, cdclk_config->vco,
1884                          cdclk_config->ref, cdclk_config->bypass,
1885                          cdclk_config->voltage_level);
1886 }
1887
1888 /**
1889  * intel_set_cdclk - Push the CDCLK configuration to the hardware
1890  * @dev_priv: i915 device
1891  * @cdclk_config: new CDCLK configuration
1892  * @pipe: pipe with which to synchronize the update
1893  *
1894  * Program the hardware based on the passed in CDCLK state,
1895  * if necessary.
1896  */
1897 static void intel_set_cdclk(struct drm_i915_private *dev_priv,
1898                             const struct intel_cdclk_config *cdclk_config,
1899                             enum pipe pipe)
1900 {
1901         struct intel_encoder *encoder;
1902
1903         if (!intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_config))
1904                 return;
1905
1906         if (drm_WARN_ON_ONCE(&dev_priv->drm, !dev_priv->display.set_cdclk))
1907                 return;
1908
1909         intel_dump_cdclk_config(cdclk_config, "Changing CDCLK to");
1910
1911         /*
1912          * Lock aux/gmbus while we change cdclk in case those
1913          * functions use cdclk. Not all platforms/ports do,
1914          * but we'll lock them all for simplicity.
1915          */
1916         mutex_lock(&dev_priv->gmbus_mutex);
1917         for_each_intel_dp(&dev_priv->drm, encoder) {
1918                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
1919
1920                 mutex_lock_nest_lock(&intel_dp->aux.hw_mutex,
1921                                      &dev_priv->gmbus_mutex);
1922         }
1923
1924         dev_priv->display.set_cdclk(dev_priv, cdclk_config, pipe);
1925
1926         for_each_intel_dp(&dev_priv->drm, encoder) {
1927                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
1928
1929                 mutex_unlock(&intel_dp->aux.hw_mutex);
1930         }
1931         mutex_unlock(&dev_priv->gmbus_mutex);
1932
1933         if (drm_WARN(&dev_priv->drm,
1934                      intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_config),
1935                      "cdclk state doesn't match!\n")) {
1936                 intel_dump_cdclk_config(&dev_priv->cdclk.hw, "[hw state]");
1937                 intel_dump_cdclk_config(cdclk_config, "[sw state]");
1938         }
1939 }
1940
1941 /**
1942  * intel_set_cdclk_pre_plane_update - Push the CDCLK state to the hardware
1943  * @state: intel atomic state
1944  *
1945  * Program the hardware before updating the HW plane state based on the
1946  * new CDCLK state, if necessary.
1947  */
1948 void
1949 intel_set_cdclk_pre_plane_update(struct intel_atomic_state *state)
1950 {
1951         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
1952         const struct intel_cdclk_state *old_cdclk_state =
1953                 intel_atomic_get_old_cdclk_state(state);
1954         const struct intel_cdclk_state *new_cdclk_state =
1955                 intel_atomic_get_new_cdclk_state(state);
1956         enum pipe pipe = new_cdclk_state->pipe;
1957
1958         if (!intel_cdclk_changed(&old_cdclk_state->actual,
1959                                  &new_cdclk_state->actual))
1960                 return;
1961
1962         if (pipe == INVALID_PIPE ||
1963             old_cdclk_state->actual.cdclk <= new_cdclk_state->actual.cdclk) {
1964                 drm_WARN_ON(&dev_priv->drm, !new_cdclk_state->base.changed);
1965
1966                 intel_set_cdclk(dev_priv, &new_cdclk_state->actual, pipe);
1967         }
1968 }
1969
1970 /**
1971  * intel_set_cdclk_post_plane_update - Push the CDCLK state to the hardware
1972  * @state: intel atomic state
1973  *
1974  * Program the hardware after updating the HW plane state based on the
1975  * new CDCLK state, if necessary.
1976  */
1977 void
1978 intel_set_cdclk_post_plane_update(struct intel_atomic_state *state)
1979 {
1980         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
1981         const struct intel_cdclk_state *old_cdclk_state =
1982                 intel_atomic_get_old_cdclk_state(state);
1983         const struct intel_cdclk_state *new_cdclk_state =
1984                 intel_atomic_get_new_cdclk_state(state);
1985         enum pipe pipe = new_cdclk_state->pipe;
1986
1987         if (!intel_cdclk_changed(&old_cdclk_state->actual,
1988                                  &new_cdclk_state->actual))
1989                 return;
1990
1991         if (pipe != INVALID_PIPE &&
1992             old_cdclk_state->actual.cdclk > new_cdclk_state->actual.cdclk) {
1993                 drm_WARN_ON(&dev_priv->drm, !new_cdclk_state->base.changed);
1994
1995                 intel_set_cdclk(dev_priv, &new_cdclk_state->actual, pipe);
1996         }
1997 }
1998
1999 static int intel_pixel_rate_to_cdclk(const struct intel_crtc_state *crtc_state)
2000 {
2001         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
2002         int pixel_rate = crtc_state->pixel_rate;
2003
2004         if (DISPLAY_VER(dev_priv) >= 10)
2005                 return DIV_ROUND_UP(pixel_rate, 2);
2006         else if (DISPLAY_VER(dev_priv) == 9 ||
2007                  IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2008                 return pixel_rate;
2009         else if (IS_CHERRYVIEW(dev_priv))
2010                 return DIV_ROUND_UP(pixel_rate * 100, 95);
2011         else if (crtc_state->double_wide)
2012                 return DIV_ROUND_UP(pixel_rate * 100, 90 * 2);
2013         else
2014                 return DIV_ROUND_UP(pixel_rate * 100, 90);
2015 }
2016
2017 static int intel_planes_min_cdclk(const struct intel_crtc_state *crtc_state)
2018 {
2019         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2020         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2021         struct intel_plane *plane;
2022         int min_cdclk = 0;
2023
2024         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
2025                 min_cdclk = max(crtc_state->min_cdclk[plane->id], min_cdclk);
2026
2027         return min_cdclk;
2028 }
2029
2030 int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state)
2031 {
2032         struct drm_i915_private *dev_priv =
2033                 to_i915(crtc_state->uapi.crtc->dev);
2034         int min_cdclk;
2035
2036         if (!crtc_state->hw.enable)
2037                 return 0;
2038
2039         min_cdclk = intel_pixel_rate_to_cdclk(crtc_state);
2040
2041         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
2042         if (IS_BROADWELL(dev_priv) && hsw_crtc_state_ips_capable(crtc_state))
2043                 min_cdclk = DIV_ROUND_UP(min_cdclk * 100, 95);
2044
2045         /* BSpec says "Do not use DisplayPort with CDCLK less than 432 MHz,
2046          * audio enabled, port width x4, and link rate HBR2 (5.4 GHz), or else
2047          * there may be audio corruption or screen corruption." This cdclk
2048          * restriction for GLK is 316.8 MHz.
2049          */
2050         if (intel_crtc_has_dp_encoder(crtc_state) &&
2051             crtc_state->has_audio &&
2052             crtc_state->port_clock >= 540000 &&
2053             crtc_state->lane_count == 4) {
2054                 if (DISPLAY_VER(dev_priv) == 10) {
2055                         /* Display WA #1145: glk,cnl */
2056                         min_cdclk = max(316800, min_cdclk);
2057                 } else if (DISPLAY_VER(dev_priv) == 9 || IS_BROADWELL(dev_priv)) {
2058                         /* Display WA #1144: skl,bxt */
2059                         min_cdclk = max(432000, min_cdclk);
2060                 }
2061         }
2062
2063         /*
2064          * According to BSpec, "The CD clock frequency must be at least twice
2065          * the frequency of the Azalia BCLK." and BCLK is 96 MHz by default.
2066          */
2067         if (crtc_state->has_audio && DISPLAY_VER(dev_priv) >= 9)
2068                 min_cdclk = max(2 * 96000, min_cdclk);
2069
2070         /*
2071          * "For DP audio configuration, cdclk frequency shall be set to
2072          *  meet the following requirements:
2073          *  DP Link Frequency(MHz) | Cdclk frequency(MHz)
2074          *  270                    | 320 or higher
2075          *  162                    | 200 or higher"
2076          */
2077         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
2078             intel_crtc_has_dp_encoder(crtc_state) && crtc_state->has_audio)
2079                 min_cdclk = max(crtc_state->port_clock, min_cdclk);
2080
2081         /*
2082          * On Valleyview some DSI panels lose (v|h)sync when the clock is lower
2083          * than 320000KHz.
2084          */
2085         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2086             IS_VALLEYVIEW(dev_priv))
2087                 min_cdclk = max(320000, min_cdclk);
2088
2089         /*
2090          * On Geminilake once the CDCLK gets as low as 79200
2091          * picture gets unstable, despite that values are
2092          * correct for DSI PLL and DE PLL.
2093          */
2094         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2095             IS_GEMINILAKE(dev_priv))
2096                 min_cdclk = max(158400, min_cdclk);
2097
2098         /* Account for additional needs from the planes */
2099         min_cdclk = max(intel_planes_min_cdclk(crtc_state), min_cdclk);
2100
2101         /*
2102          * HACK. Currently for TGL platforms we calculate
2103          * min_cdclk initially based on pixel_rate divided
2104          * by 2, accounting for also plane requirements,
2105          * however in some cases the lowest possible CDCLK
2106          * doesn't work and causing the underruns.
2107          * Explicitly stating here that this seems to be currently
2108          * rather a Hack, than final solution.
2109          */
2110         if (IS_TIGERLAKE(dev_priv)) {
2111                 /*
2112                  * Clamp to max_cdclk_freq in case pixel rate is higher,
2113                  * in order not to break an 8K, but still leave W/A at place.
2114                  */
2115                 min_cdclk = max_t(int, min_cdclk,
2116                                   min_t(int, crtc_state->pixel_rate,
2117                                         dev_priv->max_cdclk_freq));
2118         }
2119
2120         if (min_cdclk > dev_priv->max_cdclk_freq) {
2121                 drm_dbg_kms(&dev_priv->drm,
2122                             "required cdclk (%d kHz) exceeds max (%d kHz)\n",
2123                             min_cdclk, dev_priv->max_cdclk_freq);
2124                 return -EINVAL;
2125         }
2126
2127         return min_cdclk;
2128 }
2129
2130 static int intel_compute_min_cdclk(struct intel_cdclk_state *cdclk_state)
2131 {
2132         struct intel_atomic_state *state = cdclk_state->base.state;
2133         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2134         struct intel_bw_state *bw_state = NULL;
2135         struct intel_crtc *crtc;
2136         struct intel_crtc_state *crtc_state;
2137         int min_cdclk, i;
2138         enum pipe pipe;
2139
2140         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2141                 int ret;
2142
2143                 min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
2144                 if (min_cdclk < 0)
2145                         return min_cdclk;
2146
2147                 bw_state = intel_atomic_get_bw_state(state);
2148                 if (IS_ERR(bw_state))
2149                         return PTR_ERR(bw_state);
2150
2151                 if (cdclk_state->min_cdclk[crtc->pipe] == min_cdclk)
2152                         continue;
2153
2154                 cdclk_state->min_cdclk[crtc->pipe] = min_cdclk;
2155
2156                 ret = intel_atomic_lock_global_state(&cdclk_state->base);
2157                 if (ret)
2158                         return ret;
2159         }
2160
2161         min_cdclk = cdclk_state->force_min_cdclk;
2162         for_each_pipe(dev_priv, pipe) {
2163                 min_cdclk = max(cdclk_state->min_cdclk[pipe], min_cdclk);
2164
2165                 if (!bw_state)
2166                         continue;
2167
2168                 min_cdclk = max(bw_state->min_cdclk, min_cdclk);
2169         }
2170
2171         return min_cdclk;
2172 }
2173
2174 /*
2175  * Account for port clock min voltage level requirements.
2176  * This only really does something on CNL+ but can be
2177  * called on earlier platforms as well.
2178  *
2179  * Note that this functions assumes that 0 is
2180  * the lowest voltage value, and higher values
2181  * correspond to increasingly higher voltages.
2182  *
2183  * Should that relationship no longer hold on
2184  * future platforms this code will need to be
2185  * adjusted.
2186  */
2187 static int bxt_compute_min_voltage_level(struct intel_cdclk_state *cdclk_state)
2188 {
2189         struct intel_atomic_state *state = cdclk_state->base.state;
2190         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2191         struct intel_crtc *crtc;
2192         struct intel_crtc_state *crtc_state;
2193         u8 min_voltage_level;
2194         int i;
2195         enum pipe pipe;
2196
2197         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2198                 int ret;
2199
2200                 if (crtc_state->hw.enable)
2201                         min_voltage_level = crtc_state->min_voltage_level;
2202                 else
2203                         min_voltage_level = 0;
2204
2205                 if (cdclk_state->min_voltage_level[crtc->pipe] == min_voltage_level)
2206                         continue;
2207
2208                 cdclk_state->min_voltage_level[crtc->pipe] = min_voltage_level;
2209
2210                 ret = intel_atomic_lock_global_state(&cdclk_state->base);
2211                 if (ret)
2212                         return ret;
2213         }
2214
2215         min_voltage_level = 0;
2216         for_each_pipe(dev_priv, pipe)
2217                 min_voltage_level = max(cdclk_state->min_voltage_level[pipe],
2218                                         min_voltage_level);
2219
2220         return min_voltage_level;
2221 }
2222
2223 static int vlv_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2224 {
2225         struct intel_atomic_state *state = cdclk_state->base.state;
2226         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2227         int min_cdclk, cdclk;
2228
2229         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2230         if (min_cdclk < 0)
2231                 return min_cdclk;
2232
2233         cdclk = vlv_calc_cdclk(dev_priv, min_cdclk);
2234
2235         cdclk_state->logical.cdclk = cdclk;
2236         cdclk_state->logical.voltage_level =
2237                 vlv_calc_voltage_level(dev_priv, cdclk);
2238
2239         if (!cdclk_state->active_pipes) {
2240                 cdclk = vlv_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
2241
2242                 cdclk_state->actual.cdclk = cdclk;
2243                 cdclk_state->actual.voltage_level =
2244                         vlv_calc_voltage_level(dev_priv, cdclk);
2245         } else {
2246                 cdclk_state->actual = cdclk_state->logical;
2247         }
2248
2249         return 0;
2250 }
2251
2252 static int bdw_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2253 {
2254         int min_cdclk, cdclk;
2255
2256         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2257         if (min_cdclk < 0)
2258                 return min_cdclk;
2259
2260         /*
2261          * FIXME should also account for plane ratio
2262          * once 64bpp pixel formats are supported.
2263          */
2264         cdclk = bdw_calc_cdclk(min_cdclk);
2265
2266         cdclk_state->logical.cdclk = cdclk;
2267         cdclk_state->logical.voltage_level =
2268                 bdw_calc_voltage_level(cdclk);
2269
2270         if (!cdclk_state->active_pipes) {
2271                 cdclk = bdw_calc_cdclk(cdclk_state->force_min_cdclk);
2272
2273                 cdclk_state->actual.cdclk = cdclk;
2274                 cdclk_state->actual.voltage_level =
2275                         bdw_calc_voltage_level(cdclk);
2276         } else {
2277                 cdclk_state->actual = cdclk_state->logical;
2278         }
2279
2280         return 0;
2281 }
2282
2283 static int skl_dpll0_vco(struct intel_cdclk_state *cdclk_state)
2284 {
2285         struct intel_atomic_state *state = cdclk_state->base.state;
2286         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2287         struct intel_crtc *crtc;
2288         struct intel_crtc_state *crtc_state;
2289         int vco, i;
2290
2291         vco = cdclk_state->logical.vco;
2292         if (!vco)
2293                 vco = dev_priv->skl_preferred_vco_freq;
2294
2295         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2296                 if (!crtc_state->hw.enable)
2297                         continue;
2298
2299                 if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP))
2300                         continue;
2301
2302                 /*
2303                  * DPLL0 VCO may need to be adjusted to get the correct
2304                  * clock for eDP. This will affect cdclk as well.
2305                  */
2306                 switch (crtc_state->port_clock / 2) {
2307                 case 108000:
2308                 case 216000:
2309                         vco = 8640000;
2310                         break;
2311                 default:
2312                         vco = 8100000;
2313                         break;
2314                 }
2315         }
2316
2317         return vco;
2318 }
2319
2320 static int skl_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2321 {
2322         int min_cdclk, cdclk, vco;
2323
2324         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2325         if (min_cdclk < 0)
2326                 return min_cdclk;
2327
2328         vco = skl_dpll0_vco(cdclk_state);
2329
2330         /*
2331          * FIXME should also account for plane ratio
2332          * once 64bpp pixel formats are supported.
2333          */
2334         cdclk = skl_calc_cdclk(min_cdclk, vco);
2335
2336         cdclk_state->logical.vco = vco;
2337         cdclk_state->logical.cdclk = cdclk;
2338         cdclk_state->logical.voltage_level =
2339                 skl_calc_voltage_level(cdclk);
2340
2341         if (!cdclk_state->active_pipes) {
2342                 cdclk = skl_calc_cdclk(cdclk_state->force_min_cdclk, vco);
2343
2344                 cdclk_state->actual.vco = vco;
2345                 cdclk_state->actual.cdclk = cdclk;
2346                 cdclk_state->actual.voltage_level =
2347                         skl_calc_voltage_level(cdclk);
2348         } else {
2349                 cdclk_state->actual = cdclk_state->logical;
2350         }
2351
2352         return 0;
2353 }
2354
2355 static int bxt_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2356 {
2357         struct intel_atomic_state *state = cdclk_state->base.state;
2358         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2359         int min_cdclk, min_voltage_level, cdclk, vco;
2360
2361         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2362         if (min_cdclk < 0)
2363                 return min_cdclk;
2364
2365         min_voltage_level = bxt_compute_min_voltage_level(cdclk_state);
2366         if (min_voltage_level < 0)
2367                 return min_voltage_level;
2368
2369         cdclk = bxt_calc_cdclk(dev_priv, min_cdclk);
2370         vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2371
2372         cdclk_state->logical.vco = vco;
2373         cdclk_state->logical.cdclk = cdclk;
2374         cdclk_state->logical.voltage_level =
2375                 max_t(int, min_voltage_level,
2376                       dev_priv->display.calc_voltage_level(cdclk));
2377
2378         if (!cdclk_state->active_pipes) {
2379                 cdclk = bxt_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
2380                 vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2381
2382                 cdclk_state->actual.vco = vco;
2383                 cdclk_state->actual.cdclk = cdclk;
2384                 cdclk_state->actual.voltage_level =
2385                         dev_priv->display.calc_voltage_level(cdclk);
2386         } else {
2387                 cdclk_state->actual = cdclk_state->logical;
2388         }
2389
2390         return 0;
2391 }
2392
2393 static int fixed_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2394 {
2395         int min_cdclk;
2396
2397         /*
2398          * We can't change the cdclk frequency, but we still want to
2399          * check that the required minimum frequency doesn't exceed
2400          * the actual cdclk frequency.
2401          */
2402         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2403         if (min_cdclk < 0)
2404                 return min_cdclk;
2405
2406         return 0;
2407 }
2408
2409 static struct intel_global_state *intel_cdclk_duplicate_state(struct intel_global_obj *obj)
2410 {
2411         struct intel_cdclk_state *cdclk_state;
2412
2413         cdclk_state = kmemdup(obj->state, sizeof(*cdclk_state), GFP_KERNEL);
2414         if (!cdclk_state)
2415                 return NULL;
2416
2417         cdclk_state->pipe = INVALID_PIPE;
2418
2419         return &cdclk_state->base;
2420 }
2421
2422 static void intel_cdclk_destroy_state(struct intel_global_obj *obj,
2423                                       struct intel_global_state *state)
2424 {
2425         kfree(state);
2426 }
2427
2428 static const struct intel_global_state_funcs intel_cdclk_funcs = {
2429         .atomic_duplicate_state = intel_cdclk_duplicate_state,
2430         .atomic_destroy_state = intel_cdclk_destroy_state,
2431 };
2432
2433 struct intel_cdclk_state *
2434 intel_atomic_get_cdclk_state(struct intel_atomic_state *state)
2435 {
2436         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2437         struct intel_global_state *cdclk_state;
2438
2439         cdclk_state = intel_atomic_get_global_obj_state(state, &dev_priv->cdclk.obj);
2440         if (IS_ERR(cdclk_state))
2441                 return ERR_CAST(cdclk_state);
2442
2443         return to_intel_cdclk_state(cdclk_state);
2444 }
2445
2446 int intel_cdclk_init(struct drm_i915_private *dev_priv)
2447 {
2448         struct intel_cdclk_state *cdclk_state;
2449
2450         cdclk_state = kzalloc(sizeof(*cdclk_state), GFP_KERNEL);
2451         if (!cdclk_state)
2452                 return -ENOMEM;
2453
2454         intel_atomic_global_obj_init(dev_priv, &dev_priv->cdclk.obj,
2455                                      &cdclk_state->base, &intel_cdclk_funcs);
2456
2457         return 0;
2458 }
2459
2460 int intel_modeset_calc_cdclk(struct intel_atomic_state *state)
2461 {
2462         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2463         const struct intel_cdclk_state *old_cdclk_state;
2464         struct intel_cdclk_state *new_cdclk_state;
2465         enum pipe pipe;
2466         int ret;
2467
2468         new_cdclk_state = intel_atomic_get_cdclk_state(state);
2469         if (IS_ERR(new_cdclk_state))
2470                 return PTR_ERR(new_cdclk_state);
2471
2472         old_cdclk_state = intel_atomic_get_old_cdclk_state(state);
2473
2474         new_cdclk_state->active_pipes =
2475                 intel_calc_active_pipes(state, old_cdclk_state->active_pipes);
2476
2477         ret = dev_priv->display.modeset_calc_cdclk(new_cdclk_state);
2478         if (ret)
2479                 return ret;
2480
2481         if (intel_cdclk_changed(&old_cdclk_state->actual,
2482                                 &new_cdclk_state->actual)) {
2483                 /*
2484                  * Also serialize commits across all crtcs
2485                  * if the actual hw needs to be poked.
2486                  */
2487                 ret = intel_atomic_serialize_global_state(&new_cdclk_state->base);
2488                 if (ret)
2489                         return ret;
2490         } else if (old_cdclk_state->active_pipes != new_cdclk_state->active_pipes ||
2491                    old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk ||
2492                    intel_cdclk_changed(&old_cdclk_state->logical,
2493                                        &new_cdclk_state->logical)) {
2494                 ret = intel_atomic_lock_global_state(&new_cdclk_state->base);
2495                 if (ret)
2496                         return ret;
2497         } else {
2498                 return 0;
2499         }
2500
2501         if (is_power_of_2(new_cdclk_state->active_pipes) &&
2502             intel_cdclk_can_cd2x_update(dev_priv,
2503                                         &old_cdclk_state->actual,
2504                                         &new_cdclk_state->actual)) {
2505                 struct intel_crtc *crtc;
2506                 struct intel_crtc_state *crtc_state;
2507
2508                 pipe = ilog2(new_cdclk_state->active_pipes);
2509                 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
2510
2511                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
2512                 if (IS_ERR(crtc_state))
2513                         return PTR_ERR(crtc_state);
2514
2515                 if (drm_atomic_crtc_needs_modeset(&crtc_state->uapi))
2516                         pipe = INVALID_PIPE;
2517         } else {
2518                 pipe = INVALID_PIPE;
2519         }
2520
2521         if (pipe != INVALID_PIPE) {
2522                 new_cdclk_state->pipe = pipe;
2523
2524                 drm_dbg_kms(&dev_priv->drm,
2525                             "Can change cdclk with pipe %c active\n",
2526                             pipe_name(pipe));
2527         } else if (intel_cdclk_needs_modeset(&old_cdclk_state->actual,
2528                                              &new_cdclk_state->actual)) {
2529                 /* All pipes must be switched off while we change the cdclk. */
2530                 ret = intel_modeset_all_pipes(state);
2531                 if (ret)
2532                         return ret;
2533
2534                 new_cdclk_state->pipe = INVALID_PIPE;
2535
2536                 drm_dbg_kms(&dev_priv->drm,
2537                             "Modeset required for cdclk change\n");
2538         }
2539
2540         drm_dbg_kms(&dev_priv->drm,
2541                     "New cdclk calculated to be logical %u kHz, actual %u kHz\n",
2542                     new_cdclk_state->logical.cdclk,
2543                     new_cdclk_state->actual.cdclk);
2544         drm_dbg_kms(&dev_priv->drm,
2545                     "New voltage level calculated to be logical %u, actual %u\n",
2546                     new_cdclk_state->logical.voltage_level,
2547                     new_cdclk_state->actual.voltage_level);
2548
2549         return 0;
2550 }
2551
2552 static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
2553 {
2554         int max_cdclk_freq = dev_priv->max_cdclk_freq;
2555
2556         if (DISPLAY_VER(dev_priv) >= 10)
2557                 return 2 * max_cdclk_freq;
2558         else if (DISPLAY_VER(dev_priv) == 9 ||
2559                  IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2560                 return max_cdclk_freq;
2561         else if (IS_CHERRYVIEW(dev_priv))
2562                 return max_cdclk_freq*95/100;
2563         else if (DISPLAY_VER(dev_priv) < 4)
2564                 return 2*max_cdclk_freq*90/100;
2565         else
2566                 return max_cdclk_freq*90/100;
2567 }
2568
2569 /**
2570  * intel_update_max_cdclk - Determine the maximum support CDCLK frequency
2571  * @dev_priv: i915 device
2572  *
2573  * Determine the maximum CDCLK frequency the platform supports, and also
2574  * derive the maximum dot clock frequency the maximum CDCLK frequency
2575  * allows.
2576  */
2577 void intel_update_max_cdclk(struct drm_i915_private *dev_priv)
2578 {
2579         if (IS_JSL_EHL(dev_priv)) {
2580                 if (dev_priv->cdclk.hw.ref == 24000)
2581                         dev_priv->max_cdclk_freq = 552000;
2582                 else
2583                         dev_priv->max_cdclk_freq = 556800;
2584         } else if (DISPLAY_VER(dev_priv) >= 11) {
2585                 if (dev_priv->cdclk.hw.ref == 24000)
2586                         dev_priv->max_cdclk_freq = 648000;
2587                 else
2588                         dev_priv->max_cdclk_freq = 652800;
2589         } else if (IS_CANNONLAKE(dev_priv)) {
2590                 dev_priv->max_cdclk_freq = 528000;
2591         } else if (IS_GEMINILAKE(dev_priv)) {
2592                 dev_priv->max_cdclk_freq = 316800;
2593         } else if (IS_BROXTON(dev_priv)) {
2594                 dev_priv->max_cdclk_freq = 624000;
2595         } else if (DISPLAY_VER(dev_priv) == 9) {
2596                 u32 limit = intel_de_read(dev_priv, SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
2597                 int max_cdclk, vco;
2598
2599                 vco = dev_priv->skl_preferred_vco_freq;
2600                 drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
2601
2602                 /*
2603                  * Use the lower (vco 8640) cdclk values as a
2604                  * first guess. skl_calc_cdclk() will correct it
2605                  * if the preferred vco is 8100 instead.
2606                  */
2607                 if (limit == SKL_DFSM_CDCLK_LIMIT_675)
2608                         max_cdclk = 617143;
2609                 else if (limit == SKL_DFSM_CDCLK_LIMIT_540)
2610                         max_cdclk = 540000;
2611                 else if (limit == SKL_DFSM_CDCLK_LIMIT_450)
2612                         max_cdclk = 432000;
2613                 else
2614                         max_cdclk = 308571;
2615
2616                 dev_priv->max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco);
2617         } else if (IS_BROADWELL(dev_priv))  {
2618                 /*
2619                  * FIXME with extra cooling we can allow
2620                  * 540 MHz for ULX and 675 Mhz for ULT.
2621                  * How can we know if extra cooling is
2622                  * available? PCI ID, VTB, something else?
2623                  */
2624                 if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
2625                         dev_priv->max_cdclk_freq = 450000;
2626                 else if (IS_BDW_ULX(dev_priv))
2627                         dev_priv->max_cdclk_freq = 450000;
2628                 else if (IS_BDW_ULT(dev_priv))
2629                         dev_priv->max_cdclk_freq = 540000;
2630                 else
2631                         dev_priv->max_cdclk_freq = 675000;
2632         } else if (IS_CHERRYVIEW(dev_priv)) {
2633                 dev_priv->max_cdclk_freq = 320000;
2634         } else if (IS_VALLEYVIEW(dev_priv)) {
2635                 dev_priv->max_cdclk_freq = 400000;
2636         } else {
2637                 /* otherwise assume cdclk is fixed */
2638                 dev_priv->max_cdclk_freq = dev_priv->cdclk.hw.cdclk;
2639         }
2640
2641         dev_priv->max_dotclk_freq = intel_compute_max_dotclk(dev_priv);
2642
2643         drm_dbg(&dev_priv->drm, "Max CD clock rate: %d kHz\n",
2644                 dev_priv->max_cdclk_freq);
2645
2646         drm_dbg(&dev_priv->drm, "Max dotclock rate: %d kHz\n",
2647                 dev_priv->max_dotclk_freq);
2648 }
2649
2650 /**
2651  * intel_update_cdclk - Determine the current CDCLK frequency
2652  * @dev_priv: i915 device
2653  *
2654  * Determine the current CDCLK frequency.
2655  */
2656 void intel_update_cdclk(struct drm_i915_private *dev_priv)
2657 {
2658         dev_priv->display.get_cdclk(dev_priv, &dev_priv->cdclk.hw);
2659
2660         /*
2661          * 9:0 CMBUS [sic] CDCLK frequency (cdfreq):
2662          * Programmng [sic] note: bit[9:2] should be programmed to the number
2663          * of cdclk that generates 4MHz reference clock freq which is used to
2664          * generate GMBus clock. This will vary with the cdclk freq.
2665          */
2666         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2667                 intel_de_write(dev_priv, GMBUSFREQ_VLV,
2668                                DIV_ROUND_UP(dev_priv->cdclk.hw.cdclk, 1000));
2669 }
2670
2671 static int dg1_rawclk(struct drm_i915_private *dev_priv)
2672 {
2673         /*
2674          * DG1 always uses a 38.4 MHz rawclk.  The bspec tells us
2675          * "Program Numerator=2, Denominator=4, Divider=37 decimal."
2676          */
2677         intel_de_write(dev_priv, PCH_RAWCLK_FREQ,
2678                        CNP_RAWCLK_DEN(4) | CNP_RAWCLK_DIV(37) | ICP_RAWCLK_NUM(2));
2679
2680         return 38400;
2681 }
2682
2683 static int cnp_rawclk(struct drm_i915_private *dev_priv)
2684 {
2685         u32 rawclk;
2686         int divider, fraction;
2687
2688         if (intel_de_read(dev_priv, SFUSE_STRAP) & SFUSE_STRAP_RAW_FREQUENCY) {
2689                 /* 24 MHz */
2690                 divider = 24000;
2691                 fraction = 0;
2692         } else {
2693                 /* 19.2 MHz */
2694                 divider = 19000;
2695                 fraction = 200;
2696         }
2697
2698         rawclk = CNP_RAWCLK_DIV(divider / 1000);
2699         if (fraction) {
2700                 int numerator = 1;
2701
2702                 rawclk |= CNP_RAWCLK_DEN(DIV_ROUND_CLOSEST(numerator * 1000,
2703                                                            fraction) - 1);
2704                 if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
2705                         rawclk |= ICP_RAWCLK_NUM(numerator);
2706         }
2707
2708         intel_de_write(dev_priv, PCH_RAWCLK_FREQ, rawclk);
2709         return divider + fraction;
2710 }
2711
2712 static int pch_rawclk(struct drm_i915_private *dev_priv)
2713 {
2714         return (intel_de_read(dev_priv, PCH_RAWCLK_FREQ) & RAWCLK_FREQ_MASK) * 1000;
2715 }
2716
2717 static int vlv_hrawclk(struct drm_i915_private *dev_priv)
2718 {
2719         /* RAWCLK_FREQ_VLV register updated from power well code */
2720         return vlv_get_cck_clock_hpll(dev_priv, "hrawclk",
2721                                       CCK_DISPLAY_REF_CLOCK_CONTROL);
2722 }
2723
2724 static int i9xx_hrawclk(struct drm_i915_private *dev_priv)
2725 {
2726         u32 clkcfg;
2727
2728         /*
2729          * hrawclock is 1/4 the FSB frequency
2730          *
2731          * Note that this only reads the state of the FSB
2732          * straps, not the actual FSB frequency. Some BIOSen
2733          * let you configure each independently. Ideally we'd
2734          * read out the actual FSB frequency but sadly we
2735          * don't know which registers have that information,
2736          * and all the relevant docs have gone to bit heaven :(
2737          */
2738         clkcfg = intel_de_read(dev_priv, CLKCFG) & CLKCFG_FSB_MASK;
2739
2740         if (IS_MOBILE(dev_priv)) {
2741                 switch (clkcfg) {
2742                 case CLKCFG_FSB_400:
2743                         return 100000;
2744                 case CLKCFG_FSB_533:
2745                         return 133333;
2746                 case CLKCFG_FSB_667:
2747                         return 166667;
2748                 case CLKCFG_FSB_800:
2749                         return 200000;
2750                 case CLKCFG_FSB_1067:
2751                         return 266667;
2752                 case CLKCFG_FSB_1333:
2753                         return 333333;
2754                 default:
2755                         MISSING_CASE(clkcfg);
2756                         return 133333;
2757                 }
2758         } else {
2759                 switch (clkcfg) {
2760                 case CLKCFG_FSB_400_ALT:
2761                         return 100000;
2762                 case CLKCFG_FSB_533:
2763                         return 133333;
2764                 case CLKCFG_FSB_667:
2765                         return 166667;
2766                 case CLKCFG_FSB_800:
2767                         return 200000;
2768                 case CLKCFG_FSB_1067_ALT:
2769                         return 266667;
2770                 case CLKCFG_FSB_1333_ALT:
2771                         return 333333;
2772                 case CLKCFG_FSB_1600_ALT:
2773                         return 400000;
2774                 default:
2775                         return 133333;
2776                 }
2777         }
2778 }
2779
2780 /**
2781  * intel_read_rawclk - Determine the current RAWCLK frequency
2782  * @dev_priv: i915 device
2783  *
2784  * Determine the current RAWCLK frequency. RAWCLK is a fixed
2785  * frequency clock so this needs to done only once.
2786  */
2787 u32 intel_read_rawclk(struct drm_i915_private *dev_priv)
2788 {
2789         u32 freq;
2790
2791         if (INTEL_PCH_TYPE(dev_priv) >= PCH_DG1)
2792                 freq = dg1_rawclk(dev_priv);
2793         else if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP)
2794                 freq = cnp_rawclk(dev_priv);
2795         else if (HAS_PCH_SPLIT(dev_priv))
2796                 freq = pch_rawclk(dev_priv);
2797         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2798                 freq = vlv_hrawclk(dev_priv);
2799         else if (DISPLAY_VER(dev_priv) >= 3)
2800                 freq = i9xx_hrawclk(dev_priv);
2801         else
2802                 /* no rawclk on other platforms, or no need to know it */
2803                 return 0;
2804
2805         return freq;
2806 }
2807
2808 /**
2809  * intel_init_cdclk_hooks - Initialize CDCLK related modesetting hooks
2810  * @dev_priv: i915 device
2811  */
2812 void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv)
2813 {
2814         if (IS_ALDERLAKE_P(dev_priv)) {
2815                 dev_priv->display.set_cdclk = bxt_set_cdclk;
2816                 dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk;
2817                 dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk;
2818                 dev_priv->display.calc_voltage_level = tgl_calc_voltage_level;
2819                 /* Wa_22011320316:adlp[a0] */
2820                 if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_A0))
2821                         dev_priv->cdclk.table = adlp_a_step_cdclk_table;
2822                 else
2823                         dev_priv->cdclk.table = adlp_cdclk_table;
2824         } else if (IS_ROCKETLAKE(dev_priv)) {
2825                 dev_priv->display.set_cdclk = bxt_set_cdclk;
2826                 dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk;
2827                 dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk;
2828                 dev_priv->display.calc_voltage_level = tgl_calc_voltage_level;
2829                 dev_priv->cdclk.table = rkl_cdclk_table;
2830         } else if (DISPLAY_VER(dev_priv) >= 12) {
2831                 dev_priv->display.set_cdclk = bxt_set_cdclk;
2832                 dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk;
2833                 dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk;
2834                 dev_priv->display.calc_voltage_level = tgl_calc_voltage_level;
2835                 dev_priv->cdclk.table = icl_cdclk_table;
2836         } else if (IS_JSL_EHL(dev_priv)) {
2837                 dev_priv->display.set_cdclk = bxt_set_cdclk;
2838                 dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk;
2839                 dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk;
2840                 dev_priv->display.calc_voltage_level = ehl_calc_voltage_level;
2841                 dev_priv->cdclk.table = icl_cdclk_table;
2842         } else if (DISPLAY_VER(dev_priv) >= 11) {
2843                 dev_priv->display.set_cdclk = bxt_set_cdclk;
2844                 dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk;
2845                 dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk;
2846                 dev_priv->display.calc_voltage_level = icl_calc_voltage_level;
2847                 dev_priv->cdclk.table = icl_cdclk_table;
2848         } else if (IS_CANNONLAKE(dev_priv)) {
2849                 dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk;
2850                 dev_priv->display.set_cdclk = bxt_set_cdclk;
2851                 dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk;
2852                 dev_priv->display.calc_voltage_level = cnl_calc_voltage_level;
2853                 dev_priv->cdclk.table = cnl_cdclk_table;
2854         } else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
2855                 dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk;
2856                 dev_priv->display.set_cdclk = bxt_set_cdclk;
2857                 dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk;
2858                 dev_priv->display.calc_voltage_level = bxt_calc_voltage_level;
2859                 if (IS_GEMINILAKE(dev_priv))
2860                         dev_priv->cdclk.table = glk_cdclk_table;
2861                 else
2862                         dev_priv->cdclk.table = bxt_cdclk_table;
2863         } else if (DISPLAY_VER(dev_priv) == 9) {
2864                 dev_priv->display.bw_calc_min_cdclk = skl_bw_calc_min_cdclk;
2865                 dev_priv->display.set_cdclk = skl_set_cdclk;
2866                 dev_priv->display.modeset_calc_cdclk = skl_modeset_calc_cdclk;
2867         } else if (IS_BROADWELL(dev_priv)) {
2868                 dev_priv->display.bw_calc_min_cdclk = intel_bw_calc_min_cdclk;
2869                 dev_priv->display.set_cdclk = bdw_set_cdclk;
2870                 dev_priv->display.modeset_calc_cdclk = bdw_modeset_calc_cdclk;
2871         } else if (IS_CHERRYVIEW(dev_priv)) {
2872                 dev_priv->display.bw_calc_min_cdclk = intel_bw_calc_min_cdclk;
2873                 dev_priv->display.set_cdclk = chv_set_cdclk;
2874                 dev_priv->display.modeset_calc_cdclk = vlv_modeset_calc_cdclk;
2875         } else if (IS_VALLEYVIEW(dev_priv)) {
2876                 dev_priv->display.bw_calc_min_cdclk = intel_bw_calc_min_cdclk;
2877                 dev_priv->display.set_cdclk = vlv_set_cdclk;
2878                 dev_priv->display.modeset_calc_cdclk = vlv_modeset_calc_cdclk;
2879         } else {
2880                 dev_priv->display.bw_calc_min_cdclk = intel_bw_calc_min_cdclk;
2881                 dev_priv->display.modeset_calc_cdclk = fixed_modeset_calc_cdclk;
2882         }
2883
2884         if (DISPLAY_VER(dev_priv) >= 10 || IS_BROXTON(dev_priv))
2885                 dev_priv->display.get_cdclk = bxt_get_cdclk;
2886         else if (DISPLAY_VER(dev_priv) == 9)
2887                 dev_priv->display.get_cdclk = skl_get_cdclk;
2888         else if (IS_BROADWELL(dev_priv))
2889                 dev_priv->display.get_cdclk = bdw_get_cdclk;
2890         else if (IS_HASWELL(dev_priv))
2891                 dev_priv->display.get_cdclk = hsw_get_cdclk;
2892         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2893                 dev_priv->display.get_cdclk = vlv_get_cdclk;
2894         else if (IS_SANDYBRIDGE(dev_priv) || IS_IVYBRIDGE(dev_priv))
2895                 dev_priv->display.get_cdclk = fixed_400mhz_get_cdclk;
2896         else if (IS_IRONLAKE(dev_priv))
2897                 dev_priv->display.get_cdclk = fixed_450mhz_get_cdclk;
2898         else if (IS_GM45(dev_priv))
2899                 dev_priv->display.get_cdclk = gm45_get_cdclk;
2900         else if (IS_G45(dev_priv))
2901                 dev_priv->display.get_cdclk = g33_get_cdclk;
2902         else if (IS_I965GM(dev_priv))
2903                 dev_priv->display.get_cdclk = i965gm_get_cdclk;
2904         else if (IS_I965G(dev_priv))
2905                 dev_priv->display.get_cdclk = fixed_400mhz_get_cdclk;
2906         else if (IS_PINEVIEW(dev_priv))
2907                 dev_priv->display.get_cdclk = pnv_get_cdclk;
2908         else if (IS_G33(dev_priv))
2909                 dev_priv->display.get_cdclk = g33_get_cdclk;
2910         else if (IS_I945GM(dev_priv))
2911                 dev_priv->display.get_cdclk = i945gm_get_cdclk;
2912         else if (IS_I945G(dev_priv))
2913                 dev_priv->display.get_cdclk = fixed_400mhz_get_cdclk;
2914         else if (IS_I915GM(dev_priv))
2915                 dev_priv->display.get_cdclk = i915gm_get_cdclk;
2916         else if (IS_I915G(dev_priv))
2917                 dev_priv->display.get_cdclk = fixed_333mhz_get_cdclk;
2918         else if (IS_I865G(dev_priv))
2919                 dev_priv->display.get_cdclk = fixed_266mhz_get_cdclk;
2920         else if (IS_I85X(dev_priv))
2921                 dev_priv->display.get_cdclk = i85x_get_cdclk;
2922         else if (IS_I845G(dev_priv))
2923                 dev_priv->display.get_cdclk = fixed_200mhz_get_cdclk;
2924         else if (IS_I830(dev_priv))
2925                 dev_priv->display.get_cdclk = fixed_133mhz_get_cdclk;
2926
2927         if (drm_WARN(&dev_priv->drm, !dev_priv->display.get_cdclk,
2928                      "Unknown platform. Assuming 133 MHz CDCLK\n"))
2929                 dev_priv->display.get_cdclk = fixed_133mhz_get_cdclk;
2930 }