drivers/gpu/drm/i915/display/intel_cdclk.c

   1 /*
   2  * Copyright © 2006-2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include <linux/time.h>
  25
  26 #include "intel_atomic.h"
  27 #include "intel_bw.h"
  28 #include "intel_cdclk.h"
  29 #include "intel_de.h"
  30 #include "intel_display_types.h"
  31 #include "intel_pcode.h"
  32 #include "intel_psr.h"
  33 #include "vlv_sideband.h"
  34
  35 /**
  36  * DOC: CDCLK / RAWCLK
  37  *
  38  * The display engine uses several different clocks to do its work. There
  39  * are two main clocks involved that aren't directly related to the actual
  40  * pixel clock or any symbol/bit clock of the actual output port. These
  41  * are the core display clock (CDCLK) and RAWCLK.
  42  *
  43  * CDCLK clocks most of the display pipe logic, and thus its frequency
  44  * must be high enough to support the rate at which pixels are flowing
  45  * through the pipes. Downscaling must also be accounted as that increases
  46  * the effective pixel rate.
  47  *
  48  * On several platforms the CDCLK frequency can be changed dynamically
  49  * to minimize power consumption for a given display configuration.
  50  * Typically changes to the CDCLK frequency require all the display pipes
  51  * to be shut down while the frequency is being changed.
  52  *
  53  * On SKL+ the DMC will toggle the CDCLK off/on during DC5/6 entry/exit.
  54  * DMC will not change the active CDCLK frequency however, so that part
  55  * will still be performed by the driver directly.
  56  *
  57  * RAWCLK is a fixed frequency clock, often used by various auxiliary
  58  * blocks such as AUX CH or backlight PWM. Hence the only thing we
  59  * really need to know about RAWCLK is its frequency so that various
  60  * dividers can be programmed correctly.
  61  */
  62
  63 void intel_cdclk_get_cdclk(struct drm_i915_private *dev_priv,
  64                            struct intel_cdclk_config *cdclk_config)
  65 {
  66         dev_priv->cdclk_funcs->get_cdclk(dev_priv, cdclk_config);
  67 }
  68
  69 int intel_cdclk_bw_calc_min_cdclk(struct intel_atomic_state *state)
  70 {
  71         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
  72         return dev_priv->cdclk_funcs->bw_calc_min_cdclk(state);
  73 }
  74
  75 static void intel_cdclk_set_cdclk(struct drm_i915_private *dev_priv,
  76                                   const struct intel_cdclk_config *cdclk_config,
  77                                   enum pipe pipe)
  78 {
  79         dev_priv->cdclk_funcs->set_cdclk(dev_priv, cdclk_config, pipe);
  80 }
  81
  82 static int intel_cdclk_modeset_calc_cdclk(struct drm_i915_private *dev_priv,
  83                                           struct intel_cdclk_state *cdclk_config)
  84 {
  85         return dev_priv->cdclk_funcs->modeset_calc_cdclk(cdclk_config);
  86 }
  87
  88 static u8 intel_cdclk_calc_voltage_level(struct drm_i915_private *dev_priv,
  89                                          int cdclk)
  90 {
  91         return dev_priv->cdclk_funcs->calc_voltage_level(cdclk);
  92 }
  93
  94 static void fixed_133mhz_get_cdclk(struct drm_i915_private *dev_priv,
  95                                    struct intel_cdclk_config *cdclk_config)
  96 {
  97         cdclk_config->cdclk = 133333;
  98 }
  99
 100 static void fixed_200mhz_get_cdclk(struct drm_i915_private *dev_priv,
 101                                    struct intel_cdclk_config *cdclk_config)
 102 {
 103         cdclk_config->cdclk = 200000;
 104 }
 105
 106 static void fixed_266mhz_get_cdclk(struct drm_i915_private *dev_priv,
 107                                    struct intel_cdclk_config *cdclk_config)
 108 {
 109         cdclk_config->cdclk = 266667;
 110 }
 111
 112 static void fixed_333mhz_get_cdclk(struct drm_i915_private *dev_priv,
 113                                    struct intel_cdclk_config *cdclk_config)
 114 {
 115         cdclk_config->cdclk = 333333;
 116 }
 117
 118 static void fixed_400mhz_get_cdclk(struct drm_i915_private *dev_priv,
 119                                    struct intel_cdclk_config *cdclk_config)
 120 {
 121         cdclk_config->cdclk = 400000;
 122 }
 123
 124 static void fixed_450mhz_get_cdclk(struct drm_i915_private *dev_priv,
 125                                    struct intel_cdclk_config *cdclk_config)
 126 {
 127         cdclk_config->cdclk = 450000;
 128 }
 129
 130 static void i85x_get_cdclk(struct drm_i915_private *dev_priv,
 131                            struct intel_cdclk_config *cdclk_config)
 132 {
 133         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 134         u16 hpllcc = 0;
 135
 136         /*
 137          * 852GM/852GMV only supports 133 MHz and the HPLLCC
 138          * encoding is different :(
 139          * FIXME is this the right way to detect 852GM/852GMV?
 140          */
 141         if (pdev->revision == 0x1) {
 142                 cdclk_config->cdclk = 133333;
 143                 return;
 144         }
 145
 146         pci_bus_read_config_word(pdev->bus,
 147                                  PCI_DEVFN(0, 3), HPLLCC, &hpllcc);
 148
 149         /* Assume that the hardware is in the high speed state.  This
 150          * should be the default.
 151          */
 152         switch (hpllcc & GC_CLOCK_CONTROL_MASK) {
 153         case GC_CLOCK_133_200:
 154         case GC_CLOCK_133_200_2:
 155         case GC_CLOCK_100_200:
 156                 cdclk_config->cdclk = 200000;
 157                 break;
 158         case GC_CLOCK_166_250:
 159                 cdclk_config->cdclk = 250000;
 160                 break;
 161         case GC_CLOCK_100_133:
 162                 cdclk_config->cdclk = 133333;
 163                 break;
 164         case GC_CLOCK_133_266:
 165         case GC_CLOCK_133_266_2:
 166         case GC_CLOCK_166_266:
 167                 cdclk_config->cdclk = 266667;
 168                 break;
 169         }
 170 }
 171
 172 static void i915gm_get_cdclk(struct drm_i915_private *dev_priv,
 173                              struct intel_cdclk_config *cdclk_config)
 174 {
 175         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 176         u16 gcfgc = 0;
 177
 178         pci_read_config_word(pdev, GCFGC, &gcfgc);
 179
 180         if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
 181                 cdclk_config->cdclk = 133333;
 182                 return;
 183         }
 184
 185         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 186         case GC_DISPLAY_CLOCK_333_320_MHZ:
 187                 cdclk_config->cdclk = 333333;
 188                 break;
 189         default:
 190         case GC_DISPLAY_CLOCK_190_200_MHZ:
 191                 cdclk_config->cdclk = 190000;
 192                 break;
 193         }
 194 }
 195
 196 static void i945gm_get_cdclk(struct drm_i915_private *dev_priv,
 197                              struct intel_cdclk_config *cdclk_config)
 198 {
 199         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 200         u16 gcfgc = 0;
 201
 202         pci_read_config_word(pdev, GCFGC, &gcfgc);
 203
 204         if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
 205                 cdclk_config->cdclk = 133333;
 206                 return;
 207         }
 208
 209         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 210         case GC_DISPLAY_CLOCK_333_320_MHZ:
 211                 cdclk_config->cdclk = 320000;
 212                 break;
 213         default:
 214         case GC_DISPLAY_CLOCK_190_200_MHZ:
 215                 cdclk_config->cdclk = 200000;
 216                 break;
 217         }
 218 }
 219
 220 static unsigned int intel_hpll_vco(struct drm_i915_private *dev_priv)
 221 {
 222         static const unsigned int blb_vco[8] = {
 223                 [0] = 3200000,
 224                 [1] = 4000000,
 225                 [2] = 5333333,
 226                 [3] = 4800000,
 227                 [4] = 6400000,
 228         };
 229         static const unsigned int pnv_vco[8] = {
 230                 [0] = 3200000,
 231                 [1] = 4000000,
 232                 [2] = 5333333,
 233                 [3] = 4800000,
 234                 [4] = 2666667,
 235         };
 236         static const unsigned int cl_vco[8] = {
 237                 [0] = 3200000,
 238                 [1] = 4000000,
 239                 [2] = 5333333,
 240                 [3] = 6400000,
 241                 [4] = 3333333,
 242                 [5] = 3566667,
 243                 [6] = 4266667,
 244         };
 245         static const unsigned int elk_vco[8] = {
 246                 [0] = 3200000,
 247                 [1] = 4000000,
 248                 [2] = 5333333,
 249                 [3] = 4800000,
 250         };
 251         static const unsigned int ctg_vco[8] = {
 252                 [0] = 3200000,
 253                 [1] = 4000000,
 254                 [2] = 5333333,
 255                 [3] = 6400000,
 256                 [4] = 2666667,
 257                 [5] = 4266667,
 258         };
 259         const unsigned int *vco_table;
 260         unsigned int vco;
 261         u8 tmp = 0;
 262
 263         /* FIXME other chipsets? */
 264         if (IS_GM45(dev_priv))
 265                 vco_table = ctg_vco;
 266         else if (IS_G45(dev_priv))
 267                 vco_table = elk_vco;
 268         else if (IS_I965GM(dev_priv))
 269                 vco_table = cl_vco;
 270         else if (IS_PINEVIEW(dev_priv))
 271                 vco_table = pnv_vco;
 272         else if (IS_G33(dev_priv))
 273                 vco_table = blb_vco;
 274         else
 275                 return 0;
 276
 277         tmp = intel_de_read(dev_priv,
 278                             IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv) ? HPLLVCO_MOBILE : HPLLVCO);
 279
 280         vco = vco_table[tmp & 0x7];
 281         if (vco == 0)
 282                 drm_err(&dev_priv->drm, "Bad HPLL VCO (HPLLVCO=0x%02x)\n",
 283                         tmp);
 284         else
 285                 drm_dbg_kms(&dev_priv->drm, "HPLL VCO %u kHz\n", vco);
 286
 287         return vco;
 288 }
 289
 290 static void g33_get_cdclk(struct drm_i915_private *dev_priv,
 291                           struct intel_cdclk_config *cdclk_config)
 292 {
 293         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 294         static const u8 div_3200[] = { 12, 10,  8,  7, 5, 16 };
 295         static const u8 div_4000[] = { 14, 12, 10,  8, 6, 20 };
 296         static const u8 div_4800[] = { 20, 14, 12, 10, 8, 24 };
 297         static const u8 div_5333[] = { 20, 16, 12, 12, 8, 28 };
 298         const u8 *div_table;
 299         unsigned int cdclk_sel;
 300         u16 tmp = 0;
 301
 302         cdclk_config->vco = intel_hpll_vco(dev_priv);
 303
 304         pci_read_config_word(pdev, GCFGC, &tmp);
 305
 306         cdclk_sel = (tmp >> 4) & 0x7;
 307
 308         if (cdclk_sel >= ARRAY_SIZE(div_3200))
 309                 goto fail;
 310
 311         switch (cdclk_config->vco) {
 312         case 3200000:
 313                 div_table = div_3200;
 314                 break;
 315         case 4000000:
 316                 div_table = div_4000;
 317                 break;
 318         case 4800000:
 319                 div_table = div_4800;
 320                 break;
 321         case 5333333:
 322                 div_table = div_5333;
 323                 break;
 324         default:
 325                 goto fail;
 326         }
 327
 328         cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
 329                                                 div_table[cdclk_sel]);
 330         return;
 331
 332 fail:
 333         drm_err(&dev_priv->drm,
 334                 "Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%08x\n",
 335                 cdclk_config->vco, tmp);
 336         cdclk_config->cdclk = 190476;
 337 }
 338
 339 static void pnv_get_cdclk(struct drm_i915_private *dev_priv,
 340                           struct intel_cdclk_config *cdclk_config)
 341 {
 342         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 343         u16 gcfgc = 0;
 344
 345         pci_read_config_word(pdev, GCFGC, &gcfgc);
 346
 347         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 348         case GC_DISPLAY_CLOCK_267_MHZ_PNV:
 349                 cdclk_config->cdclk = 266667;
 350                 break;
 351         case GC_DISPLAY_CLOCK_333_MHZ_PNV:
 352                 cdclk_config->cdclk = 333333;
 353                 break;
 354         case GC_DISPLAY_CLOCK_444_MHZ_PNV:
 355                 cdclk_config->cdclk = 444444;
 356                 break;
 357         case GC_DISPLAY_CLOCK_200_MHZ_PNV:
 358                 cdclk_config->cdclk = 200000;
 359                 break;
 360         default:
 361                 drm_err(&dev_priv->drm,
 362                         "Unknown pnv display core clock 0x%04x\n", gcfgc);
 363                 fallthrough;
 364         case GC_DISPLAY_CLOCK_133_MHZ_PNV:
 365                 cdclk_config->cdclk = 133333;
 366                 break;
 367         case GC_DISPLAY_CLOCK_167_MHZ_PNV:
 368                 cdclk_config->cdclk = 166667;
 369                 break;
 370         }
 371 }
 372
 373 static void i965gm_get_cdclk(struct drm_i915_private *dev_priv,
 374                              struct intel_cdclk_config *cdclk_config)
 375 {
 376         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 377         static const u8 div_3200[] = { 16, 10,  8 };
 378         static const u8 div_4000[] = { 20, 12, 10 };
 379         static const u8 div_5333[] = { 24, 16, 14 };
 380         const u8 *div_table;
 381         unsigned int cdclk_sel;
 382         u16 tmp = 0;
 383
 384         cdclk_config->vco = intel_hpll_vco(dev_priv);
 385
 386         pci_read_config_word(pdev, GCFGC, &tmp);
 387
 388         cdclk_sel = ((tmp >> 8) & 0x1f) - 1;
 389
 390         if (cdclk_sel >= ARRAY_SIZE(div_3200))
 391                 goto fail;
 392
 393         switch (cdclk_config->vco) {
 394         case 3200000:
 395                 div_table = div_3200;
 396                 break;
 397         case 4000000:
 398                 div_table = div_4000;
 399                 break;
 400         case 5333333:
 401                 div_table = div_5333;
 402                 break;
 403         default:
 404                 goto fail;
 405         }
 406
 407         cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
 408                                                 div_table[cdclk_sel]);
 409         return;
 410
 411 fail:
 412         drm_err(&dev_priv->drm,
 413                 "Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%04x\n",
 414                 cdclk_config->vco, tmp);
 415         cdclk_config->cdclk = 200000;
 416 }
 417
 418 static void gm45_get_cdclk(struct drm_i915_private *dev_priv,
 419                            struct intel_cdclk_config *cdclk_config)
 420 {
 421         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 422         unsigned int cdclk_sel;
 423         u16 tmp = 0;
 424
 425         cdclk_config->vco = intel_hpll_vco(dev_priv);
 426
 427         pci_read_config_word(pdev, GCFGC, &tmp);
 428
 429         cdclk_sel = (tmp >> 12) & 0x1;
 430
 431         switch (cdclk_config->vco) {
 432         case 2666667:
 433         case 4000000:
 434         case 5333333:
 435                 cdclk_config->cdclk = cdclk_sel ? 333333 : 222222;
 436                 break;
 437         case 3200000:
 438                 cdclk_config->cdclk = cdclk_sel ? 320000 : 228571;
 439                 break;
 440         default:
 441                 drm_err(&dev_priv->drm,
 442                         "Unable to determine CDCLK. HPLL VCO=%u, CFGC=0x%04x\n",
 443                         cdclk_config->vco, tmp);
 444                 cdclk_config->cdclk = 222222;
 445                 break;
 446         }
 447 }
 448
 449 static void hsw_get_cdclk(struct drm_i915_private *dev_priv,
 450                           struct intel_cdclk_config *cdclk_config)
 451 {
 452         u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
 453         u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
 454
 455         if (lcpll & LCPLL_CD_SOURCE_FCLK)
 456                 cdclk_config->cdclk = 800000;
 457         else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
 458                 cdclk_config->cdclk = 450000;
 459         else if (freq == LCPLL_CLK_FREQ_450)
 460                 cdclk_config->cdclk = 450000;
 461         else if (IS_HSW_ULT(dev_priv))
 462                 cdclk_config->cdclk = 337500;
 463         else
 464                 cdclk_config->cdclk = 540000;
 465 }
 466
 467 static int vlv_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
 468 {
 469         int freq_320 = (dev_priv->hpll_freq <<  1) % 320000 != 0 ?
 470                 333333 : 320000;
 471
 472         /*
 473          * We seem to get an unstable or solid color picture at 200MHz.
 474          * Not sure what's wrong. For now use 200MHz only when all pipes
 475          * are off.
 476          */
 477         if (IS_VALLEYVIEW(dev_priv) && min_cdclk > freq_320)
 478                 return 400000;
 479         else if (min_cdclk > 266667)
 480                 return freq_320;
 481         else if (min_cdclk > 0)
 482                 return 266667;
 483         else
 484                 return 200000;
 485 }
 486
 487 static u8 vlv_calc_voltage_level(struct drm_i915_private *dev_priv, int cdclk)
 488 {
 489         if (IS_VALLEYVIEW(dev_priv)) {
 490                 if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */
 491                         return 2;
 492                 else if (cdclk >= 266667)
 493                         return 1;
 494                 else
 495                         return 0;
 496         } else {
 497                 /*
 498                  * Specs are full of misinformation, but testing on actual
 499                  * hardware has shown that we just need to write the desired
 500                  * CCK divider into the Punit register.
 501                  */
 502                 return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
 503         }
 504 }
 505
 506 static void vlv_get_cdclk(struct drm_i915_private *dev_priv,
 507                           struct intel_cdclk_config *cdclk_config)
 508 {
 509         u32 val;
 510
 511         vlv_iosf_sb_get(dev_priv,
 512                         BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
 513
 514         cdclk_config->vco = vlv_get_hpll_vco(dev_priv);
 515         cdclk_config->cdclk = vlv_get_cck_clock(dev_priv, "cdclk",
 516                                                 CCK_DISPLAY_CLOCK_CONTROL,
 517                                                 cdclk_config->vco);
 518
 519         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 520
 521         vlv_iosf_sb_put(dev_priv,
 522                         BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
 523
 524         if (IS_VALLEYVIEW(dev_priv))
 525                 cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK) >>
 526                         DSPFREQGUAR_SHIFT;
 527         else
 528                 cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK_CHV) >>
 529                         DSPFREQGUAR_SHIFT_CHV;
 530 }
 531
 532 static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
 533 {
 534         unsigned int credits, default_credits;
 535
 536         if (IS_CHERRYVIEW(dev_priv))
 537                 default_credits = PFI_CREDIT(12);
 538         else
 539                 default_credits = PFI_CREDIT(8);
 540
 541         if (dev_priv->cdclk.hw.cdclk >= dev_priv->czclk_freq) {
 542                 /* CHV suggested value is 31 or 63 */
 543                 if (IS_CHERRYVIEW(dev_priv))
 544                         credits = PFI_CREDIT_63;
 545                 else
 546                         credits = PFI_CREDIT(15);
 547         } else {
 548                 credits = default_credits;
 549         }
 550
 551         /*
 552          * WA - write default credits before re-programming
 553          * FIXME: should we also set the resend bit here?
 554          */
 555         intel_de_write(dev_priv, GCI_CONTROL,
 556                        VGA_FAST_MODE_DISABLE | default_credits);
 557
 558         intel_de_write(dev_priv, GCI_CONTROL,
 559                        VGA_FAST_MODE_DISABLE | credits | PFI_CREDIT_RESEND);
 560
 561         /*
 562          * FIXME is this guaranteed to clear
 563          * immediately or should we poll for it?
 564          */
 565         drm_WARN_ON(&dev_priv->drm,
 566                     intel_de_read(dev_priv, GCI_CONTROL) & PFI_CREDIT_RESEND);
 567 }
 568
 569 static void vlv_set_cdclk(struct drm_i915_private *dev_priv,
 570                           const struct intel_cdclk_config *cdclk_config,
 571                           enum pipe pipe)
 572 {
 573         int cdclk = cdclk_config->cdclk;
 574         u32 val, cmd = cdclk_config->voltage_level;
 575         intel_wakeref_t wakeref;
 576
 577         switch (cdclk) {
 578         case 400000:
 579         case 333333:
 580         case 320000:
 581         case 266667:
 582         case 200000:
 583                 break;
 584         default:
 585                 MISSING_CASE(cdclk);
 586                 return;
 587         }
 588
 589         /* There are cases where we can end up here with power domains
 590          * off and a CDCLK frequency other than the minimum, like when
 591          * issuing a modeset without actually changing any display after
 592          * a system suspend.  So grab the display core domain, which covers
 593          * the HW blocks needed for the following programming.
 594          */
 595         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
 596
 597         vlv_iosf_sb_get(dev_priv,
 598                         BIT(VLV_IOSF_SB_CCK) |
 599                         BIT(VLV_IOSF_SB_BUNIT) |
 600                         BIT(VLV_IOSF_SB_PUNIT));
 601
 602         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 603         val &= ~DSPFREQGUAR_MASK;
 604         val |= (cmd << DSPFREQGUAR_SHIFT);
 605         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 606         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
 607                       DSPFREQSTAT_MASK) == (cmd << DSPFREQSTAT_SHIFT),
 608                      50)) {
 609                 drm_err(&dev_priv->drm,
 610                         "timed out waiting for CDclk change\n");
 611         }
 612
 613         if (cdclk == 400000) {
 614                 u32 divider;
 615
 616                 divider = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1,
 617                                             cdclk) - 1;
 618
 619                 /* adjust cdclk divider */
 620                 val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL);
 621                 val &= ~CCK_FREQUENCY_VALUES;
 622                 val |= divider;
 623                 vlv_cck_write(dev_priv, CCK_DISPLAY_CLOCK_CONTROL, val);
 624
 625                 if (wait_for((vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL) &
 626                               CCK_FREQUENCY_STATUS) == (divider << CCK_FREQUENCY_STATUS_SHIFT),
 627                              50))
 628                         drm_err(&dev_priv->drm,
 629                                 "timed out waiting for CDclk change\n");
 630         }
 631
 632         /* adjust self-refresh exit latency value */
 633         val = vlv_bunit_read(dev_priv, BUNIT_REG_BISOC);
 634         val &= ~0x7f;
 635
 636         /*
 637          * For high bandwidth configs, we set a higher latency in the bunit
 638          * so that the core display fetch happens in time to avoid underruns.
 639          */
 640         if (cdclk == 400000)
 641                 val |= 4500 / 250; /* 4.5 usec */
 642         else
 643                 val |= 3000 / 250; /* 3.0 usec */
 644         vlv_bunit_write(dev_priv, BUNIT_REG_BISOC, val);
 645
 646         vlv_iosf_sb_put(dev_priv,
 647                         BIT(VLV_IOSF_SB_CCK) |
 648                         BIT(VLV_IOSF_SB_BUNIT) |
 649                         BIT(VLV_IOSF_SB_PUNIT));
 650
 651         intel_update_cdclk(dev_priv);
 652
 653         vlv_program_pfi_credits(dev_priv);
 654
 655         intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
 656 }
 657
 658 static void chv_set_cdclk(struct drm_i915_private *dev_priv,
 659                           const struct intel_cdclk_config *cdclk_config,
 660                           enum pipe pipe)
 661 {
 662         int cdclk = cdclk_config->cdclk;
 663         u32 val, cmd = cdclk_config->voltage_level;
 664         intel_wakeref_t wakeref;
 665
 666         switch (cdclk) {
 667         case 333333:
 668         case 320000:
 669         case 266667:
 670         case 200000:
 671                 break;
 672         default:
 673                 MISSING_CASE(cdclk);
 674                 return;
 675         }
 676
 677         /* There are cases where we can end up here with power domains
 678          * off and a CDCLK frequency other than the minimum, like when
 679          * issuing a modeset without actually changing any display after
 680          * a system suspend.  So grab the display core domain, which covers
 681          * the HW blocks needed for the following programming.
 682          */
 683         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
 684
 685         vlv_punit_get(dev_priv);
 686         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 687         val &= ~DSPFREQGUAR_MASK_CHV;
 688         val |= (cmd << DSPFREQGUAR_SHIFT_CHV);
 689         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 690         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
 691                       DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV),
 692                      50)) {
 693                 drm_err(&dev_priv->drm,
 694                         "timed out waiting for CDclk change\n");
 695         }
 696
 697         vlv_punit_put(dev_priv);
 698
 699         intel_update_cdclk(dev_priv);
 700
 701         vlv_program_pfi_credits(dev_priv);
 702
 703         intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
 704 }
 705
 706 static int bdw_calc_cdclk(int min_cdclk)
 707 {
 708         if (min_cdclk > 540000)
 709                 return 675000;
 710         else if (min_cdclk > 450000)
 711                 return 540000;
 712         else if (min_cdclk > 337500)
 713                 return 450000;
 714         else
 715                 return 337500;
 716 }
 717
 718 static u8 bdw_calc_voltage_level(int cdclk)
 719 {
 720         switch (cdclk) {
 721         default:
 722         case 337500:
 723                 return 2;
 724         case 450000:
 725                 return 0;
 726         case 540000:
 727                 return 1;
 728         case 675000:
 729                 return 3;
 730         }
 731 }
 732
 733 static void bdw_get_cdclk(struct drm_i915_private *dev_priv,
 734                           struct intel_cdclk_config *cdclk_config)
 735 {
 736         u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
 737         u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
 738
 739         if (lcpll & LCPLL_CD_SOURCE_FCLK)
 740                 cdclk_config->cdclk = 800000;
 741         else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
 742                 cdclk_config->cdclk = 450000;
 743         else if (freq == LCPLL_CLK_FREQ_450)
 744                 cdclk_config->cdclk = 450000;
 745         else if (freq == LCPLL_CLK_FREQ_54O_BDW)
 746                 cdclk_config->cdclk = 540000;
 747         else if (freq == LCPLL_CLK_FREQ_337_5_BDW)
 748                 cdclk_config->cdclk = 337500;
 749         else
 750                 cdclk_config->cdclk = 675000;
 751
 752         /*
 753          * Can't read this out :( Let's assume it's
 754          * at least what the CDCLK frequency requires.
 755          */
 756         cdclk_config->voltage_level =
 757                 bdw_calc_voltage_level(cdclk_config->cdclk);
 758 }
 759
 760 static u32 bdw_cdclk_freq_sel(int cdclk)
 761 {
 762         switch (cdclk) {
 763         default:
 764                 MISSING_CASE(cdclk);
 765                 fallthrough;
 766         case 337500:
 767                 return LCPLL_CLK_FREQ_337_5_BDW;
 768         case 450000:
 769                 return LCPLL_CLK_FREQ_450;
 770         case 540000:
 771                 return LCPLL_CLK_FREQ_54O_BDW;
 772         case 675000:
 773                 return LCPLL_CLK_FREQ_675_BDW;
 774         }
 775 }
 776
 777 static void bdw_set_cdclk(struct drm_i915_private *dev_priv,
 778                           const struct intel_cdclk_config *cdclk_config,
 779                           enum pipe pipe)
 780 {
 781         int cdclk = cdclk_config->cdclk;
 782         int ret;
 783
 784         if (drm_WARN(&dev_priv->drm,
 785                      (intel_de_read(dev_priv, LCPLL_CTL) &
 786                       (LCPLL_PLL_DISABLE | LCPLL_PLL_LOCK |
 787                        LCPLL_CD_CLOCK_DISABLE | LCPLL_ROOT_CD_CLOCK_DISABLE |
 788                        LCPLL_CD2X_CLOCK_DISABLE | LCPLL_POWER_DOWN_ALLOW |
 789                        LCPLL_CD_SOURCE_FCLK)) != LCPLL_PLL_LOCK,
 790                      "trying to change cdclk frequency with cdclk not enabled\n"))
 791                 return;
 792
 793         ret = sandybridge_pcode_write(dev_priv,
 794                                       BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0);
 795         if (ret) {
 796                 drm_err(&dev_priv->drm,
 797                         "failed to inform pcode about cdclk change\n");
 798                 return;
 799         }
 800
 801         intel_de_rmw(dev_priv, LCPLL_CTL,
 802                      0, LCPLL_CD_SOURCE_FCLK);
 803
 804         /*
 805          * According to the spec, it should be enough to poll for this 1 us.
 806          * However, extensive testing shows that this can take longer.
 807          */
 808         if (wait_for_us(intel_de_read(dev_priv, LCPLL_CTL) &
 809                         LCPLL_CD_SOURCE_FCLK_DONE, 100))
 810                 drm_err(&dev_priv->drm, "Switching to FCLK failed\n");
 811
 812         intel_de_rmw(dev_priv, LCPLL_CTL,
 813                      LCPLL_CLK_FREQ_MASK, bdw_cdclk_freq_sel(cdclk));
 814
 815         intel_de_rmw(dev_priv, LCPLL_CTL,
 816                      LCPLL_CD_SOURCE_FCLK, 0);
 817
 818         if (wait_for_us((intel_de_read(dev_priv, LCPLL_CTL) &
 819                          LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
 820                 drm_err(&dev_priv->drm, "Switching back to LCPLL failed\n");
 821
 822         sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ,
 823                                 cdclk_config->voltage_level);
 824
 825         intel_de_write(dev_priv, CDCLK_FREQ,
 826                        DIV_ROUND_CLOSEST(cdclk, 1000) - 1);
 827
 828         intel_update_cdclk(dev_priv);
 829 }
 830
 831 static int skl_calc_cdclk(int min_cdclk, int vco)
 832 {
 833         if (vco == 8640000) {
 834                 if (min_cdclk > 540000)
 835                         return 617143;
 836                 else if (min_cdclk > 432000)
 837                         return 540000;
 838                 else if (min_cdclk > 308571)
 839                         return 432000;
 840                 else
 841                         return 308571;
 842         } else {
 843                 if (min_cdclk > 540000)
 844                         return 675000;
 845                 else if (min_cdclk > 450000)
 846                         return 540000;
 847                 else if (min_cdclk > 337500)
 848                         return 450000;
 849                 else
 850                         return 337500;
 851         }
 852 }
 853
 854 static u8 skl_calc_voltage_level(int cdclk)
 855 {
 856         if (cdclk > 540000)
 857                 return 3;
 858         else if (cdclk > 450000)
 859                 return 2;
 860         else if (cdclk > 337500)
 861                 return 1;
 862         else
 863                 return 0;
 864 }
 865
 866 static void skl_dpll0_update(struct drm_i915_private *dev_priv,
 867                              struct intel_cdclk_config *cdclk_config)
 868 {
 869         u32 val;
 870
 871         cdclk_config->ref = 24000;
 872         cdclk_config->vco = 0;
 873
 874         val = intel_de_read(dev_priv, LCPLL1_CTL);
 875         if ((val & LCPLL_PLL_ENABLE) == 0)
 876                 return;
 877
 878         if (drm_WARN_ON(&dev_priv->drm, (val & LCPLL_PLL_LOCK) == 0))
 879                 return;
 880
 881         val = intel_de_read(dev_priv, DPLL_CTRL1);
 882
 883         if (drm_WARN_ON(&dev_priv->drm,
 884                         (val & (DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
 885                                 DPLL_CTRL1_SSC(SKL_DPLL0) |
 886                                 DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) !=
 887                         DPLL_CTRL1_OVERRIDE(SKL_DPLL0)))
 888                 return;
 889
 890         switch (val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) {
 891         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0):
 892         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1350, SKL_DPLL0):
 893         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1620, SKL_DPLL0):
 894         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2700, SKL_DPLL0):
 895                 cdclk_config->vco = 8100000;
 896                 break;
 897         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0):
 898         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2160, SKL_DPLL0):
 899                 cdclk_config->vco = 8640000;
 900                 break;
 901         default:
 902                 MISSING_CASE(val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0));
 903                 break;
 904         }
 905 }
 906
 907 static void skl_get_cdclk(struct drm_i915_private *dev_priv,
 908                           struct intel_cdclk_config *cdclk_config)
 909 {
 910         u32 cdctl;
 911
 912         skl_dpll0_update(dev_priv, cdclk_config);
 913
 914         cdclk_config->cdclk = cdclk_config->bypass = cdclk_config->ref;
 915
 916         if (cdclk_config->vco == 0)
 917                 goto out;
 918
 919         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
 920
 921         if (cdclk_config->vco == 8640000) {
 922                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
 923                 case CDCLK_FREQ_450_432:
 924                         cdclk_config->cdclk = 432000;
 925                         break;
 926                 case CDCLK_FREQ_337_308:
 927                         cdclk_config->cdclk = 308571;
 928                         break;
 929                 case CDCLK_FREQ_540:
 930                         cdclk_config->cdclk = 540000;
 931                         break;
 932                 case CDCLK_FREQ_675_617:
 933                         cdclk_config->cdclk = 617143;
 934                         break;
 935                 default:
 936                         MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
 937                         break;
 938                 }
 939         } else {
 940                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
 941                 case CDCLK_FREQ_450_432:
 942                         cdclk_config->cdclk = 450000;
 943                         break;
 944                 case CDCLK_FREQ_337_308:
 945                         cdclk_config->cdclk = 337500;
 946                         break;
 947                 case CDCLK_FREQ_540:
 948                         cdclk_config->cdclk = 540000;
 949                         break;
 950                 case CDCLK_FREQ_675_617:
 951                         cdclk_config->cdclk = 675000;
 952                         break;
 953                 default:
 954                         MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
 955                         break;
 956                 }
 957         }
 958
 959  out:
 960         /*
 961          * Can't read this out :( Let's assume it's
 962          * at least what the CDCLK frequency requires.
 963          */
 964         cdclk_config->voltage_level =
 965                 skl_calc_voltage_level(cdclk_config->cdclk);
 966 }
 967
 968 /* convert from kHz to .1 fixpoint MHz with -1MHz offset */
 969 static int skl_cdclk_decimal(int cdclk)
 970 {
 971         return DIV_ROUND_CLOSEST(cdclk - 1000, 500);
 972 }
 973
 974 static void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv,
 975                                         int vco)
 976 {
 977         bool changed = dev_priv->skl_preferred_vco_freq != vco;
 978
 979         dev_priv->skl_preferred_vco_freq = vco;
 980
 981         if (changed)
 982                 intel_update_max_cdclk(dev_priv);
 983 }
 984
 985 static u32 skl_dpll0_link_rate(struct drm_i915_private *dev_priv, int vco)
 986 {
 987         drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
 988
 989         /*
 990          * We always enable DPLL0 with the lowest link rate possible, but still
 991          * taking into account the VCO required to operate the eDP panel at the
 992          * desired frequency. The usual DP link rates operate with a VCO of
 993          * 8100 while the eDP 1.4 alternate link rates need a VCO of 8640.
 994          * The modeset code is responsible for the selection of the exact link
 995          * rate later on, with the constraint of choosing a frequency that
 996          * works with vco.
 997          */
 998         if (vco == 8640000)
 999                 return DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0);
1000         else
1001                 return DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0);
1002 }
1003
1004 static void skl_dpll0_enable(struct drm_i915_private *dev_priv, int vco)
1005 {
1006         intel_de_rmw(dev_priv, DPLL_CTRL1,
1007                      DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
1008                      DPLL_CTRL1_SSC(SKL_DPLL0) |
1009                      DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0),
1010                      DPLL_CTRL1_OVERRIDE(SKL_DPLL0) |
1011                      skl_dpll0_link_rate(dev_priv, vco));
1012         intel_de_posting_read(dev_priv, DPLL_CTRL1);
1013
1014         intel_de_rmw(dev_priv, LCPLL1_CTL,
1015                      0, LCPLL_PLL_ENABLE);
1016
1017         if (intel_de_wait_for_set(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 5))
1018                 drm_err(&dev_priv->drm, "DPLL0 not locked\n");
1019
1020         dev_priv->cdclk.hw.vco = vco;
1021
1022         /* We'll want to keep using the current vco from now on. */
1023         skl_set_preferred_cdclk_vco(dev_priv, vco);
1024 }
1025
1026 static void skl_dpll0_disable(struct drm_i915_private *dev_priv)
1027 {
1028         intel_de_rmw(dev_priv, LCPLL1_CTL,
1029                      LCPLL_PLL_ENABLE, 0);
1030
1031         if (intel_de_wait_for_clear(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 1))
1032                 drm_err(&dev_priv->drm, "Couldn't disable DPLL0\n");
1033
1034         dev_priv->cdclk.hw.vco = 0;
1035 }
1036
1037 static u32 skl_cdclk_freq_sel(struct drm_i915_private *dev_priv,
1038                               int cdclk, int vco)
1039 {
1040         switch (cdclk) {
1041         default:
1042                 drm_WARN_ON(&dev_priv->drm,
1043                             cdclk != dev_priv->cdclk.hw.bypass);
1044                 drm_WARN_ON(&dev_priv->drm, vco != 0);
1045                 fallthrough;
1046         case 308571:
1047         case 337500:
1048                 return CDCLK_FREQ_337_308;
1049         case 450000:
1050         case 432000:
1051                 return CDCLK_FREQ_450_432;
1052         case 540000:
1053                 return CDCLK_FREQ_540;
1054         case 617143:
1055         case 675000:
1056                 return CDCLK_FREQ_675_617;
1057         }
1058 }
1059
1060 static void skl_set_cdclk(struct drm_i915_private *dev_priv,
1061                           const struct intel_cdclk_config *cdclk_config,
1062                           enum pipe pipe)
1063 {
1064         int cdclk = cdclk_config->cdclk;
1065         int vco = cdclk_config->vco;
1066         u32 freq_select, cdclk_ctl;
1067         int ret;
1068
1069         /*
1070          * Based on WA#1183 CDCLK rates 308 and 617MHz CDCLK rates are
1071          * unsupported on SKL. In theory this should never happen since only
1072          * the eDP1.4 2.16 and 4.32Gbps rates require it, but eDP1.4 is not
1073          * supported on SKL either, see the above WA. WARN whenever trying to
1074          * use the corresponding VCO freq as that always leads to using the
1075          * minimum 308MHz CDCLK.
1076          */
1077         drm_WARN_ON_ONCE(&dev_priv->drm,
1078                          IS_SKYLAKE(dev_priv) && vco == 8640000);
1079
1080         ret = skl_pcode_request(dev_priv, SKL_PCODE_CDCLK_CONTROL,
1081                                 SKL_CDCLK_PREPARE_FOR_CHANGE,
1082                                 SKL_CDCLK_READY_FOR_CHANGE,
1083                                 SKL_CDCLK_READY_FOR_CHANGE, 3);
1084         if (ret) {
1085                 drm_err(&dev_priv->drm,
1086                         "Failed to inform PCU about cdclk change (%d)\n", ret);
1087                 return;
1088         }
1089
1090         freq_select = skl_cdclk_freq_sel(dev_priv, cdclk, vco);
1091
1092         if (dev_priv->cdclk.hw.vco != 0 &&
1093             dev_priv->cdclk.hw.vco != vco)
1094                 skl_dpll0_disable(dev_priv);
1095
1096         cdclk_ctl = intel_de_read(dev_priv, CDCLK_CTL);
1097
1098         if (dev_priv->cdclk.hw.vco != vco) {
1099                 /* Wa Display #1183: skl,kbl,cfl */
1100                 cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1101                 cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1102                 intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1103         }
1104
1105         /* Wa Display #1183: skl,kbl,cfl */
1106         cdclk_ctl |= CDCLK_DIVMUX_CD_OVERRIDE;
1107         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1108         intel_de_posting_read(dev_priv, CDCLK_CTL);
1109
1110         if (dev_priv->cdclk.hw.vco != vco)
1111                 skl_dpll0_enable(dev_priv, vco);
1112
1113         /* Wa Display #1183: skl,kbl,cfl */
1114         cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1115         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1116
1117         cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1118         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1119
1120         /* Wa Display #1183: skl,kbl,cfl */
1121         cdclk_ctl &= ~CDCLK_DIVMUX_CD_OVERRIDE;
1122         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1123         intel_de_posting_read(dev_priv, CDCLK_CTL);
1124
1125         /* inform PCU of the change */
1126         sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL,
1127                                 cdclk_config->voltage_level);
1128
1129         intel_update_cdclk(dev_priv);
1130 }
1131
1132 static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
1133 {
1134         u32 cdctl, expected;
1135
1136         /*
1137          * check if the pre-os initialized the display
1138          * There is SWF18 scratchpad register defined which is set by the
1139          * pre-os which can be used by the OS drivers to check the status
1140          */
1141         if ((intel_de_read(dev_priv, SWF_ILK(0x18)) & 0x00FFFFFF) == 0)
1142                 goto sanitize;
1143
1144         intel_update_cdclk(dev_priv);
1145         intel_dump_cdclk_config(&dev_priv->cdclk.hw, "Current CDCLK");
1146
1147         /* Is PLL enabled and locked ? */
1148         if (dev_priv->cdclk.hw.vco == 0 ||
1149             dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.bypass)
1150                 goto sanitize;
1151
1152         /* DPLL okay; verify the cdclock
1153          *
1154          * Noticed in some instances that the freq selection is correct but
1155          * decimal part is programmed wrong from BIOS where pre-os does not
1156          * enable display. Verify the same as well.
1157          */
1158         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
1159         expected = (cdctl & CDCLK_FREQ_SEL_MASK) |
1160                 skl_cdclk_decimal(dev_priv->cdclk.hw.cdclk);
1161         if (cdctl == expected)
1162                 /* All well; nothing to sanitize */
1163                 return;
1164
1165 sanitize:
1166         drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
1167
1168         /* force cdclk programming */
1169         dev_priv->cdclk.hw.cdclk = 0;
1170         /* force full PLL disable + enable */
1171         dev_priv->cdclk.hw.vco = -1;
1172 }
1173
1174 static void skl_cdclk_init_hw(struct drm_i915_private *dev_priv)
1175 {
1176         struct intel_cdclk_config cdclk_config;
1177
1178         skl_sanitize_cdclk(dev_priv);
1179
1180         if (dev_priv->cdclk.hw.cdclk != 0 &&
1181             dev_priv->cdclk.hw.vco != 0) {
1182                 /*
1183                  * Use the current vco as our initial
1184                  * guess as to what the preferred vco is.
1185                  */
1186                 if (dev_priv->skl_preferred_vco_freq == 0)
1187                         skl_set_preferred_cdclk_vco(dev_priv,
1188                                                     dev_priv->cdclk.hw.vco);
1189                 return;
1190         }
1191
1192         cdclk_config = dev_priv->cdclk.hw;
1193
1194         cdclk_config.vco = dev_priv->skl_preferred_vco_freq;
1195         if (cdclk_config.vco == 0)
1196                 cdclk_config.vco = 8100000;
1197         cdclk_config.cdclk = skl_calc_cdclk(0, cdclk_config.vco);
1198         cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1199
1200         skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1201 }
1202
1203 static void skl_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
1204 {
1205         struct intel_cdclk_config cdclk_config = dev_priv->cdclk.hw;
1206
1207         cdclk_config.cdclk = cdclk_config.bypass;
1208         cdclk_config.vco = 0;
1209         cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1210
1211         skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1212 }
1213
1214 static const struct intel_cdclk_vals bxt_cdclk_table[] = {
1215         { .refclk = 19200, .cdclk = 144000, .divider = 8, .ratio = 60 },
1216         { .refclk = 19200, .cdclk = 288000, .divider = 4, .ratio = 60 },
1217         { .refclk = 19200, .cdclk = 384000, .divider = 3, .ratio = 60 },
1218         { .refclk = 19200, .cdclk = 576000, .divider = 2, .ratio = 60 },
1219         { .refclk = 19200, .cdclk = 624000, .divider = 2, .ratio = 65 },
1220         {}
1221 };
1222
1223 static const struct intel_cdclk_vals glk_cdclk_table[] = {
1224         { .refclk = 19200, .cdclk =  79200, .divider = 8, .ratio = 33 },
1225         { .refclk = 19200, .cdclk = 158400, .divider = 4, .ratio = 33 },
1226         { .refclk = 19200, .cdclk = 316800, .divider = 2, .ratio = 33 },
1227         {}
1228 };
1229
1230 static const struct intel_cdclk_vals icl_cdclk_table[] = {
1231         { .refclk = 19200, .cdclk = 172800, .divider = 2, .ratio = 18 },
1232         { .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 },
1233         { .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1234         { .refclk = 19200, .cdclk = 326400, .divider = 4, .ratio = 68 },
1235         { .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1236         { .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1237
1238         { .refclk = 24000, .cdclk = 180000, .divider = 2, .ratio = 15 },
1239         { .refclk = 24000, .cdclk = 192000, .divider = 2, .ratio = 16 },
1240         { .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1241         { .refclk = 24000, .cdclk = 324000, .divider = 4, .ratio = 54 },
1242         { .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1243         { .refclk = 24000, .cdclk = 648000, .divider = 2, .ratio = 54 },
1244
1245         { .refclk = 38400, .cdclk = 172800, .divider = 2, .ratio =  9 },
1246         { .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
1247         { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1248         { .refclk = 38400, .cdclk = 326400, .divider = 4, .ratio = 34 },
1249         { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1250         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1251         {}
1252 };
1253
1254 static const struct intel_cdclk_vals rkl_cdclk_table[] = {
1255         { .refclk = 19200, .cdclk = 172800, .divider = 4, .ratio =  36 },
1256         { .refclk = 19200, .cdclk = 192000, .divider = 4, .ratio =  40 },
1257         { .refclk = 19200, .cdclk = 307200, .divider = 4, .ratio =  64 },
1258         { .refclk = 19200, .cdclk = 326400, .divider = 8, .ratio = 136 },
1259         { .refclk = 19200, .cdclk = 556800, .divider = 4, .ratio = 116 },
1260         { .refclk = 19200, .cdclk = 652800, .divider = 4, .ratio = 136 },
1261
1262         { .refclk = 24000, .cdclk = 180000, .divider = 4, .ratio =  30 },
1263         { .refclk = 24000, .cdclk = 192000, .divider = 4, .ratio =  32 },
1264         { .refclk = 24000, .cdclk = 312000, .divider = 4, .ratio =  52 },
1265         { .refclk = 24000, .cdclk = 324000, .divider = 8, .ratio = 108 },
1266         { .refclk = 24000, .cdclk = 552000, .divider = 4, .ratio =  92 },
1267         { .refclk = 24000, .cdclk = 648000, .divider = 4, .ratio = 108 },
1268
1269         { .refclk = 38400, .cdclk = 172800, .divider = 4, .ratio = 18 },
1270         { .refclk = 38400, .cdclk = 192000, .divider = 4, .ratio = 20 },
1271         { .refclk = 38400, .cdclk = 307200, .divider = 4, .ratio = 32 },
1272         { .refclk = 38400, .cdclk = 326400, .divider = 8, .ratio = 68 },
1273         { .refclk = 38400, .cdclk = 556800, .divider = 4, .ratio = 58 },
1274         { .refclk = 38400, .cdclk = 652800, .divider = 4, .ratio = 68 },
1275         {}
1276 };
1277
1278 static const struct intel_cdclk_vals adlp_a_step_cdclk_table[] = {
1279         { .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1280         { .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1281         { .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1282
1283         { .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1284         { .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1285         { .refclk = 24400, .cdclk = 648000, .divider = 2, .ratio = 54 },
1286
1287         { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1288         { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1289         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1290         {}
1291 };
1292
1293 static const struct intel_cdclk_vals adlp_cdclk_table[] = {
1294         { .refclk = 19200, .cdclk = 172800, .divider = 3, .ratio = 27 },
1295         { .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 },
1296         { .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1297         { .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1298         { .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1299
1300         { .refclk = 24000, .cdclk = 176000, .divider = 3, .ratio = 22 },
1301         { .refclk = 24000, .cdclk = 192000, .divider = 2, .ratio = 16 },
1302         { .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1303         { .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1304         { .refclk = 24400, .cdclk = 648000, .divider = 2, .ratio = 54 },
1305
1306         { .refclk = 38400, .cdclk = 179200, .divider = 3, .ratio = 14 },
1307         { .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
1308         { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1309         { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1310         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1311         {}
1312 };
1313
1314 static const struct intel_cdclk_vals dg2_cdclk_table[] = {
1315         { .refclk = 38400, .cdclk = 172800, .divider = 2, .ratio =  9 },
1316         { .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
1317         { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1318         { .refclk = 38400, .cdclk = 326400, .divider = 4, .ratio = 34 },
1319         { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1320         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1321         {}
1322 };
1323
1324 static int bxt_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
1325 {
1326         const struct intel_cdclk_vals *table = dev_priv->cdclk.table;
1327         int i;
1328
1329         for (i = 0; table[i].refclk; i++)
1330                 if (table[i].refclk == dev_priv->cdclk.hw.ref &&
1331                     table[i].cdclk >= min_cdclk)
1332                         return table[i].cdclk;
1333
1334         drm_WARN(&dev_priv->drm, 1,
1335                  "Cannot satisfy minimum cdclk %d with refclk %u\n",
1336                  min_cdclk, dev_priv->cdclk.hw.ref);
1337         return 0;
1338 }
1339
1340 static int bxt_calc_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
1341 {
1342         const struct intel_cdclk_vals *table = dev_priv->cdclk.table;
1343         int i;
1344
1345         if (cdclk == dev_priv->cdclk.hw.bypass)
1346                 return 0;
1347
1348         for (i = 0; table[i].refclk; i++)
1349                 if (table[i].refclk == dev_priv->cdclk.hw.ref &&
1350                     table[i].cdclk == cdclk)
1351                         return dev_priv->cdclk.hw.ref * table[i].ratio;
1352
1353         drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n",
1354                  cdclk, dev_priv->cdclk.hw.ref);
1355         return 0;
1356 }
1357
1358 static u8 bxt_calc_voltage_level(int cdclk)
1359 {
1360         return DIV_ROUND_UP(cdclk, 25000);
1361 }
1362
1363 static u8 icl_calc_voltage_level(int cdclk)
1364 {
1365         if (cdclk > 556800)
1366                 return 2;
1367         else if (cdclk > 312000)
1368                 return 1;
1369         else
1370                 return 0;
1371 }
1372
1373 static u8 ehl_calc_voltage_level(int cdclk)
1374 {
1375         if (cdclk > 326400)
1376                 return 3;
1377         else if (cdclk > 312000)
1378                 return 2;
1379         else if (cdclk > 180000)
1380                 return 1;
1381         else
1382                 return 0;
1383 }
1384
1385 static u8 tgl_calc_voltage_level(int cdclk)
1386 {
1387         if (cdclk > 556800)
1388                 return 3;
1389         else if (cdclk > 326400)
1390                 return 2;
1391         else if (cdclk > 312000)
1392                 return 1;
1393         else
1394                 return 0;
1395 }
1396
1397 static void icl_readout_refclk(struct drm_i915_private *dev_priv,
1398                                struct intel_cdclk_config *cdclk_config)
1399 {
1400         u32 dssm = intel_de_read(dev_priv, SKL_DSSM) & ICL_DSSM_CDCLK_PLL_REFCLK_MASK;
1401
1402         switch (dssm) {
1403         default:
1404                 MISSING_CASE(dssm);
1405                 fallthrough;
1406         case ICL_DSSM_CDCLK_PLL_REFCLK_24MHz:
1407                 cdclk_config->ref = 24000;
1408                 break;
1409         case ICL_DSSM_CDCLK_PLL_REFCLK_19_2MHz:
1410                 cdclk_config->ref = 19200;
1411                 break;
1412         case ICL_DSSM_CDCLK_PLL_REFCLK_38_4MHz:
1413                 cdclk_config->ref = 38400;
1414                 break;
1415         }
1416 }
1417
1418 static void bxt_de_pll_readout(struct drm_i915_private *dev_priv,
1419                                struct intel_cdclk_config *cdclk_config)
1420 {
1421         u32 val, ratio;
1422
1423         if (IS_DG2(dev_priv))
1424                 cdclk_config->ref = 38400;
1425         else if (DISPLAY_VER(dev_priv) >= 11)
1426                 icl_readout_refclk(dev_priv, cdclk_config);
1427         else
1428                 cdclk_config->ref = 19200;
1429
1430         val = intel_de_read(dev_priv, BXT_DE_PLL_ENABLE);
1431         if ((val & BXT_DE_PLL_PLL_ENABLE) == 0 ||
1432             (val & BXT_DE_PLL_LOCK) == 0) {
1433                 /*
1434                  * CDCLK PLL is disabled, the VCO/ratio doesn't matter, but
1435                  * setting it to zero is a way to signal that.
1436                  */
1437                 cdclk_config->vco = 0;
1438                 return;
1439         }
1440
1441         /*
1442          * DISPLAY_VER >= 11 have the ratio directly in the PLL enable register,
1443          * gen9lp had it in a separate PLL control register.
1444          */
1445         if (DISPLAY_VER(dev_priv) >= 11)
1446                 ratio = val & ICL_CDCLK_PLL_RATIO_MASK;
1447         else
1448                 ratio = intel_de_read(dev_priv, BXT_DE_PLL_CTL) & BXT_DE_PLL_RATIO_MASK;
1449
1450         cdclk_config->vco = ratio * cdclk_config->ref;
1451 }
1452
1453 static void bxt_get_cdclk(struct drm_i915_private *dev_priv,
1454                           struct intel_cdclk_config *cdclk_config)
1455 {
1456         u32 divider;
1457         int div;
1458
1459         bxt_de_pll_readout(dev_priv, cdclk_config);
1460
1461         if (DISPLAY_VER(dev_priv) >= 12)
1462                 cdclk_config->bypass = cdclk_config->ref / 2;
1463         else if (DISPLAY_VER(dev_priv) >= 11)
1464                 cdclk_config->bypass = 50000;
1465         else
1466                 cdclk_config->bypass = cdclk_config->ref;
1467
1468         if (cdclk_config->vco == 0) {
1469                 cdclk_config->cdclk = cdclk_config->bypass;
1470                 goto out;
1471         }
1472
1473         divider = intel_de_read(dev_priv, CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK;
1474
1475         switch (divider) {
1476         case BXT_CDCLK_CD2X_DIV_SEL_1:
1477                 div = 2;
1478                 break;
1479         case BXT_CDCLK_CD2X_DIV_SEL_1_5:
1480                 div = 3;
1481                 break;
1482         case BXT_CDCLK_CD2X_DIV_SEL_2:
1483                 div = 4;
1484                 break;
1485         case BXT_CDCLK_CD2X_DIV_SEL_4:
1486                 div = 8;
1487                 break;
1488         default:
1489                 MISSING_CASE(divider);
1490                 return;
1491         }
1492
1493         cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco, div);
1494
1495  out:
1496         /*
1497          * Can't read this out :( Let's assume it's
1498          * at least what the CDCLK frequency requires.
1499          */
1500         cdclk_config->voltage_level =
1501                 intel_cdclk_calc_voltage_level(dev_priv, cdclk_config->cdclk);
1502 }
1503
1504 static void bxt_de_pll_disable(struct drm_i915_private *dev_priv)
1505 {
1506         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, 0);
1507
1508         /* Timeout 200us */
1509         if (intel_de_wait_for_clear(dev_priv,
1510                                     BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1511                 drm_err(&dev_priv->drm, "timeout waiting for DE PLL unlock\n");
1512
1513         dev_priv->cdclk.hw.vco = 0;
1514 }
1515
1516 static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco)
1517 {
1518         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref);
1519
1520         intel_de_rmw(dev_priv, BXT_DE_PLL_CTL,
1521                      BXT_DE_PLL_RATIO_MASK, BXT_DE_PLL_RATIO(ratio));
1522
1523         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE);
1524
1525         /* Timeout 200us */
1526         if (intel_de_wait_for_set(dev_priv,
1527                                   BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1528                 drm_err(&dev_priv->drm, "timeout waiting for DE PLL lock\n");
1529
1530         dev_priv->cdclk.hw.vco = vco;
1531 }
1532
1533 static void icl_cdclk_pll_disable(struct drm_i915_private *dev_priv)
1534 {
1535         intel_de_rmw(dev_priv, BXT_DE_PLL_ENABLE,
1536                      BXT_DE_PLL_PLL_ENABLE, 0);
1537
1538         /* Timeout 200us */
1539         if (intel_de_wait_for_clear(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1540                 drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL unlock\n");
1541
1542         dev_priv->cdclk.hw.vco = 0;
1543 }
1544
1545 static void icl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco)
1546 {
1547         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref);
1548         u32 val;
1549
1550         val = ICL_CDCLK_PLL_RATIO(ratio);
1551         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1552
1553         val |= BXT_DE_PLL_PLL_ENABLE;
1554         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1555
1556         /* Timeout 200us */
1557         if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1558                 drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL lock\n");
1559
1560         dev_priv->cdclk.hw.vco = vco;
1561 }
1562
1563 static void adlp_cdclk_pll_crawl(struct drm_i915_private *dev_priv, int vco)
1564 {
1565         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref);
1566         u32 val;
1567
1568         /* Write PLL ratio without disabling */
1569         val = ICL_CDCLK_PLL_RATIO(ratio) | BXT_DE_PLL_PLL_ENABLE;
1570         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1571
1572         /* Submit freq change request */
1573         val |= BXT_DE_PLL_FREQ_REQ;
1574         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1575
1576         /* Timeout 200us */
1577         if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE,
1578                                   BXT_DE_PLL_LOCK | BXT_DE_PLL_FREQ_REQ_ACK, 1))
1579                 DRM_ERROR("timeout waiting for FREQ change request ack\n");
1580
1581         val &= ~BXT_DE_PLL_FREQ_REQ;
1582         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1583
1584         dev_priv->cdclk.hw.vco = vco;
1585 }
1586
1587 static u32 bxt_cdclk_cd2x_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
1588 {
1589         if (DISPLAY_VER(dev_priv) >= 12) {
1590                 if (pipe == INVALID_PIPE)
1591                         return TGL_CDCLK_CD2X_PIPE_NONE;
1592                 else
1593                         return TGL_CDCLK_CD2X_PIPE(pipe);
1594         } else if (DISPLAY_VER(dev_priv) >= 11) {
1595                 if (pipe == INVALID_PIPE)
1596                         return ICL_CDCLK_CD2X_PIPE_NONE;
1597                 else
1598                         return ICL_CDCLK_CD2X_PIPE(pipe);
1599         } else {
1600                 if (pipe == INVALID_PIPE)
1601                         return BXT_CDCLK_CD2X_PIPE_NONE;
1602                 else
1603                         return BXT_CDCLK_CD2X_PIPE(pipe);
1604         }
1605 }
1606
1607 static u32 bxt_cdclk_cd2x_div_sel(struct drm_i915_private *dev_priv,
1608                                   int cdclk, int vco)
1609 {
1610         /* cdclk = vco / 2 / div{1,1.5,2,4} */
1611         switch (DIV_ROUND_CLOSEST(vco, cdclk)) {
1612         default:
1613                 drm_WARN_ON(&dev_priv->drm,
1614                             cdclk != dev_priv->cdclk.hw.bypass);
1615                 drm_WARN_ON(&dev_priv->drm, vco != 0);
1616                 fallthrough;
1617         case 2:
1618                 return BXT_CDCLK_CD2X_DIV_SEL_1;
1619         case 3:
1620                 return BXT_CDCLK_CD2X_DIV_SEL_1_5;
1621         case 4:
1622                 return BXT_CDCLK_CD2X_DIV_SEL_2;
1623         case 8:
1624                 return BXT_CDCLK_CD2X_DIV_SEL_4;
1625         }
1626 }
1627
1628 static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
1629                           const struct intel_cdclk_config *cdclk_config,
1630                           enum pipe pipe)
1631 {
1632         int cdclk = cdclk_config->cdclk;
1633         int vco = cdclk_config->vco;
1634         u32 val;
1635         int ret;
1636
1637         /* Inform power controller of upcoming frequency change. */
1638         if (DISPLAY_VER(dev_priv) >= 11)
1639                 ret = skl_pcode_request(dev_priv, SKL_PCODE_CDCLK_CONTROL,
1640                                         SKL_CDCLK_PREPARE_FOR_CHANGE,
1641                                         SKL_CDCLK_READY_FOR_CHANGE,
1642                                         SKL_CDCLK_READY_FOR_CHANGE, 3);
1643         else
1644                 /*
1645                  * BSpec requires us to wait up to 150usec, but that leads to
1646                  * timeouts; the 2ms used here is based on experiment.
1647                  */
1648                 ret = sandybridge_pcode_write_timeout(dev_priv,
1649                                                       HSW_PCODE_DE_WRITE_FREQ_REQ,
1650                                                       0x80000000, 150, 2);
1651
1652         if (ret) {
1653                 drm_err(&dev_priv->drm,
1654                         "Failed to inform PCU about cdclk change (err %d, freq %d)\n",
1655                         ret, cdclk);
1656                 return;
1657         }
1658
1659         if (HAS_CDCLK_CRAWL(dev_priv) && dev_priv->cdclk.hw.vco > 0 && vco > 0) {
1660                 if (dev_priv->cdclk.hw.vco != vco)
1661                         adlp_cdclk_pll_crawl(dev_priv, vco);
1662         } else if (DISPLAY_VER(dev_priv) >= 11) {
1663                 if (dev_priv->cdclk.hw.vco != 0 &&
1664                     dev_priv->cdclk.hw.vco != vco)
1665                         icl_cdclk_pll_disable(dev_priv);
1666
1667                 if (dev_priv->cdclk.hw.vco != vco)
1668                         icl_cdclk_pll_enable(dev_priv, vco);
1669         } else {
1670                 if (dev_priv->cdclk.hw.vco != 0 &&
1671                     dev_priv->cdclk.hw.vco != vco)
1672                         bxt_de_pll_disable(dev_priv);
1673
1674                 if (dev_priv->cdclk.hw.vco != vco)
1675                         bxt_de_pll_enable(dev_priv, vco);
1676         }
1677
1678         val = bxt_cdclk_cd2x_div_sel(dev_priv, cdclk, vco) |
1679                 bxt_cdclk_cd2x_pipe(dev_priv, pipe) |
1680                 skl_cdclk_decimal(cdclk);
1681
1682         /*
1683          * Disable SSA Precharge when CD clock frequency < 500 MHz,
1684          * enable otherwise.
1685          */
1686         if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
1687             cdclk >= 500000)
1688                 val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
1689         intel_de_write(dev_priv, CDCLK_CTL, val);
1690
1691         if (pipe != INVALID_PIPE)
1692                 intel_wait_for_vblank(dev_priv, pipe);
1693
1694         if (DISPLAY_VER(dev_priv) >= 11) {
1695                 ret = sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL,
1696                                               cdclk_config->voltage_level);
1697         } else {
1698                 /*
1699                  * The timeout isn't specified, the 2ms used here is based on
1700                  * experiment.
1701                  * FIXME: Waiting for the request completion could be delayed
1702                  * until the next PCODE request based on BSpec.
1703                  */
1704                 ret = sandybridge_pcode_write_timeout(dev_priv,
1705                                                       HSW_PCODE_DE_WRITE_FREQ_REQ,
1706                                                       cdclk_config->voltage_level,
1707                                                       150, 2);
1708         }
1709
1710         if (ret) {
1711                 drm_err(&dev_priv->drm,
1712                         "PCode CDCLK freq set failed, (err %d, freq %d)\n",
1713                         ret, cdclk);
1714                 return;
1715         }
1716
1717         intel_update_cdclk(dev_priv);
1718
1719         if (DISPLAY_VER(dev_priv) >= 11)
1720                 /*
1721                  * Can't read out the voltage level :(
1722                  * Let's just assume everything is as expected.
1723                  */
1724                 dev_priv->cdclk.hw.voltage_level = cdclk_config->voltage_level;
1725 }
1726
1727 static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv)
1728 {
1729         u32 cdctl, expected;
1730         int cdclk, vco;
1731
1732         intel_update_cdclk(dev_priv);
1733         intel_dump_cdclk_config(&dev_priv->cdclk.hw, "Current CDCLK");
1734
1735         if (dev_priv->cdclk.hw.vco == 0 ||
1736             dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.bypass)
1737                 goto sanitize;
1738
1739         /* DPLL okay; verify the cdclock
1740          *
1741          * Some BIOS versions leave an incorrect decimal frequency value and
1742          * set reserved MBZ bits in CDCLK_CTL at least during exiting from S4,
1743          * so sanitize this register.
1744          */
1745         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
1746         /*
1747          * Let's ignore the pipe field, since BIOS could have configured the
1748          * dividers both synching to an active pipe, or asynchronously
1749          * (PIPE_NONE).
1750          */
1751         cdctl &= ~bxt_cdclk_cd2x_pipe(dev_priv, INVALID_PIPE);
1752
1753         /* Make sure this is a legal cdclk value for the platform */
1754         cdclk = bxt_calc_cdclk(dev_priv, dev_priv->cdclk.hw.cdclk);
1755         if (cdclk != dev_priv->cdclk.hw.cdclk)
1756                 goto sanitize;
1757
1758         /* Make sure the VCO is correct for the cdclk */
1759         vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
1760         if (vco != dev_priv->cdclk.hw.vco)
1761                 goto sanitize;
1762
1763         expected = skl_cdclk_decimal(cdclk);
1764
1765         /* Figure out what CD2X divider we should be using for this cdclk */
1766         expected |= bxt_cdclk_cd2x_div_sel(dev_priv,
1767                                            dev_priv->cdclk.hw.cdclk,
1768                                            dev_priv->cdclk.hw.vco);
1769
1770         /*
1771          * Disable SSA Precharge when CD clock frequency < 500 MHz,
1772          * enable otherwise.
1773          */
1774         if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
1775             dev_priv->cdclk.hw.cdclk >= 500000)
1776                 expected |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
1777
1778         if (cdctl == expected)
1779                 /* All well; nothing to sanitize */
1780                 return;
1781
1782 sanitize:
1783         drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
1784
1785         /* force cdclk programming */
1786         dev_priv->cdclk.hw.cdclk = 0;
1787
1788         /* force full PLL disable + enable */
1789         dev_priv->cdclk.hw.vco = -1;
1790 }
1791
1792 static void bxt_cdclk_init_hw(struct drm_i915_private *dev_priv)
1793 {
1794         struct intel_cdclk_config cdclk_config;
1795
1796         bxt_sanitize_cdclk(dev_priv);
1797
1798         if (dev_priv->cdclk.hw.cdclk != 0 &&
1799             dev_priv->cdclk.hw.vco != 0)
1800                 return;
1801
1802         cdclk_config = dev_priv->cdclk.hw;
1803
1804         /*
1805          * FIXME:
1806          * - The initial CDCLK needs to be read from VBT.
1807          *   Need to make this change after VBT has changes for BXT.
1808          */
1809         cdclk_config.cdclk = bxt_calc_cdclk(dev_priv, 0);
1810         cdclk_config.vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk_config.cdclk);
1811         cdclk_config.voltage_level =
1812                 intel_cdclk_calc_voltage_level(dev_priv, cdclk_config.cdclk);
1813
1814         bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1815 }
1816
1817 static void bxt_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
1818 {
1819         struct intel_cdclk_config cdclk_config = dev_priv->cdclk.hw;
1820
1821         cdclk_config.cdclk = cdclk_config.bypass;
1822         cdclk_config.vco = 0;
1823         cdclk_config.voltage_level =
1824                 intel_cdclk_calc_voltage_level(dev_priv, cdclk_config.cdclk);
1825
1826         bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1827 }
1828
1829 /**
1830  * intel_cdclk_init_hw - Initialize CDCLK hardware
1831  * @i915: i915 device
1832  *
1833  * Initialize CDCLK. This consists mainly of initializing dev_priv->cdclk.hw and
1834  * sanitizing the state of the hardware if needed. This is generally done only
1835  * during the display core initialization sequence, after which the DMC will
1836  * take care of turning CDCLK off/on as needed.
1837  */
1838 void intel_cdclk_init_hw(struct drm_i915_private *i915)
1839 {
1840         if (DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915))
1841                 bxt_cdclk_init_hw(i915);
1842         else if (DISPLAY_VER(i915) == 9)
1843                 skl_cdclk_init_hw(i915);
1844 }
1845
1846 /**
1847  * intel_cdclk_uninit_hw - Uninitialize CDCLK hardware
1848  * @i915: i915 device
1849  *
1850  * Uninitialize CDCLK. This is done only during the display core
1851  * uninitialization sequence.
1852  */
1853 void intel_cdclk_uninit_hw(struct drm_i915_private *i915)
1854 {
1855         if (DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915))
1856                 bxt_cdclk_uninit_hw(i915);
1857         else if (DISPLAY_VER(i915) == 9)
1858                 skl_cdclk_uninit_hw(i915);
1859 }
1860
1861 static bool intel_cdclk_can_crawl(struct drm_i915_private *dev_priv,
1862                                   const struct intel_cdclk_config *a,
1863                                   const struct intel_cdclk_config *b)
1864 {
1865         int a_div, b_div;
1866
1867         if (!HAS_CDCLK_CRAWL(dev_priv))
1868                 return false;
1869
1870         /*
1871          * The vco and cd2x divider will change independently
1872          * from each, so we disallow cd2x change when crawling.
1873          */
1874         a_div = DIV_ROUND_CLOSEST(a->vco, a->cdclk);
1875         b_div = DIV_ROUND_CLOSEST(b->vco, b->cdclk);
1876
1877         return a->vco != 0 && b->vco != 0 &&
1878                 a->vco != b->vco &&
1879                 a_div == b_div &&
1880                 a->ref == b->ref;
1881 }
1882
1883 /**
1884  * intel_cdclk_needs_modeset - Determine if changong between the CDCLK
1885  *                             configurations requires a modeset on all pipes
1886  * @a: first CDCLK configuration
1887  * @b: second CDCLK configuration
1888  *
1889  * Returns:
1890  * True if changing between the two CDCLK configurations
1891  * requires all pipes to be off, false if not.
1892  */
1893 bool intel_cdclk_needs_modeset(const struct intel_cdclk_config *a,
1894                                const struct intel_cdclk_config *b)
1895 {
1896         return a->cdclk != b->cdclk ||
1897                 a->vco != b->vco ||
1898                 a->ref != b->ref;
1899 }
1900
1901 /**
1902  * intel_cdclk_can_cd2x_update - Determine if changing between the two CDCLK
1903  *                               configurations requires only a cd2x divider update
1904  * @dev_priv: i915 device
1905  * @a: first CDCLK configuration
1906  * @b: second CDCLK configuration
1907  *
1908  * Returns:
1909  * True if changing between the two CDCLK configurations
1910  * can be done with just a cd2x divider update, false if not.
1911  */
1912 static bool intel_cdclk_can_cd2x_update(struct drm_i915_private *dev_priv,
1913                                         const struct intel_cdclk_config *a,
1914                                         const struct intel_cdclk_config *b)
1915 {
1916         /* Older hw doesn't have the capability */
1917         if (DISPLAY_VER(dev_priv) < 10 && !IS_BROXTON(dev_priv))
1918                 return false;
1919
1920         return a->cdclk != b->cdclk &&
1921                 a->vco == b->vco &&
1922                 a->ref == b->ref;
1923 }
1924
1925 /**
1926  * intel_cdclk_changed - Determine if two CDCLK configurations are different
1927  * @a: first CDCLK configuration
1928  * @b: second CDCLK configuration
1929  *
1930  * Returns:
1931  * True if the CDCLK configurations don't match, false if they do.
1932  */
1933 static bool intel_cdclk_changed(const struct intel_cdclk_config *a,
1934                                 const struct intel_cdclk_config *b)
1935 {
1936         return intel_cdclk_needs_modeset(a, b) ||
1937                 a->voltage_level != b->voltage_level;
1938 }
1939
1940 void intel_dump_cdclk_config(const struct intel_cdclk_config *cdclk_config,
1941                              const char *context)
1942 {
1943         DRM_DEBUG_DRIVER("%s %d kHz, VCO %d kHz, ref %d kHz, bypass %d kHz, voltage level %d\n",
1944                          context, cdclk_config->cdclk, cdclk_config->vco,
1945                          cdclk_config->ref, cdclk_config->bypass,
1946                          cdclk_config->voltage_level);
1947 }
1948
1949 /**
1950  * intel_set_cdclk - Push the CDCLK configuration to the hardware
1951  * @dev_priv: i915 device
1952  * @cdclk_config: new CDCLK configuration
1953  * @pipe: pipe with which to synchronize the update
1954  *
1955  * Program the hardware based on the passed in CDCLK state,
1956  * if necessary.
1957  */
1958 static void intel_set_cdclk(struct drm_i915_private *dev_priv,
1959                             const struct intel_cdclk_config *cdclk_config,
1960                             enum pipe pipe)
1961 {
1962         struct intel_encoder *encoder;
1963
1964         if (!intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_config))
1965                 return;
1966
1967         if (drm_WARN_ON_ONCE(&dev_priv->drm, !dev_priv->cdclk_funcs->set_cdclk))
1968                 return;
1969
1970         intel_dump_cdclk_config(cdclk_config, "Changing CDCLK to");
1971
1972         for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
1973                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
1974
1975                 intel_psr_pause(intel_dp);
1976         }
1977
1978         /*
1979          * Lock aux/gmbus while we change cdclk in case those
1980          * functions use cdclk. Not all platforms/ports do,
1981          * but we'll lock them all for simplicity.
1982          */
1983         mutex_lock(&dev_priv->gmbus_mutex);
1984         for_each_intel_dp(&dev_priv->drm, encoder) {
1985                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
1986
1987                 mutex_lock_nest_lock(&intel_dp->aux.hw_mutex,
1988                                      &dev_priv->gmbus_mutex);
1989         }
1990
1991         intel_cdclk_set_cdclk(dev_priv, cdclk_config, pipe);
1992
1993         for_each_intel_dp(&dev_priv->drm, encoder) {
1994                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
1995
1996                 mutex_unlock(&intel_dp->aux.hw_mutex);
1997         }
1998         mutex_unlock(&dev_priv->gmbus_mutex);
1999
2000         for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2001                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2002
2003                 intel_psr_resume(intel_dp);
2004         }
2005
2006         if (drm_WARN(&dev_priv->drm,
2007                      intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_config),
2008                      "cdclk state doesn't match!\n")) {
2009                 intel_dump_cdclk_config(&dev_priv->cdclk.hw, "[hw state]");
2010                 intel_dump_cdclk_config(cdclk_config, "[sw state]");
2011         }
2012 }
2013
2014 /**
2015  * intel_set_cdclk_pre_plane_update - Push the CDCLK state to the hardware
2016  * @state: intel atomic state
2017  *
2018  * Program the hardware before updating the HW plane state based on the
2019  * new CDCLK state, if necessary.
2020  */
2021 void
2022 intel_set_cdclk_pre_plane_update(struct intel_atomic_state *state)
2023 {
2024         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2025         const struct intel_cdclk_state *old_cdclk_state =
2026                 intel_atomic_get_old_cdclk_state(state);
2027         const struct intel_cdclk_state *new_cdclk_state =
2028                 intel_atomic_get_new_cdclk_state(state);
2029         enum pipe pipe = new_cdclk_state->pipe;
2030
2031         if (!intel_cdclk_changed(&old_cdclk_state->actual,
2032                                  &new_cdclk_state->actual))
2033                 return;
2034
2035         if (pipe == INVALID_PIPE ||
2036             old_cdclk_state->actual.cdclk <= new_cdclk_state->actual.cdclk) {
2037                 drm_WARN_ON(&dev_priv->drm, !new_cdclk_state->base.changed);
2038
2039                 intel_set_cdclk(dev_priv, &new_cdclk_state->actual, pipe);
2040         }
2041 }
2042
2043 /**
2044  * intel_set_cdclk_post_plane_update - Push the CDCLK state to the hardware
2045  * @state: intel atomic state
2046  *
2047  * Program the hardware after updating the HW plane state based on the
2048  * new CDCLK state, if necessary.
2049  */
2050 void
2051 intel_set_cdclk_post_plane_update(struct intel_atomic_state *state)
2052 {
2053         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2054         const struct intel_cdclk_state *old_cdclk_state =
2055                 intel_atomic_get_old_cdclk_state(state);
2056         const struct intel_cdclk_state *new_cdclk_state =
2057                 intel_atomic_get_new_cdclk_state(state);
2058         enum pipe pipe = new_cdclk_state->pipe;
2059
2060         if (!intel_cdclk_changed(&old_cdclk_state->actual,
2061                                  &new_cdclk_state->actual))
2062                 return;
2063
2064         if (pipe != INVALID_PIPE &&
2065             old_cdclk_state->actual.cdclk > new_cdclk_state->actual.cdclk) {
2066                 drm_WARN_ON(&dev_priv->drm, !new_cdclk_state->base.changed);
2067
2068                 intel_set_cdclk(dev_priv, &new_cdclk_state->actual, pipe);
2069         }
2070 }
2071
2072 static int intel_pixel_rate_to_cdclk(const struct intel_crtc_state *crtc_state)
2073 {
2074         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
2075         int pixel_rate = crtc_state->pixel_rate;
2076
2077         if (DISPLAY_VER(dev_priv) >= 10)
2078                 return DIV_ROUND_UP(pixel_rate, 2);
2079         else if (DISPLAY_VER(dev_priv) == 9 ||
2080                  IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2081                 return pixel_rate;
2082         else if (IS_CHERRYVIEW(dev_priv))
2083                 return DIV_ROUND_UP(pixel_rate * 100, 95);
2084         else if (crtc_state->double_wide)
2085                 return DIV_ROUND_UP(pixel_rate * 100, 90 * 2);
2086         else
2087                 return DIV_ROUND_UP(pixel_rate * 100, 90);
2088 }
2089
2090 static int intel_planes_min_cdclk(const struct intel_crtc_state *crtc_state)
2091 {
2092         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2093         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2094         struct intel_plane *plane;
2095         int min_cdclk = 0;
2096
2097         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
2098                 min_cdclk = max(crtc_state->min_cdclk[plane->id], min_cdclk);
2099
2100         return min_cdclk;
2101 }
2102
2103 int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state)
2104 {
2105         struct drm_i915_private *dev_priv =
2106                 to_i915(crtc_state->uapi.crtc->dev);
2107         int min_cdclk;
2108
2109         if (!crtc_state->hw.enable)
2110                 return 0;
2111
2112         min_cdclk = intel_pixel_rate_to_cdclk(crtc_state);
2113
2114         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
2115         if (IS_BROADWELL(dev_priv) && hsw_crtc_state_ips_capable(crtc_state))
2116                 min_cdclk = DIV_ROUND_UP(min_cdclk * 100, 95);
2117
2118         /* BSpec says "Do not use DisplayPort with CDCLK less than 432 MHz,
2119          * audio enabled, port width x4, and link rate HBR2 (5.4 GHz), or else
2120          * there may be audio corruption or screen corruption." This cdclk
2121          * restriction for GLK is 316.8 MHz.
2122          */
2123         if (intel_crtc_has_dp_encoder(crtc_state) &&
2124             crtc_state->has_audio &&
2125             crtc_state->port_clock >= 540000 &&
2126             crtc_state->lane_count == 4) {
2127                 if (DISPLAY_VER(dev_priv) == 10) {
2128                         /* Display WA #1145: glk */
2129                         min_cdclk = max(316800, min_cdclk);
2130                 } else if (DISPLAY_VER(dev_priv) == 9 || IS_BROADWELL(dev_priv)) {
2131                         /* Display WA #1144: skl,bxt */
2132                         min_cdclk = max(432000, min_cdclk);
2133                 }
2134         }
2135
2136         /*
2137          * According to BSpec, "The CD clock frequency must be at least twice
2138          * the frequency of the Azalia BCLK." and BCLK is 96 MHz by default.
2139          */
2140         if (crtc_state->has_audio && DISPLAY_VER(dev_priv) >= 9)
2141                 min_cdclk = max(2 * 96000, min_cdclk);
2142
2143         /*
2144          * "For DP audio configuration, cdclk frequency shall be set to
2145          *  meet the following requirements:
2146          *  DP Link Frequency(MHz) | Cdclk frequency(MHz)
2147          *  270                    | 320 or higher
2148          *  162                    | 200 or higher"
2149          */
2150         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
2151             intel_crtc_has_dp_encoder(crtc_state) && crtc_state->has_audio)
2152                 min_cdclk = max(crtc_state->port_clock, min_cdclk);
2153
2154         /*
2155          * On Valleyview some DSI panels lose (v|h)sync when the clock is lower
2156          * than 320000KHz.
2157          */
2158         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2159             IS_VALLEYVIEW(dev_priv))
2160                 min_cdclk = max(320000, min_cdclk);
2161
2162         /*
2163          * On Geminilake once the CDCLK gets as low as 79200
2164          * picture gets unstable, despite that values are
2165          * correct for DSI PLL and DE PLL.
2166          */
2167         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2168             IS_GEMINILAKE(dev_priv))
2169                 min_cdclk = max(158400, min_cdclk);
2170
2171         /* Account for additional needs from the planes */
2172         min_cdclk = max(intel_planes_min_cdclk(crtc_state), min_cdclk);
2173
2174         /*
2175          * When we decide to use only one VDSC engine, since
2176          * each VDSC operates with 1 ppc throughput, pixel clock
2177          * cannot be higher than the VDSC clock (cdclk)
2178          */
2179         if (crtc_state->dsc.compression_enable && !crtc_state->dsc.dsc_split)
2180                 min_cdclk = max(min_cdclk, (int)crtc_state->pixel_rate);
2181
2182         /*
2183          * HACK. Currently for TGL platforms we calculate
2184          * min_cdclk initially based on pixel_rate divided
2185          * by 2, accounting for also plane requirements,
2186          * however in some cases the lowest possible CDCLK
2187          * doesn't work and causing the underruns.
2188          * Explicitly stating here that this seems to be currently
2189          * rather a Hack, than final solution.
2190          */
2191         if (IS_TIGERLAKE(dev_priv)) {
2192                 /*
2193                  * Clamp to max_cdclk_freq in case pixel rate is higher,
2194                  * in order not to break an 8K, but still leave W/A at place.
2195                  */
2196                 min_cdclk = max_t(int, min_cdclk,
2197                                   min_t(int, crtc_state->pixel_rate,
2198                                         dev_priv->max_cdclk_freq));
2199         }
2200
2201         if (min_cdclk > dev_priv->max_cdclk_freq) {
2202                 drm_dbg_kms(&dev_priv->drm,
2203                             "required cdclk (%d kHz) exceeds max (%d kHz)\n",
2204                             min_cdclk, dev_priv->max_cdclk_freq);
2205                 return -EINVAL;
2206         }
2207
2208         return min_cdclk;
2209 }
2210
2211 static int intel_compute_min_cdclk(struct intel_cdclk_state *cdclk_state)
2212 {
2213         struct intel_atomic_state *state = cdclk_state->base.state;
2214         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2215         struct intel_bw_state *bw_state = NULL;
2216         struct intel_crtc *crtc;
2217         struct intel_crtc_state *crtc_state;
2218         int min_cdclk, i;
2219         enum pipe pipe;
2220
2221         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2222                 int ret;
2223
2224                 min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
2225                 if (min_cdclk < 0)
2226                         return min_cdclk;
2227
2228                 bw_state = intel_atomic_get_bw_state(state);
2229                 if (IS_ERR(bw_state))
2230                         return PTR_ERR(bw_state);
2231
2232                 if (cdclk_state->min_cdclk[crtc->pipe] == min_cdclk)
2233                         continue;
2234
2235                 cdclk_state->min_cdclk[crtc->pipe] = min_cdclk;
2236
2237                 ret = intel_atomic_lock_global_state(&cdclk_state->base);
2238                 if (ret)
2239                         return ret;
2240         }
2241
2242         min_cdclk = cdclk_state->force_min_cdclk;
2243         for_each_pipe(dev_priv, pipe) {
2244                 min_cdclk = max(cdclk_state->min_cdclk[pipe], min_cdclk);
2245
2246                 if (!bw_state)
2247                         continue;
2248
2249                 min_cdclk = max(bw_state->min_cdclk, min_cdclk);
2250         }
2251
2252         return min_cdclk;
2253 }
2254
2255 /*
2256  * Account for port clock min voltage level requirements.
2257  * This only really does something on DISPLA_VER >= 11 but can be
2258  * called on earlier platforms as well.
2259  *
2260  * Note that this functions assumes that 0 is
2261  * the lowest voltage value, and higher values
2262  * correspond to increasingly higher voltages.
2263  *
2264  * Should that relationship no longer hold on
2265  * future platforms this code will need to be
2266  * adjusted.
2267  */
2268 static int bxt_compute_min_voltage_level(struct intel_cdclk_state *cdclk_state)
2269 {
2270         struct intel_atomic_state *state = cdclk_state->base.state;
2271         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2272         struct intel_crtc *crtc;
2273         struct intel_crtc_state *crtc_state;
2274         u8 min_voltage_level;
2275         int i;
2276         enum pipe pipe;
2277
2278         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2279                 int ret;
2280
2281                 if (crtc_state->hw.enable)
2282                         min_voltage_level = crtc_state->min_voltage_level;
2283                 else
2284                         min_voltage_level = 0;
2285
2286                 if (cdclk_state->min_voltage_level[crtc->pipe] == min_voltage_level)
2287                         continue;
2288
2289                 cdclk_state->min_voltage_level[crtc->pipe] = min_voltage_level;
2290
2291                 ret = intel_atomic_lock_global_state(&cdclk_state->base);
2292                 if (ret)
2293                         return ret;
2294         }
2295
2296         min_voltage_level = 0;
2297         for_each_pipe(dev_priv, pipe)
2298                 min_voltage_level = max(cdclk_state->min_voltage_level[pipe],
2299                                         min_voltage_level);
2300
2301         return min_voltage_level;
2302 }
2303
2304 static int vlv_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2305 {
2306         struct intel_atomic_state *state = cdclk_state->base.state;
2307         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2308         int min_cdclk, cdclk;
2309
2310         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2311         if (min_cdclk < 0)
2312                 return min_cdclk;
2313
2314         cdclk = vlv_calc_cdclk(dev_priv, min_cdclk);
2315
2316         cdclk_state->logical.cdclk = cdclk;
2317         cdclk_state->logical.voltage_level =
2318                 vlv_calc_voltage_level(dev_priv, cdclk);
2319
2320         if (!cdclk_state->active_pipes) {
2321                 cdclk = vlv_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
2322
2323                 cdclk_state->actual.cdclk = cdclk;
2324                 cdclk_state->actual.voltage_level =
2325                         vlv_calc_voltage_level(dev_priv, cdclk);
2326         } else {
2327                 cdclk_state->actual = cdclk_state->logical;
2328         }
2329
2330         return 0;
2331 }
2332
2333 static int bdw_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2334 {
2335         int min_cdclk, cdclk;
2336
2337         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2338         if (min_cdclk < 0)
2339                 return min_cdclk;
2340
2341         /*
2342          * FIXME should also account for plane ratio
2343          * once 64bpp pixel formats are supported.
2344          */
2345         cdclk = bdw_calc_cdclk(min_cdclk);
2346
2347         cdclk_state->logical.cdclk = cdclk;
2348         cdclk_state->logical.voltage_level =
2349                 bdw_calc_voltage_level(cdclk);
2350
2351         if (!cdclk_state->active_pipes) {
2352                 cdclk = bdw_calc_cdclk(cdclk_state->force_min_cdclk);
2353
2354                 cdclk_state->actual.cdclk = cdclk;
2355                 cdclk_state->actual.voltage_level =
2356                         bdw_calc_voltage_level(cdclk);
2357         } else {
2358                 cdclk_state->actual = cdclk_state->logical;
2359         }
2360
2361         return 0;
2362 }
2363
2364 static int skl_dpll0_vco(struct intel_cdclk_state *cdclk_state)
2365 {
2366         struct intel_atomic_state *state = cdclk_state->base.state;
2367         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2368         struct intel_crtc *crtc;
2369         struct intel_crtc_state *crtc_state;
2370         int vco, i;
2371
2372         vco = cdclk_state->logical.vco;
2373         if (!vco)
2374                 vco = dev_priv->skl_preferred_vco_freq;
2375
2376         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2377                 if (!crtc_state->hw.enable)
2378                         continue;
2379
2380                 if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP))
2381                         continue;
2382
2383                 /*
2384                  * DPLL0 VCO may need to be adjusted to get the correct
2385                  * clock for eDP. This will affect cdclk as well.
2386                  */
2387                 switch (crtc_state->port_clock / 2) {
2388                 case 108000:
2389                 case 216000:
2390                         vco = 8640000;
2391                         break;
2392                 default:
2393                         vco = 8100000;
2394                         break;
2395                 }
2396         }
2397
2398         return vco;
2399 }
2400
2401 static int skl_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2402 {
2403         int min_cdclk, cdclk, vco;
2404
2405         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2406         if (min_cdclk < 0)
2407                 return min_cdclk;
2408
2409         vco = skl_dpll0_vco(cdclk_state);
2410
2411         /*
2412          * FIXME should also account for plane ratio
2413          * once 64bpp pixel formats are supported.
2414          */
2415         cdclk = skl_calc_cdclk(min_cdclk, vco);
2416
2417         cdclk_state->logical.vco = vco;
2418         cdclk_state->logical.cdclk = cdclk;
2419         cdclk_state->logical.voltage_level =
2420                 skl_calc_voltage_level(cdclk);
2421
2422         if (!cdclk_state->active_pipes) {
2423                 cdclk = skl_calc_cdclk(cdclk_state->force_min_cdclk, vco);
2424
2425                 cdclk_state->actual.vco = vco;
2426                 cdclk_state->actual.cdclk = cdclk;
2427                 cdclk_state->actual.voltage_level =
2428                         skl_calc_voltage_level(cdclk);
2429         } else {
2430                 cdclk_state->actual = cdclk_state->logical;
2431         }
2432
2433         return 0;
2434 }
2435
2436 static int bxt_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2437 {
2438         struct intel_atomic_state *state = cdclk_state->base.state;
2439         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2440         int min_cdclk, min_voltage_level, cdclk, vco;
2441
2442         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2443         if (min_cdclk < 0)
2444                 return min_cdclk;
2445
2446         min_voltage_level = bxt_compute_min_voltage_level(cdclk_state);
2447         if (min_voltage_level < 0)
2448                 return min_voltage_level;
2449
2450         cdclk = bxt_calc_cdclk(dev_priv, min_cdclk);
2451         vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2452
2453         cdclk_state->logical.vco = vco;
2454         cdclk_state->logical.cdclk = cdclk;
2455         cdclk_state->logical.voltage_level =
2456                 max_t(int, min_voltage_level,
2457                       intel_cdclk_calc_voltage_level(dev_priv, cdclk));
2458
2459         if (!cdclk_state->active_pipes) {
2460                 cdclk = bxt_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
2461                 vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2462
2463                 cdclk_state->actual.vco = vco;
2464                 cdclk_state->actual.cdclk = cdclk;
2465                 cdclk_state->actual.voltage_level =
2466                         intel_cdclk_calc_voltage_level(dev_priv, cdclk);
2467         } else {
2468                 cdclk_state->actual = cdclk_state->logical;
2469         }
2470
2471         return 0;
2472 }
2473
2474 static int fixed_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2475 {
2476         int min_cdclk;
2477
2478         /*
2479          * We can't change the cdclk frequency, but we still want to
2480          * check that the required minimum frequency doesn't exceed
2481          * the actual cdclk frequency.
2482          */
2483         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2484         if (min_cdclk < 0)
2485                 return min_cdclk;
2486
2487         return 0;
2488 }
2489
2490 static struct intel_global_state *intel_cdclk_duplicate_state(struct intel_global_obj *obj)
2491 {
2492         struct intel_cdclk_state *cdclk_state;
2493
2494         cdclk_state = kmemdup(obj->state, sizeof(*cdclk_state), GFP_KERNEL);
2495         if (!cdclk_state)
2496                 return NULL;
2497
2498         cdclk_state->pipe = INVALID_PIPE;
2499
2500         return &cdclk_state->base;
2501 }
2502
2503 static void intel_cdclk_destroy_state(struct intel_global_obj *obj,
2504                                       struct intel_global_state *state)
2505 {
2506         kfree(state);
2507 }
2508
2509 static const struct intel_global_state_funcs intel_cdclk_funcs = {
2510         .atomic_duplicate_state = intel_cdclk_duplicate_state,
2511         .atomic_destroy_state = intel_cdclk_destroy_state,
2512 };
2513
2514 struct intel_cdclk_state *
2515 intel_atomic_get_cdclk_state(struct intel_atomic_state *state)
2516 {
2517         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2518         struct intel_global_state *cdclk_state;
2519
2520         cdclk_state = intel_atomic_get_global_obj_state(state, &dev_priv->cdclk.obj);
2521         if (IS_ERR(cdclk_state))
2522                 return ERR_CAST(cdclk_state);
2523
2524         return to_intel_cdclk_state(cdclk_state);
2525 }
2526
2527 int intel_cdclk_init(struct drm_i915_private *dev_priv)
2528 {
2529         struct intel_cdclk_state *cdclk_state;
2530
2531         cdclk_state = kzalloc(sizeof(*cdclk_state), GFP_KERNEL);
2532         if (!cdclk_state)
2533                 return -ENOMEM;
2534
2535         intel_atomic_global_obj_init(dev_priv, &dev_priv->cdclk.obj,
2536                                      &cdclk_state->base, &intel_cdclk_funcs);
2537
2538         return 0;
2539 }
2540
2541 int intel_modeset_calc_cdclk(struct intel_atomic_state *state)
2542 {
2543         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2544         const struct intel_cdclk_state *old_cdclk_state;
2545         struct intel_cdclk_state *new_cdclk_state;
2546         enum pipe pipe = INVALID_PIPE;
2547         int ret;
2548
2549         new_cdclk_state = intel_atomic_get_cdclk_state(state);
2550         if (IS_ERR(new_cdclk_state))
2551                 return PTR_ERR(new_cdclk_state);
2552
2553         old_cdclk_state = intel_atomic_get_old_cdclk_state(state);
2554
2555         new_cdclk_state->active_pipes =
2556                 intel_calc_active_pipes(state, old_cdclk_state->active_pipes);
2557
2558         ret = intel_cdclk_modeset_calc_cdclk(dev_priv, new_cdclk_state);
2559         if (ret)
2560                 return ret;
2561
2562         if (intel_cdclk_changed(&old_cdclk_state->actual,
2563                                 &new_cdclk_state->actual)) {
2564                 /*
2565                  * Also serialize commits across all crtcs
2566                  * if the actual hw needs to be poked.
2567                  */
2568                 ret = intel_atomic_serialize_global_state(&new_cdclk_state->base);
2569                 if (ret)
2570                         return ret;
2571         } else if (old_cdclk_state->active_pipes != new_cdclk_state->active_pipes ||
2572                    old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk ||
2573                    intel_cdclk_changed(&old_cdclk_state->logical,
2574                                        &new_cdclk_state->logical)) {
2575                 ret = intel_atomic_lock_global_state(&new_cdclk_state->base);
2576                 if (ret)
2577                         return ret;
2578         } else {
2579                 return 0;
2580         }
2581
2582         if (is_power_of_2(new_cdclk_state->active_pipes) &&
2583             intel_cdclk_can_cd2x_update(dev_priv,
2584                                         &old_cdclk_state->actual,
2585                                         &new_cdclk_state->actual)) {
2586                 struct intel_crtc *crtc;
2587                 struct intel_crtc_state *crtc_state;
2588
2589                 pipe = ilog2(new_cdclk_state->active_pipes);
2590                 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
2591
2592                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
2593                 if (IS_ERR(crtc_state))
2594                         return PTR_ERR(crtc_state);
2595
2596                 if (drm_atomic_crtc_needs_modeset(&crtc_state->uapi))
2597                         pipe = INVALID_PIPE;
2598         }
2599
2600         if (intel_cdclk_can_crawl(dev_priv,
2601                                   &old_cdclk_state->actual,
2602                                   &new_cdclk_state->actual)) {
2603                 drm_dbg_kms(&dev_priv->drm,
2604                             "Can change cdclk via crawl\n");
2605         } else if (pipe != INVALID_PIPE) {
2606                 new_cdclk_state->pipe = pipe;
2607
2608                 drm_dbg_kms(&dev_priv->drm,
2609                             "Can change cdclk cd2x divider with pipe %c active\n",
2610                             pipe_name(pipe));
2611         } else if (intel_cdclk_needs_modeset(&old_cdclk_state->actual,
2612                                              &new_cdclk_state->actual)) {
2613                 /* All pipes must be switched off while we change the cdclk. */
2614                 ret = intel_modeset_all_pipes(state);
2615                 if (ret)
2616                         return ret;
2617
2618                 drm_dbg_kms(&dev_priv->drm,
2619                             "Modeset required for cdclk change\n");
2620         }
2621
2622         drm_dbg_kms(&dev_priv->drm,
2623                     "New cdclk calculated to be logical %u kHz, actual %u kHz\n",
2624                     new_cdclk_state->logical.cdclk,
2625                     new_cdclk_state->actual.cdclk);
2626         drm_dbg_kms(&dev_priv->drm,
2627                     "New voltage level calculated to be logical %u, actual %u\n",
2628                     new_cdclk_state->logical.voltage_level,
2629                     new_cdclk_state->actual.voltage_level);
2630
2631         return 0;
2632 }
2633
2634 static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
2635 {
2636         int max_cdclk_freq = dev_priv->max_cdclk_freq;
2637
2638         if (DISPLAY_VER(dev_priv) >= 10)
2639                 return 2 * max_cdclk_freq;
2640         else if (DISPLAY_VER(dev_priv) == 9 ||
2641                  IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2642                 return max_cdclk_freq;
2643         else if (IS_CHERRYVIEW(dev_priv))
2644                 return max_cdclk_freq*95/100;
2645         else if (DISPLAY_VER(dev_priv) < 4)
2646                 return 2*max_cdclk_freq*90/100;
2647         else
2648                 return max_cdclk_freq*90/100;
2649 }
2650
2651 /**
2652  * intel_update_max_cdclk - Determine the maximum support CDCLK frequency
2653  * @dev_priv: i915 device
2654  *
2655  * Determine the maximum CDCLK frequency the platform supports, and also
2656  * derive the maximum dot clock frequency the maximum CDCLK frequency
2657  * allows.
2658  */
2659 void intel_update_max_cdclk(struct drm_i915_private *dev_priv)
2660 {
2661         if (IS_JSL_EHL(dev_priv)) {
2662                 if (dev_priv->cdclk.hw.ref == 24000)
2663                         dev_priv->max_cdclk_freq = 552000;
2664                 else
2665                         dev_priv->max_cdclk_freq = 556800;
2666         } else if (DISPLAY_VER(dev_priv) >= 11) {
2667                 if (dev_priv->cdclk.hw.ref == 24000)
2668                         dev_priv->max_cdclk_freq = 648000;
2669                 else
2670                         dev_priv->max_cdclk_freq = 652800;
2671         } else if (IS_GEMINILAKE(dev_priv)) {
2672                 dev_priv->max_cdclk_freq = 316800;
2673         } else if (IS_BROXTON(dev_priv)) {
2674                 dev_priv->max_cdclk_freq = 624000;
2675         } else if (DISPLAY_VER(dev_priv) == 9) {
2676                 u32 limit = intel_de_read(dev_priv, SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
2677                 int max_cdclk, vco;
2678
2679                 vco = dev_priv->skl_preferred_vco_freq;
2680                 drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
2681
2682                 /*
2683                  * Use the lower (vco 8640) cdclk values as a
2684                  * first guess. skl_calc_cdclk() will correct it
2685                  * if the preferred vco is 8100 instead.
2686                  */
2687                 if (limit == SKL_DFSM_CDCLK_LIMIT_675)
2688                         max_cdclk = 617143;
2689                 else if (limit == SKL_DFSM_CDCLK_LIMIT_540)
2690                         max_cdclk = 540000;
2691                 else if (limit == SKL_DFSM_CDCLK_LIMIT_450)
2692                         max_cdclk = 432000;
2693                 else
2694                         max_cdclk = 308571;
2695
2696                 dev_priv->max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco);
2697         } else if (IS_BROADWELL(dev_priv))  {
2698                 /*
2699                  * FIXME with extra cooling we can allow
2700                  * 540 MHz for ULX and 675 Mhz for ULT.
2701                  * How can we know if extra cooling is
2702                  * available? PCI ID, VTB, something else?
2703                  */
2704                 if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
2705                         dev_priv->max_cdclk_freq = 450000;
2706                 else if (IS_BDW_ULX(dev_priv))
2707                         dev_priv->max_cdclk_freq = 450000;
2708                 else if (IS_BDW_ULT(dev_priv))
2709                         dev_priv->max_cdclk_freq = 540000;
2710                 else
2711                         dev_priv->max_cdclk_freq = 675000;
2712         } else if (IS_CHERRYVIEW(dev_priv)) {
2713                 dev_priv->max_cdclk_freq = 320000;
2714         } else if (IS_VALLEYVIEW(dev_priv)) {
2715                 dev_priv->max_cdclk_freq = 400000;
2716         } else {
2717                 /* otherwise assume cdclk is fixed */
2718                 dev_priv->max_cdclk_freq = dev_priv->cdclk.hw.cdclk;
2719         }
2720
2721         dev_priv->max_dotclk_freq = intel_compute_max_dotclk(dev_priv);
2722
2723         drm_dbg(&dev_priv->drm, "Max CD clock rate: %d kHz\n",
2724                 dev_priv->max_cdclk_freq);
2725
2726         drm_dbg(&dev_priv->drm, "Max dotclock rate: %d kHz\n",
2727                 dev_priv->max_dotclk_freq);
2728 }
2729
2730 /**
2731  * intel_update_cdclk - Determine the current CDCLK frequency
2732  * @dev_priv: i915 device
2733  *
2734  * Determine the current CDCLK frequency.
2735  */
2736 void intel_update_cdclk(struct drm_i915_private *dev_priv)
2737 {
2738         intel_cdclk_get_cdclk(dev_priv, &dev_priv->cdclk.hw);
2739
2740         /*
2741          * 9:0 CMBUS [sic] CDCLK frequency (cdfreq):
2742          * Programmng [sic] note: bit[9:2] should be programmed to the number
2743          * of cdclk that generates 4MHz reference clock freq which is used to
2744          * generate GMBus clock. This will vary with the cdclk freq.
2745          */
2746         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2747                 intel_de_write(dev_priv, GMBUSFREQ_VLV,
2748                                DIV_ROUND_UP(dev_priv->cdclk.hw.cdclk, 1000));
2749 }
2750
2751 static int dg1_rawclk(struct drm_i915_private *dev_priv)
2752 {
2753         /*
2754          * DG1 always uses a 38.4 MHz rawclk.  The bspec tells us
2755          * "Program Numerator=2, Denominator=4, Divider=37 decimal."
2756          */
2757         intel_de_write(dev_priv, PCH_RAWCLK_FREQ,
2758                        CNP_RAWCLK_DEN(4) | CNP_RAWCLK_DIV(37) | ICP_RAWCLK_NUM(2));
2759
2760         return 38400;
2761 }
2762
2763 static int cnp_rawclk(struct drm_i915_private *dev_priv)
2764 {
2765         u32 rawclk;
2766         int divider, fraction;
2767
2768         if (intel_de_read(dev_priv, SFUSE_STRAP) & SFUSE_STRAP_RAW_FREQUENCY) {
2769                 /* 24 MHz */
2770                 divider = 24000;
2771                 fraction = 0;
2772         } else {
2773                 /* 19.2 MHz */
2774                 divider = 19000;
2775                 fraction = 200;
2776         }
2777
2778         rawclk = CNP_RAWCLK_DIV(divider / 1000);
2779         if (fraction) {
2780                 int numerator = 1;
2781
2782                 rawclk |= CNP_RAWCLK_DEN(DIV_ROUND_CLOSEST(numerator * 1000,
2783                                                            fraction) - 1);
2784                 if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
2785                         rawclk |= ICP_RAWCLK_NUM(numerator);
2786         }
2787
2788         intel_de_write(dev_priv, PCH_RAWCLK_FREQ, rawclk);
2789         return divider + fraction;
2790 }
2791
2792 static int pch_rawclk(struct drm_i915_private *dev_priv)
2793 {
2794         return (intel_de_read(dev_priv, PCH_RAWCLK_FREQ) & RAWCLK_FREQ_MASK) * 1000;
2795 }
2796
2797 static int vlv_hrawclk(struct drm_i915_private *dev_priv)
2798 {
2799         /* RAWCLK_FREQ_VLV register updated from power well code */
2800         return vlv_get_cck_clock_hpll(dev_priv, "hrawclk",
2801                                       CCK_DISPLAY_REF_CLOCK_CONTROL);
2802 }
2803
2804 static int i9xx_hrawclk(struct drm_i915_private *dev_priv)
2805 {
2806         u32 clkcfg;
2807
2808         /*
2809          * hrawclock is 1/4 the FSB frequency
2810          *
2811          * Note that this only reads the state of the FSB
2812          * straps, not the actual FSB frequency. Some BIOSen
2813          * let you configure each independently. Ideally we'd
2814          * read out the actual FSB frequency but sadly we
2815          * don't know which registers have that information,
2816          * and all the relevant docs have gone to bit heaven :(
2817          */
2818         clkcfg = intel_de_read(dev_priv, CLKCFG) & CLKCFG_FSB_MASK;
2819
2820         if (IS_MOBILE(dev_priv)) {
2821                 switch (clkcfg) {
2822                 case CLKCFG_FSB_400:
2823                         return 100000;
2824                 case CLKCFG_FSB_533:
2825                         return 133333;
2826                 case CLKCFG_FSB_667:
2827                         return 166667;
2828                 case CLKCFG_FSB_800:
2829                         return 200000;
2830                 case CLKCFG_FSB_1067:
2831                         return 266667;
2832                 case CLKCFG_FSB_1333:
2833                         return 333333;
2834                 default:
2835                         MISSING_CASE(clkcfg);
2836                         return 133333;
2837                 }
2838         } else {
2839                 switch (clkcfg) {
2840                 case CLKCFG_FSB_400_ALT:
2841                         return 100000;
2842                 case CLKCFG_FSB_533:
2843                         return 133333;
2844                 case CLKCFG_FSB_667:
2845                         return 166667;
2846                 case CLKCFG_FSB_800:
2847                         return 200000;
2848                 case CLKCFG_FSB_1067_ALT:
2849                         return 266667;
2850                 case CLKCFG_FSB_1333_ALT:
2851                         return 333333;
2852                 case CLKCFG_FSB_1600_ALT:
2853                         return 400000;
2854                 default:
2855                         return 133333;
2856                 }
2857         }
2858 }
2859
2860 /**
2861  * intel_read_rawclk - Determine the current RAWCLK frequency
2862  * @dev_priv: i915 device
2863  *
2864  * Determine the current RAWCLK frequency. RAWCLK is a fixed
2865  * frequency clock so this needs to done only once.
2866  */
2867 u32 intel_read_rawclk(struct drm_i915_private *dev_priv)
2868 {
2869         u32 freq;
2870
2871         if (INTEL_PCH_TYPE(dev_priv) >= PCH_DG1)
2872                 freq = dg1_rawclk(dev_priv);
2873         else if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP)
2874                 freq = cnp_rawclk(dev_priv);
2875         else if (HAS_PCH_SPLIT(dev_priv))
2876                 freq = pch_rawclk(dev_priv);
2877         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2878                 freq = vlv_hrawclk(dev_priv);
2879         else if (DISPLAY_VER(dev_priv) >= 3)
2880                 freq = i9xx_hrawclk(dev_priv);
2881         else
2882                 /* no rawclk on other platforms, or no need to know it */
2883                 return 0;
2884
2885         return freq;
2886 }
2887
2888 static struct intel_cdclk_funcs tgl_cdclk_funcs = {
2889         .get_cdclk = bxt_get_cdclk,
2890         .set_cdclk = bxt_set_cdclk,
2891         .bw_calc_min_cdclk = skl_bw_calc_min_cdclk,
2892         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
2893         .calc_voltage_level = tgl_calc_voltage_level,
2894 };
2895
2896 static struct intel_cdclk_funcs ehl_cdclk_funcs = {
2897         .get_cdclk = bxt_get_cdclk,
2898         .set_cdclk = bxt_set_cdclk,
2899         .bw_calc_min_cdclk = skl_bw_calc_min_cdclk,
2900         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
2901         .calc_voltage_level = ehl_calc_voltage_level,
2902 };
2903
2904 static struct intel_cdclk_funcs icl_cdclk_funcs = {
2905         .get_cdclk = bxt_get_cdclk,
2906         .set_cdclk = bxt_set_cdclk,
2907         .bw_calc_min_cdclk = skl_bw_calc_min_cdclk,
2908         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
2909         .calc_voltage_level = icl_calc_voltage_level,
2910 };
2911
2912 static struct intel_cdclk_funcs bxt_cdclk_funcs = {
2913         .get_cdclk = bxt_get_cdclk,
2914         .set_cdclk = bxt_set_cdclk,
2915         .bw_calc_min_cdclk = skl_bw_calc_min_cdclk,
2916         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
2917         .calc_voltage_level = bxt_calc_voltage_level,
2918 };
2919
2920 static struct intel_cdclk_funcs skl_cdclk_funcs = {
2921         .get_cdclk = skl_get_cdclk,
2922         .set_cdclk = skl_set_cdclk,
2923         .bw_calc_min_cdclk = skl_bw_calc_min_cdclk,
2924         .modeset_calc_cdclk = skl_modeset_calc_cdclk,
2925 };
2926
2927 static struct intel_cdclk_funcs bdw_cdclk_funcs = {
2928         .get_cdclk = bdw_get_cdclk,
2929         .set_cdclk = bdw_set_cdclk,
2930         .bw_calc_min_cdclk = intel_bw_calc_min_cdclk,
2931         .modeset_calc_cdclk = bdw_modeset_calc_cdclk,
2932 };
2933
2934 static struct intel_cdclk_funcs chv_cdclk_funcs = {
2935         .get_cdclk = vlv_get_cdclk,
2936         .set_cdclk = chv_set_cdclk,
2937         .bw_calc_min_cdclk = intel_bw_calc_min_cdclk,
2938         .modeset_calc_cdclk = vlv_modeset_calc_cdclk,
2939 };
2940
2941 static struct intel_cdclk_funcs vlv_cdclk_funcs = {
2942         .get_cdclk = vlv_get_cdclk,
2943         .set_cdclk = vlv_set_cdclk,
2944         .bw_calc_min_cdclk = intel_bw_calc_min_cdclk,
2945         .modeset_calc_cdclk = vlv_modeset_calc_cdclk,
2946 };
2947
2948 static struct intel_cdclk_funcs hsw_cdclk_funcs = {
2949         .get_cdclk = hsw_get_cdclk,
2950         .bw_calc_min_cdclk = intel_bw_calc_min_cdclk,
2951         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
2952 };
2953
2954 /* SNB, IVB, 965G, 945G */
2955 static struct intel_cdclk_funcs fixed_400mhz_cdclk_funcs = {
2956         .get_cdclk = fixed_400mhz_get_cdclk,
2957         .bw_calc_min_cdclk = intel_bw_calc_min_cdclk,
2958         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
2959 };
2960
2961 static struct intel_cdclk_funcs ilk_cdclk_funcs = {
2962         .get_cdclk = fixed_450mhz_get_cdclk,
2963         .bw_calc_min_cdclk = intel_bw_calc_min_cdclk,
2964         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
2965 };
2966
2967 static struct intel_cdclk_funcs gm45_cdclk_funcs = {
2968         .get_cdclk = gm45_get_cdclk,
2969         .bw_calc_min_cdclk = intel_bw_calc_min_cdclk,
2970         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
2971 };
2972
2973 /* G45 uses G33 */
2974
2975 static struct intel_cdclk_funcs i965gm_cdclk_funcs = {
2976         .get_cdclk = i965gm_get_cdclk,
2977         .bw_calc_min_cdclk = intel_bw_calc_min_cdclk,
2978         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
2979 };
2980
2981 /* i965G uses fixed 400 */
2982
2983 static struct intel_cdclk_funcs pnv_cdclk_funcs = {
2984         .get_cdclk = pnv_get_cdclk,
2985         .bw_calc_min_cdclk = intel_bw_calc_min_cdclk,
2986         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
2987 };
2988
2989 static struct intel_cdclk_funcs g33_cdclk_funcs = {
2990         .get_cdclk = g33_get_cdclk,
2991         .bw_calc_min_cdclk = intel_bw_calc_min_cdclk,
2992         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
2993 };
2994
2995 static struct intel_cdclk_funcs i945gm_cdclk_funcs = {
2996         .get_cdclk = i945gm_get_cdclk,
2997         .bw_calc_min_cdclk = intel_bw_calc_min_cdclk,
2998         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
2999 };
3000
3001 /* i945G uses fixed 400 */
3002
3003 static struct intel_cdclk_funcs i915gm_cdclk_funcs = {
3004         .get_cdclk = i915gm_get_cdclk,
3005         .bw_calc_min_cdclk = intel_bw_calc_min_cdclk,
3006         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3007 };
3008
3009 static struct intel_cdclk_funcs i915g_cdclk_funcs = {
3010         .get_cdclk = fixed_333mhz_get_cdclk,
3011         .bw_calc_min_cdclk = intel_bw_calc_min_cdclk,
3012         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3013 };
3014
3015 static struct intel_cdclk_funcs i865g_cdclk_funcs = {
3016         .get_cdclk = fixed_266mhz_get_cdclk,
3017         .bw_calc_min_cdclk = intel_bw_calc_min_cdclk,
3018         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3019 };
3020
3021 static struct intel_cdclk_funcs i85x_cdclk_funcs = {
3022         .get_cdclk = i85x_get_cdclk,
3023         .bw_calc_min_cdclk = intel_bw_calc_min_cdclk,
3024         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3025 };
3026
3027 static struct intel_cdclk_funcs i845g_cdclk_funcs = {
3028         .get_cdclk = fixed_200mhz_get_cdclk,
3029         .bw_calc_min_cdclk = intel_bw_calc_min_cdclk,
3030         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3031 };
3032
3033 static struct intel_cdclk_funcs i830_cdclk_funcs = {
3034         .get_cdclk = fixed_133mhz_get_cdclk,
3035         .bw_calc_min_cdclk = intel_bw_calc_min_cdclk,
3036         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3037 };
3038
3039 /**
3040  * intel_init_cdclk_hooks - Initialize CDCLK related modesetting hooks
3041  * @dev_priv: i915 device
3042  */
3043 void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv)
3044 {
3045         if (IS_DG2(dev_priv)) {
3046                 dev_priv->cdclk_funcs = &tgl_cdclk_funcs;
3047                 dev_priv->cdclk.table = dg2_cdclk_table;
3048         } else if (IS_ALDERLAKE_P(dev_priv)) {
3049                 dev_priv->cdclk_funcs = &tgl_cdclk_funcs;
3050                 /* Wa_22011320316:adl-p[a0] */
3051                 if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
3052                         dev_priv->cdclk.table = adlp_a_step_cdclk_table;
3053                 else
3054                         dev_priv->cdclk.table = adlp_cdclk_table;
3055         } else if (IS_ROCKETLAKE(dev_priv)) {
3056                 dev_priv->cdclk_funcs = &tgl_cdclk_funcs;
3057                 dev_priv->cdclk.table = rkl_cdclk_table;
3058         } else if (DISPLAY_VER(dev_priv) >= 12) {
3059                 dev_priv->cdclk_funcs = &tgl_cdclk_funcs;
3060                 dev_priv->cdclk.table = icl_cdclk_table;
3061         } else if (IS_JSL_EHL(dev_priv)) {
3062                 dev_priv->cdclk_funcs = &ehl_cdclk_funcs;
3063                 dev_priv->cdclk.table = icl_cdclk_table;
3064         } else if (DISPLAY_VER(dev_priv) >= 11) {
3065                 dev_priv->cdclk_funcs = &icl_cdclk_funcs;
3066                 dev_priv->cdclk.table = icl_cdclk_table;
3067         } else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
3068                 dev_priv->cdclk_funcs = &bxt_cdclk_funcs;
3069                 if (IS_GEMINILAKE(dev_priv))
3070                         dev_priv->cdclk.table = glk_cdclk_table;
3071                 else
3072                         dev_priv->cdclk.table = bxt_cdclk_table;
3073         } else if (DISPLAY_VER(dev_priv) == 9) {
3074                 dev_priv->cdclk_funcs = &skl_cdclk_funcs;
3075         } else if (IS_BROADWELL(dev_priv)) {
3076                 dev_priv->cdclk_funcs = &bdw_cdclk_funcs;
3077         } else if (IS_HASWELL(dev_priv)) {
3078                 dev_priv->cdclk_funcs = &hsw_cdclk_funcs;
3079         } else if (IS_CHERRYVIEW(dev_priv)) {
3080                 dev_priv->cdclk_funcs = &chv_cdclk_funcs;
3081         } else if (IS_VALLEYVIEW(dev_priv)) {
3082                 dev_priv->cdclk_funcs = &vlv_cdclk_funcs;
3083         } else if (IS_SANDYBRIDGE(dev_priv) || IS_IVYBRIDGE(dev_priv)) {
3084                 dev_priv->cdclk_funcs = &fixed_400mhz_cdclk_funcs;
3085         } else if (IS_IRONLAKE(dev_priv)) {
3086                 dev_priv->cdclk_funcs = &ilk_cdclk_funcs;
3087         } else if (IS_GM45(dev_priv)) {
3088                 dev_priv->cdclk_funcs = &gm45_cdclk_funcs;
3089         } else if (IS_G45(dev_priv)) {
3090                 dev_priv->cdclk_funcs = &g33_cdclk_funcs;
3091         } else if (IS_I965GM(dev_priv)) {
3092                 dev_priv->cdclk_funcs = &i965gm_cdclk_funcs;
3093         } else if (IS_I965G(dev_priv)) {
3094                 dev_priv->cdclk_funcs = &fixed_400mhz_cdclk_funcs;
3095         } else if (IS_PINEVIEW(dev_priv)) {
3096                 dev_priv->cdclk_funcs = &pnv_cdclk_funcs;
3097         } else if (IS_G33(dev_priv)) {
3098                 dev_priv->cdclk_funcs = &g33_cdclk_funcs;
3099         } else if (IS_I945GM(dev_priv)) {
3100                 dev_priv->cdclk_funcs = &i945gm_cdclk_funcs;
3101         } else if (IS_I945G(dev_priv)) {
3102                 dev_priv->cdclk_funcs = &fixed_400mhz_cdclk_funcs;
3103         } else if (IS_I915GM(dev_priv)) {
3104                 dev_priv->cdclk_funcs = &i915gm_cdclk_funcs;
3105         } else if (IS_I915G(dev_priv)) {
3106                 dev_priv->cdclk_funcs = &i915g_cdclk_funcs;
3107         } else if (IS_I865G(dev_priv)) {
3108                 dev_priv->cdclk_funcs = &i865g_cdclk_funcs;
3109         } else if (IS_I85X(dev_priv)) {
3110                 dev_priv->cdclk_funcs = &i85x_cdclk_funcs;
3111         } else if (IS_I845G(dev_priv)) {
3112                 dev_priv->cdclk_funcs = &i845g_cdclk_funcs;
3113         } else if (IS_I830(dev_priv)) {
3114                 dev_priv->cdclk_funcs = &i830_cdclk_funcs;
3115         }
3116
3117         if (drm_WARN(&dev_priv->drm, !dev_priv->cdclk_funcs,
3118                      "Unknown platform. Assuming i830\n"))
3119                 dev_priv->cdclk_funcs = &i830_cdclk_funcs;
3120 }