drivers/gpu/drm/i915/display/intel_cdclk.c

   1 /*
   2  * Copyright © 2006-2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include "intel_atomic.h"
  25 #include "intel_cdclk.h"
  26 #include "intel_display_types.h"
  27 #include "intel_sideband.h"
  28
  29 /**
  30  * DOC: CDCLK / RAWCLK
  31  *
  32  * The display engine uses several different clocks to do its work. There
  33  * are two main clocks involved that aren't directly related to the actual
  34  * pixel clock or any symbol/bit clock of the actual output port. These
  35  * are the core display clock (CDCLK) and RAWCLK.
  36  *
  37  * CDCLK clocks most of the display pipe logic, and thus its frequency
  38  * must be high enough to support the rate at which pixels are flowing
  39  * through the pipes. Downscaling must also be accounted as that increases
  40  * the effective pixel rate.
  41  *
  42  * On several platforms the CDCLK frequency can be changed dynamically
  43  * to minimize power consumption for a given display configuration.
  44  * Typically changes to the CDCLK frequency require all the display pipes
  45  * to be shut down while the frequency is being changed.
  46  *
  47  * On SKL+ the DMC will toggle the CDCLK off/on during DC5/6 entry/exit.
  48  * DMC will not change the active CDCLK frequency however, so that part
  49  * will still be performed by the driver directly.
  50  *
  51  * RAWCLK is a fixed frequency clock, often used by various auxiliary
  52  * blocks such as AUX CH or backlight PWM. Hence the only thing we
  53  * really need to know about RAWCLK is its frequency so that various
  54  * dividers can be programmed correctly.
  55  */
  56
  57 static void fixed_133mhz_get_cdclk(struct drm_i915_private *dev_priv,
  58                                    struct intel_cdclk_config *cdclk_config)
  59 {
  60         cdclk_config->cdclk = 133333;
  61 }
  62
  63 static void fixed_200mhz_get_cdclk(struct drm_i915_private *dev_priv,
  64                                    struct intel_cdclk_config *cdclk_config)
  65 {
  66         cdclk_config->cdclk = 200000;
  67 }
  68
  69 static void fixed_266mhz_get_cdclk(struct drm_i915_private *dev_priv,
  70                                    struct intel_cdclk_config *cdclk_config)
  71 {
  72         cdclk_config->cdclk = 266667;
  73 }
  74
  75 static void fixed_333mhz_get_cdclk(struct drm_i915_private *dev_priv,
  76                                    struct intel_cdclk_config *cdclk_config)
  77 {
  78         cdclk_config->cdclk = 333333;
  79 }
  80
  81 static void fixed_400mhz_get_cdclk(struct drm_i915_private *dev_priv,
  82                                    struct intel_cdclk_config *cdclk_config)
  83 {
  84         cdclk_config->cdclk = 400000;
  85 }
  86
  87 static void fixed_450mhz_get_cdclk(struct drm_i915_private *dev_priv,
  88                                    struct intel_cdclk_config *cdclk_config)
  89 {
  90         cdclk_config->cdclk = 450000;
  91 }
  92
  93 static void i85x_get_cdclk(struct drm_i915_private *dev_priv,
  94                            struct intel_cdclk_config *cdclk_config)
  95 {
  96         struct pci_dev *pdev = dev_priv->drm.pdev;
  97         u16 hpllcc = 0;
  98
  99         /*
 100          * 852GM/852GMV only supports 133 MHz and the HPLLCC
 101          * encoding is different :(
 102          * FIXME is this the right way to detect 852GM/852GMV?
 103          */
 104         if (pdev->revision == 0x1) {
 105                 cdclk_config->cdclk = 133333;
 106                 return;
 107         }
 108
 109         pci_bus_read_config_word(pdev->bus,
 110                                  PCI_DEVFN(0, 3), HPLLCC, &hpllcc);
 111
 112         /* Assume that the hardware is in the high speed state.  This
 113          * should be the default.
 114          */
 115         switch (hpllcc & GC_CLOCK_CONTROL_MASK) {
 116         case GC_CLOCK_133_200:
 117         case GC_CLOCK_133_200_2:
 118         case GC_CLOCK_100_200:
 119                 cdclk_config->cdclk = 200000;
 120                 break;
 121         case GC_CLOCK_166_250:
 122                 cdclk_config->cdclk = 250000;
 123                 break;
 124         case GC_CLOCK_100_133:
 125                 cdclk_config->cdclk = 133333;
 126                 break;
 127         case GC_CLOCK_133_266:
 128         case GC_CLOCK_133_266_2:
 129         case GC_CLOCK_166_266:
 130                 cdclk_config->cdclk = 266667;
 131                 break;
 132         }
 133 }
 134
 135 static void i915gm_get_cdclk(struct drm_i915_private *dev_priv,
 136                              struct intel_cdclk_config *cdclk_config)
 137 {
 138         struct pci_dev *pdev = dev_priv->drm.pdev;
 139         u16 gcfgc = 0;
 140
 141         pci_read_config_word(pdev, GCFGC, &gcfgc);
 142
 143         if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
 144                 cdclk_config->cdclk = 133333;
 145                 return;
 146         }
 147
 148         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 149         case GC_DISPLAY_CLOCK_333_320_MHZ:
 150                 cdclk_config->cdclk = 333333;
 151                 break;
 152         default:
 153         case GC_DISPLAY_CLOCK_190_200_MHZ:
 154                 cdclk_config->cdclk = 190000;
 155                 break;
 156         }
 157 }
 158
 159 static void i945gm_get_cdclk(struct drm_i915_private *dev_priv,
 160                              struct intel_cdclk_config *cdclk_config)
 161 {
 162         struct pci_dev *pdev = dev_priv->drm.pdev;
 163         u16 gcfgc = 0;
 164
 165         pci_read_config_word(pdev, GCFGC, &gcfgc);
 166
 167         if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
 168                 cdclk_config->cdclk = 133333;
 169                 return;
 170         }
 171
 172         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 173         case GC_DISPLAY_CLOCK_333_320_MHZ:
 174                 cdclk_config->cdclk = 320000;
 175                 break;
 176         default:
 177         case GC_DISPLAY_CLOCK_190_200_MHZ:
 178                 cdclk_config->cdclk = 200000;
 179                 break;
 180         }
 181 }
 182
 183 static unsigned int intel_hpll_vco(struct drm_i915_private *dev_priv)
 184 {
 185         static const unsigned int blb_vco[8] = {
 186                 [0] = 3200000,
 187                 [1] = 4000000,
 188                 [2] = 5333333,
 189                 [3] = 4800000,
 190                 [4] = 6400000,
 191         };
 192         static const unsigned int pnv_vco[8] = {
 193                 [0] = 3200000,
 194                 [1] = 4000000,
 195                 [2] = 5333333,
 196                 [3] = 4800000,
 197                 [4] = 2666667,
 198         };
 199         static const unsigned int cl_vco[8] = {
 200                 [0] = 3200000,
 201                 [1] = 4000000,
 202                 [2] = 5333333,
 203                 [3] = 6400000,
 204                 [4] = 3333333,
 205                 [5] = 3566667,
 206                 [6] = 4266667,
 207         };
 208         static const unsigned int elk_vco[8] = {
 209                 [0] = 3200000,
 210                 [1] = 4000000,
 211                 [2] = 5333333,
 212                 [3] = 4800000,
 213         };
 214         static const unsigned int ctg_vco[8] = {
 215                 [0] = 3200000,
 216                 [1] = 4000000,
 217                 [2] = 5333333,
 218                 [3] = 6400000,
 219                 [4] = 2666667,
 220                 [5] = 4266667,
 221         };
 222         const unsigned int *vco_table;
 223         unsigned int vco;
 224         u8 tmp = 0;
 225
 226         /* FIXME other chipsets? */
 227         if (IS_GM45(dev_priv))
 228                 vco_table = ctg_vco;
 229         else if (IS_G45(dev_priv))
 230                 vco_table = elk_vco;
 231         else if (IS_I965GM(dev_priv))
 232                 vco_table = cl_vco;
 233         else if (IS_PINEVIEW(dev_priv))
 234                 vco_table = pnv_vco;
 235         else if (IS_G33(dev_priv))
 236                 vco_table = blb_vco;
 237         else
 238                 return 0;
 239
 240         tmp = intel_de_read(dev_priv,
 241                             IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv) ? HPLLVCO_MOBILE : HPLLVCO);
 242
 243         vco = vco_table[tmp & 0x7];
 244         if (vco == 0)
 245                 drm_err(&dev_priv->drm, "Bad HPLL VCO (HPLLVCO=0x%02x)\n",
 246                         tmp);
 247         else
 248                 drm_dbg_kms(&dev_priv->drm, "HPLL VCO %u kHz\n", vco);
 249
 250         return vco;
 251 }
 252
 253 static void g33_get_cdclk(struct drm_i915_private *dev_priv,
 254                           struct intel_cdclk_config *cdclk_config)
 255 {
 256         struct pci_dev *pdev = dev_priv->drm.pdev;
 257         static const u8 div_3200[] = { 12, 10,  8,  7, 5, 16 };
 258         static const u8 div_4000[] = { 14, 12, 10,  8, 6, 20 };
 259         static const u8 div_4800[] = { 20, 14, 12, 10, 8, 24 };
 260         static const u8 div_5333[] = { 20, 16, 12, 12, 8, 28 };
 261         const u8 *div_table;
 262         unsigned int cdclk_sel;
 263         u16 tmp = 0;
 264
 265         cdclk_config->vco = intel_hpll_vco(dev_priv);
 266
 267         pci_read_config_word(pdev, GCFGC, &tmp);
 268
 269         cdclk_sel = (tmp >> 4) & 0x7;
 270
 271         if (cdclk_sel >= ARRAY_SIZE(div_3200))
 272                 goto fail;
 273
 274         switch (cdclk_config->vco) {
 275         case 3200000:
 276                 div_table = div_3200;
 277                 break;
 278         case 4000000:
 279                 div_table = div_4000;
 280                 break;
 281         case 4800000:
 282                 div_table = div_4800;
 283                 break;
 284         case 5333333:
 285                 div_table = div_5333;
 286                 break;
 287         default:
 288                 goto fail;
 289         }
 290
 291         cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
 292                                                 div_table[cdclk_sel]);
 293         return;
 294
 295 fail:
 296         drm_err(&dev_priv->drm,
 297                 "Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%08x\n",
 298                 cdclk_config->vco, tmp);
 299         cdclk_config->cdclk = 190476;
 300 }
 301
 302 static void pnv_get_cdclk(struct drm_i915_private *dev_priv,
 303                           struct intel_cdclk_config *cdclk_config)
 304 {
 305         struct pci_dev *pdev = dev_priv->drm.pdev;
 306         u16 gcfgc = 0;
 307
 308         pci_read_config_word(pdev, GCFGC, &gcfgc);
 309
 310         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 311         case GC_DISPLAY_CLOCK_267_MHZ_PNV:
 312                 cdclk_config->cdclk = 266667;
 313                 break;
 314         case GC_DISPLAY_CLOCK_333_MHZ_PNV:
 315                 cdclk_config->cdclk = 333333;
 316                 break;
 317         case GC_DISPLAY_CLOCK_444_MHZ_PNV:
 318                 cdclk_config->cdclk = 444444;
 319                 break;
 320         case GC_DISPLAY_CLOCK_200_MHZ_PNV:
 321                 cdclk_config->cdclk = 200000;
 322                 break;
 323         default:
 324                 drm_err(&dev_priv->drm,
 325                         "Unknown pnv display core clock 0x%04x\n", gcfgc);
 326                 /* fall through */
 327         case GC_DISPLAY_CLOCK_133_MHZ_PNV:
 328                 cdclk_config->cdclk = 133333;
 329                 break;
 330         case GC_DISPLAY_CLOCK_167_MHZ_PNV:
 331                 cdclk_config->cdclk = 166667;
 332                 break;
 333         }
 334 }
 335
 336 static void i965gm_get_cdclk(struct drm_i915_private *dev_priv,
 337                              struct intel_cdclk_config *cdclk_config)
 338 {
 339         struct pci_dev *pdev = dev_priv->drm.pdev;
 340         static const u8 div_3200[] = { 16, 10,  8 };
 341         static const u8 div_4000[] = { 20, 12, 10 };
 342         static const u8 div_5333[] = { 24, 16, 14 };
 343         const u8 *div_table;
 344         unsigned int cdclk_sel;
 345         u16 tmp = 0;
 346
 347         cdclk_config->vco = intel_hpll_vco(dev_priv);
 348
 349         pci_read_config_word(pdev, GCFGC, &tmp);
 350
 351         cdclk_sel = ((tmp >> 8) & 0x1f) - 1;
 352
 353         if (cdclk_sel >= ARRAY_SIZE(div_3200))
 354                 goto fail;
 355
 356         switch (cdclk_config->vco) {
 357         case 3200000:
 358                 div_table = div_3200;
 359                 break;
 360         case 4000000:
 361                 div_table = div_4000;
 362                 break;
 363         case 5333333:
 364                 div_table = div_5333;
 365                 break;
 366         default:
 367                 goto fail;
 368         }
 369
 370         cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
 371                                                 div_table[cdclk_sel]);
 372         return;
 373
 374 fail:
 375         drm_err(&dev_priv->drm,
 376                 "Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%04x\n",
 377                 cdclk_config->vco, tmp);
 378         cdclk_config->cdclk = 200000;
 379 }
 380
 381 static void gm45_get_cdclk(struct drm_i915_private *dev_priv,
 382                            struct intel_cdclk_config *cdclk_config)
 383 {
 384         struct pci_dev *pdev = dev_priv->drm.pdev;
 385         unsigned int cdclk_sel;
 386         u16 tmp = 0;
 387
 388         cdclk_config->vco = intel_hpll_vco(dev_priv);
 389
 390         pci_read_config_word(pdev, GCFGC, &tmp);
 391
 392         cdclk_sel = (tmp >> 12) & 0x1;
 393
 394         switch (cdclk_config->vco) {
 395         case 2666667:
 396         case 4000000:
 397         case 5333333:
 398                 cdclk_config->cdclk = cdclk_sel ? 333333 : 222222;
 399                 break;
 400         case 3200000:
 401                 cdclk_config->cdclk = cdclk_sel ? 320000 : 228571;
 402                 break;
 403         default:
 404                 drm_err(&dev_priv->drm,
 405                         "Unable to determine CDCLK. HPLL VCO=%u, CFGC=0x%04x\n",
 406                         cdclk_config->vco, tmp);
 407                 cdclk_config->cdclk = 222222;
 408                 break;
 409         }
 410 }
 411
 412 static void hsw_get_cdclk(struct drm_i915_private *dev_priv,
 413                           struct intel_cdclk_config *cdclk_config)
 414 {
 415         u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
 416         u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
 417
 418         if (lcpll & LCPLL_CD_SOURCE_FCLK)
 419                 cdclk_config->cdclk = 800000;
 420         else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
 421                 cdclk_config->cdclk = 450000;
 422         else if (freq == LCPLL_CLK_FREQ_450)
 423                 cdclk_config->cdclk = 450000;
 424         else if (IS_HSW_ULT(dev_priv))
 425                 cdclk_config->cdclk = 337500;
 426         else
 427                 cdclk_config->cdclk = 540000;
 428 }
 429
 430 static int vlv_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
 431 {
 432         int freq_320 = (dev_priv->hpll_freq <<  1) % 320000 != 0 ?
 433                 333333 : 320000;
 434
 435         /*
 436          * We seem to get an unstable or solid color picture at 200MHz.
 437          * Not sure what's wrong. For now use 200MHz only when all pipes
 438          * are off.
 439          */
 440         if (IS_VALLEYVIEW(dev_priv) && min_cdclk > freq_320)
 441                 return 400000;
 442         else if (min_cdclk > 266667)
 443                 return freq_320;
 444         else if (min_cdclk > 0)
 445                 return 266667;
 446         else
 447                 return 200000;
 448 }
 449
 450 static u8 vlv_calc_voltage_level(struct drm_i915_private *dev_priv, int cdclk)
 451 {
 452         if (IS_VALLEYVIEW(dev_priv)) {
 453                 if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */
 454                         return 2;
 455                 else if (cdclk >= 266667)
 456                         return 1;
 457                 else
 458                         return 0;
 459         } else {
 460                 /*
 461                  * Specs are full of misinformation, but testing on actual
 462                  * hardware has shown that we just need to write the desired
 463                  * CCK divider into the Punit register.
 464                  */
 465                 return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
 466         }
 467 }
 468
 469 static void vlv_get_cdclk(struct drm_i915_private *dev_priv,
 470                           struct intel_cdclk_config *cdclk_config)
 471 {
 472         u32 val;
 473
 474         vlv_iosf_sb_get(dev_priv,
 475                         BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
 476
 477         cdclk_config->vco = vlv_get_hpll_vco(dev_priv);
 478         cdclk_config->cdclk = vlv_get_cck_clock(dev_priv, "cdclk",
 479                                                 CCK_DISPLAY_CLOCK_CONTROL,
 480                                                 cdclk_config->vco);
 481
 482         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 483
 484         vlv_iosf_sb_put(dev_priv,
 485                         BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
 486
 487         if (IS_VALLEYVIEW(dev_priv))
 488                 cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK) >>
 489                         DSPFREQGUAR_SHIFT;
 490         else
 491                 cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK_CHV) >>
 492                         DSPFREQGUAR_SHIFT_CHV;
 493 }
 494
 495 static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
 496 {
 497         unsigned int credits, default_credits;
 498
 499         if (IS_CHERRYVIEW(dev_priv))
 500                 default_credits = PFI_CREDIT(12);
 501         else
 502                 default_credits = PFI_CREDIT(8);
 503
 504         if (dev_priv->cdclk.hw.cdclk >= dev_priv->czclk_freq) {
 505                 /* CHV suggested value is 31 or 63 */
 506                 if (IS_CHERRYVIEW(dev_priv))
 507                         credits = PFI_CREDIT_63;
 508                 else
 509                         credits = PFI_CREDIT(15);
 510         } else {
 511                 credits = default_credits;
 512         }
 513
 514         /*
 515          * WA - write default credits before re-programming
 516          * FIXME: should we also set the resend bit here?
 517          */
 518         intel_de_write(dev_priv, GCI_CONTROL,
 519                        VGA_FAST_MODE_DISABLE | default_credits);
 520
 521         intel_de_write(dev_priv, GCI_CONTROL,
 522                        VGA_FAST_MODE_DISABLE | credits | PFI_CREDIT_RESEND);
 523
 524         /*
 525          * FIXME is this guaranteed to clear
 526          * immediately or should we poll for it?
 527          */
 528         WARN_ON(intel_de_read(dev_priv, GCI_CONTROL) & PFI_CREDIT_RESEND);
 529 }
 530
 531 static void vlv_set_cdclk(struct drm_i915_private *dev_priv,
 532                           const struct intel_cdclk_config *cdclk_config,
 533                           enum pipe pipe)
 534 {
 535         int cdclk = cdclk_config->cdclk;
 536         u32 val, cmd = cdclk_config->voltage_level;
 537         intel_wakeref_t wakeref;
 538
 539         switch (cdclk) {
 540         case 400000:
 541         case 333333:
 542         case 320000:
 543         case 266667:
 544         case 200000:
 545                 break;
 546         default:
 547                 MISSING_CASE(cdclk);
 548                 return;
 549         }
 550
 551         /* There are cases where we can end up here with power domains
 552          * off and a CDCLK frequency other than the minimum, like when
 553          * issuing a modeset without actually changing any display after
 554          * a system suspend.  So grab the display core domain, which covers
 555          * the HW blocks needed for the following programming.
 556          */
 557         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
 558
 559         vlv_iosf_sb_get(dev_priv,
 560                         BIT(VLV_IOSF_SB_CCK) |
 561                         BIT(VLV_IOSF_SB_BUNIT) |
 562                         BIT(VLV_IOSF_SB_PUNIT));
 563
 564         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 565         val &= ~DSPFREQGUAR_MASK;
 566         val |= (cmd << DSPFREQGUAR_SHIFT);
 567         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 568         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
 569                       DSPFREQSTAT_MASK) == (cmd << DSPFREQSTAT_SHIFT),
 570                      50)) {
 571                 drm_err(&dev_priv->drm,
 572                         "timed out waiting for CDclk change\n");
 573         }
 574
 575         if (cdclk == 400000) {
 576                 u32 divider;
 577
 578                 divider = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1,
 579                                             cdclk) - 1;
 580
 581                 /* adjust cdclk divider */
 582                 val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL);
 583                 val &= ~CCK_FREQUENCY_VALUES;
 584                 val |= divider;
 585                 vlv_cck_write(dev_priv, CCK_DISPLAY_CLOCK_CONTROL, val);
 586
 587                 if (wait_for((vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL) &
 588                               CCK_FREQUENCY_STATUS) == (divider << CCK_FREQUENCY_STATUS_SHIFT),
 589                              50))
 590                         drm_err(&dev_priv->drm,
 591                                 "timed out waiting for CDclk change\n");
 592         }
 593
 594         /* adjust self-refresh exit latency value */
 595         val = vlv_bunit_read(dev_priv, BUNIT_REG_BISOC);
 596         val &= ~0x7f;
 597
 598         /*
 599          * For high bandwidth configs, we set a higher latency in the bunit
 600          * so that the core display fetch happens in time to avoid underruns.
 601          */
 602         if (cdclk == 400000)
 603                 val |= 4500 / 250; /* 4.5 usec */
 604         else
 605                 val |= 3000 / 250; /* 3.0 usec */
 606         vlv_bunit_write(dev_priv, BUNIT_REG_BISOC, val);
 607
 608         vlv_iosf_sb_put(dev_priv,
 609                         BIT(VLV_IOSF_SB_CCK) |
 610                         BIT(VLV_IOSF_SB_BUNIT) |
 611                         BIT(VLV_IOSF_SB_PUNIT));
 612
 613         intel_update_cdclk(dev_priv);
 614
 615         vlv_program_pfi_credits(dev_priv);
 616
 617         intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
 618 }
 619
 620 static void chv_set_cdclk(struct drm_i915_private *dev_priv,
 621                           const struct intel_cdclk_config *cdclk_config,
 622                           enum pipe pipe)
 623 {
 624         int cdclk = cdclk_config->cdclk;
 625         u32 val, cmd = cdclk_config->voltage_level;
 626         intel_wakeref_t wakeref;
 627
 628         switch (cdclk) {
 629         case 333333:
 630         case 320000:
 631         case 266667:
 632         case 200000:
 633                 break;
 634         default:
 635                 MISSING_CASE(cdclk);
 636                 return;
 637         }
 638
 639         /* There are cases where we can end up here with power domains
 640          * off and a CDCLK frequency other than the minimum, like when
 641          * issuing a modeset without actually changing any display after
 642          * a system suspend.  So grab the display core domain, which covers
 643          * the HW blocks needed for the following programming.
 644          */
 645         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
 646
 647         vlv_punit_get(dev_priv);
 648         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 649         val &= ~DSPFREQGUAR_MASK_CHV;
 650         val |= (cmd << DSPFREQGUAR_SHIFT_CHV);
 651         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 652         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
 653                       DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV),
 654                      50)) {
 655                 drm_err(&dev_priv->drm,
 656                         "timed out waiting for CDclk change\n");
 657         }
 658
 659         vlv_punit_put(dev_priv);
 660
 661         intel_update_cdclk(dev_priv);
 662
 663         vlv_program_pfi_credits(dev_priv);
 664
 665         intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
 666 }
 667
 668 static int bdw_calc_cdclk(int min_cdclk)
 669 {
 670         if (min_cdclk > 540000)
 671                 return 675000;
 672         else if (min_cdclk > 450000)
 673                 return 540000;
 674         else if (min_cdclk > 337500)
 675                 return 450000;
 676         else
 677                 return 337500;
 678 }
 679
 680 static u8 bdw_calc_voltage_level(int cdclk)
 681 {
 682         switch (cdclk) {
 683         default:
 684         case 337500:
 685                 return 2;
 686         case 450000:
 687                 return 0;
 688         case 540000:
 689                 return 1;
 690         case 675000:
 691                 return 3;
 692         }
 693 }
 694
 695 static void bdw_get_cdclk(struct drm_i915_private *dev_priv,
 696                           struct intel_cdclk_config *cdclk_config)
 697 {
 698         u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
 699         u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
 700
 701         if (lcpll & LCPLL_CD_SOURCE_FCLK)
 702                 cdclk_config->cdclk = 800000;
 703         else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
 704                 cdclk_config->cdclk = 450000;
 705         else if (freq == LCPLL_CLK_FREQ_450)
 706                 cdclk_config->cdclk = 450000;
 707         else if (freq == LCPLL_CLK_FREQ_54O_BDW)
 708                 cdclk_config->cdclk = 540000;
 709         else if (freq == LCPLL_CLK_FREQ_337_5_BDW)
 710                 cdclk_config->cdclk = 337500;
 711         else
 712                 cdclk_config->cdclk = 675000;
 713
 714         /*
 715          * Can't read this out :( Let's assume it's
 716          * at least what the CDCLK frequency requires.
 717          */
 718         cdclk_config->voltage_level =
 719                 bdw_calc_voltage_level(cdclk_config->cdclk);
 720 }
 721
 722 static void bdw_set_cdclk(struct drm_i915_private *dev_priv,
 723                           const struct intel_cdclk_config *cdclk_config,
 724                           enum pipe pipe)
 725 {
 726         int cdclk = cdclk_config->cdclk;
 727         u32 val;
 728         int ret;
 729
 730         if (WARN((intel_de_read(dev_priv, LCPLL_CTL) &
 731                   (LCPLL_PLL_DISABLE | LCPLL_PLL_LOCK |
 732                    LCPLL_CD_CLOCK_DISABLE | LCPLL_ROOT_CD_CLOCK_DISABLE |
 733                    LCPLL_CD2X_CLOCK_DISABLE | LCPLL_POWER_DOWN_ALLOW |
 734                    LCPLL_CD_SOURCE_FCLK)) != LCPLL_PLL_LOCK,
 735                  "trying to change cdclk frequency with cdclk not enabled\n"))
 736                 return;
 737
 738         ret = sandybridge_pcode_write(dev_priv,
 739                                       BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0);
 740         if (ret) {
 741                 drm_err(&dev_priv->drm,
 742                         "failed to inform pcode about cdclk change\n");
 743                 return;
 744         }
 745
 746         val = intel_de_read(dev_priv, LCPLL_CTL);
 747         val |= LCPLL_CD_SOURCE_FCLK;
 748         intel_de_write(dev_priv, LCPLL_CTL, val);
 749
 750         /*
 751          * According to the spec, it should be enough to poll for this 1 us.
 752          * However, extensive testing shows that this can take longer.
 753          */
 754         if (wait_for_us(intel_de_read(dev_priv, LCPLL_CTL) &
 755                         LCPLL_CD_SOURCE_FCLK_DONE, 100))
 756                 drm_err(&dev_priv->drm, "Switching to FCLK failed\n");
 757
 758         val = intel_de_read(dev_priv, LCPLL_CTL);
 759         val &= ~LCPLL_CLK_FREQ_MASK;
 760
 761         switch (cdclk) {
 762         default:
 763                 MISSING_CASE(cdclk);
 764                 /* fall through */
 765         case 337500:
 766                 val |= LCPLL_CLK_FREQ_337_5_BDW;
 767                 break;
 768         case 450000:
 769                 val |= LCPLL_CLK_FREQ_450;
 770                 break;
 771         case 540000:
 772                 val |= LCPLL_CLK_FREQ_54O_BDW;
 773                 break;
 774         case 675000:
 775                 val |= LCPLL_CLK_FREQ_675_BDW;
 776                 break;
 777         }
 778
 779         intel_de_write(dev_priv, LCPLL_CTL, val);
 780
 781         val = intel_de_read(dev_priv, LCPLL_CTL);
 782         val &= ~LCPLL_CD_SOURCE_FCLK;
 783         intel_de_write(dev_priv, LCPLL_CTL, val);
 784
 785         if (wait_for_us((intel_de_read(dev_priv, LCPLL_CTL) &
 786                          LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
 787                 drm_err(&dev_priv->drm, "Switching back to LCPLL failed\n");
 788
 789         sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ,
 790                                 cdclk_config->voltage_level);
 791
 792         intel_de_write(dev_priv, CDCLK_FREQ,
 793                        DIV_ROUND_CLOSEST(cdclk, 1000) - 1);
 794
 795         intel_update_cdclk(dev_priv);
 796 }
 797
 798 static int skl_calc_cdclk(int min_cdclk, int vco)
 799 {
 800         if (vco == 8640000) {
 801                 if (min_cdclk > 540000)
 802                         return 617143;
 803                 else if (min_cdclk > 432000)
 804                         return 540000;
 805                 else if (min_cdclk > 308571)
 806                         return 432000;
 807                 else
 808                         return 308571;
 809         } else {
 810                 if (min_cdclk > 540000)
 811                         return 675000;
 812                 else if (min_cdclk > 450000)
 813                         return 540000;
 814                 else if (min_cdclk > 337500)
 815                         return 450000;
 816                 else
 817                         return 337500;
 818         }
 819 }
 820
 821 static u8 skl_calc_voltage_level(int cdclk)
 822 {
 823         if (cdclk > 540000)
 824                 return 3;
 825         else if (cdclk > 450000)
 826                 return 2;
 827         else if (cdclk > 337500)
 828                 return 1;
 829         else
 830                 return 0;
 831 }
 832
 833 static void skl_dpll0_update(struct drm_i915_private *dev_priv,
 834                              struct intel_cdclk_config *cdclk_config)
 835 {
 836         u32 val;
 837
 838         cdclk_config->ref = 24000;
 839         cdclk_config->vco = 0;
 840
 841         val = intel_de_read(dev_priv, LCPLL1_CTL);
 842         if ((val & LCPLL_PLL_ENABLE) == 0)
 843                 return;
 844
 845         if (WARN_ON((val & LCPLL_PLL_LOCK) == 0))
 846                 return;
 847
 848         val = intel_de_read(dev_priv, DPLL_CTRL1);
 849
 850         if (WARN_ON((val & (DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
 851                             DPLL_CTRL1_SSC(SKL_DPLL0) |
 852                             DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) !=
 853                     DPLL_CTRL1_OVERRIDE(SKL_DPLL0)))
 854                 return;
 855
 856         switch (val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) {
 857         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0):
 858         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1350, SKL_DPLL0):
 859         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1620, SKL_DPLL0):
 860         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2700, SKL_DPLL0):
 861                 cdclk_config->vco = 8100000;
 862                 break;
 863         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0):
 864         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2160, SKL_DPLL0):
 865                 cdclk_config->vco = 8640000;
 866                 break;
 867         default:
 868                 MISSING_CASE(val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0));
 869                 break;
 870         }
 871 }
 872
 873 static void skl_get_cdclk(struct drm_i915_private *dev_priv,
 874                           struct intel_cdclk_config *cdclk_config)
 875 {
 876         u32 cdctl;
 877
 878         skl_dpll0_update(dev_priv, cdclk_config);
 879
 880         cdclk_config->cdclk = cdclk_config->bypass = cdclk_config->ref;
 881
 882         if (cdclk_config->vco == 0)
 883                 goto out;
 884
 885         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
 886
 887         if (cdclk_config->vco == 8640000) {
 888                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
 889                 case CDCLK_FREQ_450_432:
 890                         cdclk_config->cdclk = 432000;
 891                         break;
 892                 case CDCLK_FREQ_337_308:
 893                         cdclk_config->cdclk = 308571;
 894                         break;
 895                 case CDCLK_FREQ_540:
 896                         cdclk_config->cdclk = 540000;
 897                         break;
 898                 case CDCLK_FREQ_675_617:
 899                         cdclk_config->cdclk = 617143;
 900                         break;
 901                 default:
 902                         MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
 903                         break;
 904                 }
 905         } else {
 906                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
 907                 case CDCLK_FREQ_450_432:
 908                         cdclk_config->cdclk = 450000;
 909                         break;
 910                 case CDCLK_FREQ_337_308:
 911                         cdclk_config->cdclk = 337500;
 912                         break;
 913                 case CDCLK_FREQ_540:
 914                         cdclk_config->cdclk = 540000;
 915                         break;
 916                 case CDCLK_FREQ_675_617:
 917                         cdclk_config->cdclk = 675000;
 918                         break;
 919                 default:
 920                         MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
 921                         break;
 922                 }
 923         }
 924
 925  out:
 926         /*
 927          * Can't read this out :( Let's assume it's
 928          * at least what the CDCLK frequency requires.
 929          */
 930         cdclk_config->voltage_level =
 931                 skl_calc_voltage_level(cdclk_config->cdclk);
 932 }
 933
 934 /* convert from kHz to .1 fixpoint MHz with -1MHz offset */
 935 static int skl_cdclk_decimal(int cdclk)
 936 {
 937         return DIV_ROUND_CLOSEST(cdclk - 1000, 500);
 938 }
 939
 940 static void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv,
 941                                         int vco)
 942 {
 943         bool changed = dev_priv->skl_preferred_vco_freq != vco;
 944
 945         dev_priv->skl_preferred_vco_freq = vco;
 946
 947         if (changed)
 948                 intel_update_max_cdclk(dev_priv);
 949 }
 950
 951 static void skl_dpll0_enable(struct drm_i915_private *dev_priv, int vco)
 952 {
 953         u32 val;
 954
 955         WARN_ON(vco != 8100000 && vco != 8640000);
 956
 957         /*
 958          * We always enable DPLL0 with the lowest link rate possible, but still
 959          * taking into account the VCO required to operate the eDP panel at the
 960          * desired frequency. The usual DP link rates operate with a VCO of
 961          * 8100 while the eDP 1.4 alternate link rates need a VCO of 8640.
 962          * The modeset code is responsible for the selection of the exact link
 963          * rate later on, with the constraint of choosing a frequency that
 964          * works with vco.
 965          */
 966         val = intel_de_read(dev_priv, DPLL_CTRL1);
 967
 968         val &= ~(DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) | DPLL_CTRL1_SSC(SKL_DPLL0) |
 969                  DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0));
 970         val |= DPLL_CTRL1_OVERRIDE(SKL_DPLL0);
 971         if (vco == 8640000)
 972                 val |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080,
 973                                             SKL_DPLL0);
 974         else
 975                 val |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810,
 976                                             SKL_DPLL0);
 977
 978         intel_de_write(dev_priv, DPLL_CTRL1, val);
 979         intel_de_posting_read(dev_priv, DPLL_CTRL1);
 980
 981         intel_de_write(dev_priv, LCPLL1_CTL,
 982                        intel_de_read(dev_priv, LCPLL1_CTL) | LCPLL_PLL_ENABLE);
 983
 984         if (intel_de_wait_for_set(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 5))
 985                 drm_err(&dev_priv->drm, "DPLL0 not locked\n");
 986
 987         dev_priv->cdclk.hw.vco = vco;
 988
 989         /* We'll want to keep using the current vco from now on. */
 990         skl_set_preferred_cdclk_vco(dev_priv, vco);
 991 }
 992
 993 static void skl_dpll0_disable(struct drm_i915_private *dev_priv)
 994 {
 995         intel_de_write(dev_priv, LCPLL1_CTL,
 996                        intel_de_read(dev_priv, LCPLL1_CTL) & ~LCPLL_PLL_ENABLE);
 997         if (intel_de_wait_for_clear(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 1))
 998                 drm_err(&dev_priv->drm, "Couldn't disable DPLL0\n");
 999
1000         dev_priv->cdclk.hw.vco = 0;
1001 }
1002
1003 static void skl_set_cdclk(struct drm_i915_private *dev_priv,
1004                           const struct intel_cdclk_config *cdclk_config,
1005                           enum pipe pipe)
1006 {
1007         int cdclk = cdclk_config->cdclk;
1008         int vco = cdclk_config->vco;
1009         u32 freq_select, cdclk_ctl;
1010         int ret;
1011
1012         /*
1013          * Based on WA#1183 CDCLK rates 308 and 617MHz CDCLK rates are
1014          * unsupported on SKL. In theory this should never happen since only
1015          * the eDP1.4 2.16 and 4.32Gbps rates require it, but eDP1.4 is not
1016          * supported on SKL either, see the above WA. WARN whenever trying to
1017          * use the corresponding VCO freq as that always leads to using the
1018          * minimum 308MHz CDCLK.
1019          */
1020         WARN_ON_ONCE(IS_SKYLAKE(dev_priv) && vco == 8640000);
1021
1022         ret = skl_pcode_request(dev_priv, SKL_PCODE_CDCLK_CONTROL,
1023                                 SKL_CDCLK_PREPARE_FOR_CHANGE,
1024                                 SKL_CDCLK_READY_FOR_CHANGE,
1025                                 SKL_CDCLK_READY_FOR_CHANGE, 3);
1026         if (ret) {
1027                 drm_err(&dev_priv->drm,
1028                         "Failed to inform PCU about cdclk change (%d)\n", ret);
1029                 return;
1030         }
1031
1032         /* Choose frequency for this cdclk */
1033         switch (cdclk) {
1034         default:
1035                 WARN_ON(cdclk != dev_priv->cdclk.hw.bypass);
1036                 WARN_ON(vco != 0);
1037                 /* fall through */
1038         case 308571:
1039         case 337500:
1040                 freq_select = CDCLK_FREQ_337_308;
1041                 break;
1042         case 450000:
1043         case 432000:
1044                 freq_select = CDCLK_FREQ_450_432;
1045                 break;
1046         case 540000:
1047                 freq_select = CDCLK_FREQ_540;
1048                 break;
1049         case 617143:
1050         case 675000:
1051                 freq_select = CDCLK_FREQ_675_617;
1052                 break;
1053         }
1054
1055         if (dev_priv->cdclk.hw.vco != 0 &&
1056             dev_priv->cdclk.hw.vco != vco)
1057                 skl_dpll0_disable(dev_priv);
1058
1059         cdclk_ctl = intel_de_read(dev_priv, CDCLK_CTL);
1060
1061         if (dev_priv->cdclk.hw.vco != vco) {
1062                 /* Wa Display #1183: skl,kbl,cfl */
1063                 cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1064                 cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1065                 intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1066         }
1067
1068         /* Wa Display #1183: skl,kbl,cfl */
1069         cdclk_ctl |= CDCLK_DIVMUX_CD_OVERRIDE;
1070         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1071         intel_de_posting_read(dev_priv, CDCLK_CTL);
1072
1073         if (dev_priv->cdclk.hw.vco != vco)
1074                 skl_dpll0_enable(dev_priv, vco);
1075
1076         /* Wa Display #1183: skl,kbl,cfl */
1077         cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1078         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1079
1080         cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1081         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1082
1083         /* Wa Display #1183: skl,kbl,cfl */
1084         cdclk_ctl &= ~CDCLK_DIVMUX_CD_OVERRIDE;
1085         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1086         intel_de_posting_read(dev_priv, CDCLK_CTL);
1087
1088         /* inform PCU of the change */
1089         sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL,
1090                                 cdclk_config->voltage_level);
1091
1092         intel_update_cdclk(dev_priv);
1093 }
1094
1095 static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
1096 {
1097         u32 cdctl, expected;
1098
1099         /*
1100          * check if the pre-os initialized the display
1101          * There is SWF18 scratchpad register defined which is set by the
1102          * pre-os which can be used by the OS drivers to check the status
1103          */
1104         if ((intel_de_read(dev_priv, SWF_ILK(0x18)) & 0x00FFFFFF) == 0)
1105                 goto sanitize;
1106
1107         intel_update_cdclk(dev_priv);
1108         intel_dump_cdclk_config(&dev_priv->cdclk.hw, "Current CDCLK");
1109
1110         /* Is PLL enabled and locked ? */
1111         if (dev_priv->cdclk.hw.vco == 0 ||
1112             dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.bypass)
1113                 goto sanitize;
1114
1115         /* DPLL okay; verify the cdclock
1116          *
1117          * Noticed in some instances that the freq selection is correct but
1118          * decimal part is programmed wrong from BIOS where pre-os does not
1119          * enable display. Verify the same as well.
1120          */
1121         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
1122         expected = (cdctl & CDCLK_FREQ_SEL_MASK) |
1123                 skl_cdclk_decimal(dev_priv->cdclk.hw.cdclk);
1124         if (cdctl == expected)
1125                 /* All well; nothing to sanitize */
1126                 return;
1127
1128 sanitize:
1129         drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
1130
1131         /* force cdclk programming */
1132         dev_priv->cdclk.hw.cdclk = 0;
1133         /* force full PLL disable + enable */
1134         dev_priv->cdclk.hw.vco = -1;
1135 }
1136
1137 static void skl_init_cdclk(struct drm_i915_private *dev_priv)
1138 {
1139         struct intel_cdclk_config cdclk_config;
1140
1141         skl_sanitize_cdclk(dev_priv);
1142
1143         if (dev_priv->cdclk.hw.cdclk != 0 &&
1144             dev_priv->cdclk.hw.vco != 0) {
1145                 /*
1146                  * Use the current vco as our initial
1147                  * guess as to what the preferred vco is.
1148                  */
1149                 if (dev_priv->skl_preferred_vco_freq == 0)
1150                         skl_set_preferred_cdclk_vco(dev_priv,
1151                                                     dev_priv->cdclk.hw.vco);
1152                 return;
1153         }
1154
1155         cdclk_config = dev_priv->cdclk.hw;
1156
1157         cdclk_config.vco = dev_priv->skl_preferred_vco_freq;
1158         if (cdclk_config.vco == 0)
1159                 cdclk_config.vco = 8100000;
1160         cdclk_config.cdclk = skl_calc_cdclk(0, cdclk_config.vco);
1161         cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1162
1163         skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1164 }
1165
1166 static void skl_uninit_cdclk(struct drm_i915_private *dev_priv)
1167 {
1168         struct intel_cdclk_config cdclk_config = dev_priv->cdclk.hw;
1169
1170         cdclk_config.cdclk = cdclk_config.bypass;
1171         cdclk_config.vco = 0;
1172         cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1173
1174         skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1175 }
1176
1177 static const struct intel_cdclk_vals bxt_cdclk_table[] = {
1178         { .refclk = 19200, .cdclk = 144000, .divider = 8, .ratio = 60 },
1179         { .refclk = 19200, .cdclk = 288000, .divider = 4, .ratio = 60 },
1180         { .refclk = 19200, .cdclk = 384000, .divider = 3, .ratio = 60 },
1181         { .refclk = 19200, .cdclk = 576000, .divider = 2, .ratio = 60 },
1182         { .refclk = 19200, .cdclk = 624000, .divider = 2, .ratio = 65 },
1183         {}
1184 };
1185
1186 static const struct intel_cdclk_vals glk_cdclk_table[] = {
1187         { .refclk = 19200, .cdclk =  79200, .divider = 8, .ratio = 33 },
1188         { .refclk = 19200, .cdclk = 158400, .divider = 4, .ratio = 33 },
1189         { .refclk = 19200, .cdclk = 316800, .divider = 2, .ratio = 33 },
1190         {}
1191 };
1192
1193 static const struct intel_cdclk_vals cnl_cdclk_table[] = {
1194         { .refclk = 19200, .cdclk = 168000, .divider = 4, .ratio = 35 },
1195         { .refclk = 19200, .cdclk = 336000, .divider = 2, .ratio = 35 },
1196         { .refclk = 19200, .cdclk = 528000, .divider = 2, .ratio = 55 },
1197
1198         { .refclk = 24000, .cdclk = 168000, .divider = 4, .ratio = 28 },
1199         { .refclk = 24000, .cdclk = 336000, .divider = 2, .ratio = 28 },
1200         { .refclk = 24000, .cdclk = 528000, .divider = 2, .ratio = 44 },
1201         {}
1202 };
1203
1204 static const struct intel_cdclk_vals icl_cdclk_table[] = {
1205         { .refclk = 19200, .cdclk = 172800, .divider = 2, .ratio = 18 },
1206         { .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 },
1207         { .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1208         { .refclk = 19200, .cdclk = 326400, .divider = 4, .ratio = 68 },
1209         { .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1210         { .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1211
1212         { .refclk = 24000, .cdclk = 180000, .divider = 2, .ratio = 15 },
1213         { .refclk = 24000, .cdclk = 192000, .divider = 2, .ratio = 16 },
1214         { .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1215         { .refclk = 24000, .cdclk = 324000, .divider = 4, .ratio = 54 },
1216         { .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1217         { .refclk = 24000, .cdclk = 648000, .divider = 2, .ratio = 54 },
1218
1219         { .refclk = 38400, .cdclk = 172800, .divider = 2, .ratio =  9 },
1220         { .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
1221         { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1222         { .refclk = 38400, .cdclk = 326400, .divider = 4, .ratio = 34 },
1223         { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1224         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1225         {}
1226 };
1227
1228 static int bxt_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
1229 {
1230         const struct intel_cdclk_vals *table = dev_priv->cdclk.table;
1231         int i;
1232
1233         for (i = 0; table[i].refclk; i++)
1234                 if (table[i].refclk == dev_priv->cdclk.hw.ref &&
1235                     table[i].cdclk >= min_cdclk)
1236                         return table[i].cdclk;
1237
1238         WARN(1, "Cannot satisfy minimum cdclk %d with refclk %u\n",
1239              min_cdclk, dev_priv->cdclk.hw.ref);
1240         return 0;
1241 }
1242
1243 static int bxt_calc_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
1244 {
1245         const struct intel_cdclk_vals *table = dev_priv->cdclk.table;
1246         int i;
1247
1248         if (cdclk == dev_priv->cdclk.hw.bypass)
1249                 return 0;
1250
1251         for (i = 0; table[i].refclk; i++)
1252                 if (table[i].refclk == dev_priv->cdclk.hw.ref &&
1253                     table[i].cdclk == cdclk)
1254                         return dev_priv->cdclk.hw.ref * table[i].ratio;
1255
1256         WARN(1, "cdclk %d not valid for refclk %u\n",
1257              cdclk, dev_priv->cdclk.hw.ref);
1258         return 0;
1259 }
1260
1261 static u8 bxt_calc_voltage_level(int cdclk)
1262 {
1263         return DIV_ROUND_UP(cdclk, 25000);
1264 }
1265
1266 static u8 cnl_calc_voltage_level(int cdclk)
1267 {
1268         if (cdclk > 336000)
1269                 return 2;
1270         else if (cdclk > 168000)
1271                 return 1;
1272         else
1273                 return 0;
1274 }
1275
1276 static u8 icl_calc_voltage_level(int cdclk)
1277 {
1278         if (cdclk > 556800)
1279                 return 2;
1280         else if (cdclk > 312000)
1281                 return 1;
1282         else
1283                 return 0;
1284 }
1285
1286 static u8 ehl_calc_voltage_level(int cdclk)
1287 {
1288         if (cdclk > 326400)
1289                 return 3;
1290         else if (cdclk > 312000)
1291                 return 2;
1292         else if (cdclk > 180000)
1293                 return 1;
1294         else
1295                 return 0;
1296 }
1297
1298 static void cnl_readout_refclk(struct drm_i915_private *dev_priv,
1299                                struct intel_cdclk_config *cdclk_config)
1300 {
1301         if (intel_de_read(dev_priv, SKL_DSSM) & CNL_DSSM_CDCLK_PLL_REFCLK_24MHz)
1302                 cdclk_config->ref = 24000;
1303         else
1304                 cdclk_config->ref = 19200;
1305 }
1306
1307 static void icl_readout_refclk(struct drm_i915_private *dev_priv,
1308                                struct intel_cdclk_config *cdclk_config)
1309 {
1310         u32 dssm = intel_de_read(dev_priv, SKL_DSSM) & ICL_DSSM_CDCLK_PLL_REFCLK_MASK;
1311
1312         switch (dssm) {
1313         default:
1314                 MISSING_CASE(dssm);
1315                 /* fall through */
1316         case ICL_DSSM_CDCLK_PLL_REFCLK_24MHz:
1317                 cdclk_config->ref = 24000;
1318                 break;
1319         case ICL_DSSM_CDCLK_PLL_REFCLK_19_2MHz:
1320                 cdclk_config->ref = 19200;
1321                 break;
1322         case ICL_DSSM_CDCLK_PLL_REFCLK_38_4MHz:
1323                 cdclk_config->ref = 38400;
1324                 break;
1325         }
1326 }
1327
1328 static void bxt_de_pll_readout(struct drm_i915_private *dev_priv,
1329                                struct intel_cdclk_config *cdclk_config)
1330 {
1331         u32 val, ratio;
1332
1333         if (INTEL_GEN(dev_priv) >= 11)
1334                 icl_readout_refclk(dev_priv, cdclk_config);
1335         else if (IS_CANNONLAKE(dev_priv))
1336                 cnl_readout_refclk(dev_priv, cdclk_config);
1337         else
1338                 cdclk_config->ref = 19200;
1339
1340         val = intel_de_read(dev_priv, BXT_DE_PLL_ENABLE);
1341         if ((val & BXT_DE_PLL_PLL_ENABLE) == 0 ||
1342             (val & BXT_DE_PLL_LOCK) == 0) {
1343                 /*
1344                  * CDCLK PLL is disabled, the VCO/ratio doesn't matter, but
1345                  * setting it to zero is a way to signal that.
1346                  */
1347                 cdclk_config->vco = 0;
1348                 return;
1349         }
1350
1351         /*
1352          * CNL+ have the ratio directly in the PLL enable register, gen9lp had
1353          * it in a separate PLL control register.
1354          */
1355         if (INTEL_GEN(dev_priv) >= 10)
1356                 ratio = val & CNL_CDCLK_PLL_RATIO_MASK;
1357         else
1358                 ratio = intel_de_read(dev_priv, BXT_DE_PLL_CTL) & BXT_DE_PLL_RATIO_MASK;
1359
1360         cdclk_config->vco = ratio * cdclk_config->ref;
1361 }
1362
1363 static void bxt_get_cdclk(struct drm_i915_private *dev_priv,
1364                           struct intel_cdclk_config *cdclk_config)
1365 {
1366         u32 divider;
1367         int div;
1368
1369         bxt_de_pll_readout(dev_priv, cdclk_config);
1370
1371         if (INTEL_GEN(dev_priv) >= 12)
1372                 cdclk_config->bypass = cdclk_config->ref / 2;
1373         else if (INTEL_GEN(dev_priv) >= 11)
1374                 cdclk_config->bypass = 50000;
1375         else
1376                 cdclk_config->bypass = cdclk_config->ref;
1377
1378         if (cdclk_config->vco == 0) {
1379                 cdclk_config->cdclk = cdclk_config->bypass;
1380                 goto out;
1381         }
1382
1383         divider = intel_de_read(dev_priv, CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK;
1384
1385         switch (divider) {
1386         case BXT_CDCLK_CD2X_DIV_SEL_1:
1387                 div = 2;
1388                 break;
1389         case BXT_CDCLK_CD2X_DIV_SEL_1_5:
1390                 WARN(IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10,
1391                      "Unsupported divider\n");
1392                 div = 3;
1393                 break;
1394         case BXT_CDCLK_CD2X_DIV_SEL_2:
1395                 div = 4;
1396                 break;
1397         case BXT_CDCLK_CD2X_DIV_SEL_4:
1398                 WARN(INTEL_GEN(dev_priv) >= 10, "Unsupported divider\n");
1399                 div = 8;
1400                 break;
1401         default:
1402                 MISSING_CASE(divider);
1403                 return;
1404         }
1405
1406         cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco, div);
1407
1408  out:
1409         /*
1410          * Can't read this out :( Let's assume it's
1411          * at least what the CDCLK frequency requires.
1412          */
1413         cdclk_config->voltage_level =
1414                 dev_priv->display.calc_voltage_level(cdclk_config->cdclk);
1415 }
1416
1417 static void bxt_de_pll_disable(struct drm_i915_private *dev_priv)
1418 {
1419         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, 0);
1420
1421         /* Timeout 200us */
1422         if (intel_de_wait_for_clear(dev_priv,
1423                                     BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1424                 drm_err(&dev_priv->drm, "timeout waiting for DE PLL unlock\n");
1425
1426         dev_priv->cdclk.hw.vco = 0;
1427 }
1428
1429 static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco)
1430 {
1431         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref);
1432         u32 val;
1433
1434         val = intel_de_read(dev_priv, BXT_DE_PLL_CTL);
1435         val &= ~BXT_DE_PLL_RATIO_MASK;
1436         val |= BXT_DE_PLL_RATIO(ratio);
1437         intel_de_write(dev_priv, BXT_DE_PLL_CTL, val);
1438
1439         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE);
1440
1441         /* Timeout 200us */
1442         if (intel_de_wait_for_set(dev_priv,
1443                                   BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1444                 drm_err(&dev_priv->drm, "timeout waiting for DE PLL lock\n");
1445
1446         dev_priv->cdclk.hw.vco = vco;
1447 }
1448
1449 static void cnl_cdclk_pll_disable(struct drm_i915_private *dev_priv)
1450 {
1451         u32 val;
1452
1453         val = intel_de_read(dev_priv, BXT_DE_PLL_ENABLE);
1454         val &= ~BXT_DE_PLL_PLL_ENABLE;
1455         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1456
1457         /* Timeout 200us */
1458         if (wait_for((intel_de_read(dev_priv, BXT_DE_PLL_ENABLE) & BXT_DE_PLL_LOCK) == 0, 1))
1459                 drm_err(&dev_priv->drm,
1460                         "timeout waiting for CDCLK PLL unlock\n");
1461
1462         dev_priv->cdclk.hw.vco = 0;
1463 }
1464
1465 static void cnl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco)
1466 {
1467         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref);
1468         u32 val;
1469
1470         val = CNL_CDCLK_PLL_RATIO(ratio);
1471         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1472
1473         val |= BXT_DE_PLL_PLL_ENABLE;
1474         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1475
1476         /* Timeout 200us */
1477         if (wait_for((intel_de_read(dev_priv, BXT_DE_PLL_ENABLE) & BXT_DE_PLL_LOCK) != 0, 1))
1478                 drm_err(&dev_priv->drm,
1479                         "timeout waiting for CDCLK PLL lock\n");
1480
1481         dev_priv->cdclk.hw.vco = vco;
1482 }
1483
1484 static u32 bxt_cdclk_cd2x_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
1485 {
1486         if (INTEL_GEN(dev_priv) >= 12) {
1487                 if (pipe == INVALID_PIPE)
1488                         return TGL_CDCLK_CD2X_PIPE_NONE;
1489                 else
1490                         return TGL_CDCLK_CD2X_PIPE(pipe);
1491         } else if (INTEL_GEN(dev_priv) >= 11) {
1492                 if (pipe == INVALID_PIPE)
1493                         return ICL_CDCLK_CD2X_PIPE_NONE;
1494                 else
1495                         return ICL_CDCLK_CD2X_PIPE(pipe);
1496         } else {
1497                 if (pipe == INVALID_PIPE)
1498                         return BXT_CDCLK_CD2X_PIPE_NONE;
1499                 else
1500                         return BXT_CDCLK_CD2X_PIPE(pipe);
1501         }
1502 }
1503
1504 static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
1505                           const struct intel_cdclk_config *cdclk_config,
1506                           enum pipe pipe)
1507 {
1508         int cdclk = cdclk_config->cdclk;
1509         int vco = cdclk_config->vco;
1510         u32 val, divider;
1511         int ret;
1512
1513         /* Inform power controller of upcoming frequency change. */
1514         if (INTEL_GEN(dev_priv) >= 10)
1515                 ret = skl_pcode_request(dev_priv, SKL_PCODE_CDCLK_CONTROL,
1516                                         SKL_CDCLK_PREPARE_FOR_CHANGE,
1517                                         SKL_CDCLK_READY_FOR_CHANGE,
1518                                         SKL_CDCLK_READY_FOR_CHANGE, 3);
1519         else
1520                 /*
1521                  * BSpec requires us to wait up to 150usec, but that leads to
1522                  * timeouts; the 2ms used here is based on experiment.
1523                  */
1524                 ret = sandybridge_pcode_write_timeout(dev_priv,
1525                                                       HSW_PCODE_DE_WRITE_FREQ_REQ,
1526                                                       0x80000000, 150, 2);
1527
1528         if (ret) {
1529                 drm_err(&dev_priv->drm,
1530                         "Failed to inform PCU about cdclk change (err %d, freq %d)\n",
1531                         ret, cdclk);
1532                 return;
1533         }
1534
1535         /* cdclk = vco / 2 / div{1,1.5,2,4} */
1536         switch (DIV_ROUND_CLOSEST(vco, cdclk)) {
1537         default:
1538                 WARN_ON(cdclk != dev_priv->cdclk.hw.bypass);
1539                 WARN_ON(vco != 0);
1540                 /* fall through */
1541         case 2:
1542                 divider = BXT_CDCLK_CD2X_DIV_SEL_1;
1543                 break;
1544         case 3:
1545                 WARN(IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10,
1546                      "Unsupported divider\n");
1547                 divider = BXT_CDCLK_CD2X_DIV_SEL_1_5;
1548                 break;
1549         case 4:
1550                 divider = BXT_CDCLK_CD2X_DIV_SEL_2;
1551                 break;
1552         case 8:
1553                 WARN(INTEL_GEN(dev_priv) >= 10, "Unsupported divider\n");
1554                 divider = BXT_CDCLK_CD2X_DIV_SEL_4;
1555                 break;
1556         }
1557
1558         if (INTEL_GEN(dev_priv) >= 10) {
1559                 if (dev_priv->cdclk.hw.vco != 0 &&
1560                     dev_priv->cdclk.hw.vco != vco)
1561                         cnl_cdclk_pll_disable(dev_priv);
1562
1563                 if (dev_priv->cdclk.hw.vco != vco)
1564                         cnl_cdclk_pll_enable(dev_priv, vco);
1565
1566         } else {
1567                 if (dev_priv->cdclk.hw.vco != 0 &&
1568                     dev_priv->cdclk.hw.vco != vco)
1569                         bxt_de_pll_disable(dev_priv);
1570
1571                 if (dev_priv->cdclk.hw.vco != vco)
1572                         bxt_de_pll_enable(dev_priv, vco);
1573         }
1574
1575         val = divider | skl_cdclk_decimal(cdclk) |
1576                 bxt_cdclk_cd2x_pipe(dev_priv, pipe);
1577
1578         /*
1579          * Disable SSA Precharge when CD clock frequency < 500 MHz,
1580          * enable otherwise.
1581          */
1582         if (IS_GEN9_LP(dev_priv) && cdclk >= 500000)
1583                 val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
1584         intel_de_write(dev_priv, CDCLK_CTL, val);
1585
1586         if (pipe != INVALID_PIPE)
1587                 intel_wait_for_vblank(dev_priv, pipe);
1588
1589         if (INTEL_GEN(dev_priv) >= 10) {
1590                 ret = sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL,
1591                                               cdclk_config->voltage_level);
1592         } else {
1593                 /*
1594                  * The timeout isn't specified, the 2ms used here is based on
1595                  * experiment.
1596                  * FIXME: Waiting for the request completion could be delayed
1597                  * until the next PCODE request based on BSpec.
1598                  */
1599                 ret = sandybridge_pcode_write_timeout(dev_priv,
1600                                                       HSW_PCODE_DE_WRITE_FREQ_REQ,
1601                                                       cdclk_config->voltage_level,
1602                                                       150, 2);
1603         }
1604
1605         if (ret) {
1606                 drm_err(&dev_priv->drm,
1607                         "PCode CDCLK freq set failed, (err %d, freq %d)\n",
1608                         ret, cdclk);
1609                 return;
1610         }
1611
1612         intel_update_cdclk(dev_priv);
1613
1614         if (INTEL_GEN(dev_priv) >= 10)
1615                 /*
1616                  * Can't read out the voltage level :(
1617                  * Let's just assume everything is as expected.
1618                  */
1619                 dev_priv->cdclk.hw.voltage_level = cdclk_config->voltage_level;
1620 }
1621
1622 static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv)
1623 {
1624         u32 cdctl, expected;
1625         int cdclk, vco;
1626
1627         intel_update_cdclk(dev_priv);
1628         intel_dump_cdclk_config(&dev_priv->cdclk.hw, "Current CDCLK");
1629
1630         if (dev_priv->cdclk.hw.vco == 0 ||
1631             dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.bypass)
1632                 goto sanitize;
1633
1634         /* DPLL okay; verify the cdclock
1635          *
1636          * Some BIOS versions leave an incorrect decimal frequency value and
1637          * set reserved MBZ bits in CDCLK_CTL at least during exiting from S4,
1638          * so sanitize this register.
1639          */
1640         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
1641         /*
1642          * Let's ignore the pipe field, since BIOS could have configured the
1643          * dividers both synching to an active pipe, or asynchronously
1644          * (PIPE_NONE).
1645          */
1646         cdctl &= ~bxt_cdclk_cd2x_pipe(dev_priv, INVALID_PIPE);
1647
1648         /* Make sure this is a legal cdclk value for the platform */
1649         cdclk = bxt_calc_cdclk(dev_priv, dev_priv->cdclk.hw.cdclk);
1650         if (cdclk != dev_priv->cdclk.hw.cdclk)
1651                 goto sanitize;
1652
1653         /* Make sure the VCO is correct for the cdclk */
1654         vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
1655         if (vco != dev_priv->cdclk.hw.vco)
1656                 goto sanitize;
1657
1658         expected = skl_cdclk_decimal(cdclk);
1659
1660         /* Figure out what CD2X divider we should be using for this cdclk */
1661         switch (DIV_ROUND_CLOSEST(dev_priv->cdclk.hw.vco,
1662                                   dev_priv->cdclk.hw.cdclk)) {
1663         case 2:
1664                 expected |= BXT_CDCLK_CD2X_DIV_SEL_1;
1665                 break;
1666         case 3:
1667                 expected |= BXT_CDCLK_CD2X_DIV_SEL_1_5;
1668                 break;
1669         case 4:
1670                 expected |= BXT_CDCLK_CD2X_DIV_SEL_2;
1671                 break;
1672         case 8:
1673                 expected |= BXT_CDCLK_CD2X_DIV_SEL_4;
1674                 break;
1675         default:
1676                 goto sanitize;
1677         }
1678
1679         /*
1680          * Disable SSA Precharge when CD clock frequency < 500 MHz,
1681          * enable otherwise.
1682          */
1683         if (IS_GEN9_LP(dev_priv) && dev_priv->cdclk.hw.cdclk >= 500000)
1684                 expected |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
1685
1686         if (cdctl == expected)
1687                 /* All well; nothing to sanitize */
1688                 return;
1689
1690 sanitize:
1691         drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
1692
1693         /* force cdclk programming */
1694         dev_priv->cdclk.hw.cdclk = 0;
1695
1696         /* force full PLL disable + enable */
1697         dev_priv->cdclk.hw.vco = -1;
1698 }
1699
1700 static void bxt_init_cdclk(struct drm_i915_private *dev_priv)
1701 {
1702         struct intel_cdclk_config cdclk_config;
1703
1704         bxt_sanitize_cdclk(dev_priv);
1705
1706         if (dev_priv->cdclk.hw.cdclk != 0 &&
1707             dev_priv->cdclk.hw.vco != 0)
1708                 return;
1709
1710         cdclk_config = dev_priv->cdclk.hw;
1711
1712         /*
1713          * FIXME:
1714          * - The initial CDCLK needs to be read from VBT.
1715          *   Need to make this change after VBT has changes for BXT.
1716          */
1717         cdclk_config.cdclk = bxt_calc_cdclk(dev_priv, 0);
1718         cdclk_config.vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk_config.cdclk);
1719         cdclk_config.voltage_level =
1720                 dev_priv->display.calc_voltage_level(cdclk_config.cdclk);
1721
1722         bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1723 }
1724
1725 static void bxt_uninit_cdclk(struct drm_i915_private *dev_priv)
1726 {
1727         struct intel_cdclk_config cdclk_config = dev_priv->cdclk.hw;
1728
1729         cdclk_config.cdclk = cdclk_config.bypass;
1730         cdclk_config.vco = 0;
1731         cdclk_config.voltage_level =
1732                 dev_priv->display.calc_voltage_level(cdclk_config.cdclk);
1733
1734         bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1735 }
1736
1737 /**
1738  * intel_cdclk_init - Initialize CDCLK
1739  * @i915: i915 device
1740  *
1741  * Initialize CDCLK. This consists mainly of initializing dev_priv->cdclk.hw and
1742  * sanitizing the state of the hardware if needed. This is generally done only
1743  * during the display core initialization sequence, after which the DMC will
1744  * take care of turning CDCLK off/on as needed.
1745  */
1746 void intel_cdclk_init(struct drm_i915_private *i915)
1747 {
1748         if (IS_GEN9_LP(i915) || INTEL_GEN(i915) >= 10)
1749                 bxt_init_cdclk(i915);
1750         else if (IS_GEN9_BC(i915))
1751                 skl_init_cdclk(i915);
1752 }
1753
1754 /**
1755  * intel_cdclk_uninit - Uninitialize CDCLK
1756  * @i915: i915 device
1757  *
1758  * Uninitialize CDCLK. This is done only during the display core
1759  * uninitialization sequence.
1760  */
1761 void intel_cdclk_uninit(struct drm_i915_private *i915)
1762 {
1763         if (INTEL_GEN(i915) >= 10 || IS_GEN9_LP(i915))
1764                 bxt_uninit_cdclk(i915);
1765         else if (IS_GEN9_BC(i915))
1766                 skl_uninit_cdclk(i915);
1767 }
1768
1769 /**
1770  * intel_cdclk_needs_modeset - Determine if changong between the CDCLK
1771  *                             configurations requires a modeset on all pipes
1772  * @a: first CDCLK configuration
1773  * @b: second CDCLK configuration
1774  *
1775  * Returns:
1776  * True if changing between the two CDCLK configurations
1777  * requires all pipes to be off, false if not.
1778  */
1779 bool intel_cdclk_needs_modeset(const struct intel_cdclk_config *a,
1780                                const struct intel_cdclk_config *b)
1781 {
1782         return a->cdclk != b->cdclk ||
1783                 a->vco != b->vco ||
1784                 a->ref != b->ref;
1785 }
1786
1787 /**
1788  * intel_cdclk_can_cd2x_update - Determine if changing between the two CDCLK
1789  *                               configurations requires only a cd2x divider update
1790  * @dev_priv: i915 device
1791  * @a: first CDCLK configuration
1792  * @b: second CDCLK configuration
1793  *
1794  * Returns:
1795  * True if changing between the two CDCLK configurations
1796  * can be done with just a cd2x divider update, false if not.
1797  */
1798 static bool intel_cdclk_can_cd2x_update(struct drm_i915_private *dev_priv,
1799                                         const struct intel_cdclk_config *a,
1800                                         const struct intel_cdclk_config *b)
1801 {
1802         /* Older hw doesn't have the capability */
1803         if (INTEL_GEN(dev_priv) < 10 && !IS_GEN9_LP(dev_priv))
1804                 return false;
1805
1806         return a->cdclk != b->cdclk &&
1807                 a->vco == b->vco &&
1808                 a->ref == b->ref;
1809 }
1810
1811 /**
1812  * intel_cdclk_changed - Determine if two CDCLK configurations are different
1813  * @a: first CDCLK configuration
1814  * @b: second CDCLK configuration
1815  *
1816  * Returns:
1817  * True if the CDCLK configurations don't match, false if they do.
1818  */
1819 static bool intel_cdclk_changed(const struct intel_cdclk_config *a,
1820                                 const struct intel_cdclk_config *b)
1821 {
1822         return intel_cdclk_needs_modeset(a, b) ||
1823                 a->voltage_level != b->voltage_level;
1824 }
1825
1826 /**
1827  * intel_cdclk_clear_state - clear the cdclk state
1828  * @state: atomic state
1829  *
1830  * Clear the cdclk state for ww_mutex backoff.
1831  */
1832 void intel_cdclk_clear_state(struct intel_atomic_state *state)
1833 {
1834         memset(&state->cdclk_state, 0, sizeof(state->cdclk_state));
1835         state->cdclk_state.pipe = INVALID_PIPE;
1836 }
1837
1838 /**
1839  * intel_cdclk_swap_state - make atomic CDCLK configuration effective
1840  * @state: atomic state
1841  *
1842  * This is the CDCLK version of drm_atomic_helper_swap_state() since the
1843  * helper does not handle driver-specific global state.
1844  *
1845  * Similarly to the atomic helpers this function does a complete swap,
1846  * i.e. it also puts the old state into @state. This is used by the commit
1847  * code to determine how CDCLK has changed (for instance did it increase or
1848  * decrease).
1849  */
1850 void intel_cdclk_swap_state(struct intel_atomic_state *state)
1851 {
1852         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
1853
1854         /* FIXME maybe swap() these too */
1855         memcpy(dev_priv->cdclk_state.min_cdclk,
1856                state->cdclk_state.min_cdclk,
1857                sizeof(state->cdclk_state.min_cdclk));
1858         memcpy(dev_priv->cdclk_state.min_voltage_level,
1859                state->cdclk_state.min_voltage_level,
1860                sizeof(state->cdclk_state.min_voltage_level));
1861
1862         dev_priv->cdclk_state.force_min_cdclk =
1863                 state->cdclk_state.force_min_cdclk;
1864
1865         swap(state->cdclk_state.logical, dev_priv->cdclk_state.logical);
1866         swap(state->cdclk_state.actual, dev_priv->cdclk_state.actual);
1867 }
1868
1869 void intel_dump_cdclk_config(const struct intel_cdclk_config *cdclk_config,
1870                              const char *context)
1871 {
1872         DRM_DEBUG_DRIVER("%s %d kHz, VCO %d kHz, ref %d kHz, bypass %d kHz, voltage level %d\n",
1873                          context, cdclk_config->cdclk, cdclk_config->vco,
1874                          cdclk_config->ref, cdclk_config->bypass,
1875                          cdclk_config->voltage_level);
1876 }
1877
1878 /**
1879  * intel_set_cdclk - Push the CDCLK configuration to the hardware
1880  * @dev_priv: i915 device
1881  * @cdclk_config: new CDCLK configuration
1882  * @pipe: pipe with which to synchronize the update
1883  *
1884  * Program the hardware based on the passed in CDCLK state,
1885  * if necessary.
1886  */
1887 static void intel_set_cdclk(struct drm_i915_private *dev_priv,
1888                             const struct intel_cdclk_config *cdclk_config,
1889                             enum pipe pipe)
1890 {
1891         if (!intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_config))
1892                 return;
1893
1894         if (WARN_ON_ONCE(!dev_priv->display.set_cdclk))
1895                 return;
1896
1897         intel_dump_cdclk_config(cdclk_config, "Changing CDCLK to");
1898
1899         dev_priv->display.set_cdclk(dev_priv, cdclk_config, pipe);
1900
1901         if (WARN(intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_config),
1902                  "cdclk state doesn't match!\n")) {
1903                 intel_dump_cdclk_config(&dev_priv->cdclk.hw, "[hw state]");
1904                 intel_dump_cdclk_config(cdclk_config, "[sw state]");
1905         }
1906 }
1907
1908 /**
1909  * intel_set_cdclk_pre_plane_update - Push the CDCLK state to the hardware
1910  * @state: intel atomic state
1911  *
1912  * Program the hardware before updating the HW plane state based on the
1913  * new CDCLK state, if necessary.
1914  */
1915 void
1916 intel_set_cdclk_pre_plane_update(struct intel_atomic_state *state)
1917 {
1918         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
1919         /* called after intel_cdclk_swap_state()! */
1920         const struct intel_cdclk_state *old_cdclk_state = &state->cdclk_state;
1921         const struct intel_cdclk_state *new_cdclk_state = &dev_priv->cdclk_state;
1922         enum pipe pipe = old_cdclk_state->pipe; /* not swapped */
1923
1924         if (pipe == INVALID_PIPE ||
1925             old_cdclk_state->actual.cdclk <= new_cdclk_state->actual.cdclk)
1926                 intel_set_cdclk(dev_priv, &new_cdclk_state->actual, pipe);
1927 }
1928
1929 /**
1930  * intel_set_cdclk_post_plane_update - Push the CDCLK state to the hardware
1931  * @state: intel atomic state
1932  *
1933  * Program the hardware before updating the HW plane state based on the
1934  * new CDCLK state, if necessary.
1935  */
1936 void
1937 intel_set_cdclk_post_plane_update(struct intel_atomic_state *state)
1938 {
1939         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
1940         /* called after intel_cdclk_swap_state()! */
1941         const struct intel_cdclk_state *old_cdclk_state = &state->cdclk_state;
1942         const struct intel_cdclk_state *new_cdclk_state = &dev_priv->cdclk_state;
1943         enum pipe pipe = old_cdclk_state->pipe; /* not swapped */
1944
1945         if (pipe != INVALID_PIPE &&
1946             old_cdclk_state->actual.cdclk > new_cdclk_state->actual.cdclk)
1947                 intel_set_cdclk(dev_priv, &new_cdclk_state->actual, pipe);
1948 }
1949
1950 static int intel_pixel_rate_to_cdclk(const struct intel_crtc_state *crtc_state)
1951 {
1952         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1953         int pixel_rate = crtc_state->pixel_rate;
1954
1955         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
1956                 return DIV_ROUND_UP(pixel_rate, 2);
1957         else if (IS_GEN(dev_priv, 9) ||
1958                  IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
1959                 return pixel_rate;
1960         else if (IS_CHERRYVIEW(dev_priv))
1961                 return DIV_ROUND_UP(pixel_rate * 100, 95);
1962         else if (crtc_state->double_wide)
1963                 return DIV_ROUND_UP(pixel_rate * 100, 90 * 2);
1964         else
1965                 return DIV_ROUND_UP(pixel_rate * 100, 90);
1966 }
1967
1968 static int intel_planes_min_cdclk(const struct intel_crtc_state *crtc_state)
1969 {
1970         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1971         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1972         struct intel_plane *plane;
1973         int min_cdclk = 0;
1974
1975         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
1976                 min_cdclk = max(crtc_state->min_cdclk[plane->id], min_cdclk);
1977
1978         return min_cdclk;
1979 }
1980
1981 int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state)
1982 {
1983         struct drm_i915_private *dev_priv =
1984                 to_i915(crtc_state->uapi.crtc->dev);
1985         int min_cdclk;
1986
1987         if (!crtc_state->hw.enable)
1988                 return 0;
1989
1990         min_cdclk = intel_pixel_rate_to_cdclk(crtc_state);
1991
1992         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
1993         if (IS_BROADWELL(dev_priv) && hsw_crtc_state_ips_capable(crtc_state))
1994                 min_cdclk = DIV_ROUND_UP(min_cdclk * 100, 95);
1995
1996         /* BSpec says "Do not use DisplayPort with CDCLK less than 432 MHz,
1997          * audio enabled, port width x4, and link rate HBR2 (5.4 GHz), or else
1998          * there may be audio corruption or screen corruption." This cdclk
1999          * restriction for GLK is 316.8 MHz.
2000          */
2001         if (intel_crtc_has_dp_encoder(crtc_state) &&
2002             crtc_state->has_audio &&
2003             crtc_state->port_clock >= 540000 &&
2004             crtc_state->lane_count == 4) {
2005                 if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv)) {
2006                         /* Display WA #1145: glk,cnl */
2007                         min_cdclk = max(316800, min_cdclk);
2008                 } else if (IS_GEN(dev_priv, 9) || IS_BROADWELL(dev_priv)) {
2009                         /* Display WA #1144: skl,bxt */
2010                         min_cdclk = max(432000, min_cdclk);
2011                 }
2012         }
2013
2014         /*
2015          * According to BSpec, "The CD clock frequency must be at least twice
2016          * the frequency of the Azalia BCLK." and BCLK is 96 MHz by default.
2017          */
2018         if (crtc_state->has_audio && INTEL_GEN(dev_priv) >= 9)
2019                 min_cdclk = max(2 * 96000, min_cdclk);
2020
2021         /*
2022          * "For DP audio configuration, cdclk frequency shall be set to
2023          *  meet the following requirements:
2024          *  DP Link Frequency(MHz) | Cdclk frequency(MHz)
2025          *  270                    | 320 or higher
2026          *  162                    | 200 or higher"
2027          */
2028         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
2029             intel_crtc_has_dp_encoder(crtc_state) && crtc_state->has_audio)
2030                 min_cdclk = max(crtc_state->port_clock, min_cdclk);
2031
2032         /*
2033          * On Valleyview some DSI panels lose (v|h)sync when the clock is lower
2034          * than 320000KHz.
2035          */
2036         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2037             IS_VALLEYVIEW(dev_priv))
2038                 min_cdclk = max(320000, min_cdclk);
2039
2040         /*
2041          * On Geminilake once the CDCLK gets as low as 79200
2042          * picture gets unstable, despite that values are
2043          * correct for DSI PLL and DE PLL.
2044          */
2045         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2046             IS_GEMINILAKE(dev_priv))
2047                 min_cdclk = max(158400, min_cdclk);
2048
2049         /* Account for additional needs from the planes */
2050         min_cdclk = max(intel_planes_min_cdclk(crtc_state), min_cdclk);
2051
2052         /*
2053          * HACK. Currently for TGL platforms we calculate
2054          * min_cdclk initially based on pixel_rate divided
2055          * by 2, accounting for also plane requirements,
2056          * however in some cases the lowest possible CDCLK
2057          * doesn't work and causing the underruns.
2058          * Explicitly stating here that this seems to be currently
2059          * rather a Hack, than final solution.
2060          */
2061         if (IS_TIGERLAKE(dev_priv))
2062                 min_cdclk = max(min_cdclk, (int)crtc_state->pixel_rate);
2063
2064         if (min_cdclk > dev_priv->max_cdclk_freq) {
2065                 drm_dbg_kms(&dev_priv->drm,
2066                             "required cdclk (%d kHz) exceeds max (%d kHz)\n",
2067                             min_cdclk, dev_priv->max_cdclk_freq);
2068                 return -EINVAL;
2069         }
2070
2071         return min_cdclk;
2072 }
2073
2074 static int intel_compute_min_cdclk(struct intel_atomic_state *state)
2075 {
2076         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2077         struct intel_cdclk_state *cdclk_state = &state->cdclk_state;
2078         struct intel_crtc *crtc;
2079         struct intel_crtc_state *crtc_state;
2080         int min_cdclk, i;
2081         enum pipe pipe;
2082
2083         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2084                 int ret;
2085
2086                 min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
2087                 if (min_cdclk < 0)
2088                         return min_cdclk;
2089
2090                 if (cdclk_state->min_cdclk[i] == min_cdclk)
2091                         continue;
2092
2093                 cdclk_state->min_cdclk[i] = min_cdclk;
2094
2095                 ret = intel_atomic_lock_global_state(state);
2096                 if (ret)
2097                         return ret;
2098         }
2099
2100         min_cdclk = cdclk_state->force_min_cdclk;
2101         for_each_pipe(dev_priv, pipe)
2102                 min_cdclk = max(cdclk_state->min_cdclk[pipe], min_cdclk);
2103
2104         return min_cdclk;
2105 }
2106
2107 /*
2108  * Account for port clock min voltage level requirements.
2109  * This only really does something on CNL+ but can be
2110  * called on earlier platforms as well.
2111  *
2112  * Note that this functions assumes that 0 is
2113  * the lowest voltage value, and higher values
2114  * correspond to increasingly higher voltages.
2115  *
2116  * Should that relationship no longer hold on
2117  * future platforms this code will need to be
2118  * adjusted.
2119  */
2120 static int bxt_compute_min_voltage_level(struct intel_atomic_state *state)
2121 {
2122         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2123         struct intel_cdclk_state *cdclk_state = &state->cdclk_state;
2124         struct intel_crtc *crtc;
2125         struct intel_crtc_state *crtc_state;
2126         u8 min_voltage_level;
2127         int i;
2128         enum pipe pipe;
2129
2130         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2131                 int ret;
2132
2133                 if (crtc_state->hw.enable)
2134                         min_voltage_level = crtc_state->min_voltage_level;
2135                 else
2136                         min_voltage_level = 0;
2137
2138                 if (cdclk_state->min_voltage_level[i] == min_voltage_level)
2139                         continue;
2140
2141                 cdclk_state->min_voltage_level[i] = min_voltage_level;
2142
2143                 ret = intel_atomic_lock_global_state(state);
2144                 if (ret)
2145                         return ret;
2146         }
2147
2148         min_voltage_level = 0;
2149         for_each_pipe(dev_priv, pipe)
2150                 min_voltage_level = max(cdclk_state->min_voltage_level[pipe],
2151                                         min_voltage_level);
2152
2153         return min_voltage_level;
2154 }
2155
2156 static int vlv_modeset_calc_cdclk(struct intel_atomic_state *state)
2157 {
2158         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2159         struct intel_cdclk_state *cdclk_state = &state->cdclk_state;
2160         int min_cdclk, cdclk;
2161
2162         min_cdclk = intel_compute_min_cdclk(state);
2163         if (min_cdclk < 0)
2164                 return min_cdclk;
2165
2166         cdclk = vlv_calc_cdclk(dev_priv, min_cdclk);
2167
2168         cdclk_state->logical.cdclk = cdclk;
2169         cdclk_state->logical.voltage_level =
2170                 vlv_calc_voltage_level(dev_priv, cdclk);
2171
2172         if (!state->active_pipes) {
2173                 cdclk = vlv_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
2174
2175                 cdclk_state->actual.cdclk = cdclk;
2176                 cdclk_state->actual.voltage_level =
2177                         vlv_calc_voltage_level(dev_priv, cdclk);
2178         } else {
2179                 cdclk_state->actual = cdclk_state->logical;
2180         }
2181
2182         return 0;
2183 }
2184
2185 static int bdw_modeset_calc_cdclk(struct intel_atomic_state *state)
2186 {
2187         struct intel_cdclk_state *cdclk_state = &state->cdclk_state;
2188         int min_cdclk, cdclk;
2189
2190         min_cdclk = intel_compute_min_cdclk(state);
2191         if (min_cdclk < 0)
2192                 return min_cdclk;
2193
2194         /*
2195          * FIXME should also account for plane ratio
2196          * once 64bpp pixel formats are supported.
2197          */
2198         cdclk = bdw_calc_cdclk(min_cdclk);
2199
2200         cdclk_state->logical.cdclk = cdclk;
2201         cdclk_state->logical.voltage_level =
2202                 bdw_calc_voltage_level(cdclk);
2203
2204         if (!state->active_pipes) {
2205                 cdclk = bdw_calc_cdclk(cdclk_state->force_min_cdclk);
2206
2207                 cdclk_state->actual.cdclk = cdclk;
2208                 cdclk_state->actual.voltage_level =
2209                         bdw_calc_voltage_level(cdclk);
2210         } else {
2211                 cdclk_state->actual = cdclk_state->logical;
2212         }
2213
2214         return 0;
2215 }
2216
2217 static int skl_dpll0_vco(struct intel_atomic_state *state)
2218 {
2219         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2220         struct intel_cdclk_state *cdclk_state = &state->cdclk_state;
2221         struct intel_crtc *crtc;
2222         struct intel_crtc_state *crtc_state;
2223         int vco, i;
2224
2225         vco = cdclk_state->logical.vco;
2226         if (!vco)
2227                 vco = dev_priv->skl_preferred_vco_freq;
2228
2229         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2230                 if (!crtc_state->hw.enable)
2231                         continue;
2232
2233                 if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP))
2234                         continue;
2235
2236                 /*
2237                  * DPLL0 VCO may need to be adjusted to get the correct
2238                  * clock for eDP. This will affect cdclk as well.
2239                  */
2240                 switch (crtc_state->port_clock / 2) {
2241                 case 108000:
2242                 case 216000:
2243                         vco = 8640000;
2244                         break;
2245                 default:
2246                         vco = 8100000;
2247                         break;
2248                 }
2249         }
2250
2251         return vco;
2252 }
2253
2254 static int skl_modeset_calc_cdclk(struct intel_atomic_state *state)
2255 {
2256         struct intel_cdclk_state *cdclk_state = &state->cdclk_state;
2257         int min_cdclk, cdclk, vco;
2258
2259         min_cdclk = intel_compute_min_cdclk(state);
2260         if (min_cdclk < 0)
2261                 return min_cdclk;
2262
2263         vco = skl_dpll0_vco(state);
2264
2265         /*
2266          * FIXME should also account for plane ratio
2267          * once 64bpp pixel formats are supported.
2268          */
2269         cdclk = skl_calc_cdclk(min_cdclk, vco);
2270
2271         cdclk_state->logical.vco = vco;
2272         cdclk_state->logical.cdclk = cdclk;
2273         cdclk_state->logical.voltage_level =
2274                 skl_calc_voltage_level(cdclk);
2275
2276         if (!state->active_pipes) {
2277                 cdclk = skl_calc_cdclk(cdclk_state->force_min_cdclk, vco);
2278
2279                 cdclk_state->actual.vco = vco;
2280                 cdclk_state->actual.cdclk = cdclk;
2281                 cdclk_state->actual.voltage_level =
2282                         skl_calc_voltage_level(cdclk);
2283         } else {
2284                 cdclk_state->actual = cdclk_state->logical;
2285         }
2286
2287         return 0;
2288 }
2289
2290 static int bxt_modeset_calc_cdclk(struct intel_atomic_state *state)
2291 {
2292         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2293         struct intel_cdclk_state *cdclk_state = &state->cdclk_state;
2294         int min_cdclk, min_voltage_level, cdclk, vco;
2295
2296         min_cdclk = intel_compute_min_cdclk(state);
2297         if (min_cdclk < 0)
2298                 return min_cdclk;
2299
2300         min_voltage_level = bxt_compute_min_voltage_level(state);
2301         if (min_voltage_level < 0)
2302                 return min_voltage_level;
2303
2304         cdclk = bxt_calc_cdclk(dev_priv, min_cdclk);
2305         vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2306
2307         cdclk_state->logical.vco = vco;
2308         cdclk_state->logical.cdclk = cdclk;
2309         cdclk_state->logical.voltage_level =
2310                 max_t(int, min_voltage_level,
2311                       dev_priv->display.calc_voltage_level(cdclk));
2312
2313         if (!state->active_pipes) {
2314                 cdclk = bxt_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
2315                 vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2316
2317                 cdclk_state->actual.vco = vco;
2318                 cdclk_state->actual.cdclk = cdclk;
2319                 cdclk_state->actual.voltage_level =
2320                         dev_priv->display.calc_voltage_level(cdclk);
2321         } else {
2322                 cdclk_state->actual = cdclk_state->logical;
2323         }
2324
2325         return 0;
2326 }
2327
2328 static int intel_modeset_all_pipes(struct intel_atomic_state *state)
2329 {
2330         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2331         struct intel_crtc *crtc;
2332
2333         /*
2334          * Add all pipes to the state, and force
2335          * a modeset on all the active ones.
2336          */
2337         for_each_intel_crtc(&dev_priv->drm, crtc) {
2338                 struct intel_crtc_state *crtc_state;
2339                 int ret;
2340
2341                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
2342                 if (IS_ERR(crtc_state))
2343                         return PTR_ERR(crtc_state);
2344
2345                 if (!crtc_state->hw.active ||
2346                     drm_atomic_crtc_needs_modeset(&crtc_state->uapi))
2347                         continue;
2348
2349                 crtc_state->uapi.mode_changed = true;
2350
2351                 ret = drm_atomic_add_affected_connectors(&state->base,
2352                                                          &crtc->base);
2353                 if (ret)
2354                         return ret;
2355
2356                 ret = drm_atomic_add_affected_planes(&state->base,
2357                                                      &crtc->base);
2358                 if (ret)
2359                         return ret;
2360
2361                 crtc_state->update_planes |= crtc_state->active_planes;
2362         }
2363
2364         return 0;
2365 }
2366
2367 static int fixed_modeset_calc_cdclk(struct intel_atomic_state *state)
2368 {
2369         int min_cdclk;
2370
2371         /*
2372          * We can't change the cdclk frequency, but we still want to
2373          * check that the required minimum frequency doesn't exceed
2374          * the actual cdclk frequency.
2375          */
2376         min_cdclk = intel_compute_min_cdclk(state);
2377         if (min_cdclk < 0)
2378                 return min_cdclk;
2379
2380         return 0;
2381 }
2382
2383 int intel_modeset_calc_cdclk(struct intel_atomic_state *state)
2384 {
2385         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2386         const struct intel_cdclk_state *old_cdclk_state = &dev_priv->cdclk_state;
2387         struct intel_cdclk_state *new_cdclk_state = &state->cdclk_state;
2388         enum pipe pipe;
2389         int ret;
2390
2391         memcpy(new_cdclk_state->min_cdclk, old_cdclk_state->min_cdclk,
2392                sizeof(new_cdclk_state->min_cdclk));
2393         memcpy(new_cdclk_state->min_voltage_level, old_cdclk_state->min_voltage_level,
2394                sizeof(new_cdclk_state->min_voltage_level));
2395
2396         /* keep the current setting */
2397         if (!new_cdclk_state->force_min_cdclk_changed)
2398                 new_cdclk_state->force_min_cdclk = old_cdclk_state->force_min_cdclk;
2399
2400         new_cdclk_state->logical = old_cdclk_state->logical;
2401         new_cdclk_state->actual = old_cdclk_state->actual;
2402
2403         ret = dev_priv->display.modeset_calc_cdclk(state);
2404         if (ret)
2405                 return ret;
2406
2407         /*
2408          * Writes to dev_priv->cdclk.{actual,logical} must protected
2409          * by holding all the crtc mutexes even if we don't end up
2410          * touching the hardware
2411          */
2412         if (intel_cdclk_changed(&old_cdclk_state->actual,
2413                                 &new_cdclk_state->actual)) {
2414                 /*
2415                  * Also serialize commits across all crtcs
2416                  * if the actual hw needs to be poked.
2417                  */
2418                 ret = intel_atomic_serialize_global_state(state);
2419                 if (ret)
2420                         return ret;
2421         } else if (intel_cdclk_changed(&old_cdclk_state->logical,
2422                                        &new_cdclk_state->logical)) {
2423                 ret = intel_atomic_lock_global_state(state);
2424                 if (ret)
2425                         return ret;
2426         } else {
2427                 return 0;
2428         }
2429
2430         if (is_power_of_2(state->active_pipes) &&
2431             intel_cdclk_can_cd2x_update(dev_priv,
2432                                         &old_cdclk_state->actual,
2433                                         &new_cdclk_state->actual)) {
2434                 struct intel_crtc *crtc;
2435                 struct intel_crtc_state *crtc_state;
2436
2437                 pipe = ilog2(state->active_pipes);
2438                 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
2439
2440                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
2441                 if (IS_ERR(crtc_state))
2442                         return PTR_ERR(crtc_state);
2443
2444                 if (drm_atomic_crtc_needs_modeset(&crtc_state->uapi))
2445                         pipe = INVALID_PIPE;
2446         } else {
2447                 pipe = INVALID_PIPE;
2448         }
2449
2450         if (pipe != INVALID_PIPE) {
2451                 new_cdclk_state->pipe = pipe;
2452
2453                 drm_dbg_kms(&dev_priv->drm,
2454                             "Can change cdclk with pipe %c active\n",
2455                             pipe_name(pipe));
2456         } else if (intel_cdclk_needs_modeset(&old_cdclk_state->actual,
2457                                              &new_cdclk_state->actual)) {
2458                 /* All pipes must be switched off while we change the cdclk. */
2459                 ret = intel_modeset_all_pipes(state);
2460                 if (ret)
2461                         return ret;
2462
2463                 new_cdclk_state->pipe = INVALID_PIPE;
2464
2465                 drm_dbg_kms(&dev_priv->drm,
2466                             "Modeset required for cdclk change\n");
2467         }
2468
2469         drm_dbg_kms(&dev_priv->drm,
2470                     "New cdclk calculated to be logical %u kHz, actual %u kHz\n",
2471                     new_cdclk_state->logical.cdclk,
2472                     new_cdclk_state->actual.cdclk);
2473         drm_dbg_kms(&dev_priv->drm,
2474                     "New voltage level calculated to be logical %u, actual %u\n",
2475                     new_cdclk_state->logical.voltage_level,
2476                     new_cdclk_state->actual.voltage_level);
2477
2478         return 0;
2479 }
2480
2481 static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
2482 {
2483         int max_cdclk_freq = dev_priv->max_cdclk_freq;
2484
2485         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
2486                 return 2 * max_cdclk_freq;
2487         else if (IS_GEN(dev_priv, 9) ||
2488                  IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2489                 return max_cdclk_freq;
2490         else if (IS_CHERRYVIEW(dev_priv))
2491                 return max_cdclk_freq*95/100;
2492         else if (INTEL_GEN(dev_priv) < 4)
2493                 return 2*max_cdclk_freq*90/100;
2494         else
2495                 return max_cdclk_freq*90/100;
2496 }
2497
2498 /**
2499  * intel_update_max_cdclk - Determine the maximum support CDCLK frequency
2500  * @dev_priv: i915 device
2501  *
2502  * Determine the maximum CDCLK frequency the platform supports, and also
2503  * derive the maximum dot clock frequency the maximum CDCLK frequency
2504  * allows.
2505  */
2506 void intel_update_max_cdclk(struct drm_i915_private *dev_priv)
2507 {
2508         if (IS_ELKHARTLAKE(dev_priv)) {
2509                 if (dev_priv->cdclk.hw.ref == 24000)
2510                         dev_priv->max_cdclk_freq = 552000;
2511                 else
2512                         dev_priv->max_cdclk_freq = 556800;
2513         } else if (INTEL_GEN(dev_priv) >= 11) {
2514                 if (dev_priv->cdclk.hw.ref == 24000)
2515                         dev_priv->max_cdclk_freq = 648000;
2516                 else
2517                         dev_priv->max_cdclk_freq = 652800;
2518         } else if (IS_CANNONLAKE(dev_priv)) {
2519                 dev_priv->max_cdclk_freq = 528000;
2520         } else if (IS_GEN9_BC(dev_priv)) {
2521                 u32 limit = intel_de_read(dev_priv, SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
2522                 int max_cdclk, vco;
2523
2524                 vco = dev_priv->skl_preferred_vco_freq;
2525                 WARN_ON(vco != 8100000 && vco != 8640000);
2526
2527                 /*
2528                  * Use the lower (vco 8640) cdclk values as a
2529                  * first guess. skl_calc_cdclk() will correct it
2530                  * if the preferred vco is 8100 instead.
2531                  */
2532                 if (limit == SKL_DFSM_CDCLK_LIMIT_675)
2533                         max_cdclk = 617143;
2534                 else if (limit == SKL_DFSM_CDCLK_LIMIT_540)
2535                         max_cdclk = 540000;
2536                 else if (limit == SKL_DFSM_CDCLK_LIMIT_450)
2537                         max_cdclk = 432000;
2538                 else
2539                         max_cdclk = 308571;
2540
2541                 dev_priv->max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco);
2542         } else if (IS_GEMINILAKE(dev_priv)) {
2543                 dev_priv->max_cdclk_freq = 316800;
2544         } else if (IS_BROXTON(dev_priv)) {
2545                 dev_priv->max_cdclk_freq = 624000;
2546         } else if (IS_BROADWELL(dev_priv))  {
2547                 /*
2548                  * FIXME with extra cooling we can allow
2549                  * 540 MHz for ULX and 675 Mhz for ULT.
2550                  * How can we know if extra cooling is
2551                  * available? PCI ID, VTB, something else?
2552                  */
2553                 if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
2554                         dev_priv->max_cdclk_freq = 450000;
2555                 else if (IS_BDW_ULX(dev_priv))
2556                         dev_priv->max_cdclk_freq = 450000;
2557                 else if (IS_BDW_ULT(dev_priv))
2558                         dev_priv->max_cdclk_freq = 540000;
2559                 else
2560                         dev_priv->max_cdclk_freq = 675000;
2561         } else if (IS_CHERRYVIEW(dev_priv)) {
2562                 dev_priv->max_cdclk_freq = 320000;
2563         } else if (IS_VALLEYVIEW(dev_priv)) {
2564                 dev_priv->max_cdclk_freq = 400000;
2565         } else {
2566                 /* otherwise assume cdclk is fixed */
2567                 dev_priv->max_cdclk_freq = dev_priv->cdclk.hw.cdclk;
2568         }
2569
2570         dev_priv->max_dotclk_freq = intel_compute_max_dotclk(dev_priv);
2571
2572         drm_dbg(&dev_priv->drm, "Max CD clock rate: %d kHz\n",
2573                 dev_priv->max_cdclk_freq);
2574
2575         drm_dbg(&dev_priv->drm, "Max dotclock rate: %d kHz\n",
2576                 dev_priv->max_dotclk_freq);
2577 }
2578
2579 /**
2580  * intel_update_cdclk - Determine the current CDCLK frequency
2581  * @dev_priv: i915 device
2582  *
2583  * Determine the current CDCLK frequency.
2584  */
2585 void intel_update_cdclk(struct drm_i915_private *dev_priv)
2586 {
2587         dev_priv->display.get_cdclk(dev_priv, &dev_priv->cdclk.hw);
2588
2589         /*
2590          * 9:0 CMBUS [sic] CDCLK frequency (cdfreq):
2591          * Programmng [sic] note: bit[9:2] should be programmed to the number
2592          * of cdclk that generates 4MHz reference clock freq which is used to
2593          * generate GMBus clock. This will vary with the cdclk freq.
2594          */
2595         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2596                 intel_de_write(dev_priv, GMBUSFREQ_VLV,
2597                                DIV_ROUND_UP(dev_priv->cdclk.hw.cdclk, 1000));
2598 }
2599
2600 static int cnp_rawclk(struct drm_i915_private *dev_priv)
2601 {
2602         u32 rawclk;
2603         int divider, fraction;
2604
2605         if (intel_de_read(dev_priv, SFUSE_STRAP) & SFUSE_STRAP_RAW_FREQUENCY) {
2606                 /* 24 MHz */
2607                 divider = 24000;
2608                 fraction = 0;
2609         } else {
2610                 /* 19.2 MHz */
2611                 divider = 19000;
2612                 fraction = 200;
2613         }
2614
2615         rawclk = CNP_RAWCLK_DIV(divider / 1000);
2616         if (fraction) {
2617                 int numerator = 1;
2618
2619                 rawclk |= CNP_RAWCLK_DEN(DIV_ROUND_CLOSEST(numerator * 1000,
2620                                                            fraction) - 1);
2621                 if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
2622                         rawclk |= ICP_RAWCLK_NUM(numerator);
2623         }
2624
2625         intel_de_write(dev_priv, PCH_RAWCLK_FREQ, rawclk);
2626         return divider + fraction;
2627 }
2628
2629 static int pch_rawclk(struct drm_i915_private *dev_priv)
2630 {
2631         return (intel_de_read(dev_priv, PCH_RAWCLK_FREQ) & RAWCLK_FREQ_MASK) * 1000;
2632 }
2633
2634 static int vlv_hrawclk(struct drm_i915_private *dev_priv)
2635 {
2636         /* RAWCLK_FREQ_VLV register updated from power well code */
2637         return vlv_get_cck_clock_hpll(dev_priv, "hrawclk",
2638                                       CCK_DISPLAY_REF_CLOCK_CONTROL);
2639 }
2640
2641 static int g4x_hrawclk(struct drm_i915_private *dev_priv)
2642 {
2643         u32 clkcfg;
2644
2645         /* hrawclock is 1/4 the FSB frequency */
2646         clkcfg = intel_de_read(dev_priv, CLKCFG);
2647         switch (clkcfg & CLKCFG_FSB_MASK) {
2648         case CLKCFG_FSB_400:
2649                 return 100000;
2650         case CLKCFG_FSB_533:
2651                 return 133333;
2652         case CLKCFG_FSB_667:
2653                 return 166667;
2654         case CLKCFG_FSB_800:
2655                 return 200000;
2656         case CLKCFG_FSB_1067:
2657         case CLKCFG_FSB_1067_ALT:
2658                 return 266667;
2659         case CLKCFG_FSB_1333:
2660         case CLKCFG_FSB_1333_ALT:
2661                 return 333333;
2662         default:
2663                 return 133333;
2664         }
2665 }
2666
2667 /**
2668  * intel_update_rawclk - Determine the current RAWCLK frequency
2669  * @dev_priv: i915 device
2670  *
2671  * Determine the current RAWCLK frequency. RAWCLK is a fixed
2672  * frequency clock so this needs to done only once.
2673  */
2674 void intel_update_rawclk(struct drm_i915_private *dev_priv)
2675 {
2676         if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP)
2677                 dev_priv->rawclk_freq = cnp_rawclk(dev_priv);
2678         else if (HAS_PCH_SPLIT(dev_priv))
2679                 dev_priv->rawclk_freq = pch_rawclk(dev_priv);
2680         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2681                 dev_priv->rawclk_freq = vlv_hrawclk(dev_priv);
2682         else if (IS_G4X(dev_priv) || IS_PINEVIEW(dev_priv))
2683                 dev_priv->rawclk_freq = g4x_hrawclk(dev_priv);
2684         else
2685                 /* no rawclk on other platforms, or no need to know it */
2686                 return;
2687
2688         drm_dbg(&dev_priv->drm, "rawclk rate: %d kHz\n",
2689                 dev_priv->rawclk_freq);
2690 }
2691
2692 /**
2693  * intel_init_cdclk_hooks - Initialize CDCLK related modesetting hooks
2694  * @dev_priv: i915 device
2695  */
2696 void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv)
2697 {
2698         if (IS_ELKHARTLAKE(dev_priv)) {
2699                 dev_priv->display.set_cdclk = bxt_set_cdclk;
2700                 dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk;
2701                 dev_priv->display.calc_voltage_level = ehl_calc_voltage_level;
2702                 dev_priv->cdclk.table = icl_cdclk_table;
2703         } else if (INTEL_GEN(dev_priv) >= 11) {
2704                 dev_priv->display.set_cdclk = bxt_set_cdclk;
2705                 dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk;
2706                 dev_priv->display.calc_voltage_level = icl_calc_voltage_level;
2707                 dev_priv->cdclk.table = icl_cdclk_table;
2708         } else if (IS_CANNONLAKE(dev_priv)) {
2709                 dev_priv->display.set_cdclk = bxt_set_cdclk;
2710                 dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk;
2711                 dev_priv->display.calc_voltage_level = cnl_calc_voltage_level;
2712                 dev_priv->cdclk.table = cnl_cdclk_table;
2713         } else if (IS_GEN9_LP(dev_priv)) {
2714                 dev_priv->display.set_cdclk = bxt_set_cdclk;
2715                 dev_priv->display.modeset_calc_cdclk = bxt_modeset_calc_cdclk;
2716                 dev_priv->display.calc_voltage_level = bxt_calc_voltage_level;
2717                 if (IS_GEMINILAKE(dev_priv))
2718                         dev_priv->cdclk.table = glk_cdclk_table;
2719                 else
2720                         dev_priv->cdclk.table = bxt_cdclk_table;
2721         } else if (IS_GEN9_BC(dev_priv)) {
2722                 dev_priv->display.set_cdclk = skl_set_cdclk;
2723                 dev_priv->display.modeset_calc_cdclk = skl_modeset_calc_cdclk;
2724         } else if (IS_BROADWELL(dev_priv)) {
2725                 dev_priv->display.set_cdclk = bdw_set_cdclk;
2726                 dev_priv->display.modeset_calc_cdclk = bdw_modeset_calc_cdclk;
2727         } else if (IS_CHERRYVIEW(dev_priv)) {
2728                 dev_priv->display.set_cdclk = chv_set_cdclk;
2729                 dev_priv->display.modeset_calc_cdclk = vlv_modeset_calc_cdclk;
2730         } else if (IS_VALLEYVIEW(dev_priv)) {
2731                 dev_priv->display.set_cdclk = vlv_set_cdclk;
2732                 dev_priv->display.modeset_calc_cdclk = vlv_modeset_calc_cdclk;
2733         } else {
2734                 dev_priv->display.modeset_calc_cdclk = fixed_modeset_calc_cdclk;
2735         }
2736
2737         if (INTEL_GEN(dev_priv) >= 10 || IS_GEN9_LP(dev_priv))
2738                 dev_priv->display.get_cdclk = bxt_get_cdclk;
2739         else if (IS_GEN9_BC(dev_priv))
2740                 dev_priv->display.get_cdclk = skl_get_cdclk;
2741         else if (IS_BROADWELL(dev_priv))
2742                 dev_priv->display.get_cdclk = bdw_get_cdclk;
2743         else if (IS_HASWELL(dev_priv))
2744                 dev_priv->display.get_cdclk = hsw_get_cdclk;
2745         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2746                 dev_priv->display.get_cdclk = vlv_get_cdclk;
2747         else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
2748                 dev_priv->display.get_cdclk = fixed_400mhz_get_cdclk;
2749         else if (IS_GEN(dev_priv, 5))
2750                 dev_priv->display.get_cdclk = fixed_450mhz_get_cdclk;
2751         else if (IS_GM45(dev_priv))
2752                 dev_priv->display.get_cdclk = gm45_get_cdclk;
2753         else if (IS_G45(dev_priv))
2754                 dev_priv->display.get_cdclk = g33_get_cdclk;
2755         else if (IS_I965GM(dev_priv))
2756                 dev_priv->display.get_cdclk = i965gm_get_cdclk;
2757         else if (IS_I965G(dev_priv))
2758                 dev_priv->display.get_cdclk = fixed_400mhz_get_cdclk;
2759         else if (IS_PINEVIEW(dev_priv))
2760                 dev_priv->display.get_cdclk = pnv_get_cdclk;
2761         else if (IS_G33(dev_priv))
2762                 dev_priv->display.get_cdclk = g33_get_cdclk;
2763         else if (IS_I945GM(dev_priv))
2764                 dev_priv->display.get_cdclk = i945gm_get_cdclk;
2765         else if (IS_I945G(dev_priv))
2766                 dev_priv->display.get_cdclk = fixed_400mhz_get_cdclk;
2767         else if (IS_I915GM(dev_priv))
2768                 dev_priv->display.get_cdclk = i915gm_get_cdclk;
2769         else if (IS_I915G(dev_priv))
2770                 dev_priv->display.get_cdclk = fixed_333mhz_get_cdclk;
2771         else if (IS_I865G(dev_priv))
2772                 dev_priv->display.get_cdclk = fixed_266mhz_get_cdclk;
2773         else if (IS_I85X(dev_priv))
2774                 dev_priv->display.get_cdclk = i85x_get_cdclk;
2775         else if (IS_I845G(dev_priv))
2776                 dev_priv->display.get_cdclk = fixed_200mhz_get_cdclk;
2777         else { /* 830 */
2778                 WARN(!IS_I830(dev_priv),
2779                      "Unknown platform. Assuming 133 MHz CDCLK\n");
2780                 dev_priv->display.get_cdclk = fixed_133mhz_get_cdclk;
2781         }
2782 }