drivers/gpu/drm/i915/display/intel_display_power.c

   1 /* SPDX-License-Identifier: MIT */
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include "display/intel_crt.h"
   7 #include "display/intel_dp.h"
   8
   9 #include "i915_drv.h"
  10 #include "i915_irq.h"
  11 #include "intel_cdclk.h"
  12 #include "intel_combo_phy.h"
  13 #include "intel_csr.h"
  14 #include "intel_display_power.h"
  15 #include "intel_display_types.h"
  16 #include "intel_dpio_phy.h"
  17 #include "intel_hotplug.h"
  18 #include "intel_pm.h"
  19 #include "intel_sideband.h"
  20 #include "intel_tc.h"
  21 #include "intel_vga.h"
  22
  23 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
  24                                          enum i915_power_well_id power_well_id);
  25
  26 const char *
  27 intel_display_power_domain_str(enum intel_display_power_domain domain)
  28 {
  29         switch (domain) {
  30         case POWER_DOMAIN_DISPLAY_CORE:
  31                 return "DISPLAY_CORE";
  32         case POWER_DOMAIN_PIPE_A:
  33                 return "PIPE_A";
  34         case POWER_DOMAIN_PIPE_B:
  35                 return "PIPE_B";
  36         case POWER_DOMAIN_PIPE_C:
  37                 return "PIPE_C";
  38         case POWER_DOMAIN_PIPE_D:
  39                 return "PIPE_D";
  40         case POWER_DOMAIN_PIPE_A_PANEL_FITTER:
  41                 return "PIPE_A_PANEL_FITTER";
  42         case POWER_DOMAIN_PIPE_B_PANEL_FITTER:
  43                 return "PIPE_B_PANEL_FITTER";
  44         case POWER_DOMAIN_PIPE_C_PANEL_FITTER:
  45                 return "PIPE_C_PANEL_FITTER";
  46         case POWER_DOMAIN_PIPE_D_PANEL_FITTER:
  47                 return "PIPE_D_PANEL_FITTER";
  48         case POWER_DOMAIN_TRANSCODER_A:
  49                 return "TRANSCODER_A";
  50         case POWER_DOMAIN_TRANSCODER_B:
  51                 return "TRANSCODER_B";
  52         case POWER_DOMAIN_TRANSCODER_C:
  53                 return "TRANSCODER_C";
  54         case POWER_DOMAIN_TRANSCODER_D:
  55                 return "TRANSCODER_D";
  56         case POWER_DOMAIN_TRANSCODER_EDP:
  57                 return "TRANSCODER_EDP";
  58         case POWER_DOMAIN_TRANSCODER_VDSC_PW2:
  59                 return "TRANSCODER_VDSC_PW2";
  60         case POWER_DOMAIN_TRANSCODER_DSI_A:
  61                 return "TRANSCODER_DSI_A";
  62         case POWER_DOMAIN_TRANSCODER_DSI_C:
  63                 return "TRANSCODER_DSI_C";
  64         case POWER_DOMAIN_PORT_DDI_A_LANES:
  65                 return "PORT_DDI_A_LANES";
  66         case POWER_DOMAIN_PORT_DDI_B_LANES:
  67                 return "PORT_DDI_B_LANES";
  68         case POWER_DOMAIN_PORT_DDI_C_LANES:
  69                 return "PORT_DDI_C_LANES";
  70         case POWER_DOMAIN_PORT_DDI_D_LANES:
  71                 return "PORT_DDI_D_LANES";
  72         case POWER_DOMAIN_PORT_DDI_E_LANES:
  73                 return "PORT_DDI_E_LANES";
  74         case POWER_DOMAIN_PORT_DDI_F_LANES:
  75                 return "PORT_DDI_F_LANES";
  76         case POWER_DOMAIN_PORT_DDI_G_LANES:
  77                 return "PORT_DDI_G_LANES";
  78         case POWER_DOMAIN_PORT_DDI_H_LANES:
  79                 return "PORT_DDI_H_LANES";
  80         case POWER_DOMAIN_PORT_DDI_I_LANES:
  81                 return "PORT_DDI_I_LANES";
  82         case POWER_DOMAIN_PORT_DDI_A_IO:
  83                 return "PORT_DDI_A_IO";
  84         case POWER_DOMAIN_PORT_DDI_B_IO:
  85                 return "PORT_DDI_B_IO";
  86         case POWER_DOMAIN_PORT_DDI_C_IO:
  87                 return "PORT_DDI_C_IO";
  88         case POWER_DOMAIN_PORT_DDI_D_IO:
  89                 return "PORT_DDI_D_IO";
  90         case POWER_DOMAIN_PORT_DDI_E_IO:
  91                 return "PORT_DDI_E_IO";
  92         case POWER_DOMAIN_PORT_DDI_F_IO:
  93                 return "PORT_DDI_F_IO";
  94         case POWER_DOMAIN_PORT_DDI_G_IO:
  95                 return "PORT_DDI_G_IO";
  96         case POWER_DOMAIN_PORT_DDI_H_IO:
  97                 return "PORT_DDI_H_IO";
  98         case POWER_DOMAIN_PORT_DDI_I_IO:
  99                 return "PORT_DDI_I_IO";
 100         case POWER_DOMAIN_PORT_DSI:
 101                 return "PORT_DSI";
 102         case POWER_DOMAIN_PORT_CRT:
 103                 return "PORT_CRT";
 104         case POWER_DOMAIN_PORT_OTHER:
 105                 return "PORT_OTHER";
 106         case POWER_DOMAIN_VGA:
 107                 return "VGA";
 108         case POWER_DOMAIN_AUDIO:
 109                 return "AUDIO";
 110         case POWER_DOMAIN_AUX_A:
 111                 return "AUX_A";
 112         case POWER_DOMAIN_AUX_B:
 113                 return "AUX_B";
 114         case POWER_DOMAIN_AUX_C:
 115                 return "AUX_C";
 116         case POWER_DOMAIN_AUX_D:
 117                 return "AUX_D";
 118         case POWER_DOMAIN_AUX_E:
 119                 return "AUX_E";
 120         case POWER_DOMAIN_AUX_F:
 121                 return "AUX_F";
 122         case POWER_DOMAIN_AUX_G:
 123                 return "AUX_G";
 124         case POWER_DOMAIN_AUX_H:
 125                 return "AUX_H";
 126         case POWER_DOMAIN_AUX_I:
 127                 return "AUX_I";
 128         case POWER_DOMAIN_AUX_IO_A:
 129                 return "AUX_IO_A";
 130         case POWER_DOMAIN_AUX_C_TBT:
 131                 return "AUX_C_TBT";
 132         case POWER_DOMAIN_AUX_D_TBT:
 133                 return "AUX_D_TBT";
 134         case POWER_DOMAIN_AUX_E_TBT:
 135                 return "AUX_E_TBT";
 136         case POWER_DOMAIN_AUX_F_TBT:
 137                 return "AUX_F_TBT";
 138         case POWER_DOMAIN_AUX_G_TBT:
 139                 return "AUX_G_TBT";
 140         case POWER_DOMAIN_AUX_H_TBT:
 141                 return "AUX_H_TBT";
 142         case POWER_DOMAIN_AUX_I_TBT:
 143                 return "AUX_I_TBT";
 144         case POWER_DOMAIN_GMBUS:
 145                 return "GMBUS";
 146         case POWER_DOMAIN_INIT:
 147                 return "INIT";
 148         case POWER_DOMAIN_MODESET:
 149                 return "MODESET";
 150         case POWER_DOMAIN_GT_IRQ:
 151                 return "GT_IRQ";
 152         case POWER_DOMAIN_DPLL_DC_OFF:
 153                 return "DPLL_DC_OFF";
 154         default:
 155                 MISSING_CASE(domain);
 156                 return "?";
 157         }
 158 }
 159
 160 static void intel_power_well_enable(struct drm_i915_private *dev_priv,
 161                                     struct i915_power_well *power_well)
 162 {
 163         drm_dbg_kms(&dev_priv->drm, "enabling %s\n", power_well->desc->name);
 164         power_well->desc->ops->enable(dev_priv, power_well);
 165         power_well->hw_enabled = true;
 166 }
 167
 168 static void intel_power_well_disable(struct drm_i915_private *dev_priv,
 169                                      struct i915_power_well *power_well)
 170 {
 171         drm_dbg_kms(&dev_priv->drm, "disabling %s\n", power_well->desc->name);
 172         power_well->hw_enabled = false;
 173         power_well->desc->ops->disable(dev_priv, power_well);
 174 }
 175
 176 static void intel_power_well_get(struct drm_i915_private *dev_priv,
 177                                  struct i915_power_well *power_well)
 178 {
 179         if (!power_well->count++)
 180                 intel_power_well_enable(dev_priv, power_well);
 181 }
 182
 183 static void intel_power_well_put(struct drm_i915_private *dev_priv,
 184                                  struct i915_power_well *power_well)
 185 {
 186         drm_WARN(&dev_priv->drm, !power_well->count,
 187                  "Use count on power well %s is already zero",
 188                  power_well->desc->name);
 189
 190         if (!--power_well->count)
 191                 intel_power_well_disable(dev_priv, power_well);
 192 }
 193
 194 /**
 195  * __intel_display_power_is_enabled - unlocked check for a power domain
 196  * @dev_priv: i915 device instance
 197  * @domain: power domain to check
 198  *
 199  * This is the unlocked version of intel_display_power_is_enabled() and should
 200  * only be used from error capture and recovery code where deadlocks are
 201  * possible.
 202  *
 203  * Returns:
 204  * True when the power domain is enabled, false otherwise.
 205  */
 206 bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 207                                       enum intel_display_power_domain domain)
 208 {
 209         struct i915_power_well *power_well;
 210         bool is_enabled;
 211
 212         if (dev_priv->runtime_pm.suspended)
 213                 return false;
 214
 215         is_enabled = true;
 216
 217         for_each_power_domain_well_reverse(dev_priv, power_well, BIT_ULL(domain)) {
 218                 if (power_well->desc->always_on)
 219                         continue;
 220
 221                 if (!power_well->hw_enabled) {
 222                         is_enabled = false;
 223                         break;
 224                 }
 225         }
 226
 227         return is_enabled;
 228 }
 229
 230 /**
 231  * intel_display_power_is_enabled - check for a power domain
 232  * @dev_priv: i915 device instance
 233  * @domain: power domain to check
 234  *
 235  * This function can be used to check the hw power domain state. It is mostly
 236  * used in hardware state readout functions. Everywhere else code should rely
 237  * upon explicit power domain reference counting to ensure that the hardware
 238  * block is powered up before accessing it.
 239  *
 240  * Callers must hold the relevant modesetting locks to ensure that concurrent
 241  * threads can't disable the power well while the caller tries to read a few
 242  * registers.
 243  *
 244  * Returns:
 245  * True when the power domain is enabled, false otherwise.
 246  */
 247 bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 248                                     enum intel_display_power_domain domain)
 249 {
 250         struct i915_power_domains *power_domains;
 251         bool ret;
 252
 253         power_domains = &dev_priv->power_domains;
 254
 255         mutex_lock(&power_domains->lock);
 256         ret = __intel_display_power_is_enabled(dev_priv, domain);
 257         mutex_unlock(&power_domains->lock);
 258
 259         return ret;
 260 }
 261
 262 /*
 263  * Starting with Haswell, we have a "Power Down Well" that can be turned off
 264  * when not needed anymore. We have 4 registers that can request the power well
 265  * to be enabled, and it will only be disabled if none of the registers is
 266  * requesting it to be enabled.
 267  */
 268 static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv,
 269                                        u8 irq_pipe_mask, bool has_vga)
 270 {
 271         if (has_vga)
 272                 intel_vga_reset_io_mem(dev_priv);
 273
 274         if (irq_pipe_mask)
 275                 gen8_irq_power_well_post_enable(dev_priv, irq_pipe_mask);
 276 }
 277
 278 static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv,
 279                                        u8 irq_pipe_mask)
 280 {
 281         if (irq_pipe_mask)
 282                 gen8_irq_power_well_pre_disable(dev_priv, irq_pipe_mask);
 283 }
 284
 285 static struct intel_digital_port *
 286 aux_ch_to_digital_port(struct drm_i915_private *dev_priv,
 287                        enum aux_ch aux_ch)
 288 {
 289         struct intel_digital_port *dig_port = NULL;
 290         struct intel_encoder *encoder;
 291
 292         for_each_intel_encoder(&dev_priv->drm, encoder) {
 293                 /* We'll check the MST primary port */
 294                 if (encoder->type == INTEL_OUTPUT_DP_MST)
 295                         continue;
 296
 297                 dig_port = enc_to_dig_port(encoder);
 298                 if (!dig_port)
 299                         continue;
 300
 301                 if (dig_port->aux_ch != aux_ch) {
 302                         dig_port = NULL;
 303                         continue;
 304                 }
 305
 306                 break;
 307         }
 308
 309         return dig_port;
 310 }
 311
 312 static void hsw_wait_for_power_well_enable(struct drm_i915_private *dev_priv,
 313                                            struct i915_power_well *power_well)
 314 {
 315         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 316         int pw_idx = power_well->desc->hsw.idx;
 317
 318         /* Timeout for PW1:10 us, AUX:not specified, other PWs:20 us. */
 319         if (intel_de_wait_for_set(dev_priv, regs->driver,
 320                                   HSW_PWR_WELL_CTL_STATE(pw_idx), 1)) {
 321                 drm_dbg_kms(&dev_priv->drm, "%s power well enable timeout\n",
 322                             power_well->desc->name);
 323
 324                 /* An AUX timeout is expected if the TBT DP tunnel is down. */
 325                 drm_WARN_ON(&dev_priv->drm, !power_well->desc->hsw.is_tc_tbt);
 326         }
 327 }
 328
 329 static u32 hsw_power_well_requesters(struct drm_i915_private *dev_priv,
 330                                      const struct i915_power_well_regs *regs,
 331                                      int pw_idx)
 332 {
 333         u32 req_mask = HSW_PWR_WELL_CTL_REQ(pw_idx);
 334         u32 ret;
 335
 336         ret = intel_de_read(dev_priv, regs->bios) & req_mask ? 1 : 0;
 337         ret |= intel_de_read(dev_priv, regs->driver) & req_mask ? 2 : 0;
 338         if (regs->kvmr.reg)
 339                 ret |= intel_de_read(dev_priv, regs->kvmr) & req_mask ? 4 : 0;
 340         ret |= intel_de_read(dev_priv, regs->debug) & req_mask ? 8 : 0;
 341
 342         return ret;
 343 }
 344
 345 static void hsw_wait_for_power_well_disable(struct drm_i915_private *dev_priv,
 346                                             struct i915_power_well *power_well)
 347 {
 348         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 349         int pw_idx = power_well->desc->hsw.idx;
 350         bool disabled;
 351         u32 reqs;
 352
 353         /*
 354          * Bspec doesn't require waiting for PWs to get disabled, but still do
 355          * this for paranoia. The known cases where a PW will be forced on:
 356          * - a KVMR request on any power well via the KVMR request register
 357          * - a DMC request on PW1 and MISC_IO power wells via the BIOS and
 358          *   DEBUG request registers
 359          * Skip the wait in case any of the request bits are set and print a
 360          * diagnostic message.
 361          */
 362         wait_for((disabled = !(intel_de_read(dev_priv, regs->driver) &
 363                                HSW_PWR_WELL_CTL_STATE(pw_idx))) ||
 364                  (reqs = hsw_power_well_requesters(dev_priv, regs, pw_idx)), 1);
 365         if (disabled)
 366                 return;
 367
 368         drm_dbg_kms(&dev_priv->drm,
 369                     "%s forced on (bios:%d driver:%d kvmr:%d debug:%d)\n",
 370                     power_well->desc->name,
 371                     !!(reqs & 1), !!(reqs & 2), !!(reqs & 4), !!(reqs & 8));
 372 }
 373
 374 static void gen9_wait_for_power_well_fuses(struct drm_i915_private *dev_priv,
 375                                            enum skl_power_gate pg)
 376 {
 377         /* Timeout 5us for PG#0, for other PGs 1us */
 378         drm_WARN_ON(&dev_priv->drm,
 379                     intel_de_wait_for_set(dev_priv, SKL_FUSE_STATUS,
 380                                           SKL_FUSE_PG_DIST_STATUS(pg), 1));
 381 }
 382
 383 static void hsw_power_well_enable_prepare(struct drm_i915_private *dev_priv,
 384                                           struct i915_power_well *power_well)
 385 {
 386         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 387         int pw_idx = power_well->desc->hsw.idx;
 388         u32 val;
 389
 390         if (power_well->desc->hsw.has_fuses) {
 391                 enum skl_power_gate pg;
 392
 393                 pg = INTEL_GEN(dev_priv) >= 11 ? ICL_PW_CTL_IDX_TO_PG(pw_idx) :
 394                                                  SKL_PW_CTL_IDX_TO_PG(pw_idx);
 395                 /*
 396                  * For PW1 we have to wait both for the PW0/PG0 fuse state
 397                  * before enabling the power well and PW1/PG1's own fuse
 398                  * state after the enabling. For all other power wells with
 399                  * fuses we only have to wait for that PW/PG's fuse state
 400                  * after the enabling.
 401                  */
 402                 if (pg == SKL_PG1)
 403                         gen9_wait_for_power_well_fuses(dev_priv, SKL_PG0);
 404         }
 405
 406         val = intel_de_read(dev_priv, regs->driver);
 407         intel_de_write(dev_priv, regs->driver,
 408                        val | HSW_PWR_WELL_CTL_REQ(pw_idx));
 409 }
 410
 411 static void hsw_power_well_enable_complete(struct drm_i915_private *dev_priv,
 412                                            struct i915_power_well *power_well)
 413 {
 414         int pw_idx = power_well->desc->hsw.idx;
 415
 416         hsw_wait_for_power_well_enable(dev_priv, power_well);
 417
 418         /* Display WA #1178: cnl */
 419         if (IS_CANNONLAKE(dev_priv) &&
 420             pw_idx >= GLK_PW_CTL_IDX_AUX_B &&
 421             pw_idx <= CNL_PW_CTL_IDX_AUX_F) {
 422                 u32 val;
 423
 424                 val = intel_de_read(dev_priv, CNL_AUX_ANAOVRD1(pw_idx));
 425                 val |= CNL_AUX_ANAOVRD1_ENABLE | CNL_AUX_ANAOVRD1_LDO_BYPASS;
 426                 intel_de_write(dev_priv, CNL_AUX_ANAOVRD1(pw_idx), val);
 427         }
 428
 429         if (power_well->desc->hsw.has_fuses) {
 430                 enum skl_power_gate pg;
 431
 432                 pg = INTEL_GEN(dev_priv) >= 11 ? ICL_PW_CTL_IDX_TO_PG(pw_idx) :
 433                                                  SKL_PW_CTL_IDX_TO_PG(pw_idx);
 434                 gen9_wait_for_power_well_fuses(dev_priv, pg);
 435         }
 436
 437         hsw_power_well_post_enable(dev_priv,
 438                                    power_well->desc->hsw.irq_pipe_mask,
 439                                    power_well->desc->hsw.has_vga);
 440 }
 441
 442 static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
 443                                   struct i915_power_well *power_well)
 444 {
 445         hsw_power_well_enable_prepare(dev_priv, power_well);
 446         hsw_power_well_enable_complete(dev_priv, power_well);
 447 }
 448
 449 static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
 450                                    struct i915_power_well *power_well)
 451 {
 452         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 453         int pw_idx = power_well->desc->hsw.idx;
 454         u32 val;
 455
 456         hsw_power_well_pre_disable(dev_priv,
 457                                    power_well->desc->hsw.irq_pipe_mask);
 458
 459         val = intel_de_read(dev_priv, regs->driver);
 460         intel_de_write(dev_priv, regs->driver,
 461                        val & ~HSW_PWR_WELL_CTL_REQ(pw_idx));
 462         hsw_wait_for_power_well_disable(dev_priv, power_well);
 463 }
 464
 465 #define ICL_AUX_PW_TO_PHY(pw_idx)       ((pw_idx) - ICL_PW_CTL_IDX_AUX_A)
 466
 467 static void
 468 icl_combo_phy_aux_power_well_enable(struct drm_i915_private *dev_priv,
 469                                     struct i915_power_well *power_well)
 470 {
 471         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 472         int pw_idx = power_well->desc->hsw.idx;
 473         enum phy phy = ICL_AUX_PW_TO_PHY(pw_idx);
 474         u32 val;
 475
 476         drm_WARN_ON(&dev_priv->drm, !IS_ICELAKE(dev_priv));
 477
 478         val = intel_de_read(dev_priv, regs->driver);
 479         intel_de_write(dev_priv, regs->driver,
 480                        val | HSW_PWR_WELL_CTL_REQ(pw_idx));
 481
 482         if (INTEL_GEN(dev_priv) < 12) {
 483                 val = intel_de_read(dev_priv, ICL_PORT_CL_DW12(phy));
 484                 intel_de_write(dev_priv, ICL_PORT_CL_DW12(phy),
 485                                val | ICL_LANE_ENABLE_AUX);
 486         }
 487
 488         hsw_wait_for_power_well_enable(dev_priv, power_well);
 489
 490         /* Display WA #1178: icl */
 491         if (pw_idx >= ICL_PW_CTL_IDX_AUX_A && pw_idx <= ICL_PW_CTL_IDX_AUX_B &&
 492             !intel_bios_is_port_edp(dev_priv, (enum port)phy)) {
 493                 val = intel_de_read(dev_priv, ICL_AUX_ANAOVRD1(pw_idx));
 494                 val |= ICL_AUX_ANAOVRD1_ENABLE | ICL_AUX_ANAOVRD1_LDO_BYPASS;
 495                 intel_de_write(dev_priv, ICL_AUX_ANAOVRD1(pw_idx), val);
 496         }
 497 }
 498
 499 static void
 500 icl_combo_phy_aux_power_well_disable(struct drm_i915_private *dev_priv,
 501                                      struct i915_power_well *power_well)
 502 {
 503         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 504         int pw_idx = power_well->desc->hsw.idx;
 505         enum phy phy = ICL_AUX_PW_TO_PHY(pw_idx);
 506         u32 val;
 507
 508         drm_WARN_ON(&dev_priv->drm, !IS_ICELAKE(dev_priv));
 509
 510         val = intel_de_read(dev_priv, ICL_PORT_CL_DW12(phy));
 511         intel_de_write(dev_priv, ICL_PORT_CL_DW12(phy),
 512                        val & ~ICL_LANE_ENABLE_AUX);
 513
 514         val = intel_de_read(dev_priv, regs->driver);
 515         intel_de_write(dev_priv, regs->driver,
 516                        val & ~HSW_PWR_WELL_CTL_REQ(pw_idx));
 517
 518         hsw_wait_for_power_well_disable(dev_priv, power_well);
 519 }
 520
 521 #define ICL_AUX_PW_TO_CH(pw_idx)        \
 522         ((pw_idx) - ICL_PW_CTL_IDX_AUX_A + AUX_CH_A)
 523
 524 #define ICL_TBT_AUX_PW_TO_CH(pw_idx)    \
 525         ((pw_idx) - ICL_PW_CTL_IDX_AUX_TBT1 + AUX_CH_C)
 526
 527 static enum aux_ch icl_tc_phy_aux_ch(struct drm_i915_private *dev_priv,
 528                                      struct i915_power_well *power_well)
 529 {
 530         int pw_idx = power_well->desc->hsw.idx;
 531
 532         return power_well->desc->hsw.is_tc_tbt ? ICL_TBT_AUX_PW_TO_CH(pw_idx) :
 533                                                  ICL_AUX_PW_TO_CH(pw_idx);
 534 }
 535
 536 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
 537
 538 static u64 async_put_domains_mask(struct i915_power_domains *power_domains);
 539
 540 static int power_well_async_ref_count(struct drm_i915_private *dev_priv,
 541                                       struct i915_power_well *power_well)
 542 {
 543         int refs = hweight64(power_well->desc->domains &
 544                              async_put_domains_mask(&dev_priv->power_domains));
 545
 546         drm_WARN_ON(&dev_priv->drm, refs > power_well->count);
 547
 548         return refs;
 549 }
 550
 551 static void icl_tc_port_assert_ref_held(struct drm_i915_private *dev_priv,
 552                                         struct i915_power_well *power_well,
 553                                         struct intel_digital_port *dig_port)
 554 {
 555         /* Bypass the check if all references are released asynchronously */
 556         if (power_well_async_ref_count(dev_priv, power_well) ==
 557             power_well->count)
 558                 return;
 559
 560         if (drm_WARN_ON(&dev_priv->drm, !dig_port))
 561                 return;
 562
 563         drm_WARN_ON(&dev_priv->drm, !intel_tc_port_ref_held(dig_port));
 564 }
 565
 566 #else
 567
 568 static void icl_tc_port_assert_ref_held(struct drm_i915_private *dev_priv,
 569                                         struct i915_power_well *power_well,
 570                                         struct intel_digital_port *dig_port)
 571 {
 572 }
 573
 574 #endif
 575
 576 #define TGL_AUX_PW_TO_TC_PORT(pw_idx)   ((pw_idx) - TGL_PW_CTL_IDX_AUX_TC1)
 577
 578 static void
 579 icl_tc_phy_aux_power_well_enable(struct drm_i915_private *dev_priv,
 580                                  struct i915_power_well *power_well)
 581 {
 582         enum aux_ch aux_ch = icl_tc_phy_aux_ch(dev_priv, power_well);
 583         struct intel_digital_port *dig_port = aux_ch_to_digital_port(dev_priv, aux_ch);
 584         u32 val;
 585
 586         icl_tc_port_assert_ref_held(dev_priv, power_well, dig_port);
 587
 588         val = intel_de_read(dev_priv, DP_AUX_CH_CTL(aux_ch));
 589         val &= ~DP_AUX_CH_CTL_TBT_IO;
 590         if (power_well->desc->hsw.is_tc_tbt)
 591                 val |= DP_AUX_CH_CTL_TBT_IO;
 592         intel_de_write(dev_priv, DP_AUX_CH_CTL(aux_ch), val);
 593
 594         hsw_power_well_enable_prepare(dev_priv, power_well);
 595
 596         /* TODO ICL TC cold handling */
 597
 598         hsw_power_well_enable_complete(dev_priv, power_well);
 599
 600         if (INTEL_GEN(dev_priv) >= 12 && !power_well->desc->hsw.is_tc_tbt) {
 601                 enum tc_port tc_port;
 602
 603                 tc_port = TGL_AUX_PW_TO_TC_PORT(power_well->desc->hsw.idx);
 604                 intel_de_write(dev_priv, HIP_INDEX_REG(tc_port),
 605                                HIP_INDEX_VAL(tc_port, 0x2));
 606
 607                 if (intel_de_wait_for_set(dev_priv, DKL_CMN_UC_DW_27(tc_port),
 608                                           DKL_CMN_UC_DW27_UC_HEALTH, 1))
 609                         drm_warn(&dev_priv->drm,
 610                                  "Timeout waiting TC uC health\n");
 611         }
 612 }
 613
 614 static void
 615 icl_tc_phy_aux_power_well_disable(struct drm_i915_private *dev_priv,
 616                                   struct i915_power_well *power_well)
 617 {
 618         enum aux_ch aux_ch = icl_tc_phy_aux_ch(dev_priv, power_well);
 619         struct intel_digital_port *dig_port = aux_ch_to_digital_port(dev_priv, aux_ch);
 620
 621         icl_tc_port_assert_ref_held(dev_priv, power_well, dig_port);
 622
 623         hsw_power_well_disable(dev_priv, power_well);
 624 }
 625
 626 /*
 627  * We should only use the power well if we explicitly asked the hardware to
 628  * enable it, so check if it's enabled and also check if we've requested it to
 629  * be enabled.
 630  */
 631 static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
 632                                    struct i915_power_well *power_well)
 633 {
 634         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 635         enum i915_power_well_id id = power_well->desc->id;
 636         int pw_idx = power_well->desc->hsw.idx;
 637         u32 mask = HSW_PWR_WELL_CTL_REQ(pw_idx) |
 638                    HSW_PWR_WELL_CTL_STATE(pw_idx);
 639         u32 val;
 640
 641         val = intel_de_read(dev_priv, regs->driver);
 642
 643         /*
 644          * On GEN9 big core due to a DMC bug the driver's request bits for PW1
 645          * and the MISC_IO PW will be not restored, so check instead for the
 646          * BIOS's own request bits, which are forced-on for these power wells
 647          * when exiting DC5/6.
 648          */
 649         if (IS_GEN(dev_priv, 9) && !IS_GEN9_LP(dev_priv) &&
 650             (id == SKL_DISP_PW_1 || id == SKL_DISP_PW_MISC_IO))
 651                 val |= intel_de_read(dev_priv, regs->bios);
 652
 653         return (val & mask) == mask;
 654 }
 655
 656 static void assert_can_enable_dc9(struct drm_i915_private *dev_priv)
 657 {
 658         drm_WARN_ONCE(&dev_priv->drm,
 659                       (intel_de_read(dev_priv, DC_STATE_EN) & DC_STATE_EN_DC9),
 660                       "DC9 already programmed to be enabled.\n");
 661         drm_WARN_ONCE(&dev_priv->drm,
 662                       intel_de_read(dev_priv, DC_STATE_EN) &
 663                       DC_STATE_EN_UPTO_DC5,
 664                       "DC5 still not disabled to enable DC9.\n");
 665         drm_WARN_ONCE(&dev_priv->drm,
 666                       intel_de_read(dev_priv, HSW_PWR_WELL_CTL2) &
 667                       HSW_PWR_WELL_CTL_REQ(SKL_PW_CTL_IDX_PW_2),
 668                       "Power well 2 on.\n");
 669         drm_WARN_ONCE(&dev_priv->drm, intel_irqs_enabled(dev_priv),
 670                       "Interrupts not disabled yet.\n");
 671
 672          /*
 673           * TODO: check for the following to verify the conditions to enter DC9
 674           * state are satisfied:
 675           * 1] Check relevant display engine registers to verify if mode set
 676           * disable sequence was followed.
 677           * 2] Check if display uninitialize sequence is initialized.
 678           */
 679 }
 680
 681 static void assert_can_disable_dc9(struct drm_i915_private *dev_priv)
 682 {
 683         drm_WARN_ONCE(&dev_priv->drm, intel_irqs_enabled(dev_priv),
 684                       "Interrupts not disabled yet.\n");
 685         drm_WARN_ONCE(&dev_priv->drm,
 686                       intel_de_read(dev_priv, DC_STATE_EN) &
 687                       DC_STATE_EN_UPTO_DC5,
 688                       "DC5 still not disabled.\n");
 689
 690          /*
 691           * TODO: check for the following to verify DC9 state was indeed
 692           * entered before programming to disable it:
 693           * 1] Check relevant display engine registers to verify if mode
 694           *  set disable sequence was followed.
 695           * 2] Check if display uninitialize sequence is initialized.
 696           */
 697 }
 698
 699 static void gen9_write_dc_state(struct drm_i915_private *dev_priv,
 700                                 u32 state)
 701 {
 702         int rewrites = 0;
 703         int rereads = 0;
 704         u32 v;
 705
 706         intel_de_write(dev_priv, DC_STATE_EN, state);
 707
 708         /* It has been observed that disabling the dc6 state sometimes
 709          * doesn't stick and dmc keeps returning old value. Make sure
 710          * the write really sticks enough times and also force rewrite until
 711          * we are confident that state is exactly what we want.
 712          */
 713         do  {
 714                 v = intel_de_read(dev_priv, DC_STATE_EN);
 715
 716                 if (v != state) {
 717                         intel_de_write(dev_priv, DC_STATE_EN, state);
 718                         rewrites++;
 719                         rereads = 0;
 720                 } else if (rereads++ > 5) {
 721                         break;
 722                 }
 723
 724         } while (rewrites < 100);
 725
 726         if (v != state)
 727                 drm_err(&dev_priv->drm,
 728                         "Writing dc state to 0x%x failed, now 0x%x\n",
 729                         state, v);
 730
 731         /* Most of the times we need one retry, avoid spam */
 732         if (rewrites > 1)
 733                 drm_dbg_kms(&dev_priv->drm,
 734                             "Rewrote dc state to 0x%x %d times\n",
 735                             state, rewrites);
 736 }
 737
 738 static u32 gen9_dc_mask(struct drm_i915_private *dev_priv)
 739 {
 740         u32 mask;
 741
 742         mask = DC_STATE_EN_UPTO_DC5;
 743
 744         if (INTEL_GEN(dev_priv) >= 12)
 745                 mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC6
 746                                           | DC_STATE_EN_DC9;
 747         else if (IS_GEN(dev_priv, 11))
 748                 mask |= DC_STATE_EN_UPTO_DC6 | DC_STATE_EN_DC9;
 749         else if (IS_GEN9_LP(dev_priv))
 750                 mask |= DC_STATE_EN_DC9;
 751         else
 752                 mask |= DC_STATE_EN_UPTO_DC6;
 753
 754         return mask;
 755 }
 756
 757 static void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv)
 758 {
 759         u32 val;
 760
 761         val = intel_de_read(dev_priv, DC_STATE_EN) & gen9_dc_mask(dev_priv);
 762
 763         drm_dbg_kms(&dev_priv->drm,
 764                     "Resetting DC state tracking from %02x to %02x\n",
 765                     dev_priv->csr.dc_state, val);
 766         dev_priv->csr.dc_state = val;
 767 }
 768
 769 /**
 770  * gen9_set_dc_state - set target display C power state
 771  * @dev_priv: i915 device instance
 772  * @state: target DC power state
 773  * - DC_STATE_DISABLE
 774  * - DC_STATE_EN_UPTO_DC5
 775  * - DC_STATE_EN_UPTO_DC6
 776  * - DC_STATE_EN_DC9
 777  *
 778  * Signal to DMC firmware/HW the target DC power state passed in @state.
 779  * DMC/HW can turn off individual display clocks and power rails when entering
 780  * a deeper DC power state (higher in number) and turns these back when exiting
 781  * that state to a shallower power state (lower in number). The HW will decide
 782  * when to actually enter a given state on an on-demand basis, for instance
 783  * depending on the active state of display pipes. The state of display
 784  * registers backed by affected power rails are saved/restored as needed.
 785  *
 786  * Based on the above enabling a deeper DC power state is asynchronous wrt.
 787  * enabling it. Disabling a deeper power state is synchronous: for instance
 788  * setting %DC_STATE_DISABLE won't complete until all HW resources are turned
 789  * back on and register state is restored. This is guaranteed by the MMIO write
 790  * to DC_STATE_EN blocking until the state is restored.
 791  */
 792 static void gen9_set_dc_state(struct drm_i915_private *dev_priv, u32 state)
 793 {
 794         u32 val;
 795         u32 mask;
 796
 797         if (drm_WARN_ON_ONCE(&dev_priv->drm,
 798                              state & ~dev_priv->csr.allowed_dc_mask))
 799                 state &= dev_priv->csr.allowed_dc_mask;
 800
 801         val = intel_de_read(dev_priv, DC_STATE_EN);
 802         mask = gen9_dc_mask(dev_priv);
 803         drm_dbg_kms(&dev_priv->drm, "Setting DC state from %02x to %02x\n",
 804                     val & mask, state);
 805
 806         /* Check if DMC is ignoring our DC state requests */
 807         if ((val & mask) != dev_priv->csr.dc_state)
 808                 drm_err(&dev_priv->drm, "DC state mismatch (0x%x -> 0x%x)\n",
 809                         dev_priv->csr.dc_state, val & mask);
 810
 811         val &= ~mask;
 812         val |= state;
 813
 814         gen9_write_dc_state(dev_priv, val);
 815
 816         dev_priv->csr.dc_state = val & mask;
 817 }
 818
 819 static u32
 820 sanitize_target_dc_state(struct drm_i915_private *dev_priv,
 821                          u32 target_dc_state)
 822 {
 823         u32 states[] = {
 824                 DC_STATE_EN_UPTO_DC6,
 825                 DC_STATE_EN_UPTO_DC5,
 826                 DC_STATE_EN_DC3CO,
 827                 DC_STATE_DISABLE,
 828         };
 829         int i;
 830
 831         for (i = 0; i < ARRAY_SIZE(states) - 1; i++) {
 832                 if (target_dc_state != states[i])
 833                         continue;
 834
 835                 if (dev_priv->csr.allowed_dc_mask & target_dc_state)
 836                         break;
 837
 838                 target_dc_state = states[i + 1];
 839         }
 840
 841         return target_dc_state;
 842 }
 843
 844 static void tgl_enable_dc3co(struct drm_i915_private *dev_priv)
 845 {
 846         drm_dbg_kms(&dev_priv->drm, "Enabling DC3CO\n");
 847         gen9_set_dc_state(dev_priv, DC_STATE_EN_DC3CO);
 848 }
 849
 850 static void tgl_disable_dc3co(struct drm_i915_private *dev_priv)
 851 {
 852         u32 val;
 853
 854         drm_dbg_kms(&dev_priv->drm, "Disabling DC3CO\n");
 855         val = intel_de_read(dev_priv, DC_STATE_EN);
 856         val &= ~DC_STATE_DC3CO_STATUS;
 857         intel_de_write(dev_priv, DC_STATE_EN, val);
 858         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 859         /*
 860          * Delay of 200us DC3CO Exit time B.Spec 49196
 861          */
 862         usleep_range(200, 210);
 863 }
 864
 865 static void bxt_enable_dc9(struct drm_i915_private *dev_priv)
 866 {
 867         assert_can_enable_dc9(dev_priv);
 868
 869         drm_dbg_kms(&dev_priv->drm, "Enabling DC9\n");
 870         /*
 871          * Power sequencer reset is not needed on
 872          * platforms with South Display Engine on PCH,
 873          * because PPS registers are always on.
 874          */
 875         if (!HAS_PCH_SPLIT(dev_priv))
 876                 intel_power_sequencer_reset(dev_priv);
 877         gen9_set_dc_state(dev_priv, DC_STATE_EN_DC9);
 878 }
 879
 880 static void bxt_disable_dc9(struct drm_i915_private *dev_priv)
 881 {
 882         assert_can_disable_dc9(dev_priv);
 883
 884         drm_dbg_kms(&dev_priv->drm, "Disabling DC9\n");
 885
 886         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 887
 888         intel_pps_unlock_regs_wa(dev_priv);
 889 }
 890
 891 static void assert_csr_loaded(struct drm_i915_private *dev_priv)
 892 {
 893         drm_WARN_ONCE(&dev_priv->drm,
 894                       !intel_de_read(dev_priv, CSR_PROGRAM(0)),
 895                       "CSR program storage start is NULL\n");
 896         drm_WARN_ONCE(&dev_priv->drm, !intel_de_read(dev_priv, CSR_SSP_BASE),
 897                       "CSR SSP Base Not fine\n");
 898         drm_WARN_ONCE(&dev_priv->drm, !intel_de_read(dev_priv, CSR_HTP_SKL),
 899                       "CSR HTP Not fine\n");
 900 }
 901
 902 static struct i915_power_well *
 903 lookup_power_well(struct drm_i915_private *dev_priv,
 904                   enum i915_power_well_id power_well_id)
 905 {
 906         struct i915_power_well *power_well;
 907
 908         for_each_power_well(dev_priv, power_well)
 909                 if (power_well->desc->id == power_well_id)
 910                         return power_well;
 911
 912         /*
 913          * It's not feasible to add error checking code to the callers since
 914          * this condition really shouldn't happen and it doesn't even make sense
 915          * to abort things like display initialization sequences. Just return
 916          * the first power well and hope the WARN gets reported so we can fix
 917          * our driver.
 918          */
 919         drm_WARN(&dev_priv->drm, 1,
 920                  "Power well %d not defined for this platform\n",
 921                  power_well_id);
 922         return &dev_priv->power_domains.power_wells[0];
 923 }
 924
 925 /**
 926  * intel_display_power_set_target_dc_state - Set target dc state.
 927  * @dev_priv: i915 device
 928  * @state: state which needs to be set as target_dc_state.
 929  *
 930  * This function set the "DC off" power well target_dc_state,
 931  * based upon this target_dc_stste, "DC off" power well will
 932  * enable desired DC state.
 933  */
 934 void intel_display_power_set_target_dc_state(struct drm_i915_private *dev_priv,
 935                                              u32 state)
 936 {
 937         struct i915_power_well *power_well;
 938         bool dc_off_enabled;
 939         struct i915_power_domains *power_domains = &dev_priv->power_domains;
 940
 941         mutex_lock(&power_domains->lock);
 942         power_well = lookup_power_well(dev_priv, SKL_DISP_DC_OFF);
 943
 944         if (drm_WARN_ON(&dev_priv->drm, !power_well))
 945                 goto unlock;
 946
 947         state = sanitize_target_dc_state(dev_priv, state);
 948
 949         if (state == dev_priv->csr.target_dc_state)
 950                 goto unlock;
 951
 952         dc_off_enabled = power_well->desc->ops->is_enabled(dev_priv,
 953                                                            power_well);
 954         /*
 955          * If DC off power well is disabled, need to enable and disable the
 956          * DC off power well to effect target DC state.
 957          */
 958         if (!dc_off_enabled)
 959                 power_well->desc->ops->enable(dev_priv, power_well);
 960
 961         dev_priv->csr.target_dc_state = state;
 962
 963         if (!dc_off_enabled)
 964                 power_well->desc->ops->disable(dev_priv, power_well);
 965
 966 unlock:
 967         mutex_unlock(&power_domains->lock);
 968 }
 969
 970 static void assert_can_enable_dc5(struct drm_i915_private *dev_priv)
 971 {
 972         enum i915_power_well_id high_pg;
 973
 974         /* Power wells at this level and above must be disabled for DC5 entry */
 975         if (INTEL_GEN(dev_priv) >= 12)
 976                 high_pg = ICL_DISP_PW_3;
 977         else
 978                 high_pg = SKL_DISP_PW_2;
 979
 980         drm_WARN_ONCE(&dev_priv->drm,
 981                       intel_display_power_well_is_enabled(dev_priv, high_pg),
 982                       "Power wells above platform's DC5 limit still enabled.\n");
 983
 984         drm_WARN_ONCE(&dev_priv->drm,
 985                       (intel_de_read(dev_priv, DC_STATE_EN) &
 986                        DC_STATE_EN_UPTO_DC5),
 987                       "DC5 already programmed to be enabled.\n");
 988         assert_rpm_wakelock_held(&dev_priv->runtime_pm);
 989
 990         assert_csr_loaded(dev_priv);
 991 }
 992
 993 static void gen9_enable_dc5(struct drm_i915_private *dev_priv)
 994 {
 995         assert_can_enable_dc5(dev_priv);
 996
 997         drm_dbg_kms(&dev_priv->drm, "Enabling DC5\n");
 998
 999         /* Wa Display #1183: skl,kbl,cfl */
1000         if (IS_GEN9_BC(dev_priv))
1001                 intel_de_write(dev_priv, GEN8_CHICKEN_DCPR_1,
1002                                intel_de_read(dev_priv, GEN8_CHICKEN_DCPR_1) | SKL_SELECT_ALTERNATE_DC_EXIT);
1003
1004         gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5);
1005 }
1006
1007 static void assert_can_enable_dc6(struct drm_i915_private *dev_priv)
1008 {
1009         drm_WARN_ONCE(&dev_priv->drm,
1010                       intel_de_read(dev_priv, UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
1011                       "Backlight is not disabled.\n");
1012         drm_WARN_ONCE(&dev_priv->drm,
1013                       (intel_de_read(dev_priv, DC_STATE_EN) &
1014                        DC_STATE_EN_UPTO_DC6),
1015                       "DC6 already programmed to be enabled.\n");
1016
1017         assert_csr_loaded(dev_priv);
1018 }
1019
1020 static void skl_enable_dc6(struct drm_i915_private *dev_priv)
1021 {
1022         assert_can_enable_dc6(dev_priv);
1023
1024         drm_dbg_kms(&dev_priv->drm, "Enabling DC6\n");
1025
1026         /* Wa Display #1183: skl,kbl,cfl */
1027         if (IS_GEN9_BC(dev_priv))
1028                 intel_de_write(dev_priv, GEN8_CHICKEN_DCPR_1,
1029                                intel_de_read(dev_priv, GEN8_CHICKEN_DCPR_1) | SKL_SELECT_ALTERNATE_DC_EXIT);
1030
1031         gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
1032 }
1033
1034 static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
1035                                    struct i915_power_well *power_well)
1036 {
1037         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
1038         int pw_idx = power_well->desc->hsw.idx;
1039         u32 mask = HSW_PWR_WELL_CTL_REQ(pw_idx);
1040         u32 bios_req = intel_de_read(dev_priv, regs->bios);
1041
1042         /* Take over the request bit if set by BIOS. */
1043         if (bios_req & mask) {
1044                 u32 drv_req = intel_de_read(dev_priv, regs->driver);
1045
1046                 if (!(drv_req & mask))
1047                         intel_de_write(dev_priv, regs->driver, drv_req | mask);
1048                 intel_de_write(dev_priv, regs->bios, bios_req & ~mask);
1049         }
1050 }
1051
1052 static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1053                                            struct i915_power_well *power_well)
1054 {
1055         bxt_ddi_phy_init(dev_priv, power_well->desc->bxt.phy);
1056 }
1057
1058 static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1059                                             struct i915_power_well *power_well)
1060 {
1061         bxt_ddi_phy_uninit(dev_priv, power_well->desc->bxt.phy);
1062 }
1063
1064 static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv,
1065                                             struct i915_power_well *power_well)
1066 {
1067         return bxt_ddi_phy_is_enabled(dev_priv, power_well->desc->bxt.phy);
1068 }
1069
1070 static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv)
1071 {
1072         struct i915_power_well *power_well;
1073
1074         power_well = lookup_power_well(dev_priv, BXT_DISP_PW_DPIO_CMN_A);
1075         if (power_well->count > 0)
1076                 bxt_ddi_phy_verify_state(dev_priv, power_well->desc->bxt.phy);
1077
1078         power_well = lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
1079         if (power_well->count > 0)
1080                 bxt_ddi_phy_verify_state(dev_priv, power_well->desc->bxt.phy);
1081
1082         if (IS_GEMINILAKE(dev_priv)) {
1083                 power_well = lookup_power_well(dev_priv,
1084                                                GLK_DISP_PW_DPIO_CMN_C);
1085                 if (power_well->count > 0)
1086                         bxt_ddi_phy_verify_state(dev_priv,
1087                                                  power_well->desc->bxt.phy);
1088         }
1089 }
1090
1091 static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv,
1092                                            struct i915_power_well *power_well)
1093 {
1094         return ((intel_de_read(dev_priv, DC_STATE_EN) & DC_STATE_EN_DC3CO) == 0 &&
1095                 (intel_de_read(dev_priv, DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0);
1096 }
1097
1098 static void gen9_assert_dbuf_enabled(struct drm_i915_private *dev_priv)
1099 {
1100         u8 hw_enabled_dbuf_slices = intel_enabled_dbuf_slices_mask(dev_priv);
1101         u8 enabled_dbuf_slices = dev_priv->enabled_dbuf_slices_mask;
1102
1103         drm_WARN(&dev_priv->drm,
1104                  hw_enabled_dbuf_slices != enabled_dbuf_slices,
1105                  "Unexpected DBuf power power state (0x%08x, expected 0x%08x)\n",
1106                  hw_enabled_dbuf_slices,
1107                  enabled_dbuf_slices);
1108 }
1109
1110 static void gen9_disable_dc_states(struct drm_i915_private *dev_priv)
1111 {
1112         struct intel_cdclk_config cdclk_config = {};
1113
1114         if (dev_priv->csr.target_dc_state == DC_STATE_EN_DC3CO) {
1115                 tgl_disable_dc3co(dev_priv);
1116                 return;
1117         }
1118
1119         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
1120
1121         dev_priv->display.get_cdclk(dev_priv, &cdclk_config);
1122         /* Can't read out voltage_level so can't use intel_cdclk_changed() */
1123         drm_WARN_ON(&dev_priv->drm,
1124                     intel_cdclk_needs_modeset(&dev_priv->cdclk.hw,
1125                                               &cdclk_config));
1126
1127         gen9_assert_dbuf_enabled(dev_priv);
1128
1129         if (IS_GEN9_LP(dev_priv))
1130                 bxt_verify_ddi_phy_power_wells(dev_priv);
1131
1132         if (INTEL_GEN(dev_priv) >= 11)
1133                 /*
1134                  * DMC retains HW context only for port A, the other combo
1135                  * PHY's HW context for port B is lost after DC transitions,
1136                  * so we need to restore it manually.
1137                  */
1138                 intel_combo_phy_init(dev_priv);
1139 }
1140
1141 static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
1142                                           struct i915_power_well *power_well)
1143 {
1144         gen9_disable_dc_states(dev_priv);
1145 }
1146
1147 static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv,
1148                                            struct i915_power_well *power_well)
1149 {
1150         if (!dev_priv->csr.dmc_payload)
1151                 return;
1152
1153         switch (dev_priv->csr.target_dc_state) {
1154         case DC_STATE_EN_DC3CO:
1155                 tgl_enable_dc3co(dev_priv);
1156                 break;
1157         case DC_STATE_EN_UPTO_DC6:
1158                 skl_enable_dc6(dev_priv);
1159                 break;
1160         case DC_STATE_EN_UPTO_DC5:
1161                 gen9_enable_dc5(dev_priv);
1162                 break;
1163         }
1164 }
1165
1166 static void i9xx_power_well_sync_hw_noop(struct drm_i915_private *dev_priv,
1167                                          struct i915_power_well *power_well)
1168 {
1169 }
1170
1171 static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
1172                                            struct i915_power_well *power_well)
1173 {
1174 }
1175
1176 static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
1177                                              struct i915_power_well *power_well)
1178 {
1179         return true;
1180 }
1181
1182 static void i830_pipes_power_well_enable(struct drm_i915_private *dev_priv,
1183                                          struct i915_power_well *power_well)
1184 {
1185         if ((intel_de_read(dev_priv, PIPECONF(PIPE_A)) & PIPECONF_ENABLE) == 0)
1186                 i830_enable_pipe(dev_priv, PIPE_A);
1187         if ((intel_de_read(dev_priv, PIPECONF(PIPE_B)) & PIPECONF_ENABLE) == 0)
1188                 i830_enable_pipe(dev_priv, PIPE_B);
1189 }
1190
1191 static void i830_pipes_power_well_disable(struct drm_i915_private *dev_priv,
1192                                           struct i915_power_well *power_well)
1193 {
1194         i830_disable_pipe(dev_priv, PIPE_B);
1195         i830_disable_pipe(dev_priv, PIPE_A);
1196 }
1197
1198 static bool i830_pipes_power_well_enabled(struct drm_i915_private *dev_priv,
1199                                           struct i915_power_well *power_well)
1200 {
1201         return intel_de_read(dev_priv, PIPECONF(PIPE_A)) & PIPECONF_ENABLE &&
1202                 intel_de_read(dev_priv, PIPECONF(PIPE_B)) & PIPECONF_ENABLE;
1203 }
1204
1205 static void i830_pipes_power_well_sync_hw(struct drm_i915_private *dev_priv,
1206                                           struct i915_power_well *power_well)
1207 {
1208         if (power_well->count > 0)
1209                 i830_pipes_power_well_enable(dev_priv, power_well);
1210         else
1211                 i830_pipes_power_well_disable(dev_priv, power_well);
1212 }
1213
1214 static void vlv_set_power_well(struct drm_i915_private *dev_priv,
1215                                struct i915_power_well *power_well, bool enable)
1216 {
1217         int pw_idx = power_well->desc->vlv.idx;
1218         u32 mask;
1219         u32 state;
1220         u32 ctrl;
1221
1222         mask = PUNIT_PWRGT_MASK(pw_idx);
1223         state = enable ? PUNIT_PWRGT_PWR_ON(pw_idx) :
1224                          PUNIT_PWRGT_PWR_GATE(pw_idx);
1225
1226         vlv_punit_get(dev_priv);
1227
1228 #define COND \
1229         ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)
1230
1231         if (COND)
1232                 goto out;
1233
1234         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
1235         ctrl &= ~mask;
1236         ctrl |= state;
1237         vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);
1238
1239         if (wait_for(COND, 100))
1240                 drm_err(&dev_priv->drm,
1241                         "timeout setting power well state %08x (%08x)\n",
1242                         state,
1243                         vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));
1244
1245 #undef COND
1246
1247 out:
1248         vlv_punit_put(dev_priv);
1249 }
1250
1251 static void vlv_power_well_enable(struct drm_i915_private *dev_priv,
1252                                   struct i915_power_well *power_well)
1253 {
1254         vlv_set_power_well(dev_priv, power_well, true);
1255 }
1256
1257 static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
1258                                    struct i915_power_well *power_well)
1259 {
1260         vlv_set_power_well(dev_priv, power_well, false);
1261 }
1262
1263 static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
1264                                    struct i915_power_well *power_well)
1265 {
1266         int pw_idx = power_well->desc->vlv.idx;
1267         bool enabled = false;
1268         u32 mask;
1269         u32 state;
1270         u32 ctrl;
1271
1272         mask = PUNIT_PWRGT_MASK(pw_idx);
1273         ctrl = PUNIT_PWRGT_PWR_ON(pw_idx);
1274
1275         vlv_punit_get(dev_priv);
1276
1277         state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
1278         /*
1279          * We only ever set the power-on and power-gate states, anything
1280          * else is unexpected.
1281          */
1282         drm_WARN_ON(&dev_priv->drm, state != PUNIT_PWRGT_PWR_ON(pw_idx) &&
1283                     state != PUNIT_PWRGT_PWR_GATE(pw_idx));
1284         if (state == ctrl)
1285                 enabled = true;
1286
1287         /*
1288          * A transient state at this point would mean some unexpected party
1289          * is poking at the power controls too.
1290          */
1291         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
1292         drm_WARN_ON(&dev_priv->drm, ctrl != state);
1293
1294         vlv_punit_put(dev_priv);
1295
1296         return enabled;
1297 }
1298
1299 static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
1300 {
1301         u32 val;
1302
1303         /*
1304          * On driver load, a pipe may be active and driving a DSI display.
1305          * Preserve DPOUNIT_CLOCK_GATE_DISABLE to avoid the pipe getting stuck
1306          * (and never recovering) in this case. intel_dsi_post_disable() will
1307          * clear it when we turn off the display.
1308          */
1309         val = intel_de_read(dev_priv, DSPCLK_GATE_D);
1310         val &= DPOUNIT_CLOCK_GATE_DISABLE;
1311         val |= VRHUNIT_CLOCK_GATE_DISABLE;
1312         intel_de_write(dev_priv, DSPCLK_GATE_D, val);
1313
1314         /*
1315          * Disable trickle feed and enable pnd deadline calculation
1316          */
1317         intel_de_write(dev_priv, MI_ARB_VLV,
1318                        MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
1319         intel_de_write(dev_priv, CBR1_VLV, 0);
1320
1321         drm_WARN_ON(&dev_priv->drm, RUNTIME_INFO(dev_priv)->rawclk_freq == 0);
1322         intel_de_write(dev_priv, RAWCLK_FREQ_VLV,
1323                        DIV_ROUND_CLOSEST(RUNTIME_INFO(dev_priv)->rawclk_freq,
1324                                          1000));
1325 }
1326
1327 static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
1328 {
1329         struct intel_encoder *encoder;
1330         enum pipe pipe;
1331
1332         /*
1333          * Enable the CRI clock source so we can get at the
1334          * display and the reference clock for VGA
1335          * hotplug / manual detection. Supposedly DSI also
1336          * needs the ref clock up and running.
1337          *
1338          * CHV DPLL B/C have some issues if VGA mode is enabled.
1339          */
1340         for_each_pipe(dev_priv, pipe) {
1341                 u32 val = intel_de_read(dev_priv, DPLL(pipe));
1342
1343                 val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1344                 if (pipe != PIPE_A)
1345                         val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1346
1347                 intel_de_write(dev_priv, DPLL(pipe), val);
1348         }
1349
1350         vlv_init_display_clock_gating(dev_priv);
1351
1352         spin_lock_irq(&dev_priv->irq_lock);
1353         valleyview_enable_display_irqs(dev_priv);
1354         spin_unlock_irq(&dev_priv->irq_lock);
1355
1356         /*
1357          * During driver initialization/resume we can avoid restoring the
1358          * part of the HW/SW state that will be inited anyway explicitly.
1359          */
1360         if (dev_priv->power_domains.initializing)
1361                 return;
1362
1363         intel_hpd_init(dev_priv);
1364
1365         /* Re-enable the ADPA, if we have one */
1366         for_each_intel_encoder(&dev_priv->drm, encoder) {
1367                 if (encoder->type == INTEL_OUTPUT_ANALOG)
1368                         intel_crt_reset(&encoder->base);
1369         }
1370
1371         intel_vga_redisable_power_on(dev_priv);
1372
1373         intel_pps_unlock_regs_wa(dev_priv);
1374 }
1375
1376 static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv)
1377 {
1378         spin_lock_irq(&dev_priv->irq_lock);
1379         valleyview_disable_display_irqs(dev_priv);
1380         spin_unlock_irq(&dev_priv->irq_lock);
1381
1382         /* make sure we're done processing display irqs */
1383         intel_synchronize_irq(dev_priv);
1384
1385         intel_power_sequencer_reset(dev_priv);
1386
1387         /* Prevent us from re-enabling polling on accident in late suspend */
1388         if (!dev_priv->drm.dev->power.is_suspended)
1389                 intel_hpd_poll_init(dev_priv);
1390 }
1391
1392 static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
1393                                           struct i915_power_well *power_well)
1394 {
1395         vlv_set_power_well(dev_priv, power_well, true);
1396
1397         vlv_display_power_well_init(dev_priv);
1398 }
1399
1400 static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
1401                                            struct i915_power_well *power_well)
1402 {
1403         vlv_display_power_well_deinit(dev_priv);
1404
1405         vlv_set_power_well(dev_priv, power_well, false);
1406 }
1407
1408 static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1409                                            struct i915_power_well *power_well)
1410 {
1411         /* since ref/cri clock was enabled */
1412         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1413
1414         vlv_set_power_well(dev_priv, power_well, true);
1415
1416         /*
1417          * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx -
1418          *  6.  De-assert cmn_reset/side_reset. Same as VLV X0.
1419          *   a. GUnit 0x2110 bit[0] set to 1 (def 0)
1420          *   b. The other bits such as sfr settings / modesel may all
1421          *      be set to 0.
1422          *
1423          * This should only be done on init and resume from S3 with
1424          * both PLLs disabled, or we risk losing DPIO and PLL
1425          * synchronization.
1426          */
1427         intel_de_write(dev_priv, DPIO_CTL,
1428                        intel_de_read(dev_priv, DPIO_CTL) | DPIO_CMNRST);
1429 }
1430
1431 static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1432                                             struct i915_power_well *power_well)
1433 {
1434         enum pipe pipe;
1435
1436         for_each_pipe(dev_priv, pipe)
1437                 assert_pll_disabled(dev_priv, pipe);
1438
1439         /* Assert common reset */
1440         intel_de_write(dev_priv, DPIO_CTL,
1441                        intel_de_read(dev_priv, DPIO_CTL) & ~DPIO_CMNRST);
1442
1443         vlv_set_power_well(dev_priv, power_well, false);
1444 }
1445
1446 #define POWER_DOMAIN_MASK (GENMASK_ULL(POWER_DOMAIN_NUM - 1, 0))
1447
1448 #define BITS_SET(val, bits) (((val) & (bits)) == (bits))
1449
1450 static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
1451 {
1452         struct i915_power_well *cmn_bc =
1453                 lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
1454         struct i915_power_well *cmn_d =
1455                 lookup_power_well(dev_priv, CHV_DISP_PW_DPIO_CMN_D);
1456         u32 phy_control = dev_priv->chv_phy_control;
1457         u32 phy_status = 0;
1458         u32 phy_status_mask = 0xffffffff;
1459
1460         /*
1461          * The BIOS can leave the PHY is some weird state
1462          * where it doesn't fully power down some parts.
1463          * Disable the asserts until the PHY has been fully
1464          * reset (ie. the power well has been disabled at
1465          * least once).
1466          */
1467         if (!dev_priv->chv_phy_assert[DPIO_PHY0])
1468                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0) |
1469                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0) |
1470                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1) |
1471                                      PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1) |
1472                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0) |
1473                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1));
1474
1475         if (!dev_priv->chv_phy_assert[DPIO_PHY1])
1476                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0) |
1477                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0) |
1478                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1));
1479
1480         if (cmn_bc->desc->ops->is_enabled(dev_priv, cmn_bc)) {
1481                 phy_status |= PHY_POWERGOOD(DPIO_PHY0);
1482
1483                 /* this assumes override is only used to enable lanes */
1484                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0)) == 0)
1485                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0);
1486
1487                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1)) == 0)
1488                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1);
1489
1490                 /* CL1 is on whenever anything is on in either channel */
1491                 if (BITS_SET(phy_control,
1492                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0) |
1493                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)))
1494                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0);
1495
1496                 /*
1497                  * The DPLLB check accounts for the pipe B + port A usage
1498                  * with CL2 powered up but all the lanes in the second channel
1499                  * powered down.
1500                  */
1501                 if (BITS_SET(phy_control,
1502                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)) &&
1503                     (intel_de_read(dev_priv, DPLL(PIPE_B)) & DPLL_VCO_ENABLE) == 0)
1504                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1);
1505
1506                 if (BITS_SET(phy_control,
1507                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH0)))
1508                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0);
1509                 if (BITS_SET(phy_control,
1510                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH0)))
1511                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1);
1512
1513                 if (BITS_SET(phy_control,
1514                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH1)))
1515                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0);
1516                 if (BITS_SET(phy_control,
1517                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH1)))
1518                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1);
1519         }
1520
1521         if (cmn_d->desc->ops->is_enabled(dev_priv, cmn_d)) {
1522                 phy_status |= PHY_POWERGOOD(DPIO_PHY1);
1523
1524                 /* this assumes override is only used to enable lanes */
1525                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0)) == 0)
1526                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0);
1527
1528                 if (BITS_SET(phy_control,
1529                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0)))
1530                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0);
1531
1532                 if (BITS_SET(phy_control,
1533                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY1, DPIO_CH0)))
1534                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0);
1535                 if (BITS_SET(phy_control,
1536                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY1, DPIO_CH0)))
1537                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1);
1538         }
1539
1540         phy_status &= phy_status_mask;
1541
1542         /*
1543          * The PHY may be busy with some initial calibration and whatnot,
1544          * so the power state can take a while to actually change.
1545          */
1546         if (intel_de_wait_for_register(dev_priv, DISPLAY_PHY_STATUS,
1547                                        phy_status_mask, phy_status, 10))
1548                 drm_err(&dev_priv->drm,
1549                         "Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n",
1550                         intel_de_read(dev_priv, DISPLAY_PHY_STATUS) & phy_status_mask,
1551                         phy_status, dev_priv->chv_phy_control);
1552 }
1553
1554 #undef BITS_SET
1555
1556 static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1557                                            struct i915_power_well *power_well)
1558 {
1559         enum dpio_phy phy;
1560         enum pipe pipe;
1561         u32 tmp;
1562
1563         drm_WARN_ON_ONCE(&dev_priv->drm,
1564                          power_well->desc->id != VLV_DISP_PW_DPIO_CMN_BC &&
1565                          power_well->desc->id != CHV_DISP_PW_DPIO_CMN_D);
1566
1567         if (power_well->desc->id == VLV_DISP_PW_DPIO_CMN_BC) {
1568                 pipe = PIPE_A;
1569                 phy = DPIO_PHY0;
1570         } else {
1571                 pipe = PIPE_C;
1572                 phy = DPIO_PHY1;
1573         }
1574
1575         /* since ref/cri clock was enabled */
1576         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1577         vlv_set_power_well(dev_priv, power_well, true);
1578
1579         /* Poll for phypwrgood signal */
1580         if (intel_de_wait_for_set(dev_priv, DISPLAY_PHY_STATUS,
1581                                   PHY_POWERGOOD(phy), 1))
1582                 drm_err(&dev_priv->drm, "Display PHY %d is not power up\n",
1583                         phy);
1584
1585         vlv_dpio_get(dev_priv);
1586
1587         /* Enable dynamic power down */
1588         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW28);
1589         tmp |= DPIO_DYNPWRDOWNEN_CH0 | DPIO_CL1POWERDOWNEN |
1590                 DPIO_SUS_CLK_CONFIG_GATE_CLKREQ;
1591         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW28, tmp);
1592
1593         if (power_well->desc->id == VLV_DISP_PW_DPIO_CMN_BC) {
1594                 tmp = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW6_CH1);
1595                 tmp |= DPIO_DYNPWRDOWNEN_CH1;
1596                 vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW6_CH1, tmp);
1597         } else {
1598                 /*
1599                  * Force the non-existing CL2 off. BXT does this
1600                  * too, so maybe it saves some power even though
1601                  * CL2 doesn't exist?
1602                  */
1603                 tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30);
1604                 tmp |= DPIO_CL2_LDOFUSE_PWRENB;
1605                 vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, tmp);
1606         }
1607
1608         vlv_dpio_put(dev_priv);
1609
1610         dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy);
1611         intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1612                        dev_priv->chv_phy_control);
1613
1614         drm_dbg_kms(&dev_priv->drm,
1615                     "Enabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1616                     phy, dev_priv->chv_phy_control);
1617
1618         assert_chv_phy_status(dev_priv);
1619 }
1620
1621 static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1622                                             struct i915_power_well *power_well)
1623 {
1624         enum dpio_phy phy;
1625
1626         drm_WARN_ON_ONCE(&dev_priv->drm,
1627                          power_well->desc->id != VLV_DISP_PW_DPIO_CMN_BC &&
1628                          power_well->desc->id != CHV_DISP_PW_DPIO_CMN_D);
1629
1630         if (power_well->desc->id == VLV_DISP_PW_DPIO_CMN_BC) {
1631                 phy = DPIO_PHY0;
1632                 assert_pll_disabled(dev_priv, PIPE_A);
1633                 assert_pll_disabled(dev_priv, PIPE_B);
1634         } else {
1635                 phy = DPIO_PHY1;
1636                 assert_pll_disabled(dev_priv, PIPE_C);
1637         }
1638
1639         dev_priv->chv_phy_control &= ~PHY_COM_LANE_RESET_DEASSERT(phy);
1640         intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1641                        dev_priv->chv_phy_control);
1642
1643         vlv_set_power_well(dev_priv, power_well, false);
1644
1645         drm_dbg_kms(&dev_priv->drm,
1646                     "Disabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1647                     phy, dev_priv->chv_phy_control);
1648
1649         /* PHY is fully reset now, so we can enable the PHY state asserts */
1650         dev_priv->chv_phy_assert[phy] = true;
1651
1652         assert_chv_phy_status(dev_priv);
1653 }
1654
1655 static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1656                                      enum dpio_channel ch, bool override, unsigned int mask)
1657 {
1658         enum pipe pipe = phy == DPIO_PHY0 ? PIPE_A : PIPE_C;
1659         u32 reg, val, expected, actual;
1660
1661         /*
1662          * The BIOS can leave the PHY is some weird state
1663          * where it doesn't fully power down some parts.
1664          * Disable the asserts until the PHY has been fully
1665          * reset (ie. the power well has been disabled at
1666          * least once).
1667          */
1668         if (!dev_priv->chv_phy_assert[phy])
1669                 return;
1670
1671         if (ch == DPIO_CH0)
1672                 reg = _CHV_CMN_DW0_CH0;
1673         else
1674                 reg = _CHV_CMN_DW6_CH1;
1675
1676         vlv_dpio_get(dev_priv);
1677         val = vlv_dpio_read(dev_priv, pipe, reg);
1678         vlv_dpio_put(dev_priv);
1679
1680         /*
1681          * This assumes !override is only used when the port is disabled.
1682          * All lanes should power down even without the override when
1683          * the port is disabled.
1684          */
1685         if (!override || mask == 0xf) {
1686                 expected = DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1687                 /*
1688                  * If CH1 common lane is not active anymore
1689                  * (eg. for pipe B DPLL) the entire channel will
1690                  * shut down, which causes the common lane registers
1691                  * to read as 0. That means we can't actually check
1692                  * the lane power down status bits, but as the entire
1693                  * register reads as 0 it's a good indication that the
1694                  * channel is indeed entirely powered down.
1695                  */
1696                 if (ch == DPIO_CH1 && val == 0)
1697                         expected = 0;
1698         } else if (mask != 0x0) {
1699                 expected = DPIO_ANYDL_POWERDOWN;
1700         } else {
1701                 expected = 0;
1702         }
1703
1704         if (ch == DPIO_CH0)
1705                 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH0;
1706         else
1707                 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH1;
1708         actual &= DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1709
1710         drm_WARN(&dev_priv->drm, actual != expected,
1711                  "Unexpected DPIO lane power down: all %d, any %d. Expected: all %d, any %d. (0x%x = 0x%08x)\n",
1712                  !!(actual & DPIO_ALLDL_POWERDOWN),
1713                  !!(actual & DPIO_ANYDL_POWERDOWN),
1714                  !!(expected & DPIO_ALLDL_POWERDOWN),
1715                  !!(expected & DPIO_ANYDL_POWERDOWN),
1716                  reg, val);
1717 }
1718
1719 bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1720                           enum dpio_channel ch, bool override)
1721 {
1722         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1723         bool was_override;
1724
1725         mutex_lock(&power_domains->lock);
1726
1727         was_override = dev_priv->chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1728
1729         if (override == was_override)
1730                 goto out;
1731
1732         if (override)
1733                 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1734         else
1735                 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1736
1737         intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1738                        dev_priv->chv_phy_control);
1739
1740         drm_dbg_kms(&dev_priv->drm,
1741                     "Power gating DPIO PHY%d CH%d (DPIO_PHY_CONTROL=0x%08x)\n",
1742                     phy, ch, dev_priv->chv_phy_control);
1743
1744         assert_chv_phy_status(dev_priv);
1745
1746 out:
1747         mutex_unlock(&power_domains->lock);
1748
1749         return was_override;
1750 }
1751
1752 void chv_phy_powergate_lanes(struct intel_encoder *encoder,
1753                              bool override, unsigned int mask)
1754 {
1755         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
1756         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1757         enum dpio_phy phy = vlv_dport_to_phy(enc_to_dig_port(encoder));
1758         enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(encoder));
1759
1760         mutex_lock(&power_domains->lock);
1761
1762         dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch);
1763         dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch);
1764
1765         if (override)
1766                 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1767         else
1768                 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1769
1770         intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1771                        dev_priv->chv_phy_control);
1772
1773         drm_dbg_kms(&dev_priv->drm,
1774                     "Power gating DPIO PHY%d CH%d lanes 0x%x (PHY_CONTROL=0x%08x)\n",
1775                     phy, ch, mask, dev_priv->chv_phy_control);
1776
1777         assert_chv_phy_status(dev_priv);
1778
1779         assert_chv_phy_powergate(dev_priv, phy, ch, override, mask);
1780
1781         mutex_unlock(&power_domains->lock);
1782 }
1783
1784 static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
1785                                         struct i915_power_well *power_well)
1786 {
1787         enum pipe pipe = PIPE_A;
1788         bool enabled;
1789         u32 state, ctrl;
1790
1791         vlv_punit_get(dev_priv);
1792
1793         state = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSS_MASK(pipe);
1794         /*
1795          * We only ever set the power-on and power-gate states, anything
1796          * else is unexpected.
1797          */
1798         drm_WARN_ON(&dev_priv->drm, state != DP_SSS_PWR_ON(pipe) &&
1799                     state != DP_SSS_PWR_GATE(pipe));
1800         enabled = state == DP_SSS_PWR_ON(pipe);
1801
1802         /*
1803          * A transient state at this point would mean some unexpected party
1804          * is poking at the power controls too.
1805          */
1806         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSC_MASK(pipe);
1807         drm_WARN_ON(&dev_priv->drm, ctrl << 16 != state);
1808
1809         vlv_punit_put(dev_priv);
1810
1811         return enabled;
1812 }
1813
1814 static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
1815                                     struct i915_power_well *power_well,
1816                                     bool enable)
1817 {
1818         enum pipe pipe = PIPE_A;
1819         u32 state;
1820         u32 ctrl;
1821
1822         state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe);
1823
1824         vlv_punit_get(dev_priv);
1825
1826 #define COND \
1827         ((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSS_MASK(pipe)) == state)
1828
1829         if (COND)
1830                 goto out;
1831
1832         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
1833         ctrl &= ~DP_SSC_MASK(pipe);
1834         ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
1835         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, ctrl);
1836
1837         if (wait_for(COND, 100))
1838                 drm_err(&dev_priv->drm,
1839                         "timeout setting power well state %08x (%08x)\n",
1840                         state,
1841                         vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM));
1842
1843 #undef COND
1844
1845 out:
1846         vlv_punit_put(dev_priv);
1847 }
1848
1849 static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv,
1850                                         struct i915_power_well *power_well)
1851 {
1852         intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1853                        dev_priv->chv_phy_control);
1854 }
1855
1856 static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
1857                                        struct i915_power_well *power_well)
1858 {
1859         chv_set_pipe_power_well(dev_priv, power_well, true);
1860
1861         vlv_display_power_well_init(dev_priv);
1862 }
1863
1864 static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv,
1865                                         struct i915_power_well *power_well)
1866 {
1867         vlv_display_power_well_deinit(dev_priv);
1868
1869         chv_set_pipe_power_well(dev_priv, power_well, false);
1870 }
1871
1872 static u64 __async_put_domains_mask(struct i915_power_domains *power_domains)
1873 {
1874         return power_domains->async_put_domains[0] |
1875                power_domains->async_put_domains[1];
1876 }
1877
1878 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
1879
1880 static bool
1881 assert_async_put_domain_masks_disjoint(struct i915_power_domains *power_domains)
1882 {
1883         return !WARN_ON(power_domains->async_put_domains[0] &
1884                         power_domains->async_put_domains[1]);
1885 }
1886
1887 static bool
1888 __async_put_domains_state_ok(struct i915_power_domains *power_domains)
1889 {
1890         enum intel_display_power_domain domain;
1891         bool err = false;
1892
1893         err |= !assert_async_put_domain_masks_disjoint(power_domains);
1894         err |= WARN_ON(!!power_domains->async_put_wakeref !=
1895                        !!__async_put_domains_mask(power_domains));
1896
1897         for_each_power_domain(domain, __async_put_domains_mask(power_domains))
1898                 err |= WARN_ON(power_domains->domain_use_count[domain] != 1);
1899
1900         return !err;
1901 }
1902
1903 static void print_power_domains(struct i915_power_domains *power_domains,
1904                                 const char *prefix, u64 mask)
1905 {
1906         struct drm_i915_private *i915 = container_of(power_domains,
1907                                                      struct drm_i915_private,
1908                                                      power_domains);
1909         enum intel_display_power_domain domain;
1910
1911         drm_dbg(&i915->drm, "%s (%lu):\n", prefix, hweight64(mask));
1912         for_each_power_domain(domain, mask)
1913                 drm_dbg(&i915->drm, "%s use_count %d\n",
1914                         intel_display_power_domain_str(domain),
1915                         power_domains->domain_use_count[domain]);
1916 }
1917
1918 static void
1919 print_async_put_domains_state(struct i915_power_domains *power_domains)
1920 {
1921         struct drm_i915_private *i915 = container_of(power_domains,
1922                                                      struct drm_i915_private,
1923                                                      power_domains);
1924
1925         drm_dbg(&i915->drm, "async_put_wakeref %u\n",
1926                 power_domains->async_put_wakeref);
1927
1928         print_power_domains(power_domains, "async_put_domains[0]",
1929                             power_domains->async_put_domains[0]);
1930         print_power_domains(power_domains, "async_put_domains[1]",
1931                             power_domains->async_put_domains[1]);
1932 }
1933
1934 static void
1935 verify_async_put_domains_state(struct i915_power_domains *power_domains)
1936 {
1937         if (!__async_put_domains_state_ok(power_domains))
1938                 print_async_put_domains_state(power_domains);
1939 }
1940
1941 #else
1942
1943 static void
1944 assert_async_put_domain_masks_disjoint(struct i915_power_domains *power_domains)
1945 {
1946 }
1947
1948 static void
1949 verify_async_put_domains_state(struct i915_power_domains *power_domains)
1950 {
1951 }
1952
1953 #endif /* CONFIG_DRM_I915_DEBUG_RUNTIME_PM */
1954
1955 static u64 async_put_domains_mask(struct i915_power_domains *power_domains)
1956 {
1957         assert_async_put_domain_masks_disjoint(power_domains);
1958
1959         return __async_put_domains_mask(power_domains);
1960 }
1961
1962 static void
1963 async_put_domains_clear_domain(struct i915_power_domains *power_domains,
1964                                enum intel_display_power_domain domain)
1965 {
1966         assert_async_put_domain_masks_disjoint(power_domains);
1967
1968         power_domains->async_put_domains[0] &= ~BIT_ULL(domain);
1969         power_domains->async_put_domains[1] &= ~BIT_ULL(domain);
1970 }
1971
1972 static bool
1973 intel_display_power_grab_async_put_ref(struct drm_i915_private *dev_priv,
1974                                        enum intel_display_power_domain domain)
1975 {
1976         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1977         bool ret = false;
1978
1979         if (!(async_put_domains_mask(power_domains) & BIT_ULL(domain)))
1980                 goto out_verify;
1981
1982         async_put_domains_clear_domain(power_domains, domain);
1983
1984         ret = true;
1985
1986         if (async_put_domains_mask(power_domains))
1987                 goto out_verify;
1988
1989         cancel_delayed_work(&power_domains->async_put_work);
1990         intel_runtime_pm_put_raw(&dev_priv->runtime_pm,
1991                                  fetch_and_zero(&power_domains->async_put_wakeref));
1992 out_verify:
1993         verify_async_put_domains_state(power_domains);
1994
1995         return ret;
1996 }
1997
1998 static void
1999 __intel_display_power_get_domain(struct drm_i915_private *dev_priv,
2000                                  enum intel_display_power_domain domain)
2001 {
2002         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2003         struct i915_power_well *power_well;
2004
2005         if (intel_display_power_grab_async_put_ref(dev_priv, domain))
2006                 return;
2007
2008         for_each_power_domain_well(dev_priv, power_well, BIT_ULL(domain))
2009                 intel_power_well_get(dev_priv, power_well);
2010
2011         power_domains->domain_use_count[domain]++;
2012 }
2013
2014 /**
2015  * intel_display_power_get - grab a power domain reference
2016  * @dev_priv: i915 device instance
2017  * @domain: power domain to reference
2018  *
2019  * This function grabs a power domain reference for @domain and ensures that the
2020  * power domain and all its parents are powered up. Therefore users should only
2021  * grab a reference to the innermost power domain they need.
2022  *
2023  * Any power domain reference obtained by this function must have a symmetric
2024  * call to intel_display_power_put() to release the reference again.
2025  */
2026 intel_wakeref_t intel_display_power_get(struct drm_i915_private *dev_priv,
2027                                         enum intel_display_power_domain domain)
2028 {
2029         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2030         intel_wakeref_t wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
2031
2032         mutex_lock(&power_domains->lock);
2033         __intel_display_power_get_domain(dev_priv, domain);
2034         mutex_unlock(&power_domains->lock);
2035
2036         return wakeref;
2037 }
2038
2039 /**
2040  * intel_display_power_get_if_enabled - grab a reference for an enabled display power domain
2041  * @dev_priv: i915 device instance
2042  * @domain: power domain to reference
2043  *
2044  * This function grabs a power domain reference for @domain and ensures that the
2045  * power domain and all its parents are powered up. Therefore users should only
2046  * grab a reference to the innermost power domain they need.
2047  *
2048  * Any power domain reference obtained by this function must have a symmetric
2049  * call to intel_display_power_put() to release the reference again.
2050  */
2051 intel_wakeref_t
2052 intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv,
2053                                    enum intel_display_power_domain domain)
2054 {
2055         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2056         intel_wakeref_t wakeref;
2057         bool is_enabled;
2058
2059         wakeref = intel_runtime_pm_get_if_in_use(&dev_priv->runtime_pm);
2060         if (!wakeref)
2061                 return false;
2062
2063         mutex_lock(&power_domains->lock);
2064
2065         if (__intel_display_power_is_enabled(dev_priv, domain)) {
2066                 __intel_display_power_get_domain(dev_priv, domain);
2067                 is_enabled = true;
2068         } else {
2069                 is_enabled = false;
2070         }
2071
2072         mutex_unlock(&power_domains->lock);
2073
2074         if (!is_enabled) {
2075                 intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
2076                 wakeref = 0;
2077         }
2078
2079         return wakeref;
2080 }
2081
2082 static void
2083 __intel_display_power_put_domain(struct drm_i915_private *dev_priv,
2084                                  enum intel_display_power_domain domain)
2085 {
2086         struct i915_power_domains *power_domains;
2087         struct i915_power_well *power_well;
2088         const char *name = intel_display_power_domain_str(domain);
2089
2090         power_domains = &dev_priv->power_domains;
2091
2092         drm_WARN(&dev_priv->drm, !power_domains->domain_use_count[domain],
2093                  "Use count on domain %s is already zero\n",
2094                  name);
2095         drm_WARN(&dev_priv->drm,
2096                  async_put_domains_mask(power_domains) & BIT_ULL(domain),
2097                  "Async disabling of domain %s is pending\n",
2098                  name);
2099
2100         power_domains->domain_use_count[domain]--;
2101
2102         for_each_power_domain_well_reverse(dev_priv, power_well, BIT_ULL(domain))
2103                 intel_power_well_put(dev_priv, power_well);
2104 }
2105
2106 static void __intel_display_power_put(struct drm_i915_private *dev_priv,
2107                                       enum intel_display_power_domain domain)
2108 {
2109         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2110
2111         mutex_lock(&power_domains->lock);
2112         __intel_display_power_put_domain(dev_priv, domain);
2113         mutex_unlock(&power_domains->lock);
2114 }
2115
2116 /**
2117  * intel_display_power_put_unchecked - release an unchecked power domain reference
2118  * @dev_priv: i915 device instance
2119  * @domain: power domain to reference
2120  *
2121  * This function drops the power domain reference obtained by
2122  * intel_display_power_get() and might power down the corresponding hardware
2123  * block right away if this is the last reference.
2124  *
2125  * This function exists only for historical reasons and should be avoided in
2126  * new code, as the correctness of its use cannot be checked. Always use
2127  * intel_display_power_put() instead.
2128  */
2129 void intel_display_power_put_unchecked(struct drm_i915_private *dev_priv,
2130                                        enum intel_display_power_domain domain)
2131 {
2132         __intel_display_power_put(dev_priv, domain);
2133         intel_runtime_pm_put_unchecked(&dev_priv->runtime_pm);
2134 }
2135
2136 static void
2137 queue_async_put_domains_work(struct i915_power_domains *power_domains,
2138                              intel_wakeref_t wakeref)
2139 {
2140         WARN_ON(power_domains->async_put_wakeref);
2141         power_domains->async_put_wakeref = wakeref;
2142         WARN_ON(!queue_delayed_work(system_unbound_wq,
2143                                     &power_domains->async_put_work,
2144                                     msecs_to_jiffies(100)));
2145 }
2146
2147 static void
2148 release_async_put_domains(struct i915_power_domains *power_domains, u64 mask)
2149 {
2150         struct drm_i915_private *dev_priv =
2151                 container_of(power_domains, struct drm_i915_private,
2152                              power_domains);
2153         struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
2154         enum intel_display_power_domain domain;
2155         intel_wakeref_t wakeref;
2156
2157         /*
2158          * The caller must hold already raw wakeref, upgrade that to a proper
2159          * wakeref to make the state checker happy about the HW access during
2160          * power well disabling.
2161          */
2162         assert_rpm_raw_wakeref_held(rpm);
2163         wakeref = intel_runtime_pm_get(rpm);
2164
2165         for_each_power_domain(domain, mask) {
2166                 /* Clear before put, so put's sanity check is happy. */
2167                 async_put_domains_clear_domain(power_domains, domain);
2168                 __intel_display_power_put_domain(dev_priv, domain);
2169         }
2170
2171         intel_runtime_pm_put(rpm, wakeref);
2172 }
2173
2174 static void
2175 intel_display_power_put_async_work(struct work_struct *work)
2176 {
2177         struct drm_i915_private *dev_priv =
2178                 container_of(work, struct drm_i915_private,
2179                              power_domains.async_put_work.work);
2180         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2181         struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
2182         intel_wakeref_t new_work_wakeref = intel_runtime_pm_get_raw(rpm);
2183         intel_wakeref_t old_work_wakeref = 0;
2184
2185         mutex_lock(&power_domains->lock);
2186
2187         /*
2188          * Bail out if all the domain refs pending to be released were grabbed
2189          * by subsequent gets or a flush_work.
2190          */
2191         old_work_wakeref = fetch_and_zero(&power_domains->async_put_wakeref);
2192         if (!old_work_wakeref)
2193                 goto out_verify;
2194
2195         release_async_put_domains(power_domains,
2196                                   power_domains->async_put_domains[0]);
2197
2198         /* Requeue the work if more domains were async put meanwhile. */
2199         if (power_domains->async_put_domains[1]) {
2200                 power_domains->async_put_domains[0] =
2201                         fetch_and_zero(&power_domains->async_put_domains[1]);
2202                 queue_async_put_domains_work(power_domains,
2203                                              fetch_and_zero(&new_work_wakeref));
2204         }
2205
2206 out_verify:
2207         verify_async_put_domains_state(power_domains);
2208
2209         mutex_unlock(&power_domains->lock);
2210
2211         if (old_work_wakeref)
2212                 intel_runtime_pm_put_raw(rpm, old_work_wakeref);
2213         if (new_work_wakeref)
2214                 intel_runtime_pm_put_raw(rpm, new_work_wakeref);
2215 }
2216
2217 /**
2218  * intel_display_power_put_async - release a power domain reference asynchronously
2219  * @i915: i915 device instance
2220  * @domain: power domain to reference
2221  * @wakeref: wakeref acquired for the reference that is being released
2222  *
2223  * This function drops the power domain reference obtained by
2224  * intel_display_power_get*() and schedules a work to power down the
2225  * corresponding hardware block if this is the last reference.
2226  */
2227 void __intel_display_power_put_async(struct drm_i915_private *i915,
2228                                      enum intel_display_power_domain domain,
2229                                      intel_wakeref_t wakeref)
2230 {
2231         struct i915_power_domains *power_domains = &i915->power_domains;
2232         struct intel_runtime_pm *rpm = &i915->runtime_pm;
2233         intel_wakeref_t work_wakeref = intel_runtime_pm_get_raw(rpm);
2234
2235         mutex_lock(&power_domains->lock);
2236
2237         if (power_domains->domain_use_count[domain] > 1) {
2238                 __intel_display_power_put_domain(i915, domain);
2239
2240                 goto out_verify;
2241         }
2242
2243         drm_WARN_ON(&i915->drm, power_domains->domain_use_count[domain] != 1);
2244
2245         /* Let a pending work requeue itself or queue a new one. */
2246         if (power_domains->async_put_wakeref) {
2247                 power_domains->async_put_domains[1] |= BIT_ULL(domain);
2248         } else {
2249                 power_domains->async_put_domains[0] |= BIT_ULL(domain);
2250                 queue_async_put_domains_work(power_domains,
2251                                              fetch_and_zero(&work_wakeref));
2252         }
2253
2254 out_verify:
2255         verify_async_put_domains_state(power_domains);
2256
2257         mutex_unlock(&power_domains->lock);
2258
2259         if (work_wakeref)
2260                 intel_runtime_pm_put_raw(rpm, work_wakeref);
2261
2262         intel_runtime_pm_put(rpm, wakeref);
2263 }
2264
2265 /**
2266  * intel_display_power_flush_work - flushes the async display power disabling work
2267  * @i915: i915 device instance
2268  *
2269  * Flushes any pending work that was scheduled by a preceding
2270  * intel_display_power_put_async() call, completing the disabling of the
2271  * corresponding power domains.
2272  *
2273  * Note that the work handler function may still be running after this
2274  * function returns; to ensure that the work handler isn't running use
2275  * intel_display_power_flush_work_sync() instead.
2276  */
2277 void intel_display_power_flush_work(struct drm_i915_private *i915)
2278 {
2279         struct i915_power_domains *power_domains = &i915->power_domains;
2280         intel_wakeref_t work_wakeref;
2281
2282         mutex_lock(&power_domains->lock);
2283
2284         work_wakeref = fetch_and_zero(&power_domains->async_put_wakeref);
2285         if (!work_wakeref)
2286                 goto out_verify;
2287
2288         release_async_put_domains(power_domains,
2289                                   async_put_domains_mask(power_domains));
2290         cancel_delayed_work(&power_domains->async_put_work);
2291
2292 out_verify:
2293         verify_async_put_domains_state(power_domains);
2294
2295         mutex_unlock(&power_domains->lock);
2296
2297         if (work_wakeref)
2298                 intel_runtime_pm_put_raw(&i915->runtime_pm, work_wakeref);
2299 }
2300
2301 /**
2302  * intel_display_power_flush_work_sync - flushes and syncs the async display power disabling work
2303  * @i915: i915 device instance
2304  *
2305  * Like intel_display_power_flush_work(), but also ensure that the work
2306  * handler function is not running any more when this function returns.
2307  */
2308 static void
2309 intel_display_power_flush_work_sync(struct drm_i915_private *i915)
2310 {
2311         struct i915_power_domains *power_domains = &i915->power_domains;
2312
2313         intel_display_power_flush_work(i915);
2314         cancel_delayed_work_sync(&power_domains->async_put_work);
2315
2316         verify_async_put_domains_state(power_domains);
2317
2318         drm_WARN_ON(&i915->drm, power_domains->async_put_wakeref);
2319 }
2320
2321 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
2322 /**
2323  * intel_display_power_put - release a power domain reference
2324  * @dev_priv: i915 device instance
2325  * @domain: power domain to reference
2326  * @wakeref: wakeref acquired for the reference that is being released
2327  *
2328  * This function drops the power domain reference obtained by
2329  * intel_display_power_get() and might power down the corresponding hardware
2330  * block right away if this is the last reference.
2331  */
2332 void intel_display_power_put(struct drm_i915_private *dev_priv,
2333                              enum intel_display_power_domain domain,
2334                              intel_wakeref_t wakeref)
2335 {
2336         __intel_display_power_put(dev_priv, domain);
2337         intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
2338 }
2339 #endif
2340
2341 #define I830_PIPES_POWER_DOMAINS (              \
2342         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
2343         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
2344         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
2345         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
2346         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
2347         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
2348         BIT_ULL(POWER_DOMAIN_INIT))
2349
2350 #define VLV_DISPLAY_POWER_DOMAINS (             \
2351         BIT_ULL(POWER_DOMAIN_DISPLAY_CORE) |    \
2352         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
2353         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
2354         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
2355         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
2356         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
2357         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
2358         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2359         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2360         BIT_ULL(POWER_DOMAIN_PORT_DSI) |                \
2361         BIT_ULL(POWER_DOMAIN_PORT_CRT) |                \
2362         BIT_ULL(POWER_DOMAIN_VGA) |                     \
2363         BIT_ULL(POWER_DOMAIN_AUDIO) |           \
2364         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2365         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2366         BIT_ULL(POWER_DOMAIN_GMBUS) |           \
2367         BIT_ULL(POWER_DOMAIN_INIT))
2368
2369 #define VLV_DPIO_CMN_BC_POWER_DOMAINS (         \
2370         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2371         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2372         BIT_ULL(POWER_DOMAIN_PORT_CRT) |                \
2373         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2374         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2375         BIT_ULL(POWER_DOMAIN_INIT))
2376
2377 #define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS (  \
2378         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2379         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2380         BIT_ULL(POWER_DOMAIN_INIT))
2381
2382 #define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS (  \
2383         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2384         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2385         BIT_ULL(POWER_DOMAIN_INIT))
2386
2387 #define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS (  \
2388         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2389         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2390         BIT_ULL(POWER_DOMAIN_INIT))
2391
2392 #define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS (  \
2393         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2394         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2395         BIT_ULL(POWER_DOMAIN_INIT))
2396
2397 #define CHV_DISPLAY_POWER_DOMAINS (             \
2398         BIT_ULL(POWER_DOMAIN_DISPLAY_CORE) |    \
2399         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
2400         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
2401         BIT_ULL(POWER_DOMAIN_PIPE_C) |          \
2402         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
2403         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
2404         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
2405         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
2406         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
2407         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |    \
2408         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2409         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2410         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
2411         BIT_ULL(POWER_DOMAIN_PORT_DSI) |                \
2412         BIT_ULL(POWER_DOMAIN_VGA) |                     \
2413         BIT_ULL(POWER_DOMAIN_AUDIO) |           \
2414         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2415         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2416         BIT_ULL(POWER_DOMAIN_AUX_D) |           \
2417         BIT_ULL(POWER_DOMAIN_GMBUS) |           \
2418         BIT_ULL(POWER_DOMAIN_INIT))
2419
2420 #define CHV_DPIO_CMN_BC_POWER_DOMAINS (         \
2421         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2422         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2423         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2424         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2425         BIT_ULL(POWER_DOMAIN_INIT))
2426
2427 #define CHV_DPIO_CMN_D_POWER_DOMAINS (          \
2428         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
2429         BIT_ULL(POWER_DOMAIN_AUX_D) |           \
2430         BIT_ULL(POWER_DOMAIN_INIT))
2431
2432 #define HSW_DISPLAY_POWER_DOMAINS (                     \
2433         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2434         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2435         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |             \
2436         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
2437         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
2438         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2439         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2440         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2441         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2442         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2443         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
2444         BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */    \
2445         BIT_ULL(POWER_DOMAIN_VGA) |                             \
2446         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2447         BIT_ULL(POWER_DOMAIN_INIT))
2448
2449 #define BDW_DISPLAY_POWER_DOMAINS (                     \
2450         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2451         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2452         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
2453         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
2454         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2455         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2456         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2457         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2458         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2459         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
2460         BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */    \
2461         BIT_ULL(POWER_DOMAIN_VGA) |                             \
2462         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2463         BIT_ULL(POWER_DOMAIN_INIT))
2464
2465 #define SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
2466         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2467         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2468         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2469         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2470         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2471         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
2472         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
2473         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2474         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2475         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
2476         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |                \
2477         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
2478         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2479         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
2480         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2481         BIT_ULL(POWER_DOMAIN_VGA) |                             \
2482         BIT_ULL(POWER_DOMAIN_INIT))
2483 #define SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS (          \
2484         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |           \
2485         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO) |           \
2486         BIT_ULL(POWER_DOMAIN_INIT))
2487 #define SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS (            \
2488         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |           \
2489         BIT_ULL(POWER_DOMAIN_INIT))
2490 #define SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS (            \
2491         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |           \
2492         BIT_ULL(POWER_DOMAIN_INIT))
2493 #define SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS (            \
2494         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |           \
2495         BIT_ULL(POWER_DOMAIN_INIT))
2496 #define SKL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
2497         SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
2498         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
2499         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
2500         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2501         BIT_ULL(POWER_DOMAIN_INIT))
2502
2503 #define BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
2504         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2505         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2506         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2507         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2508         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2509         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
2510         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
2511         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2512         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2513         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
2514         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2515         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2516         BIT_ULL(POWER_DOMAIN_VGA) |                             \
2517         BIT_ULL(POWER_DOMAIN_INIT))
2518 #define BXT_DISPLAY_DC_OFF_POWER_DOMAINS (              \
2519         BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
2520         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
2521         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
2522         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2523         BIT_ULL(POWER_DOMAIN_GMBUS) |                   \
2524         BIT_ULL(POWER_DOMAIN_INIT))
2525 #define BXT_DPIO_CMN_A_POWER_DOMAINS (                  \
2526         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
2527         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2528         BIT_ULL(POWER_DOMAIN_INIT))
2529 #define BXT_DPIO_CMN_BC_POWER_DOMAINS (                 \
2530         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2531         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2532         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
2533         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2534         BIT_ULL(POWER_DOMAIN_INIT))
2535
2536 #define GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
2537         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2538         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2539         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2540         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2541         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2542         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
2543         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
2544         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2545         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2546         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
2547         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2548         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2549         BIT_ULL(POWER_DOMAIN_VGA) |                             \
2550         BIT_ULL(POWER_DOMAIN_INIT))
2551 #define GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS (            \
2552         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO))
2553 #define GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS (            \
2554         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO))
2555 #define GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS (            \
2556         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO))
2557 #define GLK_DPIO_CMN_A_POWER_DOMAINS (                  \
2558         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
2559         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2560         BIT_ULL(POWER_DOMAIN_INIT))
2561 #define GLK_DPIO_CMN_B_POWER_DOMAINS (                  \
2562         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2563         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
2564         BIT_ULL(POWER_DOMAIN_INIT))
2565 #define GLK_DPIO_CMN_C_POWER_DOMAINS (                  \
2566         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2567         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2568         BIT_ULL(POWER_DOMAIN_INIT))
2569 #define GLK_DISPLAY_AUX_A_POWER_DOMAINS (               \
2570         BIT_ULL(POWER_DOMAIN_AUX_A) |           \
2571         BIT_ULL(POWER_DOMAIN_AUX_IO_A) |                \
2572         BIT_ULL(POWER_DOMAIN_INIT))
2573 #define GLK_DISPLAY_AUX_B_POWER_DOMAINS (               \
2574         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2575         BIT_ULL(POWER_DOMAIN_INIT))
2576 #define GLK_DISPLAY_AUX_C_POWER_DOMAINS (               \
2577         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2578         BIT_ULL(POWER_DOMAIN_INIT))
2579 #define GLK_DISPLAY_DC_OFF_POWER_DOMAINS (              \
2580         GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
2581         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
2582         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
2583         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2584         BIT_ULL(POWER_DOMAIN_GMBUS) |                   \
2585         BIT_ULL(POWER_DOMAIN_INIT))
2586
2587 #define CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
2588         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2589         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2590         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2591         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2592         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2593         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
2594         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
2595         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2596         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2597         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
2598         BIT_ULL(POWER_DOMAIN_PORT_DDI_F_LANES) |                \
2599         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
2600         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2601         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
2602         BIT_ULL(POWER_DOMAIN_AUX_F) |                   \
2603         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2604         BIT_ULL(POWER_DOMAIN_VGA) |                             \
2605         BIT_ULL(POWER_DOMAIN_INIT))
2606 #define CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS (            \
2607         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |           \
2608         BIT_ULL(POWER_DOMAIN_INIT))
2609 #define CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS (            \
2610         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |           \
2611         BIT_ULL(POWER_DOMAIN_INIT))
2612 #define CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS (            \
2613         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |           \
2614         BIT_ULL(POWER_DOMAIN_INIT))
2615 #define CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS (            \
2616         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |           \
2617         BIT_ULL(POWER_DOMAIN_INIT))
2618 #define CNL_DISPLAY_AUX_A_POWER_DOMAINS (               \
2619         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2620         BIT_ULL(POWER_DOMAIN_AUX_IO_A) |                \
2621         BIT_ULL(POWER_DOMAIN_INIT))
2622 #define CNL_DISPLAY_AUX_B_POWER_DOMAINS (               \
2623         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
2624         BIT_ULL(POWER_DOMAIN_INIT))
2625 #define CNL_DISPLAY_AUX_C_POWER_DOMAINS (               \
2626         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2627         BIT_ULL(POWER_DOMAIN_INIT))
2628 #define CNL_DISPLAY_AUX_D_POWER_DOMAINS (               \
2629         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
2630         BIT_ULL(POWER_DOMAIN_INIT))
2631 #define CNL_DISPLAY_AUX_F_POWER_DOMAINS (               \
2632         BIT_ULL(POWER_DOMAIN_AUX_F) |                   \
2633         BIT_ULL(POWER_DOMAIN_INIT))
2634 #define CNL_DISPLAY_DDI_F_IO_POWER_DOMAINS (            \
2635         BIT_ULL(POWER_DOMAIN_PORT_DDI_F_IO) |           \
2636         BIT_ULL(POWER_DOMAIN_INIT))
2637 #define CNL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
2638         CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
2639         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
2640         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
2641         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2642         BIT_ULL(POWER_DOMAIN_INIT))
2643
2644 /*
2645  * ICL PW_0/PG_0 domains (HW/DMC control):
2646  * - PCI
2647  * - clocks except port PLL
2648  * - central power except FBC
2649  * - shared functions except pipe interrupts, pipe MBUS, DBUF registers
2650  * ICL PW_1/PG_1 domains (HW/DMC control):
2651  * - DBUF function
2652  * - PIPE_A and its planes, except VGA
2653  * - transcoder EDP + PSR
2654  * - transcoder DSI
2655  * - DDI_A
2656  * - FBC
2657  */
2658 #define ICL_PW_4_POWER_DOMAINS (                        \
2659         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2660         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
2661         BIT_ULL(POWER_DOMAIN_INIT))
2662         /* VDSC/joining */
2663 #define ICL_PW_3_POWER_DOMAINS (                        \
2664         ICL_PW_4_POWER_DOMAINS |                        \
2665         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2666         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2667         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2668         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2669         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
2670         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2671         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2672         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
2673         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |        \
2674         BIT_ULL(POWER_DOMAIN_PORT_DDI_F_LANES) |        \
2675         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
2676         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2677         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
2678         BIT_ULL(POWER_DOMAIN_AUX_E) |                   \
2679         BIT_ULL(POWER_DOMAIN_AUX_F) |                   \
2680         BIT_ULL(POWER_DOMAIN_AUX_C_TBT) |               \
2681         BIT_ULL(POWER_DOMAIN_AUX_D_TBT) |               \
2682         BIT_ULL(POWER_DOMAIN_AUX_E_TBT) |               \
2683         BIT_ULL(POWER_DOMAIN_AUX_F_TBT) |               \
2684         BIT_ULL(POWER_DOMAIN_VGA) |                     \
2685         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2686         BIT_ULL(POWER_DOMAIN_INIT))
2687         /*
2688          * - transcoder WD
2689          * - KVMR (HW control)
2690          */
2691 #define ICL_PW_2_POWER_DOMAINS (                        \
2692         ICL_PW_3_POWER_DOMAINS |                        \
2693         BIT_ULL(POWER_DOMAIN_TRANSCODER_VDSC_PW2) |             \
2694         BIT_ULL(POWER_DOMAIN_INIT))
2695         /*
2696          * - KVMR (HW control)
2697          */
2698 #define ICL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
2699         ICL_PW_2_POWER_DOMAINS |                        \
2700         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
2701         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2702         BIT_ULL(POWER_DOMAIN_DPLL_DC_OFF) |                     \
2703         BIT_ULL(POWER_DOMAIN_INIT))
2704
2705 #define ICL_DDI_IO_A_POWER_DOMAINS (                    \
2706         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO))
2707 #define ICL_DDI_IO_B_POWER_DOMAINS (                    \
2708         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO))
2709 #define ICL_DDI_IO_C_POWER_DOMAINS (                    \
2710         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO))
2711 #define ICL_DDI_IO_D_POWER_DOMAINS (                    \
2712         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO))
2713 #define ICL_DDI_IO_E_POWER_DOMAINS (                    \
2714         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO))
2715 #define ICL_DDI_IO_F_POWER_DOMAINS (                    \
2716         BIT_ULL(POWER_DOMAIN_PORT_DDI_F_IO))
2717
2718 #define ICL_AUX_A_IO_POWER_DOMAINS (                    \
2719         BIT_ULL(POWER_DOMAIN_AUX_IO_A) |                \
2720         BIT_ULL(POWER_DOMAIN_AUX_A))
2721 #define ICL_AUX_B_IO_POWER_DOMAINS (                    \
2722         BIT_ULL(POWER_DOMAIN_AUX_B))
2723 #define ICL_AUX_C_TC1_IO_POWER_DOMAINS (                \
2724         BIT_ULL(POWER_DOMAIN_AUX_C))
2725 #define ICL_AUX_D_TC2_IO_POWER_DOMAINS (                \
2726         BIT_ULL(POWER_DOMAIN_AUX_D))
2727 #define ICL_AUX_E_TC3_IO_POWER_DOMAINS (                \
2728         BIT_ULL(POWER_DOMAIN_AUX_E))
2729 #define ICL_AUX_F_TC4_IO_POWER_DOMAINS (                \
2730         BIT_ULL(POWER_DOMAIN_AUX_F))
2731 #define ICL_AUX_C_TBT1_IO_POWER_DOMAINS (               \
2732         BIT_ULL(POWER_DOMAIN_AUX_C_TBT))
2733 #define ICL_AUX_D_TBT2_IO_POWER_DOMAINS (               \
2734         BIT_ULL(POWER_DOMAIN_AUX_D_TBT))
2735 #define ICL_AUX_E_TBT3_IO_POWER_DOMAINS (               \
2736         BIT_ULL(POWER_DOMAIN_AUX_E_TBT))
2737 #define ICL_AUX_F_TBT4_IO_POWER_DOMAINS (               \
2738         BIT_ULL(POWER_DOMAIN_AUX_F_TBT))
2739
2740 #define TGL_PW_5_POWER_DOMAINS (                        \
2741         BIT_ULL(POWER_DOMAIN_PIPE_D) |                  \
2742         BIT_ULL(POWER_DOMAIN_TRANSCODER_D) |            \
2743         BIT_ULL(POWER_DOMAIN_PIPE_D_PANEL_FITTER) |     \
2744         BIT_ULL(POWER_DOMAIN_INIT))
2745
2746 #define TGL_PW_4_POWER_DOMAINS (                        \
2747         TGL_PW_5_POWER_DOMAINS |                        \
2748         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2749         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2750         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
2751         BIT_ULL(POWER_DOMAIN_INIT))
2752
2753 #define TGL_PW_3_POWER_DOMAINS (                        \
2754         TGL_PW_4_POWER_DOMAINS |                        \
2755         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2756         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2757         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
2758         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
2759         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |        \
2760         BIT_ULL(POWER_DOMAIN_PORT_DDI_F_LANES) |        \
2761         BIT_ULL(POWER_DOMAIN_PORT_DDI_G_LANES) |        \
2762         BIT_ULL(POWER_DOMAIN_PORT_DDI_H_LANES) |        \
2763         BIT_ULL(POWER_DOMAIN_PORT_DDI_I_LANES) |        \
2764         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
2765         BIT_ULL(POWER_DOMAIN_AUX_E) |                   \
2766         BIT_ULL(POWER_DOMAIN_AUX_F) |                   \
2767         BIT_ULL(POWER_DOMAIN_AUX_G) |                   \
2768         BIT_ULL(POWER_DOMAIN_AUX_H) |                   \
2769         BIT_ULL(POWER_DOMAIN_AUX_I) |                   \
2770         BIT_ULL(POWER_DOMAIN_AUX_D_TBT) |               \
2771         BIT_ULL(POWER_DOMAIN_AUX_E_TBT) |               \
2772         BIT_ULL(POWER_DOMAIN_AUX_F_TBT) |               \
2773         BIT_ULL(POWER_DOMAIN_AUX_G_TBT) |               \
2774         BIT_ULL(POWER_DOMAIN_AUX_H_TBT) |               \
2775         BIT_ULL(POWER_DOMAIN_AUX_I_TBT) |               \
2776         BIT_ULL(POWER_DOMAIN_VGA) |                     \
2777         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2778         BIT_ULL(POWER_DOMAIN_INIT))
2779
2780 #define TGL_PW_2_POWER_DOMAINS (                        \
2781         TGL_PW_3_POWER_DOMAINS |                        \
2782         BIT_ULL(POWER_DOMAIN_TRANSCODER_VDSC_PW2) |     \
2783         BIT_ULL(POWER_DOMAIN_INIT))
2784
2785 #define TGL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
2786         TGL_PW_3_POWER_DOMAINS |                        \
2787         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
2788         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2789         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
2790         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2791         BIT_ULL(POWER_DOMAIN_INIT))
2792
2793 #define TGL_DDI_IO_D_TC1_POWER_DOMAINS (        \
2794         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO))
2795 #define TGL_DDI_IO_E_TC2_POWER_DOMAINS (        \
2796         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO))
2797 #define TGL_DDI_IO_F_TC3_POWER_DOMAINS (        \
2798         BIT_ULL(POWER_DOMAIN_PORT_DDI_F_IO))
2799 #define TGL_DDI_IO_G_TC4_POWER_DOMAINS (        \
2800         BIT_ULL(POWER_DOMAIN_PORT_DDI_G_IO))
2801 #define TGL_DDI_IO_H_TC5_POWER_DOMAINS (        \
2802         BIT_ULL(POWER_DOMAIN_PORT_DDI_H_IO))
2803 #define TGL_DDI_IO_I_TC6_POWER_DOMAINS (        \
2804         BIT_ULL(POWER_DOMAIN_PORT_DDI_I_IO))
2805
2806 #define TGL_AUX_A_IO_POWER_DOMAINS (            \
2807         BIT_ULL(POWER_DOMAIN_AUX_IO_A) |        \
2808         BIT_ULL(POWER_DOMAIN_AUX_A))
2809 #define TGL_AUX_B_IO_POWER_DOMAINS (            \
2810         BIT_ULL(POWER_DOMAIN_AUX_B))
2811 #define TGL_AUX_C_IO_POWER_DOMAINS (            \
2812         BIT_ULL(POWER_DOMAIN_AUX_C))
2813 #define TGL_AUX_D_TC1_IO_POWER_DOMAINS (        \
2814         BIT_ULL(POWER_DOMAIN_AUX_D))
2815 #define TGL_AUX_E_TC2_IO_POWER_DOMAINS (        \
2816         BIT_ULL(POWER_DOMAIN_AUX_E))
2817 #define TGL_AUX_F_TC3_IO_POWER_DOMAINS (        \
2818         BIT_ULL(POWER_DOMAIN_AUX_F))
2819 #define TGL_AUX_G_TC4_IO_POWER_DOMAINS (        \
2820         BIT_ULL(POWER_DOMAIN_AUX_G))
2821 #define TGL_AUX_H_TC5_IO_POWER_DOMAINS (        \
2822         BIT_ULL(POWER_DOMAIN_AUX_H))
2823 #define TGL_AUX_I_TC6_IO_POWER_DOMAINS (        \
2824         BIT_ULL(POWER_DOMAIN_AUX_I))
2825 #define TGL_AUX_D_TBT1_IO_POWER_DOMAINS (       \
2826         BIT_ULL(POWER_DOMAIN_AUX_D_TBT))
2827 #define TGL_AUX_E_TBT2_IO_POWER_DOMAINS (       \
2828         BIT_ULL(POWER_DOMAIN_AUX_E_TBT))
2829 #define TGL_AUX_F_TBT3_IO_POWER_DOMAINS (       \
2830         BIT_ULL(POWER_DOMAIN_AUX_F_TBT))
2831 #define TGL_AUX_G_TBT4_IO_POWER_DOMAINS (       \
2832         BIT_ULL(POWER_DOMAIN_AUX_G_TBT))
2833 #define TGL_AUX_H_TBT5_IO_POWER_DOMAINS (       \
2834         BIT_ULL(POWER_DOMAIN_AUX_H_TBT))
2835 #define TGL_AUX_I_TBT6_IO_POWER_DOMAINS (       \
2836         BIT_ULL(POWER_DOMAIN_AUX_I_TBT))
2837
2838 static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
2839         .sync_hw = i9xx_power_well_sync_hw_noop,
2840         .enable = i9xx_always_on_power_well_noop,
2841         .disable = i9xx_always_on_power_well_noop,
2842         .is_enabled = i9xx_always_on_power_well_enabled,
2843 };
2844
2845 static const struct i915_power_well_ops chv_pipe_power_well_ops = {
2846         .sync_hw = chv_pipe_power_well_sync_hw,
2847         .enable = chv_pipe_power_well_enable,
2848         .disable = chv_pipe_power_well_disable,
2849         .is_enabled = chv_pipe_power_well_enabled,
2850 };
2851
2852 static const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = {
2853         .sync_hw = i9xx_power_well_sync_hw_noop,
2854         .enable = chv_dpio_cmn_power_well_enable,
2855         .disable = chv_dpio_cmn_power_well_disable,
2856         .is_enabled = vlv_power_well_enabled,
2857 };
2858
2859 static const struct i915_power_well_desc i9xx_always_on_power_well[] = {
2860         {
2861                 .name = "always-on",
2862                 .always_on = true,
2863                 .domains = POWER_DOMAIN_MASK,
2864                 .ops = &i9xx_always_on_power_well_ops,
2865                 .id = DISP_PW_ID_NONE,
2866         },
2867 };
2868
2869 static const struct i915_power_well_ops i830_pipes_power_well_ops = {
2870         .sync_hw = i830_pipes_power_well_sync_hw,
2871         .enable = i830_pipes_power_well_enable,
2872         .disable = i830_pipes_power_well_disable,
2873         .is_enabled = i830_pipes_power_well_enabled,
2874 };
2875
2876 static const struct i915_power_well_desc i830_power_wells[] = {
2877         {
2878                 .name = "always-on",
2879                 .always_on = true,
2880                 .domains = POWER_DOMAIN_MASK,
2881                 .ops = &i9xx_always_on_power_well_ops,
2882                 .id = DISP_PW_ID_NONE,
2883         },
2884         {
2885                 .name = "pipes",
2886                 .domains = I830_PIPES_POWER_DOMAINS,
2887                 .ops = &i830_pipes_power_well_ops,
2888                 .id = DISP_PW_ID_NONE,
2889         },
2890 };
2891
2892 static const struct i915_power_well_ops hsw_power_well_ops = {
2893         .sync_hw = hsw_power_well_sync_hw,
2894         .enable = hsw_power_well_enable,
2895         .disable = hsw_power_well_disable,
2896         .is_enabled = hsw_power_well_enabled,
2897 };
2898
2899 static const struct i915_power_well_ops gen9_dc_off_power_well_ops = {
2900         .sync_hw = i9xx_power_well_sync_hw_noop,
2901         .enable = gen9_dc_off_power_well_enable,
2902         .disable = gen9_dc_off_power_well_disable,
2903         .is_enabled = gen9_dc_off_power_well_enabled,
2904 };
2905
2906 static const struct i915_power_well_ops bxt_dpio_cmn_power_well_ops = {
2907         .sync_hw = i9xx_power_well_sync_hw_noop,
2908         .enable = bxt_dpio_cmn_power_well_enable,
2909         .disable = bxt_dpio_cmn_power_well_disable,
2910         .is_enabled = bxt_dpio_cmn_power_well_enabled,
2911 };
2912
2913 static const struct i915_power_well_regs hsw_power_well_regs = {
2914         .bios   = HSW_PWR_WELL_CTL1,
2915         .driver = HSW_PWR_WELL_CTL2,
2916         .kvmr   = HSW_PWR_WELL_CTL3,
2917         .debug  = HSW_PWR_WELL_CTL4,
2918 };
2919
2920 static const struct i915_power_well_desc hsw_power_wells[] = {
2921         {
2922                 .name = "always-on",
2923                 .always_on = true,
2924                 .domains = POWER_DOMAIN_MASK,
2925                 .ops = &i9xx_always_on_power_well_ops,
2926                 .id = DISP_PW_ID_NONE,
2927         },
2928         {
2929                 .name = "display",
2930                 .domains = HSW_DISPLAY_POWER_DOMAINS,
2931                 .ops = &hsw_power_well_ops,
2932                 .id = HSW_DISP_PW_GLOBAL,
2933                 {
2934                         .hsw.regs = &hsw_power_well_regs,
2935                         .hsw.idx = HSW_PW_CTL_IDX_GLOBAL,
2936                         .hsw.has_vga = true,
2937                 },
2938         },
2939 };
2940
2941 static const struct i915_power_well_desc bdw_power_wells[] = {
2942         {
2943                 .name = "always-on",
2944                 .always_on = true,
2945                 .domains = POWER_DOMAIN_MASK,
2946                 .ops = &i9xx_always_on_power_well_ops,
2947                 .id = DISP_PW_ID_NONE,
2948         },
2949         {
2950                 .name = "display",
2951                 .domains = BDW_DISPLAY_POWER_DOMAINS,
2952                 .ops = &hsw_power_well_ops,
2953                 .id = HSW_DISP_PW_GLOBAL,
2954                 {
2955                         .hsw.regs = &hsw_power_well_regs,
2956                         .hsw.idx = HSW_PW_CTL_IDX_GLOBAL,
2957                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2958                         .hsw.has_vga = true,
2959                 },
2960         },
2961 };
2962
2963 static const struct i915_power_well_ops vlv_display_power_well_ops = {
2964         .sync_hw = i9xx_power_well_sync_hw_noop,
2965         .enable = vlv_display_power_well_enable,
2966         .disable = vlv_display_power_well_disable,
2967         .is_enabled = vlv_power_well_enabled,
2968 };
2969
2970 static const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = {
2971         .sync_hw = i9xx_power_well_sync_hw_noop,
2972         .enable = vlv_dpio_cmn_power_well_enable,
2973         .disable = vlv_dpio_cmn_power_well_disable,
2974         .is_enabled = vlv_power_well_enabled,
2975 };
2976
2977 static const struct i915_power_well_ops vlv_dpio_power_well_ops = {
2978         .sync_hw = i9xx_power_well_sync_hw_noop,
2979         .enable = vlv_power_well_enable,
2980         .disable = vlv_power_well_disable,
2981         .is_enabled = vlv_power_well_enabled,
2982 };
2983
2984 static const struct i915_power_well_desc vlv_power_wells[] = {
2985         {
2986                 .name = "always-on",
2987                 .always_on = true,
2988                 .domains = POWER_DOMAIN_MASK,
2989                 .ops = &i9xx_always_on_power_well_ops,
2990                 .id = DISP_PW_ID_NONE,
2991         },
2992         {
2993                 .name = "display",
2994                 .domains = VLV_DISPLAY_POWER_DOMAINS,
2995                 .ops = &vlv_display_power_well_ops,
2996                 .id = VLV_DISP_PW_DISP2D,
2997                 {
2998                         .vlv.idx = PUNIT_PWGT_IDX_DISP2D,
2999                 },
3000         },
3001         {
3002                 .name = "dpio-tx-b-01",
3003                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
3004                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
3005                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
3006                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
3007                 .ops = &vlv_dpio_power_well_ops,
3008                 .id = DISP_PW_ID_NONE,
3009                 {
3010                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_TX_B_LANES_01,
3011                 },
3012         },
3013         {
3014                 .name = "dpio-tx-b-23",
3015                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
3016                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
3017                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
3018                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
3019                 .ops = &vlv_dpio_power_well_ops,
3020                 .id = DISP_PW_ID_NONE,
3021                 {
3022                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_TX_B_LANES_23,
3023                 },
3024         },
3025         {
3026                 .name = "dpio-tx-c-01",
3027                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
3028                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
3029                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
3030                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
3031                 .ops = &vlv_dpio_power_well_ops,
3032                 .id = DISP_PW_ID_NONE,
3033                 {
3034                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_TX_C_LANES_01,
3035                 },
3036         },
3037         {
3038                 .name = "dpio-tx-c-23",
3039                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
3040                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
3041                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
3042                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
3043                 .ops = &vlv_dpio_power_well_ops,
3044                 .id = DISP_PW_ID_NONE,
3045                 {
3046                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_TX_C_LANES_23,
3047                 },
3048         },
3049         {
3050                 .name = "dpio-common",
3051                 .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS,
3052                 .ops = &vlv_dpio_cmn_power_well_ops,
3053                 .id = VLV_DISP_PW_DPIO_CMN_BC,
3054                 {
3055                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_CMN_BC,
3056                 },
3057         },
3058 };
3059
3060 static const struct i915_power_well_desc chv_power_wells[] = {
3061         {
3062                 .name = "always-on",
3063                 .always_on = true,
3064                 .domains = POWER_DOMAIN_MASK,
3065                 .ops = &i9xx_always_on_power_well_ops,
3066                 .id = DISP_PW_ID_NONE,
3067         },
3068         {
3069                 .name = "display",
3070                 /*
3071                  * Pipe A power well is the new disp2d well. Pipe B and C
3072                  * power wells don't actually exist. Pipe A power well is
3073                  * required for any pipe to work.
3074                  */
3075                 .domains = CHV_DISPLAY_POWER_DOMAINS,
3076                 .ops = &chv_pipe_power_well_ops,
3077                 .id = DISP_PW_ID_NONE,
3078         },
3079         {
3080                 .name = "dpio-common-bc",
3081                 .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS,
3082                 .ops = &chv_dpio_cmn_power_well_ops,
3083                 .id = VLV_DISP_PW_DPIO_CMN_BC,
3084                 {
3085                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_CMN_BC,
3086                 },
3087         },
3088         {
3089                 .name = "dpio-common-d",
3090                 .domains = CHV_DPIO_CMN_D_POWER_DOMAINS,
3091                 .ops = &chv_dpio_cmn_power_well_ops,
3092                 .id = CHV_DISP_PW_DPIO_CMN_D,
3093                 {
3094                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_CMN_D,
3095                 },
3096         },
3097 };
3098
3099 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
3100                                          enum i915_power_well_id power_well_id)
3101 {
3102         struct i915_power_well *power_well;
3103         bool ret;
3104
3105         power_well = lookup_power_well(dev_priv, power_well_id);
3106         ret = power_well->desc->ops->is_enabled(dev_priv, power_well);
3107
3108         return ret;
3109 }
3110
3111 static const struct i915_power_well_desc skl_power_wells[] = {
3112         {
3113                 .name = "always-on",
3114                 .always_on = true,
3115                 .domains = POWER_DOMAIN_MASK,
3116                 .ops = &i9xx_always_on_power_well_ops,
3117                 .id = DISP_PW_ID_NONE,
3118         },
3119         {
3120                 .name = "power well 1",
3121                 /* Handled by the DMC firmware */
3122                 .always_on = true,
3123                 .domains = 0,
3124                 .ops = &hsw_power_well_ops,
3125                 .id = SKL_DISP_PW_1,
3126                 {
3127                         .hsw.regs = &hsw_power_well_regs,
3128                         .hsw.idx = SKL_PW_CTL_IDX_PW_1,
3129                         .hsw.has_fuses = true,
3130                 },
3131         },
3132         {
3133                 .name = "MISC IO power well",
3134                 /* Handled by the DMC firmware */
3135                 .always_on = true,
3136                 .domains = 0,
3137                 .ops = &hsw_power_well_ops,
3138                 .id = SKL_DISP_PW_MISC_IO,
3139                 {
3140                         .hsw.regs = &hsw_power_well_regs,
3141                         .hsw.idx = SKL_PW_CTL_IDX_MISC_IO,
3142                 },
3143         },
3144         {
3145                 .name = "DC off",
3146                 .domains = SKL_DISPLAY_DC_OFF_POWER_DOMAINS,
3147                 .ops = &gen9_dc_off_power_well_ops,
3148                 .id = SKL_DISP_DC_OFF,
3149         },
3150         {
3151                 .name = "power well 2",
3152                 .domains = SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
3153                 .ops = &hsw_power_well_ops,
3154                 .id = SKL_DISP_PW_2,
3155                 {
3156                         .hsw.regs = &hsw_power_well_regs,
3157                         .hsw.idx = SKL_PW_CTL_IDX_PW_2,
3158                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
3159                         .hsw.has_vga = true,
3160                         .hsw.has_fuses = true,
3161                 },
3162         },
3163         {
3164                 .name = "DDI A/E IO power well",
3165                 .domains = SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS,
3166                 .ops = &hsw_power_well_ops,
3167                 .id = DISP_PW_ID_NONE,
3168                 {
3169                         .hsw.regs = &hsw_power_well_regs,
3170                         .hsw.idx = SKL_PW_CTL_IDX_DDI_A_E,
3171                 },
3172         },
3173         {
3174                 .name = "DDI B IO power well",
3175                 .domains = SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS,
3176                 .ops = &hsw_power_well_ops,
3177                 .id = DISP_PW_ID_NONE,
3178                 {
3179                         .hsw.regs = &hsw_power_well_regs,
3180                         .hsw.idx = SKL_PW_CTL_IDX_DDI_B,
3181                 },
3182         },
3183         {
3184                 .name = "DDI C IO power well",
3185                 .domains = SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS,
3186                 .ops = &hsw_power_well_ops,
3187                 .id = DISP_PW_ID_NONE,
3188                 {
3189                         .hsw.regs = &hsw_power_well_regs,
3190                         .hsw.idx = SKL_PW_CTL_IDX_DDI_C,
3191                 },
3192         },
3193         {
3194                 .name = "DDI D IO power well",
3195                 .domains = SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS,
3196                 .ops = &hsw_power_well_ops,
3197                 .id = DISP_PW_ID_NONE,
3198                 {
3199                         .hsw.regs = &hsw_power_well_regs,
3200                         .hsw.idx = SKL_PW_CTL_IDX_DDI_D,
3201                 },
3202         },
3203 };
3204
3205 static const struct i915_power_well_desc bxt_power_wells[] = {
3206         {
3207                 .name = "always-on",
3208                 .always_on = true,
3209                 .domains = POWER_DOMAIN_MASK,
3210                 .ops = &i9xx_always_on_power_well_ops,
3211                 .id = DISP_PW_ID_NONE,
3212         },
3213         {
3214                 .name = "power well 1",
3215                 /* Handled by the DMC firmware */
3216                 .always_on = true,
3217                 .domains = 0,
3218                 .ops = &hsw_power_well_ops,
3219                 .id = SKL_DISP_PW_1,
3220                 {
3221                         .hsw.regs = &hsw_power_well_regs,
3222                         .hsw.idx = SKL_PW_CTL_IDX_PW_1,
3223                         .hsw.has_fuses = true,
3224                 },
3225         },
3226         {
3227                 .name = "DC off",
3228                 .domains = BXT_DISPLAY_DC_OFF_POWER_DOMAINS,
3229                 .ops = &gen9_dc_off_power_well_ops,
3230                 .id = SKL_DISP_DC_OFF,
3231         },
3232         {
3233                 .name = "power well 2",
3234                 .domains = BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS,
3235                 .ops = &hsw_power_well_ops,
3236                 .id = SKL_DISP_PW_2,
3237                 {
3238                         .hsw.regs = &hsw_power_well_regs,
3239                         .hsw.idx = SKL_PW_CTL_IDX_PW_2,
3240                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
3241                         .hsw.has_vga = true,
3242                         .hsw.has_fuses = true,
3243                 },
3244         },
3245         {
3246                 .name = "dpio-common-a",
3247                 .domains = BXT_DPIO_CMN_A_POWER_DOMAINS,
3248                 .ops = &bxt_dpio_cmn_power_well_ops,
3249                 .id = BXT_DISP_PW_DPIO_CMN_A,
3250                 {
3251                         .bxt.phy = DPIO_PHY1,
3252                 },
3253         },
3254         {
3255                 .name = "dpio-common-bc",
3256                 .domains = BXT_DPIO_CMN_BC_POWER_DOMAINS,
3257                 .ops = &bxt_dpio_cmn_power_well_ops,
3258                 .id = VLV_DISP_PW_DPIO_CMN_BC,
3259                 {
3260                         .bxt.phy = DPIO_PHY0,
3261                 },
3262         },
3263 };
3264
3265 static const struct i915_power_well_desc glk_power_wells[] = {
3266         {
3267                 .name = "always-on",
3268                 .always_on = true,
3269                 .domains = POWER_DOMAIN_MASK,
3270                 .ops = &i9xx_always_on_power_well_ops,
3271                 .id = DISP_PW_ID_NONE,
3272         },
3273         {
3274                 .name = "power well 1",
3275                 /* Handled by the DMC firmware */
3276                 .always_on = true,
3277                 .domains = 0,
3278                 .ops = &hsw_power_well_ops,
3279                 .id = SKL_DISP_PW_1,
3280                 {
3281                         .hsw.regs = &hsw_power_well_regs,
3282                         .hsw.idx = SKL_PW_CTL_IDX_PW_1,
3283                         .hsw.has_fuses = true,
3284                 },
3285         },
3286         {
3287                 .name = "DC off",
3288                 .domains = GLK_DISPLAY_DC_OFF_POWER_DOMAINS,
3289                 .ops = &gen9_dc_off_power_well_ops,
3290                 .id = SKL_DISP_DC_OFF,
3291         },
3292         {
3293                 .name = "power well 2",
3294                 .domains = GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS,
3295                 .ops = &hsw_power_well_ops,
3296                 .id = SKL_DISP_PW_2,
3297                 {
3298                         .hsw.regs = &hsw_power_well_regs,
3299                         .hsw.idx = SKL_PW_CTL_IDX_PW_2,
3300                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
3301                         .hsw.has_vga = true,
3302                         .hsw.has_fuses = true,
3303                 },
3304         },
3305         {
3306                 .name = "dpio-common-a",
3307                 .domains = GLK_DPIO_CMN_A_POWER_DOMAINS,
3308                 .ops = &bxt_dpio_cmn_power_well_ops,
3309                 .id = BXT_DISP_PW_DPIO_CMN_A,
3310                 {
3311                         .bxt.phy = DPIO_PHY1,
3312                 },
3313         },
3314         {
3315                 .name = "dpio-common-b",
3316                 .domains = GLK_DPIO_CMN_B_POWER_DOMAINS,
3317                 .ops = &bxt_dpio_cmn_power_well_ops,
3318                 .id = VLV_DISP_PW_DPIO_CMN_BC,
3319                 {
3320                         .bxt.phy = DPIO_PHY0,
3321                 },
3322         },
3323         {
3324                 .name = "dpio-common-c",
3325                 .domains = GLK_DPIO_CMN_C_POWER_DOMAINS,
3326                 .ops = &bxt_dpio_cmn_power_well_ops,
3327                 .id = GLK_DISP_PW_DPIO_CMN_C,
3328                 {
3329                         .bxt.phy = DPIO_PHY2,
3330                 },
3331         },
3332         {
3333                 .name = "AUX A",
3334                 .domains = GLK_DISPLAY_AUX_A_POWER_DOMAINS,
3335                 .ops = &hsw_power_well_ops,
3336                 .id = DISP_PW_ID_NONE,
3337                 {
3338                         .hsw.regs = &hsw_power_well_regs,
3339                         .hsw.idx = GLK_PW_CTL_IDX_AUX_A,
3340                 },
3341         },
3342         {
3343                 .name = "AUX B",
3344                 .domains = GLK_DISPLAY_AUX_B_POWER_DOMAINS,
3345                 .ops = &hsw_power_well_ops,
3346                 .id = DISP_PW_ID_NONE,
3347                 {
3348                         .hsw.regs = &hsw_power_well_regs,
3349                         .hsw.idx = GLK_PW_CTL_IDX_AUX_B,
3350                 },
3351         },
3352         {
3353                 .name = "AUX C",
3354                 .domains = GLK_DISPLAY_AUX_C_POWER_DOMAINS,
3355                 .ops = &hsw_power_well_ops,
3356                 .id = DISP_PW_ID_NONE,
3357                 {
3358                         .hsw.regs = &hsw_power_well_regs,
3359                         .hsw.idx = GLK_PW_CTL_IDX_AUX_C,
3360                 },
3361         },
3362         {
3363                 .name = "DDI A IO power well",
3364                 .domains = GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS,
3365                 .ops = &hsw_power_well_ops,
3366                 .id = DISP_PW_ID_NONE,
3367                 {
3368                         .hsw.regs = &hsw_power_well_regs,
3369                         .hsw.idx = GLK_PW_CTL_IDX_DDI_A,
3370                 },
3371         },
3372         {
3373                 .name = "DDI B IO power well",
3374                 .domains = GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS,
3375                 .ops = &hsw_power_well_ops,
3376                 .id = DISP_PW_ID_NONE,
3377                 {
3378                         .hsw.regs = &hsw_power_well_regs,
3379                         .hsw.idx = SKL_PW_CTL_IDX_DDI_B,
3380                 },
3381         },
3382         {
3383                 .name = "DDI C IO power well",
3384                 .domains = GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS,
3385                 .ops = &hsw_power_well_ops,
3386                 .id = DISP_PW_ID_NONE,
3387                 {
3388                         .hsw.regs = &hsw_power_well_regs,
3389                         .hsw.idx = SKL_PW_CTL_IDX_DDI_C,
3390                 },
3391         },
3392 };
3393
3394 static const struct i915_power_well_desc cnl_power_wells[] = {
3395         {
3396                 .name = "always-on",
3397                 .always_on = true,
3398                 .domains = POWER_DOMAIN_MASK,
3399                 .ops = &i9xx_always_on_power_well_ops,
3400                 .id = DISP_PW_ID_NONE,
3401         },
3402         {
3403                 .name = "power well 1",
3404                 /* Handled by the DMC firmware */
3405                 .always_on = true,
3406                 .domains = 0,
3407                 .ops = &hsw_power_well_ops,
3408                 .id = SKL_DISP_PW_1,
3409                 {
3410                         .hsw.regs = &hsw_power_well_regs,
3411                         .hsw.idx = SKL_PW_CTL_IDX_PW_1,
3412                         .hsw.has_fuses = true,
3413                 },
3414         },
3415         {
3416                 .name = "AUX A",
3417                 .domains = CNL_DISPLAY_AUX_A_POWER_DOMAINS,
3418                 .ops = &hsw_power_well_ops,
3419                 .id = DISP_PW_ID_NONE,
3420                 {
3421                         .hsw.regs = &hsw_power_well_regs,
3422                         .hsw.idx = GLK_PW_CTL_IDX_AUX_A,
3423                 },
3424         },
3425         {
3426                 .name = "AUX B",
3427                 .domains = CNL_DISPLAY_AUX_B_POWER_DOMAINS,
3428                 .ops = &hsw_power_well_ops,
3429                 .id = DISP_PW_ID_NONE,
3430                 {
3431                         .hsw.regs = &hsw_power_well_regs,
3432                         .hsw.idx = GLK_PW_CTL_IDX_AUX_B,
3433                 },
3434         },
3435         {
3436                 .name = "AUX C",
3437                 .domains = CNL_DISPLAY_AUX_C_POWER_DOMAINS,
3438                 .ops = &hsw_power_well_ops,
3439                 .id = DISP_PW_ID_NONE,
3440                 {
3441                         .hsw.regs = &hsw_power_well_regs,
3442                         .hsw.idx = GLK_PW_CTL_IDX_AUX_C,
3443                 },
3444         },
3445         {
3446                 .name = "AUX D",
3447                 .domains = CNL_DISPLAY_AUX_D_POWER_DOMAINS,
3448                 .ops = &hsw_power_well_ops,
3449                 .id = DISP_PW_ID_NONE,
3450                 {
3451                         .hsw.regs = &hsw_power_well_regs,
3452                         .hsw.idx = CNL_PW_CTL_IDX_AUX_D,
3453                 },
3454         },
3455         {
3456                 .name = "DC off",
3457                 .domains = CNL_DISPLAY_DC_OFF_POWER_DOMAINS,
3458                 .ops = &gen9_dc_off_power_well_ops,
3459                 .id = SKL_DISP_DC_OFF,
3460         },
3461         {
3462                 .name = "power well 2",
3463                 .domains = CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
3464                 .ops = &hsw_power_well_ops,
3465                 .id = SKL_DISP_PW_2,
3466                 {
3467                         .hsw.regs = &hsw_power_well_regs,
3468                         .hsw.idx = SKL_PW_CTL_IDX_PW_2,
3469                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
3470                         .hsw.has_vga = true,
3471                         .hsw.has_fuses = true,
3472                 },
3473         },
3474         {
3475                 .name = "DDI A IO power well",
3476                 .domains = CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS,
3477                 .ops = &hsw_power_well_ops,
3478                 .id = DISP_PW_ID_NONE,
3479                 {
3480                         .hsw.regs = &hsw_power_well_regs,
3481                         .hsw.idx = GLK_PW_CTL_IDX_DDI_A,
3482                 },
3483         },
3484         {
3485                 .name = "DDI B IO power well",
3486                 .domains = CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS,
3487                 .ops = &hsw_power_well_ops,
3488                 .id = DISP_PW_ID_NONE,
3489                 {
3490                         .hsw.regs = &hsw_power_well_regs,
3491                         .hsw.idx = SKL_PW_CTL_IDX_DDI_B,
3492                 },
3493         },
3494         {
3495                 .name = "DDI C IO power well",
3496                 .domains = CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS,
3497                 .ops = &hsw_power_well_ops,
3498                 .id = DISP_PW_ID_NONE,
3499                 {
3500                         .hsw.regs = &hsw_power_well_regs,
3501                         .hsw.idx = SKL_PW_CTL_IDX_DDI_C,
3502                 },
3503         },
3504         {
3505                 .name = "DDI D IO power well",
3506                 .domains = CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS,
3507                 .ops = &hsw_power_well_ops,
3508                 .id = DISP_PW_ID_NONE,
3509                 {
3510                         .hsw.regs = &hsw_power_well_regs,
3511                         .hsw.idx = SKL_PW_CTL_IDX_DDI_D,
3512                 },
3513         },
3514         {
3515                 .name = "DDI F IO power well",
3516                 .domains = CNL_DISPLAY_DDI_F_IO_POWER_DOMAINS,
3517                 .ops = &hsw_power_well_ops,
3518                 .id = DISP_PW_ID_NONE,
3519                 {
3520                         .hsw.regs = &hsw_power_well_regs,
3521                         .hsw.idx = CNL_PW_CTL_IDX_DDI_F,
3522                 },
3523         },
3524         {
3525                 .name = "AUX F",
3526                 .domains = CNL_DISPLAY_AUX_F_POWER_DOMAINS,
3527                 .ops = &hsw_power_well_ops,
3528                 .id = DISP_PW_ID_NONE,
3529                 {
3530                         .hsw.regs = &hsw_power_well_regs,
3531                         .hsw.idx = CNL_PW_CTL_IDX_AUX_F,
3532                 },
3533         },
3534 };
3535
3536 static const struct i915_power_well_ops icl_combo_phy_aux_power_well_ops = {
3537         .sync_hw = hsw_power_well_sync_hw,
3538         .enable = icl_combo_phy_aux_power_well_enable,
3539         .disable = icl_combo_phy_aux_power_well_disable,
3540         .is_enabled = hsw_power_well_enabled,
3541 };
3542
3543 static const struct i915_power_well_ops icl_tc_phy_aux_power_well_ops = {
3544         .sync_hw = hsw_power_well_sync_hw,
3545         .enable = icl_tc_phy_aux_power_well_enable,
3546         .disable = icl_tc_phy_aux_power_well_disable,
3547         .is_enabled = hsw_power_well_enabled,
3548 };
3549
3550 static const struct i915_power_well_regs icl_aux_power_well_regs = {
3551         .bios   = ICL_PWR_WELL_CTL_AUX1,
3552         .driver = ICL_PWR_WELL_CTL_AUX2,
3553         .debug  = ICL_PWR_WELL_CTL_AUX4,
3554 };
3555
3556 static const struct i915_power_well_regs icl_ddi_power_well_regs = {
3557         .bios   = ICL_PWR_WELL_CTL_DDI1,
3558         .driver = ICL_PWR_WELL_CTL_DDI2,
3559         .debug  = ICL_PWR_WELL_CTL_DDI4,
3560 };
3561
3562 static const struct i915_power_well_desc icl_power_wells[] = {
3563         {
3564                 .name = "always-on",
3565                 .always_on = true,
3566                 .domains = POWER_DOMAIN_MASK,
3567                 .ops = &i9xx_always_on_power_well_ops,
3568                 .id = DISP_PW_ID_NONE,
3569         },
3570         {
3571                 .name = "power well 1",
3572                 /* Handled by the DMC firmware */
3573                 .always_on = true,
3574                 .domains = 0,
3575                 .ops = &hsw_power_well_ops,
3576                 .id = SKL_DISP_PW_1,
3577                 {
3578                         .hsw.regs = &hsw_power_well_regs,
3579                         .hsw.idx = ICL_PW_CTL_IDX_PW_1,
3580                         .hsw.has_fuses = true,
3581                 },
3582         },
3583         {
3584                 .name = "DC off",
3585                 .domains = ICL_DISPLAY_DC_OFF_POWER_DOMAINS,
3586                 .ops = &gen9_dc_off_power_well_ops,
3587                 .id = SKL_DISP_DC_OFF,
3588         },
3589         {
3590                 .name = "power well 2",
3591                 .domains = ICL_PW_2_POWER_DOMAINS,
3592                 .ops = &hsw_power_well_ops,
3593                 .id = SKL_DISP_PW_2,
3594                 {
3595                         .hsw.regs = &hsw_power_well_regs,
3596                         .hsw.idx = ICL_PW_CTL_IDX_PW_2,
3597                         .hsw.has_fuses = true,
3598                 },
3599         },
3600         {
3601                 .name = "power well 3",
3602                 .domains = ICL_PW_3_POWER_DOMAINS,
3603                 .ops = &hsw_power_well_ops,
3604                 .id = ICL_DISP_PW_3,
3605                 {
3606                         .hsw.regs = &hsw_power_well_regs,
3607                         .hsw.idx = ICL_PW_CTL_IDX_PW_3,
3608                         .hsw.irq_pipe_mask = BIT(PIPE_B),
3609                         .hsw.has_vga = true,
3610                         .hsw.has_fuses = true,
3611                 },
3612         },
3613         {
3614                 .name = "DDI A IO",
3615                 .domains = ICL_DDI_IO_A_POWER_DOMAINS,
3616                 .ops = &hsw_power_well_ops,
3617                 .id = DISP_PW_ID_NONE,
3618                 {
3619                         .hsw.regs = &icl_ddi_power_well_regs,
3620                         .hsw.idx = ICL_PW_CTL_IDX_DDI_A,
3621                 },
3622         },
3623         {
3624                 .name = "DDI B IO",
3625                 .domains = ICL_DDI_IO_B_POWER_DOMAINS,
3626                 .ops = &hsw_power_well_ops,
3627                 .id = DISP_PW_ID_NONE,
3628                 {
3629                         .hsw.regs = &icl_ddi_power_well_regs,
3630                         .hsw.idx = ICL_PW_CTL_IDX_DDI_B,
3631                 },
3632         },
3633         {
3634                 .name = "DDI C IO",
3635                 .domains = ICL_DDI_IO_C_POWER_DOMAINS,
3636                 .ops = &hsw_power_well_ops,
3637                 .id = DISP_PW_ID_NONE,
3638                 {
3639                         .hsw.regs = &icl_ddi_power_well_regs,
3640                         .hsw.idx = ICL_PW_CTL_IDX_DDI_C,
3641                 },
3642         },
3643         {
3644                 .name = "DDI D IO",
3645                 .domains = ICL_DDI_IO_D_POWER_DOMAINS,
3646                 .ops = &hsw_power_well_ops,
3647                 .id = DISP_PW_ID_NONE,
3648                 {
3649                         .hsw.regs = &icl_ddi_power_well_regs,
3650                         .hsw.idx = ICL_PW_CTL_IDX_DDI_D,
3651                 },
3652         },
3653         {
3654                 .name = "DDI E IO",
3655                 .domains = ICL_DDI_IO_E_POWER_DOMAINS,
3656                 .ops = &hsw_power_well_ops,
3657                 .id = DISP_PW_ID_NONE,
3658                 {
3659                         .hsw.regs = &icl_ddi_power_well_regs,
3660                         .hsw.idx = ICL_PW_CTL_IDX_DDI_E,
3661                 },
3662         },
3663         {
3664                 .name = "DDI F IO",
3665                 .domains = ICL_DDI_IO_F_POWER_DOMAINS,
3666                 .ops = &hsw_power_well_ops,
3667                 .id = DISP_PW_ID_NONE,
3668                 {
3669                         .hsw.regs = &icl_ddi_power_well_regs,
3670                         .hsw.idx = ICL_PW_CTL_IDX_DDI_F,
3671                 },
3672         },
3673         {
3674                 .name = "AUX A",
3675                 .domains = ICL_AUX_A_IO_POWER_DOMAINS,
3676                 .ops = &icl_combo_phy_aux_power_well_ops,
3677                 .id = DISP_PW_ID_NONE,
3678                 {
3679                         .hsw.regs = &icl_aux_power_well_regs,
3680                         .hsw.idx = ICL_PW_CTL_IDX_AUX_A,
3681                 },
3682         },
3683         {
3684                 .name = "AUX B",
3685                 .domains = ICL_AUX_B_IO_POWER_DOMAINS,
3686                 .ops = &icl_combo_phy_aux_power_well_ops,
3687                 .id = DISP_PW_ID_NONE,
3688                 {
3689                         .hsw.regs = &icl_aux_power_well_regs,
3690                         .hsw.idx = ICL_PW_CTL_IDX_AUX_B,
3691                 },
3692         },
3693         {
3694                 .name = "AUX C TC1",
3695                 .domains = ICL_AUX_C_TC1_IO_POWER_DOMAINS,
3696                 .ops = &icl_tc_phy_aux_power_well_ops,
3697                 .id = DISP_PW_ID_NONE,
3698                 {
3699                         .hsw.regs = &icl_aux_power_well_regs,
3700                         .hsw.idx = ICL_PW_CTL_IDX_AUX_C,
3701                         .hsw.is_tc_tbt = false,
3702                 },
3703         },
3704         {
3705                 .name = "AUX D TC2",
3706                 .domains = ICL_AUX_D_TC2_IO_POWER_DOMAINS,
3707                 .ops = &icl_tc_phy_aux_power_well_ops,
3708                 .id = DISP_PW_ID_NONE,
3709                 {
3710                         .hsw.regs = &icl_aux_power_well_regs,
3711                         .hsw.idx = ICL_PW_CTL_IDX_AUX_D,
3712                         .hsw.is_tc_tbt = false,
3713                 },
3714         },
3715         {
3716                 .name = "AUX E TC3",
3717                 .domains = ICL_AUX_E_TC3_IO_POWER_DOMAINS,
3718                 .ops = &icl_tc_phy_aux_power_well_ops,
3719                 .id = DISP_PW_ID_NONE,
3720                 {
3721                         .hsw.regs = &icl_aux_power_well_regs,
3722                         .hsw.idx = ICL_PW_CTL_IDX_AUX_E,
3723                         .hsw.is_tc_tbt = false,
3724                 },
3725         },
3726         {
3727                 .name = "AUX F TC4",
3728                 .domains = ICL_AUX_F_TC4_IO_POWER_DOMAINS,
3729                 .ops = &icl_tc_phy_aux_power_well_ops,
3730                 .id = DISP_PW_ID_NONE,
3731                 {
3732                         .hsw.regs = &icl_aux_power_well_regs,
3733                         .hsw.idx = ICL_PW_CTL_IDX_AUX_F,
3734                         .hsw.is_tc_tbt = false,
3735                 },
3736         },
3737         {
3738                 .name = "AUX C TBT1",
3739                 .domains = ICL_AUX_C_TBT1_IO_POWER_DOMAINS,
3740                 .ops = &icl_tc_phy_aux_power_well_ops,
3741                 .id = DISP_PW_ID_NONE,
3742                 {
3743                         .hsw.regs = &icl_aux_power_well_regs,
3744                         .hsw.idx = ICL_PW_CTL_IDX_AUX_TBT1,
3745                         .hsw.is_tc_tbt = true,
3746                 },
3747         },
3748         {
3749                 .name = "AUX D TBT2",
3750                 .domains = ICL_AUX_D_TBT2_IO_POWER_DOMAINS,
3751                 .ops = &icl_tc_phy_aux_power_well_ops,
3752                 .id = DISP_PW_ID_NONE,
3753                 {
3754                         .hsw.regs = &icl_aux_power_well_regs,
3755                         .hsw.idx = ICL_PW_CTL_IDX_AUX_TBT2,
3756                         .hsw.is_tc_tbt = true,
3757                 },
3758         },
3759         {
3760                 .name = "AUX E TBT3",
3761                 .domains = ICL_AUX_E_TBT3_IO_POWER_DOMAINS,
3762                 .ops = &icl_tc_phy_aux_power_well_ops,
3763                 .id = DISP_PW_ID_NONE,
3764                 {
3765                         .hsw.regs = &icl_aux_power_well_regs,
3766                         .hsw.idx = ICL_PW_CTL_IDX_AUX_TBT3,
3767                         .hsw.is_tc_tbt = true,
3768                 },
3769         },
3770         {
3771                 .name = "AUX F TBT4",
3772                 .domains = ICL_AUX_F_TBT4_IO_POWER_DOMAINS,
3773                 .ops = &icl_tc_phy_aux_power_well_ops,
3774                 .id = DISP_PW_ID_NONE,
3775                 {
3776                         .hsw.regs = &icl_aux_power_well_regs,
3777                         .hsw.idx = ICL_PW_CTL_IDX_AUX_TBT4,
3778                         .hsw.is_tc_tbt = true,
3779                 },
3780         },
3781         {
3782                 .name = "power well 4",
3783                 .domains = ICL_PW_4_POWER_DOMAINS,
3784                 .ops = &hsw_power_well_ops,
3785                 .id = DISP_PW_ID_NONE,
3786                 {
3787                         .hsw.regs = &hsw_power_well_regs,
3788                         .hsw.idx = ICL_PW_CTL_IDX_PW_4,
3789                         .hsw.has_fuses = true,
3790                         .hsw.irq_pipe_mask = BIT(PIPE_C),
3791                 },
3792         },
3793 };
3794
3795 static const struct i915_power_well_desc ehl_power_wells[] = {
3796         {
3797                 .name = "always-on",
3798                 .always_on = true,
3799                 .domains = POWER_DOMAIN_MASK,
3800                 .ops = &i9xx_always_on_power_well_ops,
3801                 .id = DISP_PW_ID_NONE,
3802         },
3803         {
3804                 .name = "power well 1",
3805                 /* Handled by the DMC firmware */
3806                 .always_on = true,
3807                 .domains = 0,
3808                 .ops = &hsw_power_well_ops,
3809                 .id = SKL_DISP_PW_1,
3810                 {
3811                         .hsw.regs = &hsw_power_well_regs,
3812                         .hsw.idx = ICL_PW_CTL_IDX_PW_1,
3813                         .hsw.has_fuses = true,
3814                 },
3815         },
3816         {
3817                 .name = "DC off",
3818                 .domains = ICL_DISPLAY_DC_OFF_POWER_DOMAINS,
3819                 .ops = &gen9_dc_off_power_well_ops,
3820                 .id = SKL_DISP_DC_OFF,
3821         },
3822         {
3823                 .name = "power well 2",
3824                 .domains = ICL_PW_2_POWER_DOMAINS,
3825                 .ops = &hsw_power_well_ops,
3826                 .id = SKL_DISP_PW_2,
3827                 {
3828                         .hsw.regs = &hsw_power_well_regs,
3829                         .hsw.idx = ICL_PW_CTL_IDX_PW_2,
3830                         .hsw.has_fuses = true,
3831                 },
3832         },
3833         {
3834                 .name = "power well 3",
3835                 .domains = ICL_PW_3_POWER_DOMAINS,
3836                 .ops = &hsw_power_well_ops,
3837                 .id = DISP_PW_ID_NONE,
3838                 {
3839                         .hsw.regs = &hsw_power_well_regs,
3840                         .hsw.idx = ICL_PW_CTL_IDX_PW_3,
3841                         .hsw.irq_pipe_mask = BIT(PIPE_B),
3842                         .hsw.has_vga = true,
3843                         .hsw.has_fuses = true,
3844                 },
3845         },
3846         {
3847                 .name = "DDI A IO",
3848                 .domains = ICL_DDI_IO_A_POWER_DOMAINS,
3849                 .ops = &hsw_power_well_ops,
3850                 .id = DISP_PW_ID_NONE,
3851                 {
3852                         .hsw.regs = &icl_ddi_power_well_regs,
3853                         .hsw.idx = ICL_PW_CTL_IDX_DDI_A,
3854                 },
3855         },
3856         {
3857                 .name = "DDI B IO",
3858                 .domains = ICL_DDI_IO_B_POWER_DOMAINS,
3859                 .ops = &hsw_power_well_ops,
3860                 .id = DISP_PW_ID_NONE,
3861                 {
3862                         .hsw.regs = &icl_ddi_power_well_regs,
3863                         .hsw.idx = ICL_PW_CTL_IDX_DDI_B,
3864                 },
3865         },
3866         {
3867                 .name = "DDI C IO",
3868                 .domains = ICL_DDI_IO_C_POWER_DOMAINS,
3869                 .ops = &hsw_power_well_ops,
3870                 .id = DISP_PW_ID_NONE,
3871                 {
3872                         .hsw.regs = &icl_ddi_power_well_regs,
3873                         .hsw.idx = ICL_PW_CTL_IDX_DDI_C,
3874                 },
3875         },
3876         {
3877                 .name = "DDI D IO",
3878                 .domains = ICL_DDI_IO_D_POWER_DOMAINS,
3879                 .ops = &hsw_power_well_ops,
3880                 .id = DISP_PW_ID_NONE,
3881                 {
3882                         .hsw.regs = &icl_ddi_power_well_regs,
3883                         .hsw.idx = ICL_PW_CTL_IDX_DDI_D,
3884                 },
3885         },
3886         {
3887                 .name = "AUX A",
3888                 .domains = ICL_AUX_A_IO_POWER_DOMAINS,
3889                 .ops = &hsw_power_well_ops,
3890                 .id = DISP_PW_ID_NONE,
3891                 {
3892                         .hsw.regs = &icl_aux_power_well_regs,
3893                         .hsw.idx = ICL_PW_CTL_IDX_AUX_A,
3894                 },
3895         },
3896         {
3897                 .name = "AUX B",
3898                 .domains = ICL_AUX_B_IO_POWER_DOMAINS,
3899                 .ops = &hsw_power_well_ops,
3900                 .id = DISP_PW_ID_NONE,
3901                 {
3902                         .hsw.regs = &icl_aux_power_well_regs,
3903                         .hsw.idx = ICL_PW_CTL_IDX_AUX_B,
3904                 },
3905         },
3906         {
3907                 .name = "AUX C",
3908                 .domains = ICL_AUX_C_TC1_IO_POWER_DOMAINS,
3909                 .ops = &hsw_power_well_ops,
3910                 .id = DISP_PW_ID_NONE,
3911                 {
3912                         .hsw.regs = &icl_aux_power_well_regs,
3913                         .hsw.idx = ICL_PW_CTL_IDX_AUX_C,
3914                 },
3915         },
3916         {
3917                 .name = "AUX D",
3918                 .domains = ICL_AUX_D_TC2_IO_POWER_DOMAINS,
3919                 .ops = &hsw_power_well_ops,
3920                 .id = DISP_PW_ID_NONE,
3921                 {
3922                         .hsw.regs = &icl_aux_power_well_regs,
3923                         .hsw.idx = ICL_PW_CTL_IDX_AUX_D,
3924                 },
3925         },
3926         {
3927                 .name = "power well 4",
3928                 .domains = ICL_PW_4_POWER_DOMAINS,
3929                 .ops = &hsw_power_well_ops,
3930                 .id = DISP_PW_ID_NONE,
3931                 {
3932                         .hsw.regs = &hsw_power_well_regs,
3933                         .hsw.idx = ICL_PW_CTL_IDX_PW_4,
3934                         .hsw.has_fuses = true,
3935                         .hsw.irq_pipe_mask = BIT(PIPE_C),
3936                 },
3937         },
3938 };
3939
3940 static const struct i915_power_well_desc tgl_power_wells[] = {
3941         {
3942                 .name = "always-on",
3943                 .always_on = true,
3944                 .domains = POWER_DOMAIN_MASK,
3945                 .ops = &i9xx_always_on_power_well_ops,
3946                 .id = DISP_PW_ID_NONE,
3947         },
3948         {
3949                 .name = "power well 1",
3950                 /* Handled by the DMC firmware */
3951                 .always_on = true,
3952                 .domains = 0,
3953                 .ops = &hsw_power_well_ops,
3954                 .id = SKL_DISP_PW_1,
3955                 {
3956                         .hsw.regs = &hsw_power_well_regs,
3957                         .hsw.idx = ICL_PW_CTL_IDX_PW_1,
3958                         .hsw.has_fuses = true,
3959                 },
3960         },
3961         {
3962                 .name = "DC off",
3963                 .domains = TGL_DISPLAY_DC_OFF_POWER_DOMAINS,
3964                 .ops = &gen9_dc_off_power_well_ops,
3965                 .id = SKL_DISP_DC_OFF,
3966         },
3967         {
3968                 .name = "power well 2",
3969                 .domains = TGL_PW_2_POWER_DOMAINS,
3970                 .ops = &hsw_power_well_ops,
3971                 .id = SKL_DISP_PW_2,
3972                 {
3973                         .hsw.regs = &hsw_power_well_regs,
3974                         .hsw.idx = ICL_PW_CTL_IDX_PW_2,
3975                         .hsw.has_fuses = true,
3976                 },
3977         },
3978         {
3979                 .name = "power well 3",
3980                 .domains = TGL_PW_3_POWER_DOMAINS,
3981                 .ops = &hsw_power_well_ops,
3982                 .id = ICL_DISP_PW_3,
3983                 {
3984                         .hsw.regs = &hsw_power_well_regs,
3985                         .hsw.idx = ICL_PW_CTL_IDX_PW_3,
3986                         .hsw.irq_pipe_mask = BIT(PIPE_B),
3987                         .hsw.has_vga = true,
3988                         .hsw.has_fuses = true,
3989                 },
3990         },
3991         {
3992                 .name = "DDI A IO",
3993                 .domains = ICL_DDI_IO_A_POWER_DOMAINS,
3994                 .ops = &hsw_power_well_ops,
3995                 .id = DISP_PW_ID_NONE,
3996                 {
3997                         .hsw.regs = &icl_ddi_power_well_regs,
3998                         .hsw.idx = ICL_PW_CTL_IDX_DDI_A,
3999                 }
4000         },
4001         {
4002                 .name = "DDI B IO",
4003                 .domains = ICL_DDI_IO_B_POWER_DOMAINS,
4004                 .ops = &hsw_power_well_ops,
4005                 .id = DISP_PW_ID_NONE,
4006                 {
4007                         .hsw.regs = &icl_ddi_power_well_regs,
4008                         .hsw.idx = ICL_PW_CTL_IDX_DDI_B,
4009                 }
4010         },
4011         {
4012                 .name = "DDI C IO",
4013                 .domains = ICL_DDI_IO_C_POWER_DOMAINS,
4014                 .ops = &hsw_power_well_ops,
4015                 .id = DISP_PW_ID_NONE,
4016                 {
4017                         .hsw.regs = &icl_ddi_power_well_regs,
4018                         .hsw.idx = ICL_PW_CTL_IDX_DDI_C,
4019                 }
4020         },
4021         {
4022                 .name = "DDI D TC1 IO",
4023                 .domains = TGL_DDI_IO_D_TC1_POWER_DOMAINS,
4024                 .ops = &hsw_power_well_ops,
4025                 .id = DISP_PW_ID_NONE,
4026                 {
4027                         .hsw.regs = &icl_ddi_power_well_regs,
4028                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC1,
4029                 },
4030         },
4031         {
4032                 .name = "DDI E TC2 IO",
4033                 .domains = TGL_DDI_IO_E_TC2_POWER_DOMAINS,
4034                 .ops = &hsw_power_well_ops,
4035                 .id = DISP_PW_ID_NONE,
4036                 {
4037                         .hsw.regs = &icl_ddi_power_well_regs,
4038                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC2,
4039                 },
4040         },
4041         {
4042                 .name = "DDI F TC3 IO",
4043                 .domains = TGL_DDI_IO_F_TC3_POWER_DOMAINS,
4044                 .ops = &hsw_power_well_ops,
4045                 .id = DISP_PW_ID_NONE,
4046                 {
4047                         .hsw.regs = &icl_ddi_power_well_regs,
4048                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC3,
4049                 },
4050         },
4051         {
4052                 .name = "DDI G TC4 IO",
4053                 .domains = TGL_DDI_IO_G_TC4_POWER_DOMAINS,
4054                 .ops = &hsw_power_well_ops,
4055                 .id = DISP_PW_ID_NONE,
4056                 {
4057                         .hsw.regs = &icl_ddi_power_well_regs,
4058                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC4,
4059                 },
4060         },
4061         {
4062                 .name = "DDI H TC5 IO",
4063                 .domains = TGL_DDI_IO_H_TC5_POWER_DOMAINS,
4064                 .ops = &hsw_power_well_ops,
4065                 .id = DISP_PW_ID_NONE,
4066                 {
4067                         .hsw.regs = &icl_ddi_power_well_regs,
4068                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC5,
4069                 },
4070         },
4071         {
4072                 .name = "DDI I TC6 IO",
4073                 .domains = TGL_DDI_IO_I_TC6_POWER_DOMAINS,
4074                 .ops = &hsw_power_well_ops,
4075                 .id = DISP_PW_ID_NONE,
4076                 {
4077                         .hsw.regs = &icl_ddi_power_well_regs,
4078                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC6,
4079                 },
4080         },
4081         {
4082                 .name = "AUX A",
4083                 .domains = TGL_AUX_A_IO_POWER_DOMAINS,
4084                 .ops = &hsw_power_well_ops,
4085                 .id = DISP_PW_ID_NONE,
4086                 {
4087                         .hsw.regs = &icl_aux_power_well_regs,
4088                         .hsw.idx = ICL_PW_CTL_IDX_AUX_A,
4089                 },
4090         },
4091         {
4092                 .name = "AUX B",
4093                 .domains = TGL_AUX_B_IO_POWER_DOMAINS,
4094                 .ops = &hsw_power_well_ops,
4095                 .id = DISP_PW_ID_NONE,
4096                 {
4097                         .hsw.regs = &icl_aux_power_well_regs,
4098                         .hsw.idx = ICL_PW_CTL_IDX_AUX_B,
4099                 },
4100         },
4101         {
4102                 .name = "AUX C",
4103                 .domains = TGL_AUX_C_IO_POWER_DOMAINS,
4104                 .ops = &hsw_power_well_ops,
4105                 .id = DISP_PW_ID_NONE,
4106                 {
4107                         .hsw.regs = &icl_aux_power_well_regs,
4108                         .hsw.idx = ICL_PW_CTL_IDX_AUX_C,
4109                 },
4110         },
4111         {
4112                 .name = "AUX D TC1",
4113                 .domains = TGL_AUX_D_TC1_IO_POWER_DOMAINS,
4114                 .ops = &icl_tc_phy_aux_power_well_ops,
4115                 .id = DISP_PW_ID_NONE,
4116                 {
4117                         .hsw.regs = &icl_aux_power_well_regs,
4118                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC1,
4119                         .hsw.is_tc_tbt = false,
4120                 },
4121         },
4122         {
4123                 .name = "AUX E TC2",
4124                 .domains = TGL_AUX_E_TC2_IO_POWER_DOMAINS,
4125                 .ops = &icl_tc_phy_aux_power_well_ops,
4126                 .id = DISP_PW_ID_NONE,
4127                 {
4128                         .hsw.regs = &icl_aux_power_well_regs,
4129                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC2,
4130                         .hsw.is_tc_tbt = false,
4131                 },
4132         },
4133         {
4134                 .name = "AUX F TC3",
4135                 .domains = TGL_AUX_F_TC3_IO_POWER_DOMAINS,
4136                 .ops = &icl_tc_phy_aux_power_well_ops,
4137                 .id = DISP_PW_ID_NONE,
4138                 {
4139                         .hsw.regs = &icl_aux_power_well_regs,
4140                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC3,
4141                         .hsw.is_tc_tbt = false,
4142                 },
4143         },
4144         {
4145                 .name = "AUX G TC4",
4146                 .domains = TGL_AUX_G_TC4_IO_POWER_DOMAINS,
4147                 .ops = &icl_tc_phy_aux_power_well_ops,
4148                 .id = DISP_PW_ID_NONE,
4149                 {
4150                         .hsw.regs = &icl_aux_power_well_regs,
4151                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC4,
4152                         .hsw.is_tc_tbt = false,
4153                 },
4154         },
4155         {
4156                 .name = "AUX H TC5",
4157                 .domains = TGL_AUX_H_TC5_IO_POWER_DOMAINS,
4158                 .ops = &icl_tc_phy_aux_power_well_ops,
4159                 .id = DISP_PW_ID_NONE,
4160                 {
4161                         .hsw.regs = &icl_aux_power_well_regs,
4162                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC5,
4163                         .hsw.is_tc_tbt = false,
4164                 },
4165         },
4166         {
4167                 .name = "AUX I TC6",
4168                 .domains = TGL_AUX_I_TC6_IO_POWER_DOMAINS,
4169                 .ops = &icl_tc_phy_aux_power_well_ops,
4170                 .id = DISP_PW_ID_NONE,
4171                 {
4172                         .hsw.regs = &icl_aux_power_well_regs,
4173                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC6,
4174                         .hsw.is_tc_tbt = false,
4175                 },
4176         },
4177         {
4178                 .name = "AUX D TBT1",
4179                 .domains = TGL_AUX_D_TBT1_IO_POWER_DOMAINS,
4180                 .ops = &icl_tc_phy_aux_power_well_ops,
4181                 .id = DISP_PW_ID_NONE,
4182                 {
4183                         .hsw.regs = &icl_aux_power_well_regs,
4184                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT1,
4185                         .hsw.is_tc_tbt = true,
4186                 },
4187         },
4188         {
4189                 .name = "AUX E TBT2",
4190                 .domains = TGL_AUX_E_TBT2_IO_POWER_DOMAINS,
4191                 .ops = &icl_tc_phy_aux_power_well_ops,
4192                 .id = DISP_PW_ID_NONE,
4193                 {
4194                         .hsw.regs = &icl_aux_power_well_regs,
4195                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT2,
4196                         .hsw.is_tc_tbt = true,
4197                 },
4198         },
4199         {
4200                 .name = "AUX F TBT3",
4201                 .domains = TGL_AUX_F_TBT3_IO_POWER_DOMAINS,
4202                 .ops = &icl_tc_phy_aux_power_well_ops,
4203                 .id = DISP_PW_ID_NONE,
4204                 {
4205                         .hsw.regs = &icl_aux_power_well_regs,
4206                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT3,
4207                         .hsw.is_tc_tbt = true,
4208                 },
4209         },
4210         {
4211                 .name = "AUX G TBT4",
4212                 .domains = TGL_AUX_G_TBT4_IO_POWER_DOMAINS,
4213                 .ops = &icl_tc_phy_aux_power_well_ops,
4214                 .id = DISP_PW_ID_NONE,
4215                 {
4216                         .hsw.regs = &icl_aux_power_well_regs,
4217                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT4,
4218                         .hsw.is_tc_tbt = true,
4219                 },
4220         },
4221         {
4222                 .name = "AUX H TBT5",
4223                 .domains = TGL_AUX_H_TBT5_IO_POWER_DOMAINS,
4224                 .ops = &icl_tc_phy_aux_power_well_ops,
4225                 .id = DISP_PW_ID_NONE,
4226                 {
4227                         .hsw.regs = &icl_aux_power_well_regs,
4228                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT5,
4229                         .hsw.is_tc_tbt = true,
4230                 },
4231         },
4232         {
4233                 .name = "AUX I TBT6",
4234                 .domains = TGL_AUX_I_TBT6_IO_POWER_DOMAINS,
4235                 .ops = &icl_tc_phy_aux_power_well_ops,
4236                 .id = DISP_PW_ID_NONE,
4237                 {
4238                         .hsw.regs = &icl_aux_power_well_regs,
4239                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT6,
4240                         .hsw.is_tc_tbt = true,
4241                 },
4242         },
4243         {
4244                 .name = "power well 4",
4245                 .domains = TGL_PW_4_POWER_DOMAINS,
4246                 .ops = &hsw_power_well_ops,
4247                 .id = DISP_PW_ID_NONE,
4248                 {
4249                         .hsw.regs = &hsw_power_well_regs,
4250                         .hsw.idx = ICL_PW_CTL_IDX_PW_4,
4251                         .hsw.has_fuses = true,
4252                         .hsw.irq_pipe_mask = BIT(PIPE_C),
4253                 }
4254         },
4255         {
4256                 .name = "power well 5",
4257                 .domains = TGL_PW_5_POWER_DOMAINS,
4258                 .ops = &hsw_power_well_ops,
4259                 .id = DISP_PW_ID_NONE,
4260                 {
4261                         .hsw.regs = &hsw_power_well_regs,
4262                         .hsw.idx = TGL_PW_CTL_IDX_PW_5,
4263                         .hsw.has_fuses = true,
4264                         .hsw.irq_pipe_mask = BIT(PIPE_D),
4265                 },
4266         },
4267 };
4268
4269 static int
4270 sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
4271                                    int disable_power_well)
4272 {
4273         if (disable_power_well >= 0)
4274                 return !!disable_power_well;
4275
4276         return 1;
4277 }
4278
4279 static u32 get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
4280                                int enable_dc)
4281 {
4282         u32 mask;
4283         int requested_dc;
4284         int max_dc;
4285
4286         if (INTEL_GEN(dev_priv) >= 12) {
4287                 max_dc = 4;
4288                 /*
4289                  * DC9 has a separate HW flow from the rest of the DC states,
4290                  * not depending on the DMC firmware. It's needed by system
4291                  * suspend/resume, so allow it unconditionally.
4292                  */
4293                 mask = DC_STATE_EN_DC9;
4294         } else if (IS_GEN(dev_priv, 11)) {
4295                 max_dc = 2;
4296                 mask = DC_STATE_EN_DC9;
4297         } else if (IS_GEN(dev_priv, 10) || IS_GEN9_BC(dev_priv)) {
4298                 max_dc = 2;
4299                 mask = 0;
4300         } else if (IS_GEN9_LP(dev_priv)) {
4301                 max_dc = 1;
4302                 mask = DC_STATE_EN_DC9;
4303         } else {
4304                 max_dc = 0;
4305                 mask = 0;
4306         }
4307
4308         if (!i915_modparams.disable_power_well)
4309                 max_dc = 0;
4310
4311         if (enable_dc >= 0 && enable_dc <= max_dc) {
4312                 requested_dc = enable_dc;
4313         } else if (enable_dc == -1) {
4314                 requested_dc = max_dc;
4315         } else if (enable_dc > max_dc && enable_dc <= 4) {
4316                 drm_dbg_kms(&dev_priv->drm,
4317                             "Adjusting requested max DC state (%d->%d)\n",
4318                             enable_dc, max_dc);
4319                 requested_dc = max_dc;
4320         } else {
4321                 drm_err(&dev_priv->drm,
4322                         "Unexpected value for enable_dc (%d)\n", enable_dc);
4323                 requested_dc = max_dc;
4324         }
4325
4326         switch (requested_dc) {
4327         case 4:
4328                 mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC6;
4329                 break;
4330         case 3:
4331                 mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC5;
4332                 break;
4333         case 2:
4334                 mask |= DC_STATE_EN_UPTO_DC6;
4335                 break;
4336         case 1:
4337                 mask |= DC_STATE_EN_UPTO_DC5;
4338                 break;
4339         }
4340
4341         drm_dbg_kms(&dev_priv->drm, "Allowed DC state mask %02x\n", mask);
4342
4343         return mask;
4344 }
4345
4346 static int
4347 __set_power_wells(struct i915_power_domains *power_domains,
4348                   const struct i915_power_well_desc *power_well_descs,
4349                   int power_well_count)
4350 {
4351         u64 power_well_ids = 0;
4352         int i;
4353
4354         power_domains->power_well_count = power_well_count;
4355         power_domains->power_wells =
4356                                 kcalloc(power_well_count,
4357                                         sizeof(*power_domains->power_wells),
4358                                         GFP_KERNEL);
4359         if (!power_domains->power_wells)
4360                 return -ENOMEM;
4361
4362         for (i = 0; i < power_well_count; i++) {
4363                 enum i915_power_well_id id = power_well_descs[i].id;
4364
4365                 power_domains->power_wells[i].desc = &power_well_descs[i];
4366
4367                 if (id == DISP_PW_ID_NONE)
4368                         continue;
4369
4370                 WARN_ON(id >= sizeof(power_well_ids) * 8);
4371                 WARN_ON(power_well_ids & BIT_ULL(id));
4372                 power_well_ids |= BIT_ULL(id);
4373         }
4374
4375         return 0;
4376 }
4377
4378 #define set_power_wells(power_domains, __power_well_descs) \
4379         __set_power_wells(power_domains, __power_well_descs, \
4380                           ARRAY_SIZE(__power_well_descs))
4381
4382 /**
4383  * intel_power_domains_init - initializes the power domain structures
4384  * @dev_priv: i915 device instance
4385  *
4386  * Initializes the power domain structures for @dev_priv depending upon the
4387  * supported platform.
4388  */
4389 int intel_power_domains_init(struct drm_i915_private *dev_priv)
4390 {
4391         struct i915_power_domains *power_domains = &dev_priv->power_domains;
4392         int err;
4393
4394         i915_modparams.disable_power_well =
4395                 sanitize_disable_power_well_option(dev_priv,
4396                                                    i915_modparams.disable_power_well);
4397         dev_priv->csr.allowed_dc_mask =
4398                 get_allowed_dc_mask(dev_priv, i915_modparams.enable_dc);
4399
4400         dev_priv->csr.target_dc_state =
4401                 sanitize_target_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
4402
4403         BUILD_BUG_ON(POWER_DOMAIN_NUM > 64);
4404
4405         mutex_init(&power_domains->lock);
4406
4407         INIT_DELAYED_WORK(&power_domains->async_put_work,
4408                           intel_display_power_put_async_work);
4409
4410         /*
4411          * The enabling order will be from lower to higher indexed wells,
4412          * the disabling order is reversed.
4413          */
4414         if (IS_GEN(dev_priv, 12)) {
4415                 err = set_power_wells(power_domains, tgl_power_wells);
4416         } else if (IS_ELKHARTLAKE(dev_priv)) {
4417                 err = set_power_wells(power_domains, ehl_power_wells);
4418         } else if (IS_GEN(dev_priv, 11)) {
4419                 err = set_power_wells(power_domains, icl_power_wells);
4420         } else if (IS_CANNONLAKE(dev_priv)) {
4421                 err = set_power_wells(power_domains, cnl_power_wells);
4422
4423                 /*
4424                  * DDI and Aux IO are getting enabled for all ports
4425                  * regardless the presence or use. So, in order to avoid
4426                  * timeouts, lets remove them from the list
4427                  * for the SKUs without port F.
4428                  */
4429                 if (!IS_CNL_WITH_PORT_F(dev_priv))
4430                         power_domains->power_well_count -= 2;
4431         } else if (IS_GEMINILAKE(dev_priv)) {
4432                 err = set_power_wells(power_domains, glk_power_wells);
4433         } else if (IS_BROXTON(dev_priv)) {
4434                 err = set_power_wells(power_domains, bxt_power_wells);
4435         } else if (IS_GEN9_BC(dev_priv)) {
4436                 err = set_power_wells(power_domains, skl_power_wells);
4437         } else if (IS_CHERRYVIEW(dev_priv)) {
4438                 err = set_power_wells(power_domains, chv_power_wells);
4439         } else if (IS_BROADWELL(dev_priv)) {
4440                 err = set_power_wells(power_domains, bdw_power_wells);
4441         } else if (IS_HASWELL(dev_priv)) {
4442                 err = set_power_wells(power_domains, hsw_power_wells);
4443         } else if (IS_VALLEYVIEW(dev_priv)) {
4444                 err = set_power_wells(power_domains, vlv_power_wells);
4445         } else if (IS_I830(dev_priv)) {
4446                 err = set_power_wells(power_domains, i830_power_wells);
4447         } else {
4448                 err = set_power_wells(power_domains, i9xx_always_on_power_well);
4449         }
4450
4451         return err;
4452 }
4453
4454 /**
4455  * intel_power_domains_cleanup - clean up power domains resources
4456  * @dev_priv: i915 device instance
4457  *
4458  * Release any resources acquired by intel_power_domains_init()
4459  */
4460 void intel_power_domains_cleanup(struct drm_i915_private *dev_priv)
4461 {
4462         kfree(dev_priv->power_domains.power_wells);
4463 }
4464
4465 static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
4466 {
4467         struct i915_power_domains *power_domains = &dev_priv->power_domains;
4468         struct i915_power_well *power_well;
4469
4470         mutex_lock(&power_domains->lock);
4471         for_each_power_well(dev_priv, power_well) {
4472                 power_well->desc->ops->sync_hw(dev_priv, power_well);
4473                 power_well->hw_enabled =
4474                         power_well->desc->ops->is_enabled(dev_priv, power_well);
4475         }
4476         mutex_unlock(&power_domains->lock);
4477 }
4478
4479 static inline
4480 bool intel_dbuf_slice_set(struct drm_i915_private *dev_priv,
4481                           i915_reg_t reg, bool enable)
4482 {
4483         u32 val, status;
4484
4485         val = intel_de_read(dev_priv, reg);
4486         val = enable ? (val | DBUF_POWER_REQUEST) : (val & ~DBUF_POWER_REQUEST);
4487         intel_de_write(dev_priv, reg, val);
4488         intel_de_posting_read(dev_priv, reg);
4489         udelay(10);
4490
4491         status = intel_de_read(dev_priv, reg) & DBUF_POWER_STATE;
4492         if ((enable && !status) || (!enable && status)) {
4493                 drm_err(&dev_priv->drm, "DBus power %s timeout!\n",
4494                         enable ? "enable" : "disable");
4495                 return false;
4496         }
4497         return true;
4498 }
4499
4500 static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
4501 {
4502         icl_dbuf_slices_update(dev_priv, BIT(DBUF_S1));
4503 }
4504
4505 static void gen9_dbuf_disable(struct drm_i915_private *dev_priv)
4506 {
4507         icl_dbuf_slices_update(dev_priv, 0);
4508 }
4509
4510 void icl_dbuf_slices_update(struct drm_i915_private *dev_priv,
4511                             u8 req_slices)
4512 {
4513         int i;
4514         int max_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4515         struct i915_power_domains *power_domains = &dev_priv->power_domains;
4516
4517         drm_WARN(&dev_priv->drm, hweight8(req_slices) > max_slices,
4518                  "Invalid number of dbuf slices requested\n");
4519
4520         drm_dbg_kms(&dev_priv->drm, "Updating dbuf slices to 0x%x\n",
4521                     req_slices);
4522
4523         /*
4524          * Might be running this in parallel to gen9_dc_off_power_well_enable
4525          * being called from intel_dp_detect for instance,
4526          * which causes assertion triggered by race condition,
4527          * as gen9_assert_dbuf_enabled might preempt this when registers
4528          * were already updated, while dev_priv was not.
4529          */
4530         mutex_lock(&power_domains->lock);
4531
4532         for (i = 0; i < max_slices; i++) {
4533                 intel_dbuf_slice_set(dev_priv,
4534                                      DBUF_CTL_S(i),
4535                                      (req_slices & BIT(i)) != 0);
4536         }
4537
4538         dev_priv->enabled_dbuf_slices_mask = req_slices;
4539
4540         mutex_unlock(&power_domains->lock);
4541 }
4542
4543 static void icl_dbuf_enable(struct drm_i915_private *dev_priv)
4544 {
4545         skl_ddb_get_hw_state(dev_priv);
4546         /*
4547          * Just power up at least 1 slice, we will
4548          * figure out later which slices we have and what we need.
4549          */
4550         icl_dbuf_slices_update(dev_priv, dev_priv->enabled_dbuf_slices_mask |
4551                                BIT(DBUF_S1));
4552 }
4553
4554 static void icl_dbuf_disable(struct drm_i915_private *dev_priv)
4555 {
4556         icl_dbuf_slices_update(dev_priv, 0);
4557 }
4558
4559 static void icl_mbus_init(struct drm_i915_private *dev_priv)
4560 {
4561         u32 mask, val;
4562
4563         mask = MBUS_ABOX_BT_CREDIT_POOL1_MASK |
4564                 MBUS_ABOX_BT_CREDIT_POOL2_MASK |
4565                 MBUS_ABOX_B_CREDIT_MASK |
4566                 MBUS_ABOX_BW_CREDIT_MASK;
4567         val = MBUS_ABOX_BT_CREDIT_POOL1(16) |
4568                 MBUS_ABOX_BT_CREDIT_POOL2(16) |
4569                 MBUS_ABOX_B_CREDIT(1) |
4570                 MBUS_ABOX_BW_CREDIT(1);
4571
4572         intel_de_rmw(dev_priv, MBUS_ABOX_CTL, mask, val);
4573         if (INTEL_GEN(dev_priv) >= 12) {
4574                 intel_de_rmw(dev_priv, MBUS_ABOX1_CTL, mask, val);
4575                 intel_de_rmw(dev_priv, MBUS_ABOX2_CTL, mask, val);
4576         }
4577 }
4578
4579 static void hsw_assert_cdclk(struct drm_i915_private *dev_priv)
4580 {
4581         u32 val = intel_de_read(dev_priv, LCPLL_CTL);
4582
4583         /*
4584          * The LCPLL register should be turned on by the BIOS. For now
4585          * let's just check its state and print errors in case
4586          * something is wrong.  Don't even try to turn it on.
4587          */
4588
4589         if (val & LCPLL_CD_SOURCE_FCLK)
4590                 drm_err(&dev_priv->drm, "CDCLK source is not LCPLL\n");
4591
4592         if (val & LCPLL_PLL_DISABLE)
4593                 drm_err(&dev_priv->drm, "LCPLL is disabled\n");
4594
4595         if ((val & LCPLL_REF_MASK) != LCPLL_REF_NON_SSC)
4596                 drm_err(&dev_priv->drm, "LCPLL not using non-SSC reference\n");
4597 }
4598
4599 static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
4600 {
4601         struct drm_device *dev = &dev_priv->drm;
4602         struct intel_crtc *crtc;
4603
4604         for_each_intel_crtc(dev, crtc)
4605                 I915_STATE_WARN(crtc->active, "CRTC for pipe %c enabled\n",
4606                                 pipe_name(crtc->pipe));
4607
4608         I915_STATE_WARN(intel_de_read(dev_priv, HSW_PWR_WELL_CTL2),
4609                         "Display power well on\n");
4610         I915_STATE_WARN(intel_de_read(dev_priv, SPLL_CTL) & SPLL_PLL_ENABLE,
4611                         "SPLL enabled\n");
4612         I915_STATE_WARN(intel_de_read(dev_priv, WRPLL_CTL(0)) & WRPLL_PLL_ENABLE,
4613                         "WRPLL1 enabled\n");
4614         I915_STATE_WARN(intel_de_read(dev_priv, WRPLL_CTL(1)) & WRPLL_PLL_ENABLE,
4615                         "WRPLL2 enabled\n");
4616         I915_STATE_WARN(intel_de_read(dev_priv, PP_STATUS(0)) & PP_ON,
4617                         "Panel power on\n");
4618         I915_STATE_WARN(intel_de_read(dev_priv, BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
4619                         "CPU PWM1 enabled\n");
4620         if (IS_HASWELL(dev_priv))
4621                 I915_STATE_WARN(intel_de_read(dev_priv, HSW_BLC_PWM2_CTL) & BLM_PWM_ENABLE,
4622                                 "CPU PWM2 enabled\n");
4623         I915_STATE_WARN(intel_de_read(dev_priv, BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE,
4624                         "PCH PWM1 enabled\n");
4625         I915_STATE_WARN(intel_de_read(dev_priv, UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
4626                         "Utility pin enabled\n");
4627         I915_STATE_WARN(intel_de_read(dev_priv, PCH_GTC_CTL) & PCH_GTC_ENABLE,
4628                         "PCH GTC enabled\n");
4629
4630         /*
4631          * In theory we can still leave IRQs enabled, as long as only the HPD
4632          * interrupts remain enabled. We used to check for that, but since it's
4633          * gen-specific and since we only disable LCPLL after we fully disable
4634          * the interrupts, the check below should be enough.
4635          */
4636         I915_STATE_WARN(intel_irqs_enabled(dev_priv), "IRQs enabled\n");
4637 }
4638
4639 static u32 hsw_read_dcomp(struct drm_i915_private *dev_priv)
4640 {
4641         if (IS_HASWELL(dev_priv))
4642                 return intel_de_read(dev_priv, D_COMP_HSW);
4643         else
4644                 return intel_de_read(dev_priv, D_COMP_BDW);
4645 }
4646
4647 static void hsw_write_dcomp(struct drm_i915_private *dev_priv, u32 val)
4648 {
4649         if (IS_HASWELL(dev_priv)) {
4650                 if (sandybridge_pcode_write(dev_priv,
4651                                             GEN6_PCODE_WRITE_D_COMP, val))
4652                         drm_dbg_kms(&dev_priv->drm,
4653                                     "Failed to write to D_COMP\n");
4654         } else {
4655                 intel_de_write(dev_priv, D_COMP_BDW, val);
4656                 intel_de_posting_read(dev_priv, D_COMP_BDW);
4657         }
4658 }
4659
4660 /*
4661  * This function implements pieces of two sequences from BSpec:
4662  * - Sequence for display software to disable LCPLL
4663  * - Sequence for display software to allow package C8+
4664  * The steps implemented here are just the steps that actually touch the LCPLL
4665  * register. Callers should take care of disabling all the display engine
4666  * functions, doing the mode unset, fixing interrupts, etc.
4667  */
4668 static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
4669                               bool switch_to_fclk, bool allow_power_down)
4670 {
4671         u32 val;
4672
4673         assert_can_disable_lcpll(dev_priv);
4674
4675         val = intel_de_read(dev_priv, LCPLL_CTL);
4676
4677         if (switch_to_fclk) {
4678                 val |= LCPLL_CD_SOURCE_FCLK;
4679                 intel_de_write(dev_priv, LCPLL_CTL, val);
4680
4681                 if (wait_for_us(intel_de_read(dev_priv, LCPLL_CTL) &
4682                                 LCPLL_CD_SOURCE_FCLK_DONE, 1))
4683                         drm_err(&dev_priv->drm, "Switching to FCLK failed\n");
4684
4685                 val = intel_de_read(dev_priv, LCPLL_CTL);
4686         }
4687
4688         val |= LCPLL_PLL_DISABLE;
4689         intel_de_write(dev_priv, LCPLL_CTL, val);
4690         intel_de_posting_read(dev_priv, LCPLL_CTL);
4691
4692         if (intel_de_wait_for_clear(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 1))
4693                 drm_err(&dev_priv->drm, "LCPLL still locked\n");
4694
4695         val = hsw_read_dcomp(dev_priv);
4696         val |= D_COMP_COMP_DISABLE;
4697         hsw_write_dcomp(dev_priv, val);
4698         ndelay(100);
4699
4700         if (wait_for((hsw_read_dcomp(dev_priv) &
4701                       D_COMP_RCOMP_IN_PROGRESS) == 0, 1))
4702                 drm_err(&dev_priv->drm, "D_COMP RCOMP still in progress\n");
4703
4704         if (allow_power_down) {
4705                 val = intel_de_read(dev_priv, LCPLL_CTL);
4706                 val |= LCPLL_POWER_DOWN_ALLOW;
4707                 intel_de_write(dev_priv, LCPLL_CTL, val);
4708                 intel_de_posting_read(dev_priv, LCPLL_CTL);
4709         }
4710 }
4711
4712 /*
4713  * Fully restores LCPLL, disallowing power down and switching back to LCPLL
4714  * source.
4715  */
4716 static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
4717 {
4718         u32 val;
4719
4720         val = intel_de_read(dev_priv, LCPLL_CTL);
4721
4722         if ((val & (LCPLL_PLL_LOCK | LCPLL_PLL_DISABLE | LCPLL_CD_SOURCE_FCLK |
4723                     LCPLL_POWER_DOWN_ALLOW)) == LCPLL_PLL_LOCK)
4724                 return;
4725
4726         /*
4727          * Make sure we're not on PC8 state before disabling PC8, otherwise
4728          * we'll hang the machine. To prevent PC8 state, just enable force_wake.
4729          */
4730         intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
4731
4732         if (val & LCPLL_POWER_DOWN_ALLOW) {
4733                 val &= ~LCPLL_POWER_DOWN_ALLOW;
4734                 intel_de_write(dev_priv, LCPLL_CTL, val);
4735                 intel_de_posting_read(dev_priv, LCPLL_CTL);
4736         }
4737
4738         val = hsw_read_dcomp(dev_priv);
4739         val |= D_COMP_COMP_FORCE;
4740         val &= ~D_COMP_COMP_DISABLE;
4741         hsw_write_dcomp(dev_priv, val);
4742
4743         val = intel_de_read(dev_priv, LCPLL_CTL);
4744         val &= ~LCPLL_PLL_DISABLE;
4745         intel_de_write(dev_priv, LCPLL_CTL, val);
4746
4747         if (intel_de_wait_for_set(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 5))
4748                 drm_err(&dev_priv->drm, "LCPLL not locked yet\n");
4749
4750         if (val & LCPLL_CD_SOURCE_FCLK) {
4751                 val = intel_de_read(dev_priv, LCPLL_CTL);
4752                 val &= ~LCPLL_CD_SOURCE_FCLK;
4753                 intel_de_write(dev_priv, LCPLL_CTL, val);
4754
4755                 if (wait_for_us((intel_de_read(dev_priv, LCPLL_CTL) &
4756                                  LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
4757                         drm_err(&dev_priv->drm,
4758                                 "Switching back to LCPLL failed\n");
4759         }
4760
4761         intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
4762
4763         intel_update_cdclk(dev_priv);
4764         intel_dump_cdclk_config(&dev_priv->cdclk.hw, "Current CDCLK");
4765 }
4766
4767 /*
4768  * Package states C8 and deeper are really deep PC states that can only be
4769  * reached when all the devices on the system allow it, so even if the graphics
4770  * device allows PC8+, it doesn't mean the system will actually get to these
4771  * states. Our driver only allows PC8+ when going into runtime PM.
4772  *
4773  * The requirements for PC8+ are that all the outputs are disabled, the power
4774  * well is disabled and most interrupts are disabled, and these are also
4775  * requirements for runtime PM. When these conditions are met, we manually do
4776  * the other conditions: disable the interrupts, clocks and switch LCPLL refclk
4777  * to Fclk. If we're in PC8+ and we get an non-hotplug interrupt, we can hard
4778  * hang the machine.
4779  *
4780  * When we really reach PC8 or deeper states (not just when we allow it) we lose
4781  * the state of some registers, so when we come back from PC8+ we need to
4782  * restore this state. We don't get into PC8+ if we're not in RC6, so we don't
4783  * need to take care of the registers kept by RC6. Notice that this happens even
4784  * if we don't put the device in PCI D3 state (which is what currently happens
4785  * because of the runtime PM support).
4786  *
4787  * For more, read "Display Sequences for Package C8" on the hardware
4788  * documentation.
4789  */
4790 static void hsw_enable_pc8(struct drm_i915_private *dev_priv)
4791 {
4792         u32 val;
4793
4794         drm_dbg_kms(&dev_priv->drm, "Enabling package C8+\n");
4795
4796         if (HAS_PCH_LPT_LP(dev_priv)) {
4797                 val = intel_de_read(dev_priv, SOUTH_DSPCLK_GATE_D);
4798                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
4799                 intel_de_write(dev_priv, SOUTH_DSPCLK_GATE_D, val);
4800         }
4801
4802         lpt_disable_clkout_dp(dev_priv);
4803         hsw_disable_lcpll(dev_priv, true, true);
4804 }
4805
4806 static void hsw_disable_pc8(struct drm_i915_private *dev_priv)
4807 {
4808         u32 val;
4809
4810         drm_dbg_kms(&dev_priv->drm, "Disabling package C8+\n");
4811
4812         hsw_restore_lcpll(dev_priv);
4813         intel_init_pch_refclk(dev_priv);
4814
4815         if (HAS_PCH_LPT_LP(dev_priv)) {
4816                 val = intel_de_read(dev_priv, SOUTH_DSPCLK_GATE_D);
4817                 val |= PCH_LP_PARTITION_LEVEL_DISABLE;
4818                 intel_de_write(dev_priv, SOUTH_DSPCLK_GATE_D, val);
4819         }
4820 }
4821
4822 static void intel_pch_reset_handshake(struct drm_i915_private *dev_priv,
4823                                       bool enable)
4824 {
4825         i915_reg_t reg;
4826         u32 reset_bits, val;
4827
4828         if (IS_IVYBRIDGE(dev_priv)) {
4829                 reg = GEN7_MSG_CTL;
4830                 reset_bits = WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK;
4831         } else {
4832                 reg = HSW_NDE_RSTWRN_OPT;
4833                 reset_bits = RESET_PCH_HANDSHAKE_ENABLE;
4834         }
4835
4836         val = intel_de_read(dev_priv, reg);
4837
4838         if (enable)
4839                 val |= reset_bits;
4840         else
4841                 val &= ~reset_bits;
4842
4843         intel_de_write(dev_priv, reg, val);
4844 }
4845
4846 static void skl_display_core_init(struct drm_i915_private *dev_priv,
4847                                   bool resume)
4848 {
4849         struct i915_power_domains *power_domains = &dev_priv->power_domains;
4850         struct i915_power_well *well;
4851
4852         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
4853
4854         /* enable PCH reset handshake */
4855         intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
4856
4857         /* enable PG1 and Misc I/O */
4858         mutex_lock(&power_domains->lock);
4859
4860         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
4861         intel_power_well_enable(dev_priv, well);
4862
4863         well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
4864         intel_power_well_enable(dev_priv, well);
4865
4866         mutex_unlock(&power_domains->lock);
4867
4868         intel_cdclk_init_hw(dev_priv);
4869
4870         gen9_dbuf_enable(dev_priv);
4871
4872         if (resume && dev_priv->csr.dmc_payload)
4873                 intel_csr_load_program(dev_priv);
4874 }
4875
4876 static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
4877 {
4878         struct i915_power_domains *power_domains = &dev_priv->power_domains;
4879         struct i915_power_well *well;
4880
4881         gen9_disable_dc_states(dev_priv);
4882
4883         gen9_dbuf_disable(dev_priv);
4884
4885         intel_cdclk_uninit_hw(dev_priv);
4886
4887         /* The spec doesn't call for removing the reset handshake flag */
4888         /* disable PG1 and Misc I/O */
4889
4890         mutex_lock(&power_domains->lock);
4891
4892         /*
4893          * BSpec says to keep the MISC IO power well enabled here, only
4894          * remove our request for power well 1.
4895          * Note that even though the driver's request is removed power well 1
4896          * may stay enabled after this due to DMC's own request on it.
4897          */
4898         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
4899         intel_power_well_disable(dev_priv, well);
4900
4901         mutex_unlock(&power_domains->lock);
4902
4903         usleep_range(10, 30);           /* 10 us delay per Bspec */
4904 }
4905
4906 static void bxt_display_core_init(struct drm_i915_private *dev_priv, bool resume)
4907 {
4908         struct i915_power_domains *power_domains = &dev_priv->power_domains;
4909         struct i915_power_well *well;
4910
4911         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
4912
4913         /*
4914          * NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
4915          * or else the reset will hang because there is no PCH to respond.
4916          * Move the handshake programming to initialization sequence.
4917          * Previously was left up to BIOS.
4918          */
4919         intel_pch_reset_handshake(dev_priv, false);
4920
4921         /* Enable PG1 */
4922         mutex_lock(&power_domains->lock);
4923
4924         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
4925         intel_power_well_enable(dev_priv, well);
4926
4927         mutex_unlock(&power_domains->lock);
4928
4929         intel_cdclk_init_hw(dev_priv);
4930
4931         gen9_dbuf_enable(dev_priv);
4932
4933         if (resume && dev_priv->csr.dmc_payload)
4934                 intel_csr_load_program(dev_priv);
4935 }
4936
4937 static void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
4938 {
4939         struct i915_power_domains *power_domains = &dev_priv->power_domains;
4940         struct i915_power_well *well;
4941
4942         gen9_disable_dc_states(dev_priv);
4943
4944         gen9_dbuf_disable(dev_priv);
4945
4946         intel_cdclk_uninit_hw(dev_priv);
4947
4948         /* The spec doesn't call for removing the reset handshake flag */
4949
4950         /*
4951          * Disable PW1 (PG1).
4952          * Note that even though the driver's request is removed power well 1
4953          * may stay enabled after this due to DMC's own request on it.
4954          */
4955         mutex_lock(&power_domains->lock);
4956
4957         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
4958         intel_power_well_disable(dev_priv, well);
4959
4960         mutex_unlock(&power_domains->lock);
4961
4962         usleep_range(10, 30);           /* 10 us delay per Bspec */
4963 }
4964
4965 static void cnl_display_core_init(struct drm_i915_private *dev_priv, bool resume)
4966 {
4967         struct i915_power_domains *power_domains = &dev_priv->power_domains;
4968         struct i915_power_well *well;
4969
4970         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
4971
4972         /* 1. Enable PCH Reset Handshake */
4973         intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
4974
4975         /* 2-3. */
4976         intel_combo_phy_init(dev_priv);
4977
4978         /*
4979          * 4. Enable Power Well 1 (PG1).
4980          *    The AUX IO power wells will be enabled on demand.
4981          */
4982         mutex_lock(&power_domains->lock);
4983         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
4984         intel_power_well_enable(dev_priv, well);
4985         mutex_unlock(&power_domains->lock);
4986
4987         /* 5. Enable CD clock */
4988         intel_cdclk_init_hw(dev_priv);
4989
4990         /* 6. Enable DBUF */
4991         gen9_dbuf_enable(dev_priv);
4992
4993         if (resume && dev_priv->csr.dmc_payload)
4994                 intel_csr_load_program(dev_priv);
4995 }
4996
4997 static void cnl_display_core_uninit(struct drm_i915_private *dev_priv)
4998 {
4999         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5000         struct i915_power_well *well;
5001
5002         gen9_disable_dc_states(dev_priv);
5003
5004         /* 1. Disable all display engine functions -> aready done */
5005
5006         /* 2. Disable DBUF */
5007         gen9_dbuf_disable(dev_priv);
5008
5009         /* 3. Disable CD clock */
5010         intel_cdclk_uninit_hw(dev_priv);
5011
5012         /*
5013          * 4. Disable Power Well 1 (PG1).
5014          *    The AUX IO power wells are toggled on demand, so they are already
5015          *    disabled at this point.
5016          */
5017         mutex_lock(&power_domains->lock);
5018         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
5019         intel_power_well_disable(dev_priv, well);
5020         mutex_unlock(&power_domains->lock);
5021
5022         usleep_range(10, 30);           /* 10 us delay per Bspec */
5023
5024         /* 5. */
5025         intel_combo_phy_uninit(dev_priv);
5026 }
5027
5028 struct buddy_page_mask {
5029         u32 page_mask;
5030         u8 type;
5031         u8 num_channels;
5032 };
5033
5034 static const struct buddy_page_mask tgl_buddy_page_masks[] = {
5035         { .num_channels = 1, .type = INTEL_DRAM_LPDDR4, .page_mask = 0xE },
5036         { .num_channels = 1, .type = INTEL_DRAM_DDR4,   .page_mask = 0xF },
5037         { .num_channels = 2, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x1C },
5038         { .num_channels = 2, .type = INTEL_DRAM_DDR4,   .page_mask = 0x1F },
5039         {}
5040 };
5041
5042 static const struct buddy_page_mask wa_1409767108_buddy_page_masks[] = {
5043         { .num_channels = 1, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x1 },
5044         { .num_channels = 1, .type = INTEL_DRAM_DDR4,   .page_mask = 0x1 },
5045         { .num_channels = 2, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x3 },
5046         { .num_channels = 2, .type = INTEL_DRAM_DDR4,   .page_mask = 0x3 },
5047         {}
5048 };
5049
5050 static void tgl_bw_buddy_init(struct drm_i915_private *dev_priv)
5051 {
5052         enum intel_dram_type type = dev_priv->dram_info.type;
5053         u8 num_channels = dev_priv->dram_info.num_channels;
5054         const struct buddy_page_mask *table;
5055         int i;
5056
5057         if (IS_TGL_REVID(dev_priv, TGL_REVID_A0, TGL_REVID_B0))
5058                 /* Wa_1409767108: tgl */
5059                 table = wa_1409767108_buddy_page_masks;
5060         else
5061                 table = tgl_buddy_page_masks;
5062
5063         for (i = 0; table[i].page_mask != 0; i++)
5064                 if (table[i].num_channels == num_channels &&
5065                     table[i].type == type)
5066                         break;
5067
5068         if (table[i].page_mask == 0) {
5069                 drm_dbg(&dev_priv->drm,
5070                         "Unknown memory configuration; disabling address buddy logic.\n");
5071                 intel_de_write(dev_priv, BW_BUDDY1_CTL, BW_BUDDY_DISABLE);
5072                 intel_de_write(dev_priv, BW_BUDDY2_CTL, BW_BUDDY_DISABLE);
5073         } else {
5074                 intel_de_write(dev_priv, BW_BUDDY1_PAGE_MASK,
5075                                table[i].page_mask);
5076                 intel_de_write(dev_priv, BW_BUDDY2_PAGE_MASK,
5077                                table[i].page_mask);
5078
5079                 /* Wa_22010178259:tgl */
5080                 intel_de_rmw(dev_priv, BW_BUDDY1_CTL,
5081                              BW_BUDDY_TLB_REQ_TIMER_MASK,
5082                              REG_FIELD_PREP(BW_BUDDY_TLB_REQ_TIMER_MASK, 0x8));
5083                 intel_de_rmw(dev_priv, BW_BUDDY2_CTL,
5084                              BW_BUDDY_TLB_REQ_TIMER_MASK,
5085                              REG_FIELD_PREP(BW_BUDDY_TLB_REQ_TIMER_MASK, 0x8));
5086         }
5087 }
5088
5089 static void icl_display_core_init(struct drm_i915_private *dev_priv,
5090                                   bool resume)
5091 {
5092         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5093         struct i915_power_well *well;
5094
5095         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
5096
5097         /* 1. Enable PCH reset handshake. */
5098         intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
5099
5100         /* 2. Initialize all combo phys */
5101         intel_combo_phy_init(dev_priv);
5102
5103         /*
5104          * 3. Enable Power Well 1 (PG1).
5105          *    The AUX IO power wells will be enabled on demand.
5106          */
5107         mutex_lock(&power_domains->lock);
5108         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
5109         intel_power_well_enable(dev_priv, well);
5110         mutex_unlock(&power_domains->lock);
5111
5112         /* 4. Enable CDCLK. */
5113         intel_cdclk_init_hw(dev_priv);
5114
5115         /* 5. Enable DBUF. */
5116         icl_dbuf_enable(dev_priv);
5117
5118         /* 6. Setup MBUS. */
5119         icl_mbus_init(dev_priv);
5120
5121         /* 7. Program arbiter BW_BUDDY registers */
5122         if (INTEL_GEN(dev_priv) >= 12)
5123                 tgl_bw_buddy_init(dev_priv);
5124
5125         if (resume && dev_priv->csr.dmc_payload)
5126                 intel_csr_load_program(dev_priv);
5127 }
5128
5129 static void icl_display_core_uninit(struct drm_i915_private *dev_priv)
5130 {
5131         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5132         struct i915_power_well *well;
5133
5134         gen9_disable_dc_states(dev_priv);
5135
5136         /* 1. Disable all display engine functions -> aready done */
5137
5138         /* 2. Disable DBUF */
5139         icl_dbuf_disable(dev_priv);
5140
5141         /* 3. Disable CD clock */
5142         intel_cdclk_uninit_hw(dev_priv);
5143
5144         /*
5145          * 4. Disable Power Well 1 (PG1).
5146          *    The AUX IO power wells are toggled on demand, so they are already
5147          *    disabled at this point.
5148          */
5149         mutex_lock(&power_domains->lock);
5150         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
5151         intel_power_well_disable(dev_priv, well);
5152         mutex_unlock(&power_domains->lock);
5153
5154         /* 5. */
5155         intel_combo_phy_uninit(dev_priv);
5156 }
5157
5158 static void chv_phy_control_init(struct drm_i915_private *dev_priv)
5159 {
5160         struct i915_power_well *cmn_bc =
5161                 lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
5162         struct i915_power_well *cmn_d =
5163                 lookup_power_well(dev_priv, CHV_DISP_PW_DPIO_CMN_D);
5164
5165         /*
5166          * DISPLAY_PHY_CONTROL can get corrupted if read. As a
5167          * workaround never ever read DISPLAY_PHY_CONTROL, and
5168          * instead maintain a shadow copy ourselves. Use the actual
5169          * power well state and lane status to reconstruct the
5170          * expected initial value.
5171          */
5172         dev_priv->chv_phy_control =
5173                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
5174                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
5175                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) |
5176                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH1) |
5177                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY1, DPIO_CH0);
5178
5179         /*
5180          * If all lanes are disabled we leave the override disabled
5181          * with all power down bits cleared to match the state we
5182          * would use after disabling the port. Otherwise enable the
5183          * override and set the lane powerdown bits accding to the
5184          * current lane status.
5185          */
5186         if (cmn_bc->desc->ops->is_enabled(dev_priv, cmn_bc)) {
5187                 u32 status = intel_de_read(dev_priv, DPLL(PIPE_A));
5188                 unsigned int mask;
5189
5190                 mask = status & DPLL_PORTB_READY_MASK;
5191                 if (mask == 0xf)
5192                         mask = 0x0;
5193                 else
5194                         dev_priv->chv_phy_control |=
5195                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0);
5196
5197                 dev_priv->chv_phy_control |=
5198                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0);
5199
5200                 mask = (status & DPLL_PORTC_READY_MASK) >> 4;
5201                 if (mask == 0xf)
5202                         mask = 0x0;
5203                 else
5204                         dev_priv->chv_phy_control |=
5205                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1);
5206
5207                 dev_priv->chv_phy_control |=
5208                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1);
5209
5210                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
5211
5212                 dev_priv->chv_phy_assert[DPIO_PHY0] = false;
5213         } else {
5214                 dev_priv->chv_phy_assert[DPIO_PHY0] = true;
5215         }
5216
5217         if (cmn_d->desc->ops->is_enabled(dev_priv, cmn_d)) {
5218                 u32 status = intel_de_read(dev_priv, DPIO_PHY_STATUS);
5219                 unsigned int mask;
5220
5221                 mask = status & DPLL_PORTD_READY_MASK;
5222
5223                 if (mask == 0xf)
5224                         mask = 0x0;
5225                 else
5226                         dev_priv->chv_phy_control |=
5227                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0);
5228
5229                 dev_priv->chv_phy_control |=
5230                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0);
5231
5232                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
5233
5234                 dev_priv->chv_phy_assert[DPIO_PHY1] = false;
5235         } else {
5236                 dev_priv->chv_phy_assert[DPIO_PHY1] = true;
5237         }
5238
5239         drm_dbg_kms(&dev_priv->drm, "Initial PHY_CONTROL=0x%08x\n",
5240                     dev_priv->chv_phy_control);
5241
5242         /* Defer application of initial phy_control to enabling the powerwell */
5243 }
5244
5245 static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
5246 {
5247         struct i915_power_well *cmn =
5248                 lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
5249         struct i915_power_well *disp2d =
5250                 lookup_power_well(dev_priv, VLV_DISP_PW_DISP2D);
5251
5252         /* If the display might be already active skip this */
5253         if (cmn->desc->ops->is_enabled(dev_priv, cmn) &&
5254             disp2d->desc->ops->is_enabled(dev_priv, disp2d) &&
5255             intel_de_read(dev_priv, DPIO_CTL) & DPIO_CMNRST)
5256                 return;
5257
5258         drm_dbg_kms(&dev_priv->drm, "toggling display PHY side reset\n");
5259
5260         /* cmnlane needs DPLL registers */
5261         disp2d->desc->ops->enable(dev_priv, disp2d);
5262
5263         /*
5264          * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
5265          * Need to assert and de-assert PHY SB reset by gating the
5266          * common lane power, then un-gating it.
5267          * Simply ungating isn't enough to reset the PHY enough to get
5268          * ports and lanes running.
5269          */
5270         cmn->desc->ops->disable(dev_priv, cmn);
5271 }
5272
5273 static bool vlv_punit_is_power_gated(struct drm_i915_private *dev_priv, u32 reg0)
5274 {
5275         bool ret;
5276
5277         vlv_punit_get(dev_priv);
5278         ret = (vlv_punit_read(dev_priv, reg0) & SSPM0_SSC_MASK) == SSPM0_SSC_PWR_GATE;
5279         vlv_punit_put(dev_priv);
5280
5281         return ret;
5282 }
5283
5284 static void assert_ved_power_gated(struct drm_i915_private *dev_priv)
5285 {
5286         drm_WARN(&dev_priv->drm,
5287                  !vlv_punit_is_power_gated(dev_priv, PUNIT_REG_VEDSSPM0),
5288                  "VED not power gated\n");
5289 }
5290
5291 static void assert_isp_power_gated(struct drm_i915_private *dev_priv)
5292 {
5293         static const struct pci_device_id isp_ids[] = {
5294                 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x0f38)},
5295                 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x22b8)},
5296                 {}
5297         };
5298
5299         drm_WARN(&dev_priv->drm, !pci_dev_present(isp_ids) &&
5300                  !vlv_punit_is_power_gated(dev_priv, PUNIT_REG_ISPSSPM0),
5301                  "ISP not power gated\n");
5302 }
5303
5304 static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv);
5305
5306 /**
5307  * intel_power_domains_init_hw - initialize hardware power domain state
5308  * @i915: i915 device instance
5309  * @resume: Called from resume code paths or not
5310  *
5311  * This function initializes the hardware power domain state and enables all
5312  * power wells belonging to the INIT power domain. Power wells in other
5313  * domains (and not in the INIT domain) are referenced or disabled by
5314  * intel_modeset_readout_hw_state(). After that the reference count of each
5315  * power well must match its HW enabled state, see
5316  * intel_power_domains_verify_state().
5317  *
5318  * It will return with power domains disabled (to be enabled later by
5319  * intel_power_domains_enable()) and must be paired with
5320  * intel_power_domains_driver_remove().
5321  */
5322 void intel_power_domains_init_hw(struct drm_i915_private *i915, bool resume)
5323 {
5324         struct i915_power_domains *power_domains = &i915->power_domains;
5325
5326         power_domains->initializing = true;
5327
5328         if (INTEL_GEN(i915) >= 11) {
5329                 icl_display_core_init(i915, resume);
5330         } else if (IS_CANNONLAKE(i915)) {
5331                 cnl_display_core_init(i915, resume);
5332         } else if (IS_GEN9_BC(i915)) {
5333                 skl_display_core_init(i915, resume);
5334         } else if (IS_GEN9_LP(i915)) {
5335                 bxt_display_core_init(i915, resume);
5336         } else if (IS_CHERRYVIEW(i915)) {
5337                 mutex_lock(&power_domains->lock);
5338                 chv_phy_control_init(i915);
5339                 mutex_unlock(&power_domains->lock);
5340                 assert_isp_power_gated(i915);
5341         } else if (IS_VALLEYVIEW(i915)) {
5342                 mutex_lock(&power_domains->lock);
5343                 vlv_cmnlane_wa(i915);
5344                 mutex_unlock(&power_domains->lock);
5345                 assert_ved_power_gated(i915);
5346                 assert_isp_power_gated(i915);
5347         } else if (IS_BROADWELL(i915) || IS_HASWELL(i915)) {
5348                 hsw_assert_cdclk(i915);
5349                 intel_pch_reset_handshake(i915, !HAS_PCH_NOP(i915));
5350         } else if (IS_IVYBRIDGE(i915)) {
5351                 intel_pch_reset_handshake(i915, !HAS_PCH_NOP(i915));
5352         }
5353
5354         /*
5355          * Keep all power wells enabled for any dependent HW access during
5356          * initialization and to make sure we keep BIOS enabled display HW
5357          * resources powered until display HW readout is complete. We drop
5358          * this reference in intel_power_domains_enable().
5359          */
5360         power_domains->wakeref =
5361                 intel_display_power_get(i915, POWER_DOMAIN_INIT);
5362
5363         /* Disable power support if the user asked so. */
5364         if (!i915_modparams.disable_power_well)
5365                 intel_display_power_get(i915, POWER_DOMAIN_INIT);
5366         intel_power_domains_sync_hw(i915);
5367
5368         power_domains->initializing = false;
5369 }
5370
5371 /**
5372  * intel_power_domains_driver_remove - deinitialize hw power domain state
5373  * @i915: i915 device instance
5374  *
5375  * De-initializes the display power domain HW state. It also ensures that the
5376  * device stays powered up so that the driver can be reloaded.
5377  *
5378  * It must be called with power domains already disabled (after a call to
5379  * intel_power_domains_disable()) and must be paired with
5380  * intel_power_domains_init_hw().
5381  */
5382 void intel_power_domains_driver_remove(struct drm_i915_private *i915)
5383 {
5384         intel_wakeref_t wakeref __maybe_unused =
5385                 fetch_and_zero(&i915->power_domains.wakeref);
5386
5387         /* Remove the refcount we took to keep power well support disabled. */
5388         if (!i915_modparams.disable_power_well)
5389                 intel_display_power_put_unchecked(i915, POWER_DOMAIN_INIT);
5390
5391         intel_display_power_flush_work_sync(i915);
5392
5393         intel_power_domains_verify_state(i915);
5394
5395         /* Keep the power well enabled, but cancel its rpm wakeref. */
5396         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
5397 }
5398
5399 /**
5400  * intel_power_domains_enable - enable toggling of display power wells
5401  * @i915: i915 device instance
5402  *
5403  * Enable the ondemand enabling/disabling of the display power wells. Note that
5404  * power wells not belonging to POWER_DOMAIN_INIT are allowed to be toggled
5405  * only at specific points of the display modeset sequence, thus they are not
5406  * affected by the intel_power_domains_enable()/disable() calls. The purpose
5407  * of these function is to keep the rest of power wells enabled until the end
5408  * of display HW readout (which will acquire the power references reflecting
5409  * the current HW state).
5410  */
5411 void intel_power_domains_enable(struct drm_i915_private *i915)
5412 {
5413         intel_wakeref_t wakeref __maybe_unused =
5414                 fetch_and_zero(&i915->power_domains.wakeref);
5415
5416         intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref);
5417         intel_power_domains_verify_state(i915);
5418 }
5419
5420 /**
5421  * intel_power_domains_disable - disable toggling of display power wells
5422  * @i915: i915 device instance
5423  *
5424  * Disable the ondemand enabling/disabling of the display power wells. See
5425  * intel_power_domains_enable() for which power wells this call controls.
5426  */
5427 void intel_power_domains_disable(struct drm_i915_private *i915)
5428 {
5429         struct i915_power_domains *power_domains = &i915->power_domains;
5430
5431         drm_WARN_ON(&i915->drm, power_domains->wakeref);
5432         power_domains->wakeref =
5433                 intel_display_power_get(i915, POWER_DOMAIN_INIT);
5434
5435         intel_power_domains_verify_state(i915);
5436 }
5437
5438 /**
5439  * intel_power_domains_suspend - suspend power domain state
5440  * @i915: i915 device instance
5441  * @suspend_mode: specifies the target suspend state (idle, mem, hibernation)
5442  *
5443  * This function prepares the hardware power domain state before entering
5444  * system suspend.
5445  *
5446  * It must be called with power domains already disabled (after a call to
5447  * intel_power_domains_disable()) and paired with intel_power_domains_resume().
5448  */
5449 void intel_power_domains_suspend(struct drm_i915_private *i915,
5450                                  enum i915_drm_suspend_mode suspend_mode)
5451 {
5452         struct i915_power_domains *power_domains = &i915->power_domains;
5453         intel_wakeref_t wakeref __maybe_unused =
5454                 fetch_and_zero(&power_domains->wakeref);
5455
5456         intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref);
5457
5458         /*
5459          * In case of suspend-to-idle (aka S0ix) on a DMC platform without DC9
5460          * support don't manually deinit the power domains. This also means the
5461          * CSR/DMC firmware will stay active, it will power down any HW
5462          * resources as required and also enable deeper system power states
5463          * that would be blocked if the firmware was inactive.
5464          */
5465         if (!(i915->csr.allowed_dc_mask & DC_STATE_EN_DC9) &&
5466             suspend_mode == I915_DRM_SUSPEND_IDLE &&
5467             i915->csr.dmc_payload) {
5468                 intel_display_power_flush_work(i915);
5469                 intel_power_domains_verify_state(i915);
5470                 return;
5471         }
5472
5473         /*
5474          * Even if power well support was disabled we still want to disable
5475          * power wells if power domains must be deinitialized for suspend.
5476          */
5477         if (!i915_modparams.disable_power_well)
5478                 intel_display_power_put_unchecked(i915, POWER_DOMAIN_INIT);
5479
5480         intel_display_power_flush_work(i915);
5481         intel_power_domains_verify_state(i915);
5482
5483         if (INTEL_GEN(i915) >= 11)
5484                 icl_display_core_uninit(i915);
5485         else if (IS_CANNONLAKE(i915))
5486                 cnl_display_core_uninit(i915);
5487         else if (IS_GEN9_BC(i915))
5488                 skl_display_core_uninit(i915);
5489         else if (IS_GEN9_LP(i915))
5490                 bxt_display_core_uninit(i915);
5491
5492         power_domains->display_core_suspended = true;
5493 }
5494
5495 /**
5496  * intel_power_domains_resume - resume power domain state
5497  * @i915: i915 device instance
5498  *
5499  * This function resume the hardware power domain state during system resume.
5500  *
5501  * It will return with power domain support disabled (to be enabled later by
5502  * intel_power_domains_enable()) and must be paired with
5503  * intel_power_domains_suspend().
5504  */
5505 void intel_power_domains_resume(struct drm_i915_private *i915)
5506 {
5507         struct i915_power_domains *power_domains = &i915->power_domains;
5508
5509         if (power_domains->display_core_suspended) {
5510                 intel_power_domains_init_hw(i915, true);
5511                 power_domains->display_core_suspended = false;
5512         } else {
5513                 drm_WARN_ON(&i915->drm, power_domains->wakeref);
5514                 power_domains->wakeref =
5515                         intel_display_power_get(i915, POWER_DOMAIN_INIT);
5516         }
5517
5518         intel_power_domains_verify_state(i915);
5519 }
5520
5521 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
5522
5523 static void intel_power_domains_dump_info(struct drm_i915_private *i915)
5524 {
5525         struct i915_power_domains *power_domains = &i915->power_domains;
5526         struct i915_power_well *power_well;
5527
5528         for_each_power_well(i915, power_well) {
5529                 enum intel_display_power_domain domain;
5530
5531                 drm_dbg(&i915->drm, "%-25s %d\n",
5532                         power_well->desc->name, power_well->count);
5533
5534                 for_each_power_domain(domain, power_well->desc->domains)
5535                         drm_dbg(&i915->drm, "  %-23s %d\n",
5536                                 intel_display_power_domain_str(domain),
5537                                 power_domains->domain_use_count[domain]);
5538         }
5539 }
5540
5541 /**
5542  * intel_power_domains_verify_state - verify the HW/SW state for all power wells
5543  * @i915: i915 device instance
5544  *
5545  * Verify if the reference count of each power well matches its HW enabled
5546  * state and the total refcount of the domains it belongs to. This must be
5547  * called after modeset HW state sanitization, which is responsible for
5548  * acquiring reference counts for any power wells in use and disabling the
5549  * ones left on by BIOS but not required by any active output.
5550  */
5551 static void intel_power_domains_verify_state(struct drm_i915_private *i915)
5552 {
5553         struct i915_power_domains *power_domains = &i915->power_domains;
5554         struct i915_power_well *power_well;
5555         bool dump_domain_info;
5556
5557         mutex_lock(&power_domains->lock);
5558
5559         verify_async_put_domains_state(power_domains);
5560
5561         dump_domain_info = false;
5562         for_each_power_well(i915, power_well) {
5563                 enum intel_display_power_domain domain;
5564                 int domains_count;
5565                 bool enabled;
5566
5567                 enabled = power_well->desc->ops->is_enabled(i915, power_well);
5568                 if ((power_well->count || power_well->desc->always_on) !=
5569                     enabled)
5570                         drm_err(&i915->drm,
5571                                 "power well %s state mismatch (refcount %d/enabled %d)",
5572                                 power_well->desc->name,
5573                                 power_well->count, enabled);
5574
5575                 domains_count = 0;
5576                 for_each_power_domain(domain, power_well->desc->domains)
5577                         domains_count += power_domains->domain_use_count[domain];
5578
5579                 if (power_well->count != domains_count) {
5580                         drm_err(&i915->drm,
5581                                 "power well %s refcount/domain refcount mismatch "
5582                                 "(refcount %d/domains refcount %d)\n",
5583                                 power_well->desc->name, power_well->count,
5584                                 domains_count);
5585                         dump_domain_info = true;
5586                 }
5587         }
5588
5589         if (dump_domain_info) {
5590                 static bool dumped;
5591
5592                 if (!dumped) {
5593                         intel_power_domains_dump_info(i915);
5594                         dumped = true;
5595                 }
5596         }
5597
5598         mutex_unlock(&power_domains->lock);
5599 }
5600
5601 #else
5602
5603 static void intel_power_domains_verify_state(struct drm_i915_private *i915)
5604 {
5605 }
5606
5607 #endif
5608
5609 void intel_display_power_suspend_late(struct drm_i915_private *i915)
5610 {
5611         if (INTEL_GEN(i915) >= 11 || IS_GEN9_LP(i915))
5612                 bxt_enable_dc9(i915);
5613         else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
5614                 hsw_enable_pc8(i915);
5615 }
5616
5617 void intel_display_power_resume_early(struct drm_i915_private *i915)
5618 {
5619         if (INTEL_GEN(i915) >= 11 || IS_GEN9_LP(i915)) {
5620                 gen9_sanitize_dc_state(i915);
5621                 bxt_disable_dc9(i915);
5622         } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
5623                 hsw_disable_pc8(i915);
5624         }
5625 }
5626
5627 void intel_display_power_suspend(struct drm_i915_private *i915)
5628 {
5629         if (INTEL_GEN(i915) >= 11) {
5630                 icl_display_core_uninit(i915);
5631                 bxt_enable_dc9(i915);
5632         } else if (IS_GEN9_LP(i915)) {
5633                 bxt_display_core_uninit(i915);
5634                 bxt_enable_dc9(i915);
5635         } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
5636                 hsw_enable_pc8(i915);
5637         }
5638 }
5639
5640 void intel_display_power_resume(struct drm_i915_private *i915)
5641 {
5642         if (INTEL_GEN(i915) >= 11) {
5643                 bxt_disable_dc9(i915);
5644                 icl_display_core_init(i915, true);
5645                 if (i915->csr.dmc_payload) {
5646                         if (i915->csr.allowed_dc_mask &
5647                             DC_STATE_EN_UPTO_DC6)
5648                                 skl_enable_dc6(i915);
5649                         else if (i915->csr.allowed_dc_mask &
5650                                  DC_STATE_EN_UPTO_DC5)
5651                                 gen9_enable_dc5(i915);
5652                 }
5653         } else if (IS_GEN9_LP(i915)) {
5654                 bxt_disable_dc9(i915);
5655                 bxt_display_core_init(i915, true);
5656                 if (i915->csr.dmc_payload &&
5657                     (i915->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5))
5658                         gen9_enable_dc5(i915);
5659         } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
5660                 hsw_disable_pc8(i915);
5661         }
5662 }