drivers/gpu/drm/i915/display/intel_display_power.c

   1 /* SPDX-License-Identifier: MIT */
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include "display/intel_crt.h"
   7
   8 #include "i915_drv.h"
   9 #include "i915_irq.h"
  10 #include "intel_cdclk.h"
  11 #include "intel_combo_phy.h"
  12 #include "intel_display_power.h"
  13 #include "intel_de.h"
  14 #include "intel_display_types.h"
  15 #include "intel_dmc.h"
  16 #include "intel_dpio_phy.h"
  17 #include "intel_hotplug.h"
  18 #include "intel_pm.h"
  19 #include "intel_pps.h"
  20 #include "intel_sideband.h"
  21 #include "intel_tc.h"
  22 #include "intel_vga.h"
  23
  24 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
  25                                          enum i915_power_well_id power_well_id);
  26
  27 const char *
  28 intel_display_power_domain_str(enum intel_display_power_domain domain)
  29 {
  30         switch (domain) {
  31         case POWER_DOMAIN_DISPLAY_CORE:
  32                 return "DISPLAY_CORE";
  33         case POWER_DOMAIN_PIPE_A:
  34                 return "PIPE_A";
  35         case POWER_DOMAIN_PIPE_B:
  36                 return "PIPE_B";
  37         case POWER_DOMAIN_PIPE_C:
  38                 return "PIPE_C";
  39         case POWER_DOMAIN_PIPE_D:
  40                 return "PIPE_D";
  41         case POWER_DOMAIN_PIPE_A_PANEL_FITTER:
  42                 return "PIPE_A_PANEL_FITTER";
  43         case POWER_DOMAIN_PIPE_B_PANEL_FITTER:
  44                 return "PIPE_B_PANEL_FITTER";
  45         case POWER_DOMAIN_PIPE_C_PANEL_FITTER:
  46                 return "PIPE_C_PANEL_FITTER";
  47         case POWER_DOMAIN_PIPE_D_PANEL_FITTER:
  48                 return "PIPE_D_PANEL_FITTER";
  49         case POWER_DOMAIN_TRANSCODER_A:
  50                 return "TRANSCODER_A";
  51         case POWER_DOMAIN_TRANSCODER_B:
  52                 return "TRANSCODER_B";
  53         case POWER_DOMAIN_TRANSCODER_C:
  54                 return "TRANSCODER_C";
  55         case POWER_DOMAIN_TRANSCODER_D:
  56                 return "TRANSCODER_D";
  57         case POWER_DOMAIN_TRANSCODER_EDP:
  58                 return "TRANSCODER_EDP";
  59         case POWER_DOMAIN_TRANSCODER_VDSC_PW2:
  60                 return "TRANSCODER_VDSC_PW2";
  61         case POWER_DOMAIN_TRANSCODER_DSI_A:
  62                 return "TRANSCODER_DSI_A";
  63         case POWER_DOMAIN_TRANSCODER_DSI_C:
  64                 return "TRANSCODER_DSI_C";
  65         case POWER_DOMAIN_PORT_DDI_A_LANES:
  66                 return "PORT_DDI_A_LANES";
  67         case POWER_DOMAIN_PORT_DDI_B_LANES:
  68                 return "PORT_DDI_B_LANES";
  69         case POWER_DOMAIN_PORT_DDI_C_LANES:
  70                 return "PORT_DDI_C_LANES";
  71         case POWER_DOMAIN_PORT_DDI_D_LANES:
  72                 return "PORT_DDI_D_LANES";
  73         case POWER_DOMAIN_PORT_DDI_E_LANES:
  74                 return "PORT_DDI_E_LANES";
  75         case POWER_DOMAIN_PORT_DDI_F_LANES:
  76                 return "PORT_DDI_F_LANES";
  77         case POWER_DOMAIN_PORT_DDI_G_LANES:
  78                 return "PORT_DDI_G_LANES";
  79         case POWER_DOMAIN_PORT_DDI_H_LANES:
  80                 return "PORT_DDI_H_LANES";
  81         case POWER_DOMAIN_PORT_DDI_I_LANES:
  82                 return "PORT_DDI_I_LANES";
  83         case POWER_DOMAIN_PORT_DDI_A_IO:
  84                 return "PORT_DDI_A_IO";
  85         case POWER_DOMAIN_PORT_DDI_B_IO:
  86                 return "PORT_DDI_B_IO";
  87         case POWER_DOMAIN_PORT_DDI_C_IO:
  88                 return "PORT_DDI_C_IO";
  89         case POWER_DOMAIN_PORT_DDI_D_IO:
  90                 return "PORT_DDI_D_IO";
  91         case POWER_DOMAIN_PORT_DDI_E_IO:
  92                 return "PORT_DDI_E_IO";
  93         case POWER_DOMAIN_PORT_DDI_F_IO:
  94                 return "PORT_DDI_F_IO";
  95         case POWER_DOMAIN_PORT_DDI_G_IO:
  96                 return "PORT_DDI_G_IO";
  97         case POWER_DOMAIN_PORT_DDI_H_IO:
  98                 return "PORT_DDI_H_IO";
  99         case POWER_DOMAIN_PORT_DDI_I_IO:
 100                 return "PORT_DDI_I_IO";
 101         case POWER_DOMAIN_PORT_DSI:
 102                 return "PORT_DSI";
 103         case POWER_DOMAIN_PORT_CRT:
 104                 return "PORT_CRT";
 105         case POWER_DOMAIN_PORT_OTHER:
 106                 return "PORT_OTHER";
 107         case POWER_DOMAIN_VGA:
 108                 return "VGA";
 109         case POWER_DOMAIN_AUDIO:
 110                 return "AUDIO";
 111         case POWER_DOMAIN_AUX_A:
 112                 return "AUX_A";
 113         case POWER_DOMAIN_AUX_B:
 114                 return "AUX_B";
 115         case POWER_DOMAIN_AUX_C:
 116                 return "AUX_C";
 117         case POWER_DOMAIN_AUX_D:
 118                 return "AUX_D";
 119         case POWER_DOMAIN_AUX_E:
 120                 return "AUX_E";
 121         case POWER_DOMAIN_AUX_F:
 122                 return "AUX_F";
 123         case POWER_DOMAIN_AUX_G:
 124                 return "AUX_G";
 125         case POWER_DOMAIN_AUX_H:
 126                 return "AUX_H";
 127         case POWER_DOMAIN_AUX_I:
 128                 return "AUX_I";
 129         case POWER_DOMAIN_AUX_IO_A:
 130                 return "AUX_IO_A";
 131         case POWER_DOMAIN_AUX_C_TBT:
 132                 return "AUX_C_TBT";
 133         case POWER_DOMAIN_AUX_D_TBT:
 134                 return "AUX_D_TBT";
 135         case POWER_DOMAIN_AUX_E_TBT:
 136                 return "AUX_E_TBT";
 137         case POWER_DOMAIN_AUX_F_TBT:
 138                 return "AUX_F_TBT";
 139         case POWER_DOMAIN_AUX_G_TBT:
 140                 return "AUX_G_TBT";
 141         case POWER_DOMAIN_AUX_H_TBT:
 142                 return "AUX_H_TBT";
 143         case POWER_DOMAIN_AUX_I_TBT:
 144                 return "AUX_I_TBT";
 145         case POWER_DOMAIN_GMBUS:
 146                 return "GMBUS";
 147         case POWER_DOMAIN_INIT:
 148                 return "INIT";
 149         case POWER_DOMAIN_MODESET:
 150                 return "MODESET";
 151         case POWER_DOMAIN_GT_IRQ:
 152                 return "GT_IRQ";
 153         case POWER_DOMAIN_DPLL_DC_OFF:
 154                 return "DPLL_DC_OFF";
 155         case POWER_DOMAIN_TC_COLD_OFF:
 156                 return "TC_COLD_OFF";
 157         default:
 158                 MISSING_CASE(domain);
 159                 return "?";
 160         }
 161 }
 162
 163 static void intel_power_well_enable(struct drm_i915_private *dev_priv,
 164                                     struct i915_power_well *power_well)
 165 {
 166         drm_dbg_kms(&dev_priv->drm, "enabling %s\n", power_well->desc->name);
 167         power_well->desc->ops->enable(dev_priv, power_well);
 168         power_well->hw_enabled = true;
 169 }
 170
 171 static void intel_power_well_disable(struct drm_i915_private *dev_priv,
 172                                      struct i915_power_well *power_well)
 173 {
 174         drm_dbg_kms(&dev_priv->drm, "disabling %s\n", power_well->desc->name);
 175         power_well->hw_enabled = false;
 176         power_well->desc->ops->disable(dev_priv, power_well);
 177 }
 178
 179 static void intel_power_well_get(struct drm_i915_private *dev_priv,
 180                                  struct i915_power_well *power_well)
 181 {
 182         if (!power_well->count++)
 183                 intel_power_well_enable(dev_priv, power_well);
 184 }
 185
 186 static void intel_power_well_put(struct drm_i915_private *dev_priv,
 187                                  struct i915_power_well *power_well)
 188 {
 189         drm_WARN(&dev_priv->drm, !power_well->count,
 190                  "Use count on power well %s is already zero",
 191                  power_well->desc->name);
 192
 193         if (!--power_well->count)
 194                 intel_power_well_disable(dev_priv, power_well);
 195 }
 196
 197 /**
 198  * __intel_display_power_is_enabled - unlocked check for a power domain
 199  * @dev_priv: i915 device instance
 200  * @domain: power domain to check
 201  *
 202  * This is the unlocked version of intel_display_power_is_enabled() and should
 203  * only be used from error capture and recovery code where deadlocks are
 204  * possible.
 205  *
 206  * Returns:
 207  * True when the power domain is enabled, false otherwise.
 208  */
 209 bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 210                                       enum intel_display_power_domain domain)
 211 {
 212         struct i915_power_well *power_well;
 213         bool is_enabled;
 214
 215         if (dev_priv->runtime_pm.suspended)
 216                 return false;
 217
 218         is_enabled = true;
 219
 220         for_each_power_domain_well_reverse(dev_priv, power_well, BIT_ULL(domain)) {
 221                 if (power_well->desc->always_on)
 222                         continue;
 223
 224                 if (!power_well->hw_enabled) {
 225                         is_enabled = false;
 226                         break;
 227                 }
 228         }
 229
 230         return is_enabled;
 231 }
 232
 233 /**
 234  * intel_display_power_is_enabled - check for a power domain
 235  * @dev_priv: i915 device instance
 236  * @domain: power domain to check
 237  *
 238  * This function can be used to check the hw power domain state. It is mostly
 239  * used in hardware state readout functions. Everywhere else code should rely
 240  * upon explicit power domain reference counting to ensure that the hardware
 241  * block is powered up before accessing it.
 242  *
 243  * Callers must hold the relevant modesetting locks to ensure that concurrent
 244  * threads can't disable the power well while the caller tries to read a few
 245  * registers.
 246  *
 247  * Returns:
 248  * True when the power domain is enabled, false otherwise.
 249  */
 250 bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 251                                     enum intel_display_power_domain domain)
 252 {
 253         struct i915_power_domains *power_domains;
 254         bool ret;
 255
 256         power_domains = &dev_priv->power_domains;
 257
 258         mutex_lock(&power_domains->lock);
 259         ret = __intel_display_power_is_enabled(dev_priv, domain);
 260         mutex_unlock(&power_domains->lock);
 261
 262         return ret;
 263 }
 264
 265 /*
 266  * Starting with Haswell, we have a "Power Down Well" that can be turned off
 267  * when not needed anymore. We have 4 registers that can request the power well
 268  * to be enabled, and it will only be disabled if none of the registers is
 269  * requesting it to be enabled.
 270  */
 271 static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv,
 272                                        u8 irq_pipe_mask, bool has_vga)
 273 {
 274         if (has_vga)
 275                 intel_vga_reset_io_mem(dev_priv);
 276
 277         if (irq_pipe_mask)
 278                 gen8_irq_power_well_post_enable(dev_priv, irq_pipe_mask);
 279 }
 280
 281 static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv,
 282                                        u8 irq_pipe_mask)
 283 {
 284         if (irq_pipe_mask)
 285                 gen8_irq_power_well_pre_disable(dev_priv, irq_pipe_mask);
 286 }
 287
 288 #define ICL_AUX_PW_TO_CH(pw_idx)        \
 289         ((pw_idx) - ICL_PW_CTL_IDX_AUX_A + AUX_CH_A)
 290
 291 #define ICL_TBT_AUX_PW_TO_CH(pw_idx)    \
 292         ((pw_idx) - ICL_PW_CTL_IDX_AUX_TBT1 + AUX_CH_C)
 293
 294 static enum aux_ch icl_tc_phy_aux_ch(struct drm_i915_private *dev_priv,
 295                                      struct i915_power_well *power_well)
 296 {
 297         int pw_idx = power_well->desc->hsw.idx;
 298
 299         return power_well->desc->hsw.is_tc_tbt ? ICL_TBT_AUX_PW_TO_CH(pw_idx) :
 300                                                  ICL_AUX_PW_TO_CH(pw_idx);
 301 }
 302
 303 static struct intel_digital_port *
 304 aux_ch_to_digital_port(struct drm_i915_private *dev_priv,
 305                        enum aux_ch aux_ch)
 306 {
 307         struct intel_digital_port *dig_port = NULL;
 308         struct intel_encoder *encoder;
 309
 310         for_each_intel_encoder(&dev_priv->drm, encoder) {
 311                 /* We'll check the MST primary port */
 312                 if (encoder->type == INTEL_OUTPUT_DP_MST)
 313                         continue;
 314
 315                 dig_port = enc_to_dig_port(encoder);
 316                 if (!dig_port)
 317                         continue;
 318
 319                 if (dig_port->aux_ch != aux_ch) {
 320                         dig_port = NULL;
 321                         continue;
 322                 }
 323
 324                 break;
 325         }
 326
 327         return dig_port;
 328 }
 329
 330 static void hsw_wait_for_power_well_enable(struct drm_i915_private *dev_priv,
 331                                            struct i915_power_well *power_well,
 332                                            bool timeout_expected)
 333 {
 334         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 335         int pw_idx = power_well->desc->hsw.idx;
 336
 337         /* Timeout for PW1:10 us, AUX:not specified, other PWs:20 us. */
 338         if (intel_de_wait_for_set(dev_priv, regs->driver,
 339                                   HSW_PWR_WELL_CTL_STATE(pw_idx), 1)) {
 340                 drm_dbg_kms(&dev_priv->drm, "%s power well enable timeout\n",
 341                             power_well->desc->name);
 342
 343                 drm_WARN_ON(&dev_priv->drm, !timeout_expected);
 344
 345         }
 346 }
 347
 348 static u32 hsw_power_well_requesters(struct drm_i915_private *dev_priv,
 349                                      const struct i915_power_well_regs *regs,
 350                                      int pw_idx)
 351 {
 352         u32 req_mask = HSW_PWR_WELL_CTL_REQ(pw_idx);
 353         u32 ret;
 354
 355         ret = intel_de_read(dev_priv, regs->bios) & req_mask ? 1 : 0;
 356         ret |= intel_de_read(dev_priv, regs->driver) & req_mask ? 2 : 0;
 357         if (regs->kvmr.reg)
 358                 ret |= intel_de_read(dev_priv, regs->kvmr) & req_mask ? 4 : 0;
 359         ret |= intel_de_read(dev_priv, regs->debug) & req_mask ? 8 : 0;
 360
 361         return ret;
 362 }
 363
 364 static void hsw_wait_for_power_well_disable(struct drm_i915_private *dev_priv,
 365                                             struct i915_power_well *power_well)
 366 {
 367         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 368         int pw_idx = power_well->desc->hsw.idx;
 369         bool disabled;
 370         u32 reqs;
 371
 372         /*
 373          * Bspec doesn't require waiting for PWs to get disabled, but still do
 374          * this for paranoia. The known cases where a PW will be forced on:
 375          * - a KVMR request on any power well via the KVMR request register
 376          * - a DMC request on PW1 and MISC_IO power wells via the BIOS and
 377          *   DEBUG request registers
 378          * Skip the wait in case any of the request bits are set and print a
 379          * diagnostic message.
 380          */
 381         wait_for((disabled = !(intel_de_read(dev_priv, regs->driver) &
 382                                HSW_PWR_WELL_CTL_STATE(pw_idx))) ||
 383                  (reqs = hsw_power_well_requesters(dev_priv, regs, pw_idx)), 1);
 384         if (disabled)
 385                 return;
 386
 387         drm_dbg_kms(&dev_priv->drm,
 388                     "%s forced on (bios:%d driver:%d kvmr:%d debug:%d)\n",
 389                     power_well->desc->name,
 390                     !!(reqs & 1), !!(reqs & 2), !!(reqs & 4), !!(reqs & 8));
 391 }
 392
 393 static void gen9_wait_for_power_well_fuses(struct drm_i915_private *dev_priv,
 394                                            enum skl_power_gate pg)
 395 {
 396         /* Timeout 5us for PG#0, for other PGs 1us */
 397         drm_WARN_ON(&dev_priv->drm,
 398                     intel_de_wait_for_set(dev_priv, SKL_FUSE_STATUS,
 399                                           SKL_FUSE_PG_DIST_STATUS(pg), 1));
 400 }
 401
 402 static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
 403                                   struct i915_power_well *power_well)
 404 {
 405         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 406         int pw_idx = power_well->desc->hsw.idx;
 407         u32 val;
 408
 409         if (power_well->desc->hsw.has_fuses) {
 410                 enum skl_power_gate pg;
 411
 412                 pg = DISPLAY_VER(dev_priv) >= 11 ? ICL_PW_CTL_IDX_TO_PG(pw_idx) :
 413                                                  SKL_PW_CTL_IDX_TO_PG(pw_idx);
 414                 /*
 415                  * For PW1 we have to wait both for the PW0/PG0 fuse state
 416                  * before enabling the power well and PW1/PG1's own fuse
 417                  * state after the enabling. For all other power wells with
 418                  * fuses we only have to wait for that PW/PG's fuse state
 419                  * after the enabling.
 420                  */
 421                 if (pg == SKL_PG1)
 422                         gen9_wait_for_power_well_fuses(dev_priv, SKL_PG0);
 423         }
 424
 425         val = intel_de_read(dev_priv, regs->driver);
 426         intel_de_write(dev_priv, regs->driver,
 427                        val | HSW_PWR_WELL_CTL_REQ(pw_idx));
 428
 429         hsw_wait_for_power_well_enable(dev_priv, power_well, false);
 430
 431         /* Display WA #1178: cnl */
 432         if (IS_CANNONLAKE(dev_priv) &&
 433             pw_idx >= GLK_PW_CTL_IDX_AUX_B &&
 434             pw_idx <= CNL_PW_CTL_IDX_AUX_F) {
 435                 u32 val;
 436
 437                 val = intel_de_read(dev_priv, CNL_AUX_ANAOVRD1(pw_idx));
 438                 val |= CNL_AUX_ANAOVRD1_ENABLE | CNL_AUX_ANAOVRD1_LDO_BYPASS;
 439                 intel_de_write(dev_priv, CNL_AUX_ANAOVRD1(pw_idx), val);
 440         }
 441
 442         if (power_well->desc->hsw.has_fuses) {
 443                 enum skl_power_gate pg;
 444
 445                 pg = DISPLAY_VER(dev_priv) >= 11 ? ICL_PW_CTL_IDX_TO_PG(pw_idx) :
 446                                                  SKL_PW_CTL_IDX_TO_PG(pw_idx);
 447                 gen9_wait_for_power_well_fuses(dev_priv, pg);
 448         }
 449
 450         hsw_power_well_post_enable(dev_priv,
 451                                    power_well->desc->hsw.irq_pipe_mask,
 452                                    power_well->desc->hsw.has_vga);
 453 }
 454
 455 static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
 456                                    struct i915_power_well *power_well)
 457 {
 458         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 459         int pw_idx = power_well->desc->hsw.idx;
 460         u32 val;
 461
 462         hsw_power_well_pre_disable(dev_priv,
 463                                    power_well->desc->hsw.irq_pipe_mask);
 464
 465         val = intel_de_read(dev_priv, regs->driver);
 466         intel_de_write(dev_priv, regs->driver,
 467                        val & ~HSW_PWR_WELL_CTL_REQ(pw_idx));
 468         hsw_wait_for_power_well_disable(dev_priv, power_well);
 469 }
 470
 471 #define ICL_AUX_PW_TO_PHY(pw_idx)       ((pw_idx) - ICL_PW_CTL_IDX_AUX_A)
 472
 473 static void
 474 icl_combo_phy_aux_power_well_enable(struct drm_i915_private *dev_priv,
 475                                     struct i915_power_well *power_well)
 476 {
 477         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 478         int pw_idx = power_well->desc->hsw.idx;
 479         enum phy phy = ICL_AUX_PW_TO_PHY(pw_idx);
 480         u32 val;
 481
 482         drm_WARN_ON(&dev_priv->drm, !IS_ICELAKE(dev_priv));
 483
 484         val = intel_de_read(dev_priv, regs->driver);
 485         intel_de_write(dev_priv, regs->driver,
 486                        val | HSW_PWR_WELL_CTL_REQ(pw_idx));
 487
 488         if (DISPLAY_VER(dev_priv) < 12) {
 489                 val = intel_de_read(dev_priv, ICL_PORT_CL_DW12(phy));
 490                 intel_de_write(dev_priv, ICL_PORT_CL_DW12(phy),
 491                                val | ICL_LANE_ENABLE_AUX);
 492         }
 493
 494         hsw_wait_for_power_well_enable(dev_priv, power_well, false);
 495
 496         /* Display WA #1178: icl */
 497         if (pw_idx >= ICL_PW_CTL_IDX_AUX_A && pw_idx <= ICL_PW_CTL_IDX_AUX_B &&
 498             !intel_bios_is_port_edp(dev_priv, (enum port)phy)) {
 499                 val = intel_de_read(dev_priv, ICL_AUX_ANAOVRD1(pw_idx));
 500                 val |= ICL_AUX_ANAOVRD1_ENABLE | ICL_AUX_ANAOVRD1_LDO_BYPASS;
 501                 intel_de_write(dev_priv, ICL_AUX_ANAOVRD1(pw_idx), val);
 502         }
 503 }
 504
 505 static void
 506 icl_combo_phy_aux_power_well_disable(struct drm_i915_private *dev_priv,
 507                                      struct i915_power_well *power_well)
 508 {
 509         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 510         int pw_idx = power_well->desc->hsw.idx;
 511         enum phy phy = ICL_AUX_PW_TO_PHY(pw_idx);
 512         u32 val;
 513
 514         drm_WARN_ON(&dev_priv->drm, !IS_ICELAKE(dev_priv));
 515
 516         val = intel_de_read(dev_priv, ICL_PORT_CL_DW12(phy));
 517         intel_de_write(dev_priv, ICL_PORT_CL_DW12(phy),
 518                        val & ~ICL_LANE_ENABLE_AUX);
 519
 520         val = intel_de_read(dev_priv, regs->driver);
 521         intel_de_write(dev_priv, regs->driver,
 522                        val & ~HSW_PWR_WELL_CTL_REQ(pw_idx));
 523
 524         hsw_wait_for_power_well_disable(dev_priv, power_well);
 525 }
 526
 527 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
 528
 529 static u64 async_put_domains_mask(struct i915_power_domains *power_domains);
 530
 531 static int power_well_async_ref_count(struct drm_i915_private *dev_priv,
 532                                       struct i915_power_well *power_well)
 533 {
 534         int refs = hweight64(power_well->desc->domains &
 535                              async_put_domains_mask(&dev_priv->power_domains));
 536
 537         drm_WARN_ON(&dev_priv->drm, refs > power_well->count);
 538
 539         return refs;
 540 }
 541
 542 static void icl_tc_port_assert_ref_held(struct drm_i915_private *dev_priv,
 543                                         struct i915_power_well *power_well,
 544                                         struct intel_digital_port *dig_port)
 545 {
 546         /* Bypass the check if all references are released asynchronously */
 547         if (power_well_async_ref_count(dev_priv, power_well) ==
 548             power_well->count)
 549                 return;
 550
 551         if (drm_WARN_ON(&dev_priv->drm, !dig_port))
 552                 return;
 553
 554         if (DISPLAY_VER(dev_priv) == 11 && dig_port->tc_legacy_port)
 555                 return;
 556
 557         drm_WARN_ON(&dev_priv->drm, !intel_tc_port_ref_held(dig_port));
 558 }
 559
 560 #else
 561
 562 static void icl_tc_port_assert_ref_held(struct drm_i915_private *dev_priv,
 563                                         struct i915_power_well *power_well,
 564                                         struct intel_digital_port *dig_port)
 565 {
 566 }
 567
 568 #endif
 569
 570 #define TGL_AUX_PW_TO_TC_PORT(pw_idx)   ((pw_idx) - TGL_PW_CTL_IDX_AUX_TC1)
 571
 572 static void icl_tc_cold_exit(struct drm_i915_private *i915)
 573 {
 574         int ret, tries = 0;
 575
 576         while (1) {
 577                 ret = sandybridge_pcode_write_timeout(i915,
 578                                                       ICL_PCODE_EXIT_TCCOLD,
 579                                                       0, 250, 1);
 580                 if (ret != -EAGAIN || ++tries == 3)
 581                         break;
 582                 msleep(1);
 583         }
 584
 585         /* Spec states that TC cold exit can take up to 1ms to complete */
 586         if (!ret)
 587                 msleep(1);
 588
 589         /* TODO: turn failure into a error as soon i915 CI updates ICL IFWI */
 590         drm_dbg_kms(&i915->drm, "TC cold block %s\n", ret ? "failed" :
 591                     "succeeded");
 592 }
 593
 594 static void
 595 icl_tc_phy_aux_power_well_enable(struct drm_i915_private *dev_priv,
 596                                  struct i915_power_well *power_well)
 597 {
 598         enum aux_ch aux_ch = icl_tc_phy_aux_ch(dev_priv, power_well);
 599         struct intel_digital_port *dig_port = aux_ch_to_digital_port(dev_priv, aux_ch);
 600         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 601         bool is_tbt = power_well->desc->hsw.is_tc_tbt;
 602         bool timeout_expected;
 603         u32 val;
 604
 605         icl_tc_port_assert_ref_held(dev_priv, power_well, dig_port);
 606
 607         val = intel_de_read(dev_priv, DP_AUX_CH_CTL(aux_ch));
 608         val &= ~DP_AUX_CH_CTL_TBT_IO;
 609         if (is_tbt)
 610                 val |= DP_AUX_CH_CTL_TBT_IO;
 611         intel_de_write(dev_priv, DP_AUX_CH_CTL(aux_ch), val);
 612
 613         val = intel_de_read(dev_priv, regs->driver);
 614         intel_de_write(dev_priv, regs->driver,
 615                        val | HSW_PWR_WELL_CTL_REQ(power_well->desc->hsw.idx));
 616
 617         /*
 618          * An AUX timeout is expected if the TBT DP tunnel is down,
 619          * or need to enable AUX on a legacy TypeC port as part of the TC-cold
 620          * exit sequence.
 621          */
 622         timeout_expected = is_tbt;
 623         if (DISPLAY_VER(dev_priv) == 11 && dig_port->tc_legacy_port) {
 624                 icl_tc_cold_exit(dev_priv);
 625                 timeout_expected = true;
 626         }
 627
 628         hsw_wait_for_power_well_enable(dev_priv, power_well, timeout_expected);
 629
 630         if (DISPLAY_VER(dev_priv) >= 12 && !is_tbt) {
 631                 enum tc_port tc_port;
 632
 633                 tc_port = TGL_AUX_PW_TO_TC_PORT(power_well->desc->hsw.idx);
 634                 intel_de_write(dev_priv, HIP_INDEX_REG(tc_port),
 635                                HIP_INDEX_VAL(tc_port, 0x2));
 636
 637                 if (intel_de_wait_for_set(dev_priv, DKL_CMN_UC_DW_27(tc_port),
 638                                           DKL_CMN_UC_DW27_UC_HEALTH, 1))
 639                         drm_warn(&dev_priv->drm,
 640                                  "Timeout waiting TC uC health\n");
 641         }
 642 }
 643
 644 static void
 645 icl_tc_phy_aux_power_well_disable(struct drm_i915_private *dev_priv,
 646                                   struct i915_power_well *power_well)
 647 {
 648         enum aux_ch aux_ch = icl_tc_phy_aux_ch(dev_priv, power_well);
 649         struct intel_digital_port *dig_port = aux_ch_to_digital_port(dev_priv, aux_ch);
 650
 651         icl_tc_port_assert_ref_held(dev_priv, power_well, dig_port);
 652
 653         hsw_power_well_disable(dev_priv, power_well);
 654 }
 655
 656 static void
 657 icl_aux_power_well_enable(struct drm_i915_private *dev_priv,
 658                           struct i915_power_well *power_well)
 659 {
 660         int pw_idx = power_well->desc->hsw.idx;
 661         enum phy phy = ICL_AUX_PW_TO_PHY(pw_idx);  /* non-TBT only */
 662         bool is_tbt = power_well->desc->hsw.is_tc_tbt;
 663
 664         if (is_tbt || intel_phy_is_tc(dev_priv, phy))
 665                 return icl_tc_phy_aux_power_well_enable(dev_priv, power_well);
 666         else if (IS_ICELAKE(dev_priv))
 667                 return icl_combo_phy_aux_power_well_enable(dev_priv,
 668                                                            power_well);
 669         else
 670                 return hsw_power_well_enable(dev_priv, power_well);
 671 }
 672
 673 static void
 674 icl_aux_power_well_disable(struct drm_i915_private *dev_priv,
 675                            struct i915_power_well *power_well)
 676 {
 677         int pw_idx = power_well->desc->hsw.idx;
 678         enum phy phy = ICL_AUX_PW_TO_PHY(pw_idx);  /* non-TBT only */
 679         bool is_tbt = power_well->desc->hsw.is_tc_tbt;
 680
 681         if (is_tbt || intel_phy_is_tc(dev_priv, phy))
 682                 return icl_tc_phy_aux_power_well_disable(dev_priv, power_well);
 683         else if (IS_ICELAKE(dev_priv))
 684                 return icl_combo_phy_aux_power_well_disable(dev_priv,
 685                                                             power_well);
 686         else
 687                 return hsw_power_well_disable(dev_priv, power_well);
 688 }
 689
 690 /*
 691  * We should only use the power well if we explicitly asked the hardware to
 692  * enable it, so check if it's enabled and also check if we've requested it to
 693  * be enabled.
 694  */
 695 static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
 696                                    struct i915_power_well *power_well)
 697 {
 698         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 699         enum i915_power_well_id id = power_well->desc->id;
 700         int pw_idx = power_well->desc->hsw.idx;
 701         u32 mask = HSW_PWR_WELL_CTL_REQ(pw_idx) |
 702                    HSW_PWR_WELL_CTL_STATE(pw_idx);
 703         u32 val;
 704
 705         val = intel_de_read(dev_priv, regs->driver);
 706
 707         /*
 708          * On GEN9 big core due to a DMC bug the driver's request bits for PW1
 709          * and the MISC_IO PW will be not restored, so check instead for the
 710          * BIOS's own request bits, which are forced-on for these power wells
 711          * when exiting DC5/6.
 712          */
 713         if (DISPLAY_VER(dev_priv) == 9 && !IS_BROXTON(dev_priv) &&
 714             (id == SKL_DISP_PW_1 || id == SKL_DISP_PW_MISC_IO))
 715                 val |= intel_de_read(dev_priv, regs->bios);
 716
 717         return (val & mask) == mask;
 718 }
 719
 720 static void assert_can_enable_dc9(struct drm_i915_private *dev_priv)
 721 {
 722         drm_WARN_ONCE(&dev_priv->drm,
 723                       (intel_de_read(dev_priv, DC_STATE_EN) & DC_STATE_EN_DC9),
 724                       "DC9 already programmed to be enabled.\n");
 725         drm_WARN_ONCE(&dev_priv->drm,
 726                       intel_de_read(dev_priv, DC_STATE_EN) &
 727                       DC_STATE_EN_UPTO_DC5,
 728                       "DC5 still not disabled to enable DC9.\n");
 729         drm_WARN_ONCE(&dev_priv->drm,
 730                       intel_de_read(dev_priv, HSW_PWR_WELL_CTL2) &
 731                       HSW_PWR_WELL_CTL_REQ(SKL_PW_CTL_IDX_PW_2),
 732                       "Power well 2 on.\n");
 733         drm_WARN_ONCE(&dev_priv->drm, intel_irqs_enabled(dev_priv),
 734                       "Interrupts not disabled yet.\n");
 735
 736          /*
 737           * TODO: check for the following to verify the conditions to enter DC9
 738           * state are satisfied:
 739           * 1] Check relevant display engine registers to verify if mode set
 740           * disable sequence was followed.
 741           * 2] Check if display uninitialize sequence is initialized.
 742           */
 743 }
 744
 745 static void assert_can_disable_dc9(struct drm_i915_private *dev_priv)
 746 {
 747         drm_WARN_ONCE(&dev_priv->drm, intel_irqs_enabled(dev_priv),
 748                       "Interrupts not disabled yet.\n");
 749         drm_WARN_ONCE(&dev_priv->drm,
 750                       intel_de_read(dev_priv, DC_STATE_EN) &
 751                       DC_STATE_EN_UPTO_DC5,
 752                       "DC5 still not disabled.\n");
 753
 754          /*
 755           * TODO: check for the following to verify DC9 state was indeed
 756           * entered before programming to disable it:
 757           * 1] Check relevant display engine registers to verify if mode
 758           *  set disable sequence was followed.
 759           * 2] Check if display uninitialize sequence is initialized.
 760           */
 761 }
 762
 763 static void gen9_write_dc_state(struct drm_i915_private *dev_priv,
 764                                 u32 state)
 765 {
 766         int rewrites = 0;
 767         int rereads = 0;
 768         u32 v;
 769
 770         intel_de_write(dev_priv, DC_STATE_EN, state);
 771
 772         /* It has been observed that disabling the dc6 state sometimes
 773          * doesn't stick and dmc keeps returning old value. Make sure
 774          * the write really sticks enough times and also force rewrite until
 775          * we are confident that state is exactly what we want.
 776          */
 777         do  {
 778                 v = intel_de_read(dev_priv, DC_STATE_EN);
 779
 780                 if (v != state) {
 781                         intel_de_write(dev_priv, DC_STATE_EN, state);
 782                         rewrites++;
 783                         rereads = 0;
 784                 } else if (rereads++ > 5) {
 785                         break;
 786                 }
 787
 788         } while (rewrites < 100);
 789
 790         if (v != state)
 791                 drm_err(&dev_priv->drm,
 792                         "Writing dc state to 0x%x failed, now 0x%x\n",
 793                         state, v);
 794
 795         /* Most of the times we need one retry, avoid spam */
 796         if (rewrites > 1)
 797                 drm_dbg_kms(&dev_priv->drm,
 798                             "Rewrote dc state to 0x%x %d times\n",
 799                             state, rewrites);
 800 }
 801
 802 static u32 gen9_dc_mask(struct drm_i915_private *dev_priv)
 803 {
 804         u32 mask;
 805
 806         mask = DC_STATE_EN_UPTO_DC5;
 807
 808         if (DISPLAY_VER(dev_priv) >= 12)
 809                 mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC6
 810                                           | DC_STATE_EN_DC9;
 811         else if (DISPLAY_VER(dev_priv) == 11)
 812                 mask |= DC_STATE_EN_UPTO_DC6 | DC_STATE_EN_DC9;
 813         else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
 814                 mask |= DC_STATE_EN_DC9;
 815         else
 816                 mask |= DC_STATE_EN_UPTO_DC6;
 817
 818         return mask;
 819 }
 820
 821 static void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv)
 822 {
 823         u32 val;
 824
 825         if (!HAS_DISPLAY(dev_priv))
 826                 return;
 827
 828         val = intel_de_read(dev_priv, DC_STATE_EN) & gen9_dc_mask(dev_priv);
 829
 830         drm_dbg_kms(&dev_priv->drm,
 831                     "Resetting DC state tracking from %02x to %02x\n",
 832                     dev_priv->dmc.dc_state, val);
 833         dev_priv->dmc.dc_state = val;
 834 }
 835
 836 /**
 837  * gen9_set_dc_state - set target display C power state
 838  * @dev_priv: i915 device instance
 839  * @state: target DC power state
 840  * - DC_STATE_DISABLE
 841  * - DC_STATE_EN_UPTO_DC5
 842  * - DC_STATE_EN_UPTO_DC6
 843  * - DC_STATE_EN_DC9
 844  *
 845  * Signal to DMC firmware/HW the target DC power state passed in @state.
 846  * DMC/HW can turn off individual display clocks and power rails when entering
 847  * a deeper DC power state (higher in number) and turns these back when exiting
 848  * that state to a shallower power state (lower in number). The HW will decide
 849  * when to actually enter a given state on an on-demand basis, for instance
 850  * depending on the active state of display pipes. The state of display
 851  * registers backed by affected power rails are saved/restored as needed.
 852  *
 853  * Based on the above enabling a deeper DC power state is asynchronous wrt.
 854  * enabling it. Disabling a deeper power state is synchronous: for instance
 855  * setting %DC_STATE_DISABLE won't complete until all HW resources are turned
 856  * back on and register state is restored. This is guaranteed by the MMIO write
 857  * to DC_STATE_EN blocking until the state is restored.
 858  */
 859 static void gen9_set_dc_state(struct drm_i915_private *dev_priv, u32 state)
 860 {
 861         u32 val;
 862         u32 mask;
 863
 864         if (!HAS_DISPLAY(dev_priv))
 865                 return;
 866
 867         if (drm_WARN_ON_ONCE(&dev_priv->drm,
 868                              state & ~dev_priv->dmc.allowed_dc_mask))
 869                 state &= dev_priv->dmc.allowed_dc_mask;
 870
 871         val = intel_de_read(dev_priv, DC_STATE_EN);
 872         mask = gen9_dc_mask(dev_priv);
 873         drm_dbg_kms(&dev_priv->drm, "Setting DC state from %02x to %02x\n",
 874                     val & mask, state);
 875
 876         /* Check if DMC is ignoring our DC state requests */
 877         if ((val & mask) != dev_priv->dmc.dc_state)
 878                 drm_err(&dev_priv->drm, "DC state mismatch (0x%x -> 0x%x)\n",
 879                         dev_priv->dmc.dc_state, val & mask);
 880
 881         val &= ~mask;
 882         val |= state;
 883
 884         gen9_write_dc_state(dev_priv, val);
 885
 886         dev_priv->dmc.dc_state = val & mask;
 887 }
 888
 889 static u32
 890 sanitize_target_dc_state(struct drm_i915_private *dev_priv,
 891                          u32 target_dc_state)
 892 {
 893         u32 states[] = {
 894                 DC_STATE_EN_UPTO_DC6,
 895                 DC_STATE_EN_UPTO_DC5,
 896                 DC_STATE_EN_DC3CO,
 897                 DC_STATE_DISABLE,
 898         };
 899         int i;
 900
 901         for (i = 0; i < ARRAY_SIZE(states) - 1; i++) {
 902                 if (target_dc_state != states[i])
 903                         continue;
 904
 905                 if (dev_priv->dmc.allowed_dc_mask & target_dc_state)
 906                         break;
 907
 908                 target_dc_state = states[i + 1];
 909         }
 910
 911         return target_dc_state;
 912 }
 913
 914 static void tgl_enable_dc3co(struct drm_i915_private *dev_priv)
 915 {
 916         drm_dbg_kms(&dev_priv->drm, "Enabling DC3CO\n");
 917         gen9_set_dc_state(dev_priv, DC_STATE_EN_DC3CO);
 918 }
 919
 920 static void tgl_disable_dc3co(struct drm_i915_private *dev_priv)
 921 {
 922         u32 val;
 923
 924         drm_dbg_kms(&dev_priv->drm, "Disabling DC3CO\n");
 925         val = intel_de_read(dev_priv, DC_STATE_EN);
 926         val &= ~DC_STATE_DC3CO_STATUS;
 927         intel_de_write(dev_priv, DC_STATE_EN, val);
 928         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 929         /*
 930          * Delay of 200us DC3CO Exit time B.Spec 49196
 931          */
 932         usleep_range(200, 210);
 933 }
 934
 935 static void bxt_enable_dc9(struct drm_i915_private *dev_priv)
 936 {
 937         assert_can_enable_dc9(dev_priv);
 938
 939         drm_dbg_kms(&dev_priv->drm, "Enabling DC9\n");
 940         /*
 941          * Power sequencer reset is not needed on
 942          * platforms with South Display Engine on PCH,
 943          * because PPS registers are always on.
 944          */
 945         if (!HAS_PCH_SPLIT(dev_priv))
 946                 intel_pps_reset_all(dev_priv);
 947         gen9_set_dc_state(dev_priv, DC_STATE_EN_DC9);
 948 }
 949
 950 static void bxt_disable_dc9(struct drm_i915_private *dev_priv)
 951 {
 952         assert_can_disable_dc9(dev_priv);
 953
 954         drm_dbg_kms(&dev_priv->drm, "Disabling DC9\n");
 955
 956         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 957
 958         intel_pps_unlock_regs_wa(dev_priv);
 959 }
 960
 961 static void assert_dmc_loaded(struct drm_i915_private *dev_priv)
 962 {
 963         drm_WARN_ONCE(&dev_priv->drm,
 964                       !intel_de_read(dev_priv, DMC_PROGRAM(0)),
 965                       "DMC program storage start is NULL\n");
 966         drm_WARN_ONCE(&dev_priv->drm, !intel_de_read(dev_priv, DMC_SSP_BASE),
 967                       "DMC SSP Base Not fine\n");
 968         drm_WARN_ONCE(&dev_priv->drm, !intel_de_read(dev_priv, DMC_HTP_SKL),
 969                       "DMC HTP Not fine\n");
 970 }
 971
 972 static struct i915_power_well *
 973 lookup_power_well(struct drm_i915_private *dev_priv,
 974                   enum i915_power_well_id power_well_id)
 975 {
 976         struct i915_power_well *power_well;
 977
 978         for_each_power_well(dev_priv, power_well)
 979                 if (power_well->desc->id == power_well_id)
 980                         return power_well;
 981
 982         /*
 983          * It's not feasible to add error checking code to the callers since
 984          * this condition really shouldn't happen and it doesn't even make sense
 985          * to abort things like display initialization sequences. Just return
 986          * the first power well and hope the WARN gets reported so we can fix
 987          * our driver.
 988          */
 989         drm_WARN(&dev_priv->drm, 1,
 990                  "Power well %d not defined for this platform\n",
 991                  power_well_id);
 992         return &dev_priv->power_domains.power_wells[0];
 993 }
 994
 995 /**
 996  * intel_display_power_set_target_dc_state - Set target dc state.
 997  * @dev_priv: i915 device
 998  * @state: state which needs to be set as target_dc_state.
 999  *
1000  * This function set the "DC off" power well target_dc_state,
1001  * based upon this target_dc_stste, "DC off" power well will
1002  * enable desired DC state.
1003  */
1004 void intel_display_power_set_target_dc_state(struct drm_i915_private *dev_priv,
1005                                              u32 state)
1006 {
1007         struct i915_power_well *power_well;
1008         bool dc_off_enabled;
1009         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1010
1011         mutex_lock(&power_domains->lock);
1012         power_well = lookup_power_well(dev_priv, SKL_DISP_DC_OFF);
1013
1014         if (drm_WARN_ON(&dev_priv->drm, !power_well))
1015                 goto unlock;
1016
1017         state = sanitize_target_dc_state(dev_priv, state);
1018
1019         if (state == dev_priv->dmc.target_dc_state)
1020                 goto unlock;
1021
1022         dc_off_enabled = power_well->desc->ops->is_enabled(dev_priv,
1023                                                            power_well);
1024         /*
1025          * If DC off power well is disabled, need to enable and disable the
1026          * DC off power well to effect target DC state.
1027          */
1028         if (!dc_off_enabled)
1029                 power_well->desc->ops->enable(dev_priv, power_well);
1030
1031         dev_priv->dmc.target_dc_state = state;
1032
1033         if (!dc_off_enabled)
1034                 power_well->desc->ops->disable(dev_priv, power_well);
1035
1036 unlock:
1037         mutex_unlock(&power_domains->lock);
1038 }
1039
1040 static void assert_can_enable_dc5(struct drm_i915_private *dev_priv)
1041 {
1042         enum i915_power_well_id high_pg;
1043
1044         /* Power wells at this level and above must be disabled for DC5 entry */
1045         if (DISPLAY_VER(dev_priv) == 12)
1046                 high_pg = ICL_DISP_PW_3;
1047         else
1048                 high_pg = SKL_DISP_PW_2;
1049
1050         drm_WARN_ONCE(&dev_priv->drm,
1051                       intel_display_power_well_is_enabled(dev_priv, high_pg),
1052                       "Power wells above platform's DC5 limit still enabled.\n");
1053
1054         drm_WARN_ONCE(&dev_priv->drm,
1055                       (intel_de_read(dev_priv, DC_STATE_EN) &
1056                        DC_STATE_EN_UPTO_DC5),
1057                       "DC5 already programmed to be enabled.\n");
1058         assert_rpm_wakelock_held(&dev_priv->runtime_pm);
1059
1060         assert_dmc_loaded(dev_priv);
1061 }
1062
1063 static void gen9_enable_dc5(struct drm_i915_private *dev_priv)
1064 {
1065         assert_can_enable_dc5(dev_priv);
1066
1067         drm_dbg_kms(&dev_priv->drm, "Enabling DC5\n");
1068
1069         /* Wa Display #1183: skl,kbl,cfl */
1070         if (DISPLAY_VER(dev_priv) == 9 && !IS_BROXTON(dev_priv))
1071                 intel_de_write(dev_priv, GEN8_CHICKEN_DCPR_1,
1072                                intel_de_read(dev_priv, GEN8_CHICKEN_DCPR_1) | SKL_SELECT_ALTERNATE_DC_EXIT);
1073
1074         gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5);
1075 }
1076
1077 static void assert_can_enable_dc6(struct drm_i915_private *dev_priv)
1078 {
1079         drm_WARN_ONCE(&dev_priv->drm,
1080                       intel_de_read(dev_priv, UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
1081                       "Backlight is not disabled.\n");
1082         drm_WARN_ONCE(&dev_priv->drm,
1083                       (intel_de_read(dev_priv, DC_STATE_EN) &
1084                        DC_STATE_EN_UPTO_DC6),
1085                       "DC6 already programmed to be enabled.\n");
1086
1087         assert_dmc_loaded(dev_priv);
1088 }
1089
1090 static void skl_enable_dc6(struct drm_i915_private *dev_priv)
1091 {
1092         assert_can_enable_dc6(dev_priv);
1093
1094         drm_dbg_kms(&dev_priv->drm, "Enabling DC6\n");
1095
1096         /* Wa Display #1183: skl,kbl,cfl */
1097         if (DISPLAY_VER(dev_priv) == 9 && !IS_BROXTON(dev_priv))
1098                 intel_de_write(dev_priv, GEN8_CHICKEN_DCPR_1,
1099                                intel_de_read(dev_priv, GEN8_CHICKEN_DCPR_1) | SKL_SELECT_ALTERNATE_DC_EXIT);
1100
1101         gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
1102 }
1103
1104 static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
1105                                    struct i915_power_well *power_well)
1106 {
1107         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
1108         int pw_idx = power_well->desc->hsw.idx;
1109         u32 mask = HSW_PWR_WELL_CTL_REQ(pw_idx);
1110         u32 bios_req = intel_de_read(dev_priv, regs->bios);
1111
1112         /* Take over the request bit if set by BIOS. */
1113         if (bios_req & mask) {
1114                 u32 drv_req = intel_de_read(dev_priv, regs->driver);
1115
1116                 if (!(drv_req & mask))
1117                         intel_de_write(dev_priv, regs->driver, drv_req | mask);
1118                 intel_de_write(dev_priv, regs->bios, bios_req & ~mask);
1119         }
1120 }
1121
1122 static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1123                                            struct i915_power_well *power_well)
1124 {
1125         bxt_ddi_phy_init(dev_priv, power_well->desc->bxt.phy);
1126 }
1127
1128 static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1129                                             struct i915_power_well *power_well)
1130 {
1131         bxt_ddi_phy_uninit(dev_priv, power_well->desc->bxt.phy);
1132 }
1133
1134 static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv,
1135                                             struct i915_power_well *power_well)
1136 {
1137         return bxt_ddi_phy_is_enabled(dev_priv, power_well->desc->bxt.phy);
1138 }
1139
1140 static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv)
1141 {
1142         struct i915_power_well *power_well;
1143
1144         power_well = lookup_power_well(dev_priv, BXT_DISP_PW_DPIO_CMN_A);
1145         if (power_well->count > 0)
1146                 bxt_ddi_phy_verify_state(dev_priv, power_well->desc->bxt.phy);
1147
1148         power_well = lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
1149         if (power_well->count > 0)
1150                 bxt_ddi_phy_verify_state(dev_priv, power_well->desc->bxt.phy);
1151
1152         if (IS_GEMINILAKE(dev_priv)) {
1153                 power_well = lookup_power_well(dev_priv,
1154                                                GLK_DISP_PW_DPIO_CMN_C);
1155                 if (power_well->count > 0)
1156                         bxt_ddi_phy_verify_state(dev_priv,
1157                                                  power_well->desc->bxt.phy);
1158         }
1159 }
1160
1161 static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv,
1162                                            struct i915_power_well *power_well)
1163 {
1164         return ((intel_de_read(dev_priv, DC_STATE_EN) & DC_STATE_EN_DC3CO) == 0 &&
1165                 (intel_de_read(dev_priv, DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0);
1166 }
1167
1168 static void gen9_assert_dbuf_enabled(struct drm_i915_private *dev_priv)
1169 {
1170         u8 hw_enabled_dbuf_slices = intel_enabled_dbuf_slices_mask(dev_priv);
1171         u8 enabled_dbuf_slices = dev_priv->dbuf.enabled_slices;
1172
1173         drm_WARN(&dev_priv->drm,
1174                  hw_enabled_dbuf_slices != enabled_dbuf_slices,
1175                  "Unexpected DBuf power power state (0x%08x, expected 0x%08x)\n",
1176                  hw_enabled_dbuf_slices,
1177                  enabled_dbuf_slices);
1178 }
1179
1180 static void gen9_disable_dc_states(struct drm_i915_private *dev_priv)
1181 {
1182         struct intel_cdclk_config cdclk_config = {};
1183
1184         if (dev_priv->dmc.target_dc_state == DC_STATE_EN_DC3CO) {
1185                 tgl_disable_dc3co(dev_priv);
1186                 return;
1187         }
1188
1189         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
1190
1191         if (!HAS_DISPLAY(dev_priv))
1192                 return;
1193
1194         dev_priv->display.get_cdclk(dev_priv, &cdclk_config);
1195         /* Can't read out voltage_level so can't use intel_cdclk_changed() */
1196         drm_WARN_ON(&dev_priv->drm,
1197                     intel_cdclk_needs_modeset(&dev_priv->cdclk.hw,
1198                                               &cdclk_config));
1199
1200         gen9_assert_dbuf_enabled(dev_priv);
1201
1202         if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
1203                 bxt_verify_ddi_phy_power_wells(dev_priv);
1204
1205         if (DISPLAY_VER(dev_priv) >= 11)
1206                 /*
1207                  * DMC retains HW context only for port A, the other combo
1208                  * PHY's HW context for port B is lost after DC transitions,
1209                  * so we need to restore it manually.
1210                  */
1211                 intel_combo_phy_init(dev_priv);
1212 }
1213
1214 static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
1215                                           struct i915_power_well *power_well)
1216 {
1217         gen9_disable_dc_states(dev_priv);
1218 }
1219
1220 static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv,
1221                                            struct i915_power_well *power_well)
1222 {
1223         if (!dev_priv->dmc.dmc_payload)
1224                 return;
1225
1226         switch (dev_priv->dmc.target_dc_state) {
1227         case DC_STATE_EN_DC3CO:
1228                 tgl_enable_dc3co(dev_priv);
1229                 break;
1230         case DC_STATE_EN_UPTO_DC6:
1231                 skl_enable_dc6(dev_priv);
1232                 break;
1233         case DC_STATE_EN_UPTO_DC5:
1234                 gen9_enable_dc5(dev_priv);
1235                 break;
1236         }
1237 }
1238
1239 static void i9xx_power_well_sync_hw_noop(struct drm_i915_private *dev_priv,
1240                                          struct i915_power_well *power_well)
1241 {
1242 }
1243
1244 static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
1245                                            struct i915_power_well *power_well)
1246 {
1247 }
1248
1249 static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
1250                                              struct i915_power_well *power_well)
1251 {
1252         return true;
1253 }
1254
1255 static void i830_pipes_power_well_enable(struct drm_i915_private *dev_priv,
1256                                          struct i915_power_well *power_well)
1257 {
1258         if ((intel_de_read(dev_priv, PIPECONF(PIPE_A)) & PIPECONF_ENABLE) == 0)
1259                 i830_enable_pipe(dev_priv, PIPE_A);
1260         if ((intel_de_read(dev_priv, PIPECONF(PIPE_B)) & PIPECONF_ENABLE) == 0)
1261                 i830_enable_pipe(dev_priv, PIPE_B);
1262 }
1263
1264 static void i830_pipes_power_well_disable(struct drm_i915_private *dev_priv,
1265                                           struct i915_power_well *power_well)
1266 {
1267         i830_disable_pipe(dev_priv, PIPE_B);
1268         i830_disable_pipe(dev_priv, PIPE_A);
1269 }
1270
1271 static bool i830_pipes_power_well_enabled(struct drm_i915_private *dev_priv,
1272                                           struct i915_power_well *power_well)
1273 {
1274         return intel_de_read(dev_priv, PIPECONF(PIPE_A)) & PIPECONF_ENABLE &&
1275                 intel_de_read(dev_priv, PIPECONF(PIPE_B)) & PIPECONF_ENABLE;
1276 }
1277
1278 static void i830_pipes_power_well_sync_hw(struct drm_i915_private *dev_priv,
1279                                           struct i915_power_well *power_well)
1280 {
1281         if (power_well->count > 0)
1282                 i830_pipes_power_well_enable(dev_priv, power_well);
1283         else
1284                 i830_pipes_power_well_disable(dev_priv, power_well);
1285 }
1286
1287 static void vlv_set_power_well(struct drm_i915_private *dev_priv,
1288                                struct i915_power_well *power_well, bool enable)
1289 {
1290         int pw_idx = power_well->desc->vlv.idx;
1291         u32 mask;
1292         u32 state;
1293         u32 ctrl;
1294
1295         mask = PUNIT_PWRGT_MASK(pw_idx);
1296         state = enable ? PUNIT_PWRGT_PWR_ON(pw_idx) :
1297                          PUNIT_PWRGT_PWR_GATE(pw_idx);
1298
1299         vlv_punit_get(dev_priv);
1300
1301 #define COND \
1302         ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)
1303
1304         if (COND)
1305                 goto out;
1306
1307         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
1308         ctrl &= ~mask;
1309         ctrl |= state;
1310         vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);
1311
1312         if (wait_for(COND, 100))
1313                 drm_err(&dev_priv->drm,
1314                         "timeout setting power well state %08x (%08x)\n",
1315                         state,
1316                         vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));
1317
1318 #undef COND
1319
1320 out:
1321         vlv_punit_put(dev_priv);
1322 }
1323
1324 static void vlv_power_well_enable(struct drm_i915_private *dev_priv,
1325                                   struct i915_power_well *power_well)
1326 {
1327         vlv_set_power_well(dev_priv, power_well, true);
1328 }
1329
1330 static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
1331                                    struct i915_power_well *power_well)
1332 {
1333         vlv_set_power_well(dev_priv, power_well, false);
1334 }
1335
1336 static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
1337                                    struct i915_power_well *power_well)
1338 {
1339         int pw_idx = power_well->desc->vlv.idx;
1340         bool enabled = false;
1341         u32 mask;
1342         u32 state;
1343         u32 ctrl;
1344
1345         mask = PUNIT_PWRGT_MASK(pw_idx);
1346         ctrl = PUNIT_PWRGT_PWR_ON(pw_idx);
1347
1348         vlv_punit_get(dev_priv);
1349
1350         state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
1351         /*
1352          * We only ever set the power-on and power-gate states, anything
1353          * else is unexpected.
1354          */
1355         drm_WARN_ON(&dev_priv->drm, state != PUNIT_PWRGT_PWR_ON(pw_idx) &&
1356                     state != PUNIT_PWRGT_PWR_GATE(pw_idx));
1357         if (state == ctrl)
1358                 enabled = true;
1359
1360         /*
1361          * A transient state at this point would mean some unexpected party
1362          * is poking at the power controls too.
1363          */
1364         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
1365         drm_WARN_ON(&dev_priv->drm, ctrl != state);
1366
1367         vlv_punit_put(dev_priv);
1368
1369         return enabled;
1370 }
1371
1372 static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
1373 {
1374         u32 val;
1375
1376         /*
1377          * On driver load, a pipe may be active and driving a DSI display.
1378          * Preserve DPOUNIT_CLOCK_GATE_DISABLE to avoid the pipe getting stuck
1379          * (and never recovering) in this case. intel_dsi_post_disable() will
1380          * clear it when we turn off the display.
1381          */
1382         val = intel_de_read(dev_priv, DSPCLK_GATE_D);
1383         val &= DPOUNIT_CLOCK_GATE_DISABLE;
1384         val |= VRHUNIT_CLOCK_GATE_DISABLE;
1385         intel_de_write(dev_priv, DSPCLK_GATE_D, val);
1386
1387         /*
1388          * Disable trickle feed and enable pnd deadline calculation
1389          */
1390         intel_de_write(dev_priv, MI_ARB_VLV,
1391                        MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
1392         intel_de_write(dev_priv, CBR1_VLV, 0);
1393
1394         drm_WARN_ON(&dev_priv->drm, RUNTIME_INFO(dev_priv)->rawclk_freq == 0);
1395         intel_de_write(dev_priv, RAWCLK_FREQ_VLV,
1396                        DIV_ROUND_CLOSEST(RUNTIME_INFO(dev_priv)->rawclk_freq,
1397                                          1000));
1398 }
1399
1400 static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
1401 {
1402         struct intel_encoder *encoder;
1403         enum pipe pipe;
1404
1405         /*
1406          * Enable the CRI clock source so we can get at the
1407          * display and the reference clock for VGA
1408          * hotplug / manual detection. Supposedly DSI also
1409          * needs the ref clock up and running.
1410          *
1411          * CHV DPLL B/C have some issues if VGA mode is enabled.
1412          */
1413         for_each_pipe(dev_priv, pipe) {
1414                 u32 val = intel_de_read(dev_priv, DPLL(pipe));
1415
1416                 val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1417                 if (pipe != PIPE_A)
1418                         val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1419
1420                 intel_de_write(dev_priv, DPLL(pipe), val);
1421         }
1422
1423         vlv_init_display_clock_gating(dev_priv);
1424
1425         spin_lock_irq(&dev_priv->irq_lock);
1426         valleyview_enable_display_irqs(dev_priv);
1427         spin_unlock_irq(&dev_priv->irq_lock);
1428
1429         /*
1430          * During driver initialization/resume we can avoid restoring the
1431          * part of the HW/SW state that will be inited anyway explicitly.
1432          */
1433         if (dev_priv->power_domains.initializing)
1434                 return;
1435
1436         intel_hpd_init(dev_priv);
1437         intel_hpd_poll_disable(dev_priv);
1438
1439         /* Re-enable the ADPA, if we have one */
1440         for_each_intel_encoder(&dev_priv->drm, encoder) {
1441                 if (encoder->type == INTEL_OUTPUT_ANALOG)
1442                         intel_crt_reset(&encoder->base);
1443         }
1444
1445         intel_vga_redisable_power_on(dev_priv);
1446
1447         intel_pps_unlock_regs_wa(dev_priv);
1448 }
1449
1450 static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv)
1451 {
1452         spin_lock_irq(&dev_priv->irq_lock);
1453         valleyview_disable_display_irqs(dev_priv);
1454         spin_unlock_irq(&dev_priv->irq_lock);
1455
1456         /* make sure we're done processing display irqs */
1457         intel_synchronize_irq(dev_priv);
1458
1459         intel_pps_reset_all(dev_priv);
1460
1461         /* Prevent us from re-enabling polling on accident in late suspend */
1462         if (!dev_priv->drm.dev->power.is_suspended)
1463                 intel_hpd_poll_enable(dev_priv);
1464 }
1465
1466 static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
1467                                           struct i915_power_well *power_well)
1468 {
1469         vlv_set_power_well(dev_priv, power_well, true);
1470
1471         vlv_display_power_well_init(dev_priv);
1472 }
1473
1474 static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
1475                                            struct i915_power_well *power_well)
1476 {
1477         vlv_display_power_well_deinit(dev_priv);
1478
1479         vlv_set_power_well(dev_priv, power_well, false);
1480 }
1481
1482 static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1483                                            struct i915_power_well *power_well)
1484 {
1485         /* since ref/cri clock was enabled */
1486         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1487
1488         vlv_set_power_well(dev_priv, power_well, true);
1489
1490         /*
1491          * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx -
1492          *  6.  De-assert cmn_reset/side_reset. Same as VLV X0.
1493          *   a. GUnit 0x2110 bit[0] set to 1 (def 0)
1494          *   b. The other bits such as sfr settings / modesel may all
1495          *      be set to 0.
1496          *
1497          * This should only be done on init and resume from S3 with
1498          * both PLLs disabled, or we risk losing DPIO and PLL
1499          * synchronization.
1500          */
1501         intel_de_write(dev_priv, DPIO_CTL,
1502                        intel_de_read(dev_priv, DPIO_CTL) | DPIO_CMNRST);
1503 }
1504
1505 static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1506                                             struct i915_power_well *power_well)
1507 {
1508         enum pipe pipe;
1509
1510         for_each_pipe(dev_priv, pipe)
1511                 assert_pll_disabled(dev_priv, pipe);
1512
1513         /* Assert common reset */
1514         intel_de_write(dev_priv, DPIO_CTL,
1515                        intel_de_read(dev_priv, DPIO_CTL) & ~DPIO_CMNRST);
1516
1517         vlv_set_power_well(dev_priv, power_well, false);
1518 }
1519
1520 #define POWER_DOMAIN_MASK (GENMASK_ULL(POWER_DOMAIN_NUM - 1, 0))
1521
1522 #define BITS_SET(val, bits) (((val) & (bits)) == (bits))
1523
1524 static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
1525 {
1526         struct i915_power_well *cmn_bc =
1527                 lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
1528         struct i915_power_well *cmn_d =
1529                 lookup_power_well(dev_priv, CHV_DISP_PW_DPIO_CMN_D);
1530         u32 phy_control = dev_priv->chv_phy_control;
1531         u32 phy_status = 0;
1532         u32 phy_status_mask = 0xffffffff;
1533
1534         /*
1535          * The BIOS can leave the PHY is some weird state
1536          * where it doesn't fully power down some parts.
1537          * Disable the asserts until the PHY has been fully
1538          * reset (ie. the power well has been disabled at
1539          * least once).
1540          */
1541         if (!dev_priv->chv_phy_assert[DPIO_PHY0])
1542                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0) |
1543                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0) |
1544                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1) |
1545                                      PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1) |
1546                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0) |
1547                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1));
1548
1549         if (!dev_priv->chv_phy_assert[DPIO_PHY1])
1550                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0) |
1551                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0) |
1552                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1));
1553
1554         if (cmn_bc->desc->ops->is_enabled(dev_priv, cmn_bc)) {
1555                 phy_status |= PHY_POWERGOOD(DPIO_PHY0);
1556
1557                 /* this assumes override is only used to enable lanes */
1558                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0)) == 0)
1559                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0);
1560
1561                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1)) == 0)
1562                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1);
1563
1564                 /* CL1 is on whenever anything is on in either channel */
1565                 if (BITS_SET(phy_control,
1566                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0) |
1567                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)))
1568                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0);
1569
1570                 /*
1571                  * The DPLLB check accounts for the pipe B + port A usage
1572                  * with CL2 powered up but all the lanes in the second channel
1573                  * powered down.
1574                  */
1575                 if (BITS_SET(phy_control,
1576                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)) &&
1577                     (intel_de_read(dev_priv, DPLL(PIPE_B)) & DPLL_VCO_ENABLE) == 0)
1578                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1);
1579
1580                 if (BITS_SET(phy_control,
1581                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH0)))
1582                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0);
1583                 if (BITS_SET(phy_control,
1584                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH0)))
1585                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1);
1586
1587                 if (BITS_SET(phy_control,
1588                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH1)))
1589                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0);
1590                 if (BITS_SET(phy_control,
1591                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH1)))
1592                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1);
1593         }
1594
1595         if (cmn_d->desc->ops->is_enabled(dev_priv, cmn_d)) {
1596                 phy_status |= PHY_POWERGOOD(DPIO_PHY1);
1597
1598                 /* this assumes override is only used to enable lanes */
1599                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0)) == 0)
1600                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0);
1601
1602                 if (BITS_SET(phy_control,
1603                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0)))
1604                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0);
1605
1606                 if (BITS_SET(phy_control,
1607                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY1, DPIO_CH0)))
1608                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0);
1609                 if (BITS_SET(phy_control,
1610                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY1, DPIO_CH0)))
1611                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1);
1612         }
1613
1614         phy_status &= phy_status_mask;
1615
1616         /*
1617          * The PHY may be busy with some initial calibration and whatnot,
1618          * so the power state can take a while to actually change.
1619          */
1620         if (intel_de_wait_for_register(dev_priv, DISPLAY_PHY_STATUS,
1621                                        phy_status_mask, phy_status, 10))
1622                 drm_err(&dev_priv->drm,
1623                         "Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n",
1624                         intel_de_read(dev_priv, DISPLAY_PHY_STATUS) & phy_status_mask,
1625                         phy_status, dev_priv->chv_phy_control);
1626 }
1627
1628 #undef BITS_SET
1629
1630 static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1631                                            struct i915_power_well *power_well)
1632 {
1633         enum dpio_phy phy;
1634         enum pipe pipe;
1635         u32 tmp;
1636
1637         drm_WARN_ON_ONCE(&dev_priv->drm,
1638                          power_well->desc->id != VLV_DISP_PW_DPIO_CMN_BC &&
1639                          power_well->desc->id != CHV_DISP_PW_DPIO_CMN_D);
1640
1641         if (power_well->desc->id == VLV_DISP_PW_DPIO_CMN_BC) {
1642                 pipe = PIPE_A;
1643                 phy = DPIO_PHY0;
1644         } else {
1645                 pipe = PIPE_C;
1646                 phy = DPIO_PHY1;
1647         }
1648
1649         /* since ref/cri clock was enabled */
1650         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1651         vlv_set_power_well(dev_priv, power_well, true);
1652
1653         /* Poll for phypwrgood signal */
1654         if (intel_de_wait_for_set(dev_priv, DISPLAY_PHY_STATUS,
1655                                   PHY_POWERGOOD(phy), 1))
1656                 drm_err(&dev_priv->drm, "Display PHY %d is not power up\n",
1657                         phy);
1658
1659         vlv_dpio_get(dev_priv);
1660
1661         /* Enable dynamic power down */
1662         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW28);
1663         tmp |= DPIO_DYNPWRDOWNEN_CH0 | DPIO_CL1POWERDOWNEN |
1664                 DPIO_SUS_CLK_CONFIG_GATE_CLKREQ;
1665         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW28, tmp);
1666
1667         if (power_well->desc->id == VLV_DISP_PW_DPIO_CMN_BC) {
1668                 tmp = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW6_CH1);
1669                 tmp |= DPIO_DYNPWRDOWNEN_CH1;
1670                 vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW6_CH1, tmp);
1671         } else {
1672                 /*
1673                  * Force the non-existing CL2 off. BXT does this
1674                  * too, so maybe it saves some power even though
1675                  * CL2 doesn't exist?
1676                  */
1677                 tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30);
1678                 tmp |= DPIO_CL2_LDOFUSE_PWRENB;
1679                 vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, tmp);
1680         }
1681
1682         vlv_dpio_put(dev_priv);
1683
1684         dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy);
1685         intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1686                        dev_priv->chv_phy_control);
1687
1688         drm_dbg_kms(&dev_priv->drm,
1689                     "Enabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1690                     phy, dev_priv->chv_phy_control);
1691
1692         assert_chv_phy_status(dev_priv);
1693 }
1694
1695 static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1696                                             struct i915_power_well *power_well)
1697 {
1698         enum dpio_phy phy;
1699
1700         drm_WARN_ON_ONCE(&dev_priv->drm,
1701                          power_well->desc->id != VLV_DISP_PW_DPIO_CMN_BC &&
1702                          power_well->desc->id != CHV_DISP_PW_DPIO_CMN_D);
1703
1704         if (power_well->desc->id == VLV_DISP_PW_DPIO_CMN_BC) {
1705                 phy = DPIO_PHY0;
1706                 assert_pll_disabled(dev_priv, PIPE_A);
1707                 assert_pll_disabled(dev_priv, PIPE_B);
1708         } else {
1709                 phy = DPIO_PHY1;
1710                 assert_pll_disabled(dev_priv, PIPE_C);
1711         }
1712
1713         dev_priv->chv_phy_control &= ~PHY_COM_LANE_RESET_DEASSERT(phy);
1714         intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1715                        dev_priv->chv_phy_control);
1716
1717         vlv_set_power_well(dev_priv, power_well, false);
1718
1719         drm_dbg_kms(&dev_priv->drm,
1720                     "Disabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1721                     phy, dev_priv->chv_phy_control);
1722
1723         /* PHY is fully reset now, so we can enable the PHY state asserts */
1724         dev_priv->chv_phy_assert[phy] = true;
1725
1726         assert_chv_phy_status(dev_priv);
1727 }
1728
1729 static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1730                                      enum dpio_channel ch, bool override, unsigned int mask)
1731 {
1732         enum pipe pipe = phy == DPIO_PHY0 ? PIPE_A : PIPE_C;
1733         u32 reg, val, expected, actual;
1734
1735         /*
1736          * The BIOS can leave the PHY is some weird state
1737          * where it doesn't fully power down some parts.
1738          * Disable the asserts until the PHY has been fully
1739          * reset (ie. the power well has been disabled at
1740          * least once).
1741          */
1742         if (!dev_priv->chv_phy_assert[phy])
1743                 return;
1744
1745         if (ch == DPIO_CH0)
1746                 reg = _CHV_CMN_DW0_CH0;
1747         else
1748                 reg = _CHV_CMN_DW6_CH1;
1749
1750         vlv_dpio_get(dev_priv);
1751         val = vlv_dpio_read(dev_priv, pipe, reg);
1752         vlv_dpio_put(dev_priv);
1753
1754         /*
1755          * This assumes !override is only used when the port is disabled.
1756          * All lanes should power down even without the override when
1757          * the port is disabled.
1758          */
1759         if (!override || mask == 0xf) {
1760                 expected = DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1761                 /*
1762                  * If CH1 common lane is not active anymore
1763                  * (eg. for pipe B DPLL) the entire channel will
1764                  * shut down, which causes the common lane registers
1765                  * to read as 0. That means we can't actually check
1766                  * the lane power down status bits, but as the entire
1767                  * register reads as 0 it's a good indication that the
1768                  * channel is indeed entirely powered down.
1769                  */
1770                 if (ch == DPIO_CH1 && val == 0)
1771                         expected = 0;
1772         } else if (mask != 0x0) {
1773                 expected = DPIO_ANYDL_POWERDOWN;
1774         } else {
1775                 expected = 0;
1776         }
1777
1778         if (ch == DPIO_CH0)
1779                 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH0;
1780         else
1781                 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH1;
1782         actual &= DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1783
1784         drm_WARN(&dev_priv->drm, actual != expected,
1785                  "Unexpected DPIO lane power down: all %d, any %d. Expected: all %d, any %d. (0x%x = 0x%08x)\n",
1786                  !!(actual & DPIO_ALLDL_POWERDOWN),
1787                  !!(actual & DPIO_ANYDL_POWERDOWN),
1788                  !!(expected & DPIO_ALLDL_POWERDOWN),
1789                  !!(expected & DPIO_ANYDL_POWERDOWN),
1790                  reg, val);
1791 }
1792
1793 bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1794                           enum dpio_channel ch, bool override)
1795 {
1796         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1797         bool was_override;
1798
1799         mutex_lock(&power_domains->lock);
1800
1801         was_override = dev_priv->chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1802
1803         if (override == was_override)
1804                 goto out;
1805
1806         if (override)
1807                 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1808         else
1809                 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1810
1811         intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1812                        dev_priv->chv_phy_control);
1813
1814         drm_dbg_kms(&dev_priv->drm,
1815                     "Power gating DPIO PHY%d CH%d (DPIO_PHY_CONTROL=0x%08x)\n",
1816                     phy, ch, dev_priv->chv_phy_control);
1817
1818         assert_chv_phy_status(dev_priv);
1819
1820 out:
1821         mutex_unlock(&power_domains->lock);
1822
1823         return was_override;
1824 }
1825
1826 void chv_phy_powergate_lanes(struct intel_encoder *encoder,
1827                              bool override, unsigned int mask)
1828 {
1829         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
1830         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1831         enum dpio_phy phy = vlv_dig_port_to_phy(enc_to_dig_port(encoder));
1832         enum dpio_channel ch = vlv_dig_port_to_channel(enc_to_dig_port(encoder));
1833
1834         mutex_lock(&power_domains->lock);
1835
1836         dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch);
1837         dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch);
1838
1839         if (override)
1840                 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1841         else
1842                 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1843
1844         intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1845                        dev_priv->chv_phy_control);
1846
1847         drm_dbg_kms(&dev_priv->drm,
1848                     "Power gating DPIO PHY%d CH%d lanes 0x%x (PHY_CONTROL=0x%08x)\n",
1849                     phy, ch, mask, dev_priv->chv_phy_control);
1850
1851         assert_chv_phy_status(dev_priv);
1852
1853         assert_chv_phy_powergate(dev_priv, phy, ch, override, mask);
1854
1855         mutex_unlock(&power_domains->lock);
1856 }
1857
1858 static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
1859                                         struct i915_power_well *power_well)
1860 {
1861         enum pipe pipe = PIPE_A;
1862         bool enabled;
1863         u32 state, ctrl;
1864
1865         vlv_punit_get(dev_priv);
1866
1867         state = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSS_MASK(pipe);
1868         /*
1869          * We only ever set the power-on and power-gate states, anything
1870          * else is unexpected.
1871          */
1872         drm_WARN_ON(&dev_priv->drm, state != DP_SSS_PWR_ON(pipe) &&
1873                     state != DP_SSS_PWR_GATE(pipe));
1874         enabled = state == DP_SSS_PWR_ON(pipe);
1875
1876         /*
1877          * A transient state at this point would mean some unexpected party
1878          * is poking at the power controls too.
1879          */
1880         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSC_MASK(pipe);
1881         drm_WARN_ON(&dev_priv->drm, ctrl << 16 != state);
1882
1883         vlv_punit_put(dev_priv);
1884
1885         return enabled;
1886 }
1887
1888 static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
1889                                     struct i915_power_well *power_well,
1890                                     bool enable)
1891 {
1892         enum pipe pipe = PIPE_A;
1893         u32 state;
1894         u32 ctrl;
1895
1896         state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe);
1897
1898         vlv_punit_get(dev_priv);
1899
1900 #define COND \
1901         ((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSS_MASK(pipe)) == state)
1902
1903         if (COND)
1904                 goto out;
1905
1906         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
1907         ctrl &= ~DP_SSC_MASK(pipe);
1908         ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
1909         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, ctrl);
1910
1911         if (wait_for(COND, 100))
1912                 drm_err(&dev_priv->drm,
1913                         "timeout setting power well state %08x (%08x)\n",
1914                         state,
1915                         vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM));
1916
1917 #undef COND
1918
1919 out:
1920         vlv_punit_put(dev_priv);
1921 }
1922
1923 static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv,
1924                                         struct i915_power_well *power_well)
1925 {
1926         intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1927                        dev_priv->chv_phy_control);
1928 }
1929
1930 static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
1931                                        struct i915_power_well *power_well)
1932 {
1933         chv_set_pipe_power_well(dev_priv, power_well, true);
1934
1935         vlv_display_power_well_init(dev_priv);
1936 }
1937
1938 static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv,
1939                                         struct i915_power_well *power_well)
1940 {
1941         vlv_display_power_well_deinit(dev_priv);
1942
1943         chv_set_pipe_power_well(dev_priv, power_well, false);
1944 }
1945
1946 static u64 __async_put_domains_mask(struct i915_power_domains *power_domains)
1947 {
1948         return power_domains->async_put_domains[0] |
1949                power_domains->async_put_domains[1];
1950 }
1951
1952 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
1953
1954 static bool
1955 assert_async_put_domain_masks_disjoint(struct i915_power_domains *power_domains)
1956 {
1957         struct drm_i915_private *i915 = container_of(power_domains,
1958                                                      struct drm_i915_private,
1959                                                      power_domains);
1960         return !drm_WARN_ON(&i915->drm, power_domains->async_put_domains[0] &
1961                             power_domains->async_put_domains[1]);
1962 }
1963
1964 static bool
1965 __async_put_domains_state_ok(struct i915_power_domains *power_domains)
1966 {
1967         struct drm_i915_private *i915 = container_of(power_domains,
1968                                                      struct drm_i915_private,
1969                                                      power_domains);
1970         enum intel_display_power_domain domain;
1971         bool err = false;
1972
1973         err |= !assert_async_put_domain_masks_disjoint(power_domains);
1974         err |= drm_WARN_ON(&i915->drm, !!power_domains->async_put_wakeref !=
1975                            !!__async_put_domains_mask(power_domains));
1976
1977         for_each_power_domain(domain, __async_put_domains_mask(power_domains))
1978                 err |= drm_WARN_ON(&i915->drm,
1979                                    power_domains->domain_use_count[domain] != 1);
1980
1981         return !err;
1982 }
1983
1984 static void print_power_domains(struct i915_power_domains *power_domains,
1985                                 const char *prefix, u64 mask)
1986 {
1987         struct drm_i915_private *i915 = container_of(power_domains,
1988                                                      struct drm_i915_private,
1989                                                      power_domains);
1990         enum intel_display_power_domain domain;
1991
1992         drm_dbg(&i915->drm, "%s (%lu):\n", prefix, hweight64(mask));
1993         for_each_power_domain(domain, mask)
1994                 drm_dbg(&i915->drm, "%s use_count %d\n",
1995                         intel_display_power_domain_str(domain),
1996                         power_domains->domain_use_count[domain]);
1997 }
1998
1999 static void
2000 print_async_put_domains_state(struct i915_power_domains *power_domains)
2001 {
2002         struct drm_i915_private *i915 = container_of(power_domains,
2003                                                      struct drm_i915_private,
2004                                                      power_domains);
2005
2006         drm_dbg(&i915->drm, "async_put_wakeref %u\n",
2007                 power_domains->async_put_wakeref);
2008
2009         print_power_domains(power_domains, "async_put_domains[0]",
2010                             power_domains->async_put_domains[0]);
2011         print_power_domains(power_domains, "async_put_domains[1]",
2012                             power_domains->async_put_domains[1]);
2013 }
2014
2015 static void
2016 verify_async_put_domains_state(struct i915_power_domains *power_domains)
2017 {
2018         if (!__async_put_domains_state_ok(power_domains))
2019                 print_async_put_domains_state(power_domains);
2020 }
2021
2022 #else
2023
2024 static void
2025 assert_async_put_domain_masks_disjoint(struct i915_power_domains *power_domains)
2026 {
2027 }
2028
2029 static void
2030 verify_async_put_domains_state(struct i915_power_domains *power_domains)
2031 {
2032 }
2033
2034 #endif /* CONFIG_DRM_I915_DEBUG_RUNTIME_PM */
2035
2036 static u64 async_put_domains_mask(struct i915_power_domains *power_domains)
2037 {
2038         assert_async_put_domain_masks_disjoint(power_domains);
2039
2040         return __async_put_domains_mask(power_domains);
2041 }
2042
2043 static void
2044 async_put_domains_clear_domain(struct i915_power_domains *power_domains,
2045                                enum intel_display_power_domain domain)
2046 {
2047         assert_async_put_domain_masks_disjoint(power_domains);
2048
2049         power_domains->async_put_domains[0] &= ~BIT_ULL(domain);
2050         power_domains->async_put_domains[1] &= ~BIT_ULL(domain);
2051 }
2052
2053 static bool
2054 intel_display_power_grab_async_put_ref(struct drm_i915_private *dev_priv,
2055                                        enum intel_display_power_domain domain)
2056 {
2057         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2058         bool ret = false;
2059
2060         if (!(async_put_domains_mask(power_domains) & BIT_ULL(domain)))
2061                 goto out_verify;
2062
2063         async_put_domains_clear_domain(power_domains, domain);
2064
2065         ret = true;
2066
2067         if (async_put_domains_mask(power_domains))
2068                 goto out_verify;
2069
2070         cancel_delayed_work(&power_domains->async_put_work);
2071         intel_runtime_pm_put_raw(&dev_priv->runtime_pm,
2072                                  fetch_and_zero(&power_domains->async_put_wakeref));
2073 out_verify:
2074         verify_async_put_domains_state(power_domains);
2075
2076         return ret;
2077 }
2078
2079 static void
2080 __intel_display_power_get_domain(struct drm_i915_private *dev_priv,
2081                                  enum intel_display_power_domain domain)
2082 {
2083         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2084         struct i915_power_well *power_well;
2085
2086         if (intel_display_power_grab_async_put_ref(dev_priv, domain))
2087                 return;
2088
2089         for_each_power_domain_well(dev_priv, power_well, BIT_ULL(domain))
2090                 intel_power_well_get(dev_priv, power_well);
2091
2092         power_domains->domain_use_count[domain]++;
2093 }
2094
2095 /**
2096  * intel_display_power_get - grab a power domain reference
2097  * @dev_priv: i915 device instance
2098  * @domain: power domain to reference
2099  *
2100  * This function grabs a power domain reference for @domain and ensures that the
2101  * power domain and all its parents are powered up. Therefore users should only
2102  * grab a reference to the innermost power domain they need.
2103  *
2104  * Any power domain reference obtained by this function must have a symmetric
2105  * call to intel_display_power_put() to release the reference again.
2106  */
2107 intel_wakeref_t intel_display_power_get(struct drm_i915_private *dev_priv,
2108                                         enum intel_display_power_domain domain)
2109 {
2110         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2111         intel_wakeref_t wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
2112
2113         mutex_lock(&power_domains->lock);
2114         __intel_display_power_get_domain(dev_priv, domain);
2115         mutex_unlock(&power_domains->lock);
2116
2117         return wakeref;
2118 }
2119
2120 /**
2121  * intel_display_power_get_if_enabled - grab a reference for an enabled display power domain
2122  * @dev_priv: i915 device instance
2123  * @domain: power domain to reference
2124  *
2125  * This function grabs a power domain reference for @domain and ensures that the
2126  * power domain and all its parents are powered up. Therefore users should only
2127  * grab a reference to the innermost power domain they need.
2128  *
2129  * Any power domain reference obtained by this function must have a symmetric
2130  * call to intel_display_power_put() to release the reference again.
2131  */
2132 intel_wakeref_t
2133 intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv,
2134                                    enum intel_display_power_domain domain)
2135 {
2136         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2137         intel_wakeref_t wakeref;
2138         bool is_enabled;
2139
2140         wakeref = intel_runtime_pm_get_if_in_use(&dev_priv->runtime_pm);
2141         if (!wakeref)
2142                 return false;
2143
2144         mutex_lock(&power_domains->lock);
2145
2146         if (__intel_display_power_is_enabled(dev_priv, domain)) {
2147                 __intel_display_power_get_domain(dev_priv, domain);
2148                 is_enabled = true;
2149         } else {
2150                 is_enabled = false;
2151         }
2152
2153         mutex_unlock(&power_domains->lock);
2154
2155         if (!is_enabled) {
2156                 intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
2157                 wakeref = 0;
2158         }
2159
2160         return wakeref;
2161 }
2162
2163 static void
2164 __intel_display_power_put_domain(struct drm_i915_private *dev_priv,
2165                                  enum intel_display_power_domain domain)
2166 {
2167         struct i915_power_domains *power_domains;
2168         struct i915_power_well *power_well;
2169         const char *name = intel_display_power_domain_str(domain);
2170
2171         power_domains = &dev_priv->power_domains;
2172
2173         drm_WARN(&dev_priv->drm, !power_domains->domain_use_count[domain],
2174                  "Use count on domain %s is already zero\n",
2175                  name);
2176         drm_WARN(&dev_priv->drm,
2177                  async_put_domains_mask(power_domains) & BIT_ULL(domain),
2178                  "Async disabling of domain %s is pending\n",
2179                  name);
2180
2181         power_domains->domain_use_count[domain]--;
2182
2183         for_each_power_domain_well_reverse(dev_priv, power_well, BIT_ULL(domain))
2184                 intel_power_well_put(dev_priv, power_well);
2185 }
2186
2187 static void __intel_display_power_put(struct drm_i915_private *dev_priv,
2188                                       enum intel_display_power_domain domain)
2189 {
2190         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2191
2192         mutex_lock(&power_domains->lock);
2193         __intel_display_power_put_domain(dev_priv, domain);
2194         mutex_unlock(&power_domains->lock);
2195 }
2196
2197 static void
2198 queue_async_put_domains_work(struct i915_power_domains *power_domains,
2199                              intel_wakeref_t wakeref)
2200 {
2201         struct drm_i915_private *i915 = container_of(power_domains,
2202                                                      struct drm_i915_private,
2203                                                      power_domains);
2204         drm_WARN_ON(&i915->drm, power_domains->async_put_wakeref);
2205         power_domains->async_put_wakeref = wakeref;
2206         drm_WARN_ON(&i915->drm, !queue_delayed_work(system_unbound_wq,
2207                                                     &power_domains->async_put_work,
2208                                                     msecs_to_jiffies(100)));
2209 }
2210
2211 static void
2212 release_async_put_domains(struct i915_power_domains *power_domains, u64 mask)
2213 {
2214         struct drm_i915_private *dev_priv =
2215                 container_of(power_domains, struct drm_i915_private,
2216                              power_domains);
2217         struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
2218         enum intel_display_power_domain domain;
2219         intel_wakeref_t wakeref;
2220
2221         /*
2222          * The caller must hold already raw wakeref, upgrade that to a proper
2223          * wakeref to make the state checker happy about the HW access during
2224          * power well disabling.
2225          */
2226         assert_rpm_raw_wakeref_held(rpm);
2227         wakeref = intel_runtime_pm_get(rpm);
2228
2229         for_each_power_domain(domain, mask) {
2230                 /* Clear before put, so put's sanity check is happy. */
2231                 async_put_domains_clear_domain(power_domains, domain);
2232                 __intel_display_power_put_domain(dev_priv, domain);
2233         }
2234
2235         intel_runtime_pm_put(rpm, wakeref);
2236 }
2237
2238 static void
2239 intel_display_power_put_async_work(struct work_struct *work)
2240 {
2241         struct drm_i915_private *dev_priv =
2242                 container_of(work, struct drm_i915_private,
2243                              power_domains.async_put_work.work);
2244         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2245         struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
2246         intel_wakeref_t new_work_wakeref = intel_runtime_pm_get_raw(rpm);
2247         intel_wakeref_t old_work_wakeref = 0;
2248
2249         mutex_lock(&power_domains->lock);
2250
2251         /*
2252          * Bail out if all the domain refs pending to be released were grabbed
2253          * by subsequent gets or a flush_work.
2254          */
2255         old_work_wakeref = fetch_and_zero(&power_domains->async_put_wakeref);
2256         if (!old_work_wakeref)
2257                 goto out_verify;
2258
2259         release_async_put_domains(power_domains,
2260                                   power_domains->async_put_domains[0]);
2261
2262         /* Requeue the work if more domains were async put meanwhile. */
2263         if (power_domains->async_put_domains[1]) {
2264                 power_domains->async_put_domains[0] =
2265                         fetch_and_zero(&power_domains->async_put_domains[1]);
2266                 queue_async_put_domains_work(power_domains,
2267                                              fetch_and_zero(&new_work_wakeref));
2268         }
2269
2270 out_verify:
2271         verify_async_put_domains_state(power_domains);
2272
2273         mutex_unlock(&power_domains->lock);
2274
2275         if (old_work_wakeref)
2276                 intel_runtime_pm_put_raw(rpm, old_work_wakeref);
2277         if (new_work_wakeref)
2278                 intel_runtime_pm_put_raw(rpm, new_work_wakeref);
2279 }
2280
2281 /**
2282  * intel_display_power_put_async - release a power domain reference asynchronously
2283  * @i915: i915 device instance
2284  * @domain: power domain to reference
2285  * @wakeref: wakeref acquired for the reference that is being released
2286  *
2287  * This function drops the power domain reference obtained by
2288  * intel_display_power_get*() and schedules a work to power down the
2289  * corresponding hardware block if this is the last reference.
2290  */
2291 void __intel_display_power_put_async(struct drm_i915_private *i915,
2292                                      enum intel_display_power_domain domain,
2293                                      intel_wakeref_t wakeref)
2294 {
2295         struct i915_power_domains *power_domains = &i915->power_domains;
2296         struct intel_runtime_pm *rpm = &i915->runtime_pm;
2297         intel_wakeref_t work_wakeref = intel_runtime_pm_get_raw(rpm);
2298
2299         mutex_lock(&power_domains->lock);
2300
2301         if (power_domains->domain_use_count[domain] > 1) {
2302                 __intel_display_power_put_domain(i915, domain);
2303
2304                 goto out_verify;
2305         }
2306
2307         drm_WARN_ON(&i915->drm, power_domains->domain_use_count[domain] != 1);
2308
2309         /* Let a pending work requeue itself or queue a new one. */
2310         if (power_domains->async_put_wakeref) {
2311                 power_domains->async_put_domains[1] |= BIT_ULL(domain);
2312         } else {
2313                 power_domains->async_put_domains[0] |= BIT_ULL(domain);
2314                 queue_async_put_domains_work(power_domains,
2315                                              fetch_and_zero(&work_wakeref));
2316         }
2317
2318 out_verify:
2319         verify_async_put_domains_state(power_domains);
2320
2321         mutex_unlock(&power_domains->lock);
2322
2323         if (work_wakeref)
2324                 intel_runtime_pm_put_raw(rpm, work_wakeref);
2325
2326         intel_runtime_pm_put(rpm, wakeref);
2327 }
2328
2329 /**
2330  * intel_display_power_flush_work - flushes the async display power disabling work
2331  * @i915: i915 device instance
2332  *
2333  * Flushes any pending work that was scheduled by a preceding
2334  * intel_display_power_put_async() call, completing the disabling of the
2335  * corresponding power domains.
2336  *
2337  * Note that the work handler function may still be running after this
2338  * function returns; to ensure that the work handler isn't running use
2339  * intel_display_power_flush_work_sync() instead.
2340  */
2341 void intel_display_power_flush_work(struct drm_i915_private *i915)
2342 {
2343         struct i915_power_domains *power_domains = &i915->power_domains;
2344         intel_wakeref_t work_wakeref;
2345
2346         mutex_lock(&power_domains->lock);
2347
2348         work_wakeref = fetch_and_zero(&power_domains->async_put_wakeref);
2349         if (!work_wakeref)
2350                 goto out_verify;
2351
2352         release_async_put_domains(power_domains,
2353                                   async_put_domains_mask(power_domains));
2354         cancel_delayed_work(&power_domains->async_put_work);
2355
2356 out_verify:
2357         verify_async_put_domains_state(power_domains);
2358
2359         mutex_unlock(&power_domains->lock);
2360
2361         if (work_wakeref)
2362                 intel_runtime_pm_put_raw(&i915->runtime_pm, work_wakeref);
2363 }
2364
2365 /**
2366  * intel_display_power_flush_work_sync - flushes and syncs the async display power disabling work
2367  * @i915: i915 device instance
2368  *
2369  * Like intel_display_power_flush_work(), but also ensure that the work
2370  * handler function is not running any more when this function returns.
2371  */
2372 static void
2373 intel_display_power_flush_work_sync(struct drm_i915_private *i915)
2374 {
2375         struct i915_power_domains *power_domains = &i915->power_domains;
2376
2377         intel_display_power_flush_work(i915);
2378         cancel_delayed_work_sync(&power_domains->async_put_work);
2379
2380         verify_async_put_domains_state(power_domains);
2381
2382         drm_WARN_ON(&i915->drm, power_domains->async_put_wakeref);
2383 }
2384
2385 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
2386 /**
2387  * intel_display_power_put - release a power domain reference
2388  * @dev_priv: i915 device instance
2389  * @domain: power domain to reference
2390  * @wakeref: wakeref acquired for the reference that is being released
2391  *
2392  * This function drops the power domain reference obtained by
2393  * intel_display_power_get() and might power down the corresponding hardware
2394  * block right away if this is the last reference.
2395  */
2396 void intel_display_power_put(struct drm_i915_private *dev_priv,
2397                              enum intel_display_power_domain domain,
2398                              intel_wakeref_t wakeref)
2399 {
2400         __intel_display_power_put(dev_priv, domain);
2401         intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
2402 }
2403 #else
2404 /**
2405  * intel_display_power_put_unchecked - release an unchecked power domain reference
2406  * @dev_priv: i915 device instance
2407  * @domain: power domain to reference
2408  *
2409  * This function drops the power domain reference obtained by
2410  * intel_display_power_get() and might power down the corresponding hardware
2411  * block right away if this is the last reference.
2412  *
2413  * This function is only for the power domain code's internal use to suppress wakeref
2414  * tracking when the correspondig debug kconfig option is disabled, should not
2415  * be used otherwise.
2416  */
2417 void intel_display_power_put_unchecked(struct drm_i915_private *dev_priv,
2418                                        enum intel_display_power_domain domain)
2419 {
2420         __intel_display_power_put(dev_priv, domain);
2421         intel_runtime_pm_put_unchecked(&dev_priv->runtime_pm);
2422 }
2423 #endif
2424
2425 void
2426 intel_display_power_get_in_set(struct drm_i915_private *i915,
2427                                struct intel_display_power_domain_set *power_domain_set,
2428                                enum intel_display_power_domain domain)
2429 {
2430         intel_wakeref_t __maybe_unused wf;
2431
2432         drm_WARN_ON(&i915->drm, power_domain_set->mask & BIT_ULL(domain));
2433
2434         wf = intel_display_power_get(i915, domain);
2435 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
2436         power_domain_set->wakerefs[domain] = wf;
2437 #endif
2438         power_domain_set->mask |= BIT_ULL(domain);
2439 }
2440
2441 bool
2442 intel_display_power_get_in_set_if_enabled(struct drm_i915_private *i915,
2443                                           struct intel_display_power_domain_set *power_domain_set,
2444                                           enum intel_display_power_domain domain)
2445 {
2446         intel_wakeref_t wf;
2447
2448         drm_WARN_ON(&i915->drm, power_domain_set->mask & BIT_ULL(domain));
2449
2450         wf = intel_display_power_get_if_enabled(i915, domain);
2451         if (!wf)
2452                 return false;
2453
2454 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
2455         power_domain_set->wakerefs[domain] = wf;
2456 #endif
2457         power_domain_set->mask |= BIT_ULL(domain);
2458
2459         return true;
2460 }
2461
2462 void
2463 intel_display_power_put_mask_in_set(struct drm_i915_private *i915,
2464                                     struct intel_display_power_domain_set *power_domain_set,
2465                                     u64 mask)
2466 {
2467         enum intel_display_power_domain domain;
2468
2469         drm_WARN_ON(&i915->drm, mask & ~power_domain_set->mask);
2470
2471         for_each_power_domain(domain, mask) {
2472                 intel_wakeref_t __maybe_unused wf = -1;
2473
2474 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
2475                 wf = fetch_and_zero(&power_domain_set->wakerefs[domain]);
2476 #endif
2477                 intel_display_power_put(i915, domain, wf);
2478                 power_domain_set->mask &= ~BIT_ULL(domain);
2479         }
2480 }
2481
2482 #define I830_PIPES_POWER_DOMAINS (              \
2483         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
2484         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
2485         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
2486         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
2487         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
2488         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
2489         BIT_ULL(POWER_DOMAIN_INIT))
2490
2491 #define VLV_DISPLAY_POWER_DOMAINS (             \
2492         BIT_ULL(POWER_DOMAIN_DISPLAY_CORE) |    \
2493         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
2494         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
2495         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
2496         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
2497         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
2498         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
2499         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2500         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2501         BIT_ULL(POWER_DOMAIN_PORT_DSI) |                \
2502         BIT_ULL(POWER_DOMAIN_PORT_CRT) |                \
2503         BIT_ULL(POWER_DOMAIN_VGA) |                     \
2504         BIT_ULL(POWER_DOMAIN_AUDIO) |           \
2505         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2506         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2507         BIT_ULL(POWER_DOMAIN_GMBUS) |           \
2508         BIT_ULL(POWER_DOMAIN_INIT))
2509
2510 #define VLV_DPIO_CMN_BC_POWER_DOMAINS (         \
2511         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2512         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2513         BIT_ULL(POWER_DOMAIN_PORT_CRT) |                \
2514         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2515         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2516         BIT_ULL(POWER_DOMAIN_INIT))
2517
2518 #define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS (  \
2519         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2520         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2521         BIT_ULL(POWER_DOMAIN_INIT))
2522
2523 #define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS (  \
2524         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2525         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2526         BIT_ULL(POWER_DOMAIN_INIT))
2527
2528 #define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS (  \
2529         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2530         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2531         BIT_ULL(POWER_DOMAIN_INIT))
2532
2533 #define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS (  \
2534         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2535         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2536         BIT_ULL(POWER_DOMAIN_INIT))
2537
2538 #define CHV_DISPLAY_POWER_DOMAINS (             \
2539         BIT_ULL(POWER_DOMAIN_DISPLAY_CORE) |    \
2540         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
2541         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
2542         BIT_ULL(POWER_DOMAIN_PIPE_C) |          \
2543         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
2544         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
2545         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
2546         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
2547         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
2548         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |    \
2549         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2550         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2551         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
2552         BIT_ULL(POWER_DOMAIN_PORT_DSI) |                \
2553         BIT_ULL(POWER_DOMAIN_VGA) |                     \
2554         BIT_ULL(POWER_DOMAIN_AUDIO) |           \
2555         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2556         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2557         BIT_ULL(POWER_DOMAIN_AUX_D) |           \
2558         BIT_ULL(POWER_DOMAIN_GMBUS) |           \
2559         BIT_ULL(POWER_DOMAIN_INIT))
2560
2561 #define CHV_DPIO_CMN_BC_POWER_DOMAINS (         \
2562         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2563         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2564         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2565         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2566         BIT_ULL(POWER_DOMAIN_INIT))
2567
2568 #define CHV_DPIO_CMN_D_POWER_DOMAINS (          \
2569         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
2570         BIT_ULL(POWER_DOMAIN_AUX_D) |           \
2571         BIT_ULL(POWER_DOMAIN_INIT))
2572
2573 #define HSW_DISPLAY_POWER_DOMAINS (                     \
2574         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2575         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2576         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |             \
2577         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
2578         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
2579         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2580         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2581         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2582         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2583         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2584         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
2585         BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */    \
2586         BIT_ULL(POWER_DOMAIN_VGA) |                             \
2587         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2588         BIT_ULL(POWER_DOMAIN_INIT))
2589
2590 #define BDW_DISPLAY_POWER_DOMAINS (                     \
2591         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2592         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2593         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
2594         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
2595         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2596         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2597         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2598         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2599         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2600         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
2601         BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */    \
2602         BIT_ULL(POWER_DOMAIN_VGA) |                             \
2603         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2604         BIT_ULL(POWER_DOMAIN_INIT))
2605
2606 #define SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
2607         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2608         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2609         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2610         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2611         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2612         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
2613         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
2614         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2615         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2616         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
2617         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |                \
2618         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
2619         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2620         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
2621         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2622         BIT_ULL(POWER_DOMAIN_VGA) |                             \
2623         BIT_ULL(POWER_DOMAIN_INIT))
2624 #define SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS (          \
2625         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |           \
2626         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO) |           \
2627         BIT_ULL(POWER_DOMAIN_INIT))
2628 #define SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS (            \
2629         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |           \
2630         BIT_ULL(POWER_DOMAIN_INIT))
2631 #define SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS (            \
2632         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |           \
2633         BIT_ULL(POWER_DOMAIN_INIT))
2634 #define SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS (            \
2635         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |           \
2636         BIT_ULL(POWER_DOMAIN_INIT))
2637 #define SKL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
2638         SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
2639         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
2640         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
2641         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2642         BIT_ULL(POWER_DOMAIN_INIT))
2643
2644 #define BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
2645         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2646         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2647         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2648         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2649         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2650         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
2651         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
2652         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2653         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2654         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
2655         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2656         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2657         BIT_ULL(POWER_DOMAIN_VGA) |                             \
2658         BIT_ULL(POWER_DOMAIN_INIT))
2659 #define BXT_DISPLAY_DC_OFF_POWER_DOMAINS (              \
2660         BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
2661         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
2662         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
2663         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2664         BIT_ULL(POWER_DOMAIN_GMBUS) |                   \
2665         BIT_ULL(POWER_DOMAIN_INIT))
2666 #define BXT_DPIO_CMN_A_POWER_DOMAINS (                  \
2667         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
2668         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2669         BIT_ULL(POWER_DOMAIN_INIT))
2670 #define BXT_DPIO_CMN_BC_POWER_DOMAINS (                 \
2671         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2672         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2673         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
2674         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2675         BIT_ULL(POWER_DOMAIN_INIT))
2676
2677 #define GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
2678         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2679         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2680         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2681         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2682         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2683         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
2684         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
2685         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2686         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2687         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
2688         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2689         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2690         BIT_ULL(POWER_DOMAIN_VGA) |                             \
2691         BIT_ULL(POWER_DOMAIN_INIT))
2692 #define GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS (            \
2693         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO))
2694 #define GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS (            \
2695         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO))
2696 #define GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS (            \
2697         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO))
2698 #define GLK_DPIO_CMN_A_POWER_DOMAINS (                  \
2699         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
2700         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2701         BIT_ULL(POWER_DOMAIN_INIT))
2702 #define GLK_DPIO_CMN_B_POWER_DOMAINS (                  \
2703         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2704         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
2705         BIT_ULL(POWER_DOMAIN_INIT))
2706 #define GLK_DPIO_CMN_C_POWER_DOMAINS (                  \
2707         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2708         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2709         BIT_ULL(POWER_DOMAIN_INIT))
2710 #define GLK_DISPLAY_AUX_A_POWER_DOMAINS (               \
2711         BIT_ULL(POWER_DOMAIN_AUX_A) |           \
2712         BIT_ULL(POWER_DOMAIN_AUX_IO_A) |                \
2713         BIT_ULL(POWER_DOMAIN_INIT))
2714 #define GLK_DISPLAY_AUX_B_POWER_DOMAINS (               \
2715         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2716         BIT_ULL(POWER_DOMAIN_INIT))
2717 #define GLK_DISPLAY_AUX_C_POWER_DOMAINS (               \
2718         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2719         BIT_ULL(POWER_DOMAIN_INIT))
2720 #define GLK_DISPLAY_DC_OFF_POWER_DOMAINS (              \
2721         GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
2722         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
2723         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
2724         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2725         BIT_ULL(POWER_DOMAIN_GMBUS) |                   \
2726         BIT_ULL(POWER_DOMAIN_INIT))
2727
2728 #define CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
2729         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2730         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2731         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2732         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2733         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2734         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
2735         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
2736         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2737         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2738         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
2739         BIT_ULL(POWER_DOMAIN_PORT_DDI_F_LANES) |                \
2740         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
2741         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2742         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
2743         BIT_ULL(POWER_DOMAIN_AUX_F) |                   \
2744         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2745         BIT_ULL(POWER_DOMAIN_VGA) |                             \
2746         BIT_ULL(POWER_DOMAIN_INIT))
2747 #define CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS (            \
2748         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |           \
2749         BIT_ULL(POWER_DOMAIN_INIT))
2750 #define CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS (            \
2751         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |           \
2752         BIT_ULL(POWER_DOMAIN_INIT))
2753 #define CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS (            \
2754         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |           \
2755         BIT_ULL(POWER_DOMAIN_INIT))
2756 #define CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS (            \
2757         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |           \
2758         BIT_ULL(POWER_DOMAIN_INIT))
2759 #define CNL_DISPLAY_AUX_A_POWER_DOMAINS (               \
2760         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2761         BIT_ULL(POWER_DOMAIN_AUX_IO_A) |                \
2762         BIT_ULL(POWER_DOMAIN_INIT))
2763 #define CNL_DISPLAY_AUX_B_POWER_DOMAINS (               \
2764         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
2765         BIT_ULL(POWER_DOMAIN_INIT))
2766 #define CNL_DISPLAY_AUX_C_POWER_DOMAINS (               \
2767         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2768         BIT_ULL(POWER_DOMAIN_INIT))
2769 #define CNL_DISPLAY_AUX_D_POWER_DOMAINS (               \
2770         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
2771         BIT_ULL(POWER_DOMAIN_INIT))
2772 #define CNL_DISPLAY_AUX_F_POWER_DOMAINS (               \
2773         BIT_ULL(POWER_DOMAIN_AUX_F) |                   \
2774         BIT_ULL(POWER_DOMAIN_INIT))
2775 #define CNL_DISPLAY_DDI_F_IO_POWER_DOMAINS (            \
2776         BIT_ULL(POWER_DOMAIN_PORT_DDI_F_IO) |           \
2777         BIT_ULL(POWER_DOMAIN_INIT))
2778 #define CNL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
2779         CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
2780         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
2781         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
2782         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2783         BIT_ULL(POWER_DOMAIN_INIT))
2784
2785 /*
2786  * ICL PW_0/PG_0 domains (HW/DMC control):
2787  * - PCI
2788  * - clocks except port PLL
2789  * - central power except FBC
2790  * - shared functions except pipe interrupts, pipe MBUS, DBUF registers
2791  * ICL PW_1/PG_1 domains (HW/DMC control):
2792  * - DBUF function
2793  * - PIPE_A and its planes, except VGA
2794  * - transcoder EDP + PSR
2795  * - transcoder DSI
2796  * - DDI_A
2797  * - FBC
2798  */
2799 #define ICL_PW_4_POWER_DOMAINS (                        \
2800         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2801         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
2802         BIT_ULL(POWER_DOMAIN_INIT))
2803         /* VDSC/joining */
2804 #define ICL_PW_3_POWER_DOMAINS (                        \
2805         ICL_PW_4_POWER_DOMAINS |                        \
2806         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2807         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2808         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2809         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2810         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
2811         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2812         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2813         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
2814         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |        \
2815         BIT_ULL(POWER_DOMAIN_PORT_DDI_F_LANES) |        \
2816         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
2817         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2818         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
2819         BIT_ULL(POWER_DOMAIN_AUX_E) |                   \
2820         BIT_ULL(POWER_DOMAIN_AUX_F) |                   \
2821         BIT_ULL(POWER_DOMAIN_AUX_C_TBT) |               \
2822         BIT_ULL(POWER_DOMAIN_AUX_D_TBT) |               \
2823         BIT_ULL(POWER_DOMAIN_AUX_E_TBT) |               \
2824         BIT_ULL(POWER_DOMAIN_AUX_F_TBT) |               \
2825         BIT_ULL(POWER_DOMAIN_VGA) |                     \
2826         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2827         BIT_ULL(POWER_DOMAIN_INIT))
2828         /*
2829          * - transcoder WD
2830          * - KVMR (HW control)
2831          */
2832 #define ICL_PW_2_POWER_DOMAINS (                        \
2833         ICL_PW_3_POWER_DOMAINS |                        \
2834         BIT_ULL(POWER_DOMAIN_TRANSCODER_VDSC_PW2) |             \
2835         BIT_ULL(POWER_DOMAIN_INIT))
2836         /*
2837          * - KVMR (HW control)
2838          */
2839 #define ICL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
2840         ICL_PW_2_POWER_DOMAINS |                        \
2841         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
2842         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2843         BIT_ULL(POWER_DOMAIN_DPLL_DC_OFF) |                     \
2844         BIT_ULL(POWER_DOMAIN_INIT))
2845
2846 #define ICL_DDI_IO_A_POWER_DOMAINS (                    \
2847         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO))
2848 #define ICL_DDI_IO_B_POWER_DOMAINS (                    \
2849         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO))
2850 #define ICL_DDI_IO_C_POWER_DOMAINS (                    \
2851         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO))
2852 #define ICL_DDI_IO_D_POWER_DOMAINS (                    \
2853         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO))
2854 #define ICL_DDI_IO_E_POWER_DOMAINS (                    \
2855         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO))
2856 #define ICL_DDI_IO_F_POWER_DOMAINS (                    \
2857         BIT_ULL(POWER_DOMAIN_PORT_DDI_F_IO))
2858
2859 #define ICL_AUX_A_IO_POWER_DOMAINS (                    \
2860         BIT_ULL(POWER_DOMAIN_AUX_IO_A) |                \
2861         BIT_ULL(POWER_DOMAIN_AUX_A))
2862 #define ICL_AUX_B_IO_POWER_DOMAINS (                    \
2863         BIT_ULL(POWER_DOMAIN_AUX_B))
2864 #define ICL_AUX_C_TC1_IO_POWER_DOMAINS (                \
2865         BIT_ULL(POWER_DOMAIN_AUX_C))
2866 #define ICL_AUX_D_TC2_IO_POWER_DOMAINS (                \
2867         BIT_ULL(POWER_DOMAIN_AUX_D))
2868 #define ICL_AUX_E_TC3_IO_POWER_DOMAINS (                \
2869         BIT_ULL(POWER_DOMAIN_AUX_E))
2870 #define ICL_AUX_F_TC4_IO_POWER_DOMAINS (                \
2871         BIT_ULL(POWER_DOMAIN_AUX_F))
2872 #define ICL_AUX_C_TBT1_IO_POWER_DOMAINS (               \
2873         BIT_ULL(POWER_DOMAIN_AUX_C_TBT))
2874 #define ICL_AUX_D_TBT2_IO_POWER_DOMAINS (               \
2875         BIT_ULL(POWER_DOMAIN_AUX_D_TBT))
2876 #define ICL_AUX_E_TBT3_IO_POWER_DOMAINS (               \
2877         BIT_ULL(POWER_DOMAIN_AUX_E_TBT))
2878 #define ICL_AUX_F_TBT4_IO_POWER_DOMAINS (               \
2879         BIT_ULL(POWER_DOMAIN_AUX_F_TBT))
2880
2881 #define TGL_PW_5_POWER_DOMAINS (                        \
2882         BIT_ULL(POWER_DOMAIN_PIPE_D) |                  \
2883         BIT_ULL(POWER_DOMAIN_TRANSCODER_D) |            \
2884         BIT_ULL(POWER_DOMAIN_PIPE_D_PANEL_FITTER) |     \
2885         BIT_ULL(POWER_DOMAIN_INIT))
2886
2887 #define TGL_PW_4_POWER_DOMAINS (                        \
2888         TGL_PW_5_POWER_DOMAINS |                        \
2889         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2890         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2891         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
2892         BIT_ULL(POWER_DOMAIN_INIT))
2893
2894 #define TGL_PW_3_POWER_DOMAINS (                        \
2895         TGL_PW_4_POWER_DOMAINS |                        \
2896         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2897         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2898         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
2899         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC1) |      \
2900         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC2) |      \
2901         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC3) |      \
2902         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC4) |      \
2903         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC5) |      \
2904         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC6) |      \
2905         BIT_ULL(POWER_DOMAIN_AUX_USBC1) |               \
2906         BIT_ULL(POWER_DOMAIN_AUX_USBC2) |               \
2907         BIT_ULL(POWER_DOMAIN_AUX_USBC3) |               \
2908         BIT_ULL(POWER_DOMAIN_AUX_USBC4) |               \
2909         BIT_ULL(POWER_DOMAIN_AUX_USBC5) |               \
2910         BIT_ULL(POWER_DOMAIN_AUX_USBC6) |               \
2911         BIT_ULL(POWER_DOMAIN_AUX_TBT1) |                \
2912         BIT_ULL(POWER_DOMAIN_AUX_TBT2) |                \
2913         BIT_ULL(POWER_DOMAIN_AUX_TBT3) |                \
2914         BIT_ULL(POWER_DOMAIN_AUX_TBT4) |                \
2915         BIT_ULL(POWER_DOMAIN_AUX_TBT5) |                \
2916         BIT_ULL(POWER_DOMAIN_AUX_TBT6) |                \
2917         BIT_ULL(POWER_DOMAIN_VGA) |                     \
2918         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2919         BIT_ULL(POWER_DOMAIN_INIT))
2920
2921 #define TGL_PW_2_POWER_DOMAINS (                        \
2922         TGL_PW_3_POWER_DOMAINS |                        \
2923         BIT_ULL(POWER_DOMAIN_TRANSCODER_VDSC_PW2) |     \
2924         BIT_ULL(POWER_DOMAIN_INIT))
2925
2926 #define TGL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
2927         TGL_PW_3_POWER_DOMAINS |                        \
2928         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
2929         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2930         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
2931         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2932         BIT_ULL(POWER_DOMAIN_INIT))
2933
2934 #define TGL_DDI_IO_TC1_POWER_DOMAINS    BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_TC1)
2935 #define TGL_DDI_IO_TC2_POWER_DOMAINS    BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_TC2)
2936 #define TGL_DDI_IO_TC3_POWER_DOMAINS    BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_TC3)
2937 #define TGL_DDI_IO_TC4_POWER_DOMAINS    BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_TC4)
2938 #define TGL_DDI_IO_TC5_POWER_DOMAINS    BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_TC5)
2939 #define TGL_DDI_IO_TC6_POWER_DOMAINS    BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_TC6)
2940
2941 #define TGL_AUX_A_IO_POWER_DOMAINS (            \
2942         BIT_ULL(POWER_DOMAIN_AUX_IO_A) |        \
2943         BIT_ULL(POWER_DOMAIN_AUX_A))
2944 #define TGL_AUX_B_IO_POWER_DOMAINS (            \
2945         BIT_ULL(POWER_DOMAIN_AUX_B))
2946 #define TGL_AUX_C_IO_POWER_DOMAINS (            \
2947         BIT_ULL(POWER_DOMAIN_AUX_C))
2948
2949 #define TGL_AUX_IO_USBC1_POWER_DOMAINS  BIT_ULL(POWER_DOMAIN_AUX_USBC1)
2950 #define TGL_AUX_IO_USBC2_POWER_DOMAINS  BIT_ULL(POWER_DOMAIN_AUX_USBC2)
2951 #define TGL_AUX_IO_USBC3_POWER_DOMAINS  BIT_ULL(POWER_DOMAIN_AUX_USBC3)
2952 #define TGL_AUX_IO_USBC4_POWER_DOMAINS  BIT_ULL(POWER_DOMAIN_AUX_USBC4)
2953 #define TGL_AUX_IO_USBC5_POWER_DOMAINS  BIT_ULL(POWER_DOMAIN_AUX_USBC5)
2954 #define TGL_AUX_IO_USBC6_POWER_DOMAINS  BIT_ULL(POWER_DOMAIN_AUX_USBC6)
2955
2956 #define TGL_AUX_IO_TBT1_POWER_DOMAINS   BIT_ULL(POWER_DOMAIN_AUX_TBT1)
2957 #define TGL_AUX_IO_TBT2_POWER_DOMAINS   BIT_ULL(POWER_DOMAIN_AUX_TBT2)
2958 #define TGL_AUX_IO_TBT3_POWER_DOMAINS   BIT_ULL(POWER_DOMAIN_AUX_TBT3)
2959 #define TGL_AUX_IO_TBT4_POWER_DOMAINS   BIT_ULL(POWER_DOMAIN_AUX_TBT4)
2960 #define TGL_AUX_IO_TBT5_POWER_DOMAINS   BIT_ULL(POWER_DOMAIN_AUX_TBT5)
2961 #define TGL_AUX_IO_TBT6_POWER_DOMAINS   BIT_ULL(POWER_DOMAIN_AUX_TBT6)
2962
2963 #define TGL_TC_COLD_OFF_POWER_DOMAINS (         \
2964         BIT_ULL(POWER_DOMAIN_AUX_USBC1) |       \
2965         BIT_ULL(POWER_DOMAIN_AUX_USBC2) |       \
2966         BIT_ULL(POWER_DOMAIN_AUX_USBC3) |       \
2967         BIT_ULL(POWER_DOMAIN_AUX_USBC4) |       \
2968         BIT_ULL(POWER_DOMAIN_AUX_USBC5) |       \
2969         BIT_ULL(POWER_DOMAIN_AUX_USBC6) |       \
2970         BIT_ULL(POWER_DOMAIN_AUX_TBT1) |        \
2971         BIT_ULL(POWER_DOMAIN_AUX_TBT2) |        \
2972         BIT_ULL(POWER_DOMAIN_AUX_TBT3) |        \
2973         BIT_ULL(POWER_DOMAIN_AUX_TBT4) |        \
2974         BIT_ULL(POWER_DOMAIN_AUX_TBT5) |        \
2975         BIT_ULL(POWER_DOMAIN_AUX_TBT6) |        \
2976         BIT_ULL(POWER_DOMAIN_TC_COLD_OFF))
2977
2978 #define RKL_PW_4_POWER_DOMAINS (                        \
2979         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2980         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
2981         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2982         BIT_ULL(POWER_DOMAIN_INIT))
2983
2984 #define RKL_PW_3_POWER_DOMAINS (                        \
2985         RKL_PW_4_POWER_DOMAINS |                        \
2986         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2987         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
2988         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2989         BIT_ULL(POWER_DOMAIN_VGA) |                     \
2990         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2991         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC1) |      \
2992         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC2) |      \
2993         BIT_ULL(POWER_DOMAIN_AUX_USBC1) |               \
2994         BIT_ULL(POWER_DOMAIN_AUX_USBC2) |               \
2995         BIT_ULL(POWER_DOMAIN_INIT))
2996
2997 /*
2998  * There is no PW_2/PG_2 on RKL.
2999  *
3000  * RKL PW_1/PG_1 domains (under HW/DMC control):
3001  * - DBUF function (note: registers are in PW0)
3002  * - PIPE_A and its planes and VDSC/joining, except VGA
3003  * - transcoder A
3004  * - DDI_A and DDI_B
3005  * - FBC
3006  *
3007  * RKL PW_0/PG_0 domains (under HW/DMC control):
3008  * - PCI
3009  * - clocks except port PLL
3010  * - shared functions:
3011  *     * interrupts except pipe interrupts
3012  *     * MBus except PIPE_MBUS_DBOX_CTL
3013  *     * DBUF registers
3014  * - central power except FBC
3015  * - top-level GTC (DDI-level GTC is in the well associated with the DDI)
3016  */
3017
3018 #define RKL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
3019         RKL_PW_3_POWER_DOMAINS |                        \
3020         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
3021         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
3022         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
3023         BIT_ULL(POWER_DOMAIN_INIT))
3024
3025 /*
3026  * XE_LPD Power Domains
3027  *
3028  * Previous platforms required that PG(n-1) be enabled before PG(n).  That
3029  * dependency chain turns into a dependency tree on XE_LPD:
3030  *
3031  *       PG0
3032  *        |
3033  *     --PG1--
3034  *    /       \
3035  *  PGA     --PG2--
3036  *         /   |   \
3037  *       PGB  PGC  PGD
3038  *
3039  * Power wells must be enabled from top to bottom and disabled from bottom
3040  * to top.  This allows pipes to be power gated independently.
3041  */
3042
3043 #define XELPD_PW_D_POWER_DOMAINS (                      \
3044         BIT_ULL(POWER_DOMAIN_PIPE_D) |                  \
3045         BIT_ULL(POWER_DOMAIN_PIPE_D_PANEL_FITTER) |     \
3046         BIT_ULL(POWER_DOMAIN_TRANSCODER_D) |            \
3047         BIT_ULL(POWER_DOMAIN_INIT))
3048
3049 #define XELPD_PW_C_POWER_DOMAINS (                      \
3050         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
3051         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
3052         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
3053         BIT_ULL(POWER_DOMAIN_INIT))
3054
3055 #define XELPD_PW_B_POWER_DOMAINS (                      \
3056         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
3057         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
3058         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
3059         BIT_ULL(POWER_DOMAIN_INIT))
3060
3061 #define XELPD_PW_A_POWER_DOMAINS (                      \
3062         BIT_ULL(POWER_DOMAIN_PIPE_A) |                  \
3063         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
3064         BIT_ULL(POWER_DOMAIN_INIT))
3065
3066 #define XELPD_PW_2_POWER_DOMAINS (                      \
3067         XELPD_PW_B_POWER_DOMAINS |                      \
3068         XELPD_PW_C_POWER_DOMAINS |                      \
3069         XELPD_PW_D_POWER_DOMAINS |                      \
3070         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
3071         BIT_ULL(POWER_DOMAIN_VGA) |                     \
3072         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
3073         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_D_XELPD) |  \
3074         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_E_XELPD) |  \
3075         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
3076         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC1) |      \
3077         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC2) |      \
3078         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC3) |      \
3079         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC4) |      \
3080         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
3081         BIT_ULL(POWER_DOMAIN_AUX_D_XELPD) |             \
3082         BIT_ULL(POWER_DOMAIN_AUX_E_XELPD) |             \
3083         BIT_ULL(POWER_DOMAIN_AUX_USBC1) |                       \
3084         BIT_ULL(POWER_DOMAIN_AUX_USBC2) |                       \
3085         BIT_ULL(POWER_DOMAIN_AUX_USBC3) |                       \
3086         BIT_ULL(POWER_DOMAIN_AUX_USBC4) |                       \
3087         BIT_ULL(POWER_DOMAIN_INIT))
3088
3089 /*
3090  * XELPD PW_1/PG_1 domains (under HW/DMC control):
3091  *  - DBUF function (registers are in PW0)
3092  *  - Transcoder A
3093  *  - DDI_A and DDI_B
3094  *
3095  * XELPD PW_0/PW_1 domains (under HW/DMC control):
3096  *  - PCI
3097  *  - Clocks except port PLL
3098  *  - Shared functions:
3099  *     * interrupts except pipe interrupts
3100  *     * MBus except PIPE_MBUS_DBOX_CTL
3101  *     * DBUF registers
3102  *  - Central power except FBC
3103  *  - Top-level GTC (DDI-level GTC is in the well associated with the DDI)
3104  */
3105
3106 #define XELPD_DISPLAY_DC_OFF_POWER_DOMAINS (            \
3107         XELPD_PW_2_POWER_DOMAINS |                      \
3108         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
3109         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
3110         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
3111         BIT_ULL(POWER_DOMAIN_INIT))
3112
3113 #define XELPD_AUX_IO_D_XELPD_POWER_DOMAINS      BIT_ULL(POWER_DOMAIN_AUX_D_XELPD)
3114 #define XELPD_AUX_IO_E_XELPD_POWER_DOMAINS      BIT_ULL(POWER_DOMAIN_AUX_E_XELPD)
3115 #define XELPD_AUX_IO_USBC1_POWER_DOMAINS        BIT_ULL(POWER_DOMAIN_AUX_USBC1)
3116 #define XELPD_AUX_IO_USBC2_POWER_DOMAINS        BIT_ULL(POWER_DOMAIN_AUX_USBC2)
3117 #define XELPD_AUX_IO_USBC3_POWER_DOMAINS        BIT_ULL(POWER_DOMAIN_AUX_USBC3)
3118 #define XELPD_AUX_IO_USBC4_POWER_DOMAINS        BIT_ULL(POWER_DOMAIN_AUX_USBC4)
3119
3120 #define XELPD_AUX_IO_TBT1_POWER_DOMAINS         BIT_ULL(POWER_DOMAIN_AUX_TBT1)
3121 #define XELPD_AUX_IO_TBT2_POWER_DOMAINS         BIT_ULL(POWER_DOMAIN_AUX_TBT2)
3122 #define XELPD_AUX_IO_TBT3_POWER_DOMAINS         BIT_ULL(POWER_DOMAIN_AUX_TBT3)
3123 #define XELPD_AUX_IO_TBT4_POWER_DOMAINS         BIT_ULL(POWER_DOMAIN_AUX_TBT4)
3124
3125 #define XELPD_DDI_IO_D_XELPD_POWER_DOMAINS      BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_D_XELPD)
3126 #define XELPD_DDI_IO_E_XELPD_POWER_DOMAINS      BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_E_XELPD)
3127 #define XELPD_DDI_IO_TC1_POWER_DOMAINS          BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_TC1)
3128 #define XELPD_DDI_IO_TC2_POWER_DOMAINS          BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_TC2)
3129 #define XELPD_DDI_IO_TC3_POWER_DOMAINS          BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_TC3)
3130 #define XELPD_DDI_IO_TC4_POWER_DOMAINS          BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_TC4)
3131
3132 static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
3133         .sync_hw = i9xx_power_well_sync_hw_noop,
3134         .enable = i9xx_always_on_power_well_noop,
3135         .disable = i9xx_always_on_power_well_noop,
3136         .is_enabled = i9xx_always_on_power_well_enabled,
3137 };
3138
3139 static const struct i915_power_well_ops chv_pipe_power_well_ops = {
3140         .sync_hw = chv_pipe_power_well_sync_hw,
3141         .enable = chv_pipe_power_well_enable,
3142         .disable = chv_pipe_power_well_disable,
3143         .is_enabled = chv_pipe_power_well_enabled,
3144 };
3145
3146 static const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = {
3147         .sync_hw = i9xx_power_well_sync_hw_noop,
3148         .enable = chv_dpio_cmn_power_well_enable,
3149         .disable = chv_dpio_cmn_power_well_disable,
3150         .is_enabled = vlv_power_well_enabled,
3151 };
3152
3153 static const struct i915_power_well_desc i9xx_always_on_power_well[] = {
3154         {
3155                 .name = "always-on",
3156                 .always_on = true,
3157                 .domains = POWER_DOMAIN_MASK,
3158                 .ops = &i9xx_always_on_power_well_ops,
3159                 .id = DISP_PW_ID_NONE,
3160         },
3161 };
3162
3163 static const struct i915_power_well_ops i830_pipes_power_well_ops = {
3164         .sync_hw = i830_pipes_power_well_sync_hw,
3165         .enable = i830_pipes_power_well_enable,
3166         .disable = i830_pipes_power_well_disable,
3167         .is_enabled = i830_pipes_power_well_enabled,
3168 };
3169
3170 static const struct i915_power_well_desc i830_power_wells[] = {
3171         {
3172                 .name = "always-on",
3173                 .always_on = true,
3174                 .domains = POWER_DOMAIN_MASK,
3175                 .ops = &i9xx_always_on_power_well_ops,
3176                 .id = DISP_PW_ID_NONE,
3177         },
3178         {
3179                 .name = "pipes",
3180                 .domains = I830_PIPES_POWER_DOMAINS,
3181                 .ops = &i830_pipes_power_well_ops,
3182                 .id = DISP_PW_ID_NONE,
3183         },
3184 };
3185
3186 static const struct i915_power_well_ops hsw_power_well_ops = {
3187         .sync_hw = hsw_power_well_sync_hw,
3188         .enable = hsw_power_well_enable,
3189         .disable = hsw_power_well_disable,
3190         .is_enabled = hsw_power_well_enabled,
3191 };
3192
3193 static const struct i915_power_well_ops gen9_dc_off_power_well_ops = {
3194         .sync_hw = i9xx_power_well_sync_hw_noop,
3195         .enable = gen9_dc_off_power_well_enable,
3196         .disable = gen9_dc_off_power_well_disable,
3197         .is_enabled = gen9_dc_off_power_well_enabled,
3198 };
3199
3200 static const struct i915_power_well_ops bxt_dpio_cmn_power_well_ops = {
3201         .sync_hw = i9xx_power_well_sync_hw_noop,
3202         .enable = bxt_dpio_cmn_power_well_enable,
3203         .disable = bxt_dpio_cmn_power_well_disable,
3204         .is_enabled = bxt_dpio_cmn_power_well_enabled,
3205 };
3206
3207 static const struct i915_power_well_regs hsw_power_well_regs = {
3208         .bios   = HSW_PWR_WELL_CTL1,
3209         .driver = HSW_PWR_WELL_CTL2,
3210         .kvmr   = HSW_PWR_WELL_CTL3,
3211         .debug  = HSW_PWR_WELL_CTL4,
3212 };
3213
3214 static const struct i915_power_well_desc hsw_power_wells[] = {
3215         {
3216                 .name = "always-on",
3217                 .always_on = true,
3218                 .domains = POWER_DOMAIN_MASK,
3219                 .ops = &i9xx_always_on_power_well_ops,
3220                 .id = DISP_PW_ID_NONE,
3221         },
3222         {
3223                 .name = "display",
3224                 .domains = HSW_DISPLAY_POWER_DOMAINS,
3225                 .ops = &hsw_power_well_ops,
3226                 .id = HSW_DISP_PW_GLOBAL,
3227                 {
3228                         .hsw.regs = &hsw_power_well_regs,
3229                         .hsw.idx = HSW_PW_CTL_IDX_GLOBAL,
3230                         .hsw.has_vga = true,
3231                 },
3232         },
3233 };
3234
3235 static const struct i915_power_well_desc bdw_power_wells[] = {
3236         {
3237                 .name = "always-on",
3238                 .always_on = true,
3239                 .domains = POWER_DOMAIN_MASK,
3240                 .ops = &i9xx_always_on_power_well_ops,
3241                 .id = DISP_PW_ID_NONE,
3242         },
3243         {
3244                 .name = "display",
3245                 .domains = BDW_DISPLAY_POWER_DOMAINS,
3246                 .ops = &hsw_power_well_ops,
3247                 .id = HSW_DISP_PW_GLOBAL,
3248                 {
3249                         .hsw.regs = &hsw_power_well_regs,
3250                         .hsw.idx = HSW_PW_CTL_IDX_GLOBAL,
3251                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
3252                         .hsw.has_vga = true,
3253                 },
3254         },
3255 };
3256
3257 static const struct i915_power_well_ops vlv_display_power_well_ops = {
3258         .sync_hw = i9xx_power_well_sync_hw_noop,
3259         .enable = vlv_display_power_well_enable,
3260         .disable = vlv_display_power_well_disable,
3261         .is_enabled = vlv_power_well_enabled,
3262 };
3263
3264 static const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = {
3265         .sync_hw = i9xx_power_well_sync_hw_noop,
3266         .enable = vlv_dpio_cmn_power_well_enable,
3267         .disable = vlv_dpio_cmn_power_well_disable,
3268         .is_enabled = vlv_power_well_enabled,
3269 };
3270
3271 static const struct i915_power_well_ops vlv_dpio_power_well_ops = {
3272         .sync_hw = i9xx_power_well_sync_hw_noop,
3273         .enable = vlv_power_well_enable,
3274         .disable = vlv_power_well_disable,
3275         .is_enabled = vlv_power_well_enabled,
3276 };
3277
3278 static const struct i915_power_well_desc vlv_power_wells[] = {
3279         {
3280                 .name = "always-on",
3281                 .always_on = true,
3282                 .domains = POWER_DOMAIN_MASK,
3283                 .ops = &i9xx_always_on_power_well_ops,
3284                 .id = DISP_PW_ID_NONE,
3285         },
3286         {
3287                 .name = "display",
3288                 .domains = VLV_DISPLAY_POWER_DOMAINS,
3289                 .ops = &vlv_display_power_well_ops,
3290                 .id = VLV_DISP_PW_DISP2D,
3291                 {
3292                         .vlv.idx = PUNIT_PWGT_IDX_DISP2D,
3293                 },
3294         },
3295         {
3296                 .name = "dpio-tx-b-01",
3297                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
3298                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
3299                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
3300                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
3301                 .ops = &vlv_dpio_power_well_ops,
3302                 .id = DISP_PW_ID_NONE,
3303                 {
3304                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_TX_B_LANES_01,
3305                 },
3306         },
3307         {
3308                 .name = "dpio-tx-b-23",
3309                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
3310                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
3311                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
3312                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
3313                 .ops = &vlv_dpio_power_well_ops,
3314                 .id = DISP_PW_ID_NONE,
3315                 {
3316                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_TX_B_LANES_23,
3317                 },
3318         },
3319         {
3320                 .name = "dpio-tx-c-01",
3321                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
3322                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
3323                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
3324                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
3325                 .ops = &vlv_dpio_power_well_ops,
3326                 .id = DISP_PW_ID_NONE,
3327                 {
3328                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_TX_C_LANES_01,
3329                 },
3330         },
3331         {
3332                 .name = "dpio-tx-c-23",
3333                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
3334                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
3335                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
3336                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
3337                 .ops = &vlv_dpio_power_well_ops,
3338                 .id = DISP_PW_ID_NONE,
3339                 {
3340                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_TX_C_LANES_23,
3341                 },
3342         },
3343         {
3344                 .name = "dpio-common",
3345                 .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS,
3346                 .ops = &vlv_dpio_cmn_power_well_ops,
3347                 .id = VLV_DISP_PW_DPIO_CMN_BC,
3348                 {
3349                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_CMN_BC,
3350                 },
3351         },
3352 };
3353
3354 static const struct i915_power_well_desc chv_power_wells[] = {
3355         {
3356                 .name = "always-on",
3357                 .always_on = true,
3358                 .domains = POWER_DOMAIN_MASK,
3359                 .ops = &i9xx_always_on_power_well_ops,
3360                 .id = DISP_PW_ID_NONE,
3361         },
3362         {
3363                 .name = "display",
3364                 /*
3365                  * Pipe A power well is the new disp2d well. Pipe B and C
3366                  * power wells don't actually exist. Pipe A power well is
3367                  * required for any pipe to work.
3368                  */
3369                 .domains = CHV_DISPLAY_POWER_DOMAINS,
3370                 .ops = &chv_pipe_power_well_ops,
3371                 .id = DISP_PW_ID_NONE,
3372         },
3373         {
3374                 .name = "dpio-common-bc",
3375                 .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS,
3376                 .ops = &chv_dpio_cmn_power_well_ops,
3377                 .id = VLV_DISP_PW_DPIO_CMN_BC,
3378                 {
3379                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_CMN_BC,
3380                 },
3381         },
3382         {
3383                 .name = "dpio-common-d",
3384                 .domains = CHV_DPIO_CMN_D_POWER_DOMAINS,
3385                 .ops = &chv_dpio_cmn_power_well_ops,
3386                 .id = CHV_DISP_PW_DPIO_CMN_D,
3387                 {
3388                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_CMN_D,
3389                 },
3390         },
3391 };
3392
3393 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
3394                                          enum i915_power_well_id power_well_id)
3395 {
3396         struct i915_power_well *power_well;
3397         bool ret;
3398
3399         power_well = lookup_power_well(dev_priv, power_well_id);
3400         ret = power_well->desc->ops->is_enabled(dev_priv, power_well);
3401
3402         return ret;
3403 }
3404
3405 static const struct i915_power_well_desc skl_power_wells[] = {
3406         {
3407                 .name = "always-on",
3408                 .always_on = true,
3409                 .domains = POWER_DOMAIN_MASK,
3410                 .ops = &i9xx_always_on_power_well_ops,
3411                 .id = DISP_PW_ID_NONE,
3412         },
3413         {
3414                 .name = "power well 1",
3415                 /* Handled by the DMC firmware */
3416                 .always_on = true,
3417                 .domains = 0,
3418                 .ops = &hsw_power_well_ops,
3419                 .id = SKL_DISP_PW_1,
3420                 {
3421                         .hsw.regs = &hsw_power_well_regs,
3422                         .hsw.idx = SKL_PW_CTL_IDX_PW_1,
3423                         .hsw.has_fuses = true,
3424                 },
3425         },
3426         {
3427                 .name = "MISC IO power well",
3428                 /* Handled by the DMC firmware */
3429                 .always_on = true,
3430                 .domains = 0,
3431                 .ops = &hsw_power_well_ops,
3432                 .id = SKL_DISP_PW_MISC_IO,
3433                 {
3434                         .hsw.regs = &hsw_power_well_regs,
3435                         .hsw.idx = SKL_PW_CTL_IDX_MISC_IO,
3436                 },
3437         },
3438         {
3439                 .name = "DC off",
3440                 .domains = SKL_DISPLAY_DC_OFF_POWER_DOMAINS,
3441                 .ops = &gen9_dc_off_power_well_ops,
3442                 .id = SKL_DISP_DC_OFF,
3443         },
3444         {
3445                 .name = "power well 2",
3446                 .domains = SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
3447                 .ops = &hsw_power_well_ops,
3448                 .id = SKL_DISP_PW_2,
3449                 {
3450                         .hsw.regs = &hsw_power_well_regs,
3451                         .hsw.idx = SKL_PW_CTL_IDX_PW_2,
3452                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
3453                         .hsw.has_vga = true,
3454                         .hsw.has_fuses = true,
3455                 },
3456         },
3457         {
3458                 .name = "DDI A/E IO power well",
3459                 .domains = SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS,
3460                 .ops = &hsw_power_well_ops,
3461                 .id = DISP_PW_ID_NONE,
3462                 {
3463                         .hsw.regs = &hsw_power_well_regs,
3464                         .hsw.idx = SKL_PW_CTL_IDX_DDI_A_E,
3465                 },
3466         },
3467         {
3468                 .name = "DDI B IO power well",
3469                 .domains = SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS,
3470                 .ops = &hsw_power_well_ops,
3471                 .id = DISP_PW_ID_NONE,
3472                 {
3473                         .hsw.regs = &hsw_power_well_regs,
3474                         .hsw.idx = SKL_PW_CTL_IDX_DDI_B,
3475                 },
3476         },
3477         {
3478                 .name = "DDI C IO power well",
3479                 .domains = SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS,
3480                 .ops = &hsw_power_well_ops,
3481                 .id = DISP_PW_ID_NONE,
3482                 {
3483                         .hsw.regs = &hsw_power_well_regs,
3484                         .hsw.idx = SKL_PW_CTL_IDX_DDI_C,
3485                 },
3486         },
3487         {
3488                 .name = "DDI D IO power well",
3489                 .domains = SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS,
3490                 .ops = &hsw_power_well_ops,
3491                 .id = DISP_PW_ID_NONE,
3492                 {
3493                         .hsw.regs = &hsw_power_well_regs,
3494                         .hsw.idx = SKL_PW_CTL_IDX_DDI_D,
3495                 },
3496         },
3497 };
3498
3499 static const struct i915_power_well_desc bxt_power_wells[] = {
3500         {
3501                 .name = "always-on",
3502                 .always_on = true,
3503                 .domains = POWER_DOMAIN_MASK,
3504                 .ops = &i9xx_always_on_power_well_ops,
3505                 .id = DISP_PW_ID_NONE,
3506         },
3507         {
3508                 .name = "power well 1",
3509                 /* Handled by the DMC firmware */
3510                 .always_on = true,
3511                 .domains = 0,
3512                 .ops = &hsw_power_well_ops,
3513                 .id = SKL_DISP_PW_1,
3514                 {
3515                         .hsw.regs = &hsw_power_well_regs,
3516                         .hsw.idx = SKL_PW_CTL_IDX_PW_1,
3517                         .hsw.has_fuses = true,
3518                 },
3519         },
3520         {
3521                 .name = "DC off",
3522                 .domains = BXT_DISPLAY_DC_OFF_POWER_DOMAINS,
3523                 .ops = &gen9_dc_off_power_well_ops,
3524                 .id = SKL_DISP_DC_OFF,
3525         },
3526         {
3527                 .name = "power well 2",
3528                 .domains = BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS,
3529                 .ops = &hsw_power_well_ops,
3530                 .id = SKL_DISP_PW_2,
3531                 {
3532                         .hsw.regs = &hsw_power_well_regs,
3533                         .hsw.idx = SKL_PW_CTL_IDX_PW_2,
3534                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
3535                         .hsw.has_vga = true,
3536                         .hsw.has_fuses = true,
3537                 },
3538         },
3539         {
3540                 .name = "dpio-common-a",
3541                 .domains = BXT_DPIO_CMN_A_POWER_DOMAINS,
3542                 .ops = &bxt_dpio_cmn_power_well_ops,
3543                 .id = BXT_DISP_PW_DPIO_CMN_A,
3544                 {
3545                         .bxt.phy = DPIO_PHY1,
3546                 },
3547         },
3548         {
3549                 .name = "dpio-common-bc",
3550                 .domains = BXT_DPIO_CMN_BC_POWER_DOMAINS,
3551                 .ops = &bxt_dpio_cmn_power_well_ops,
3552                 .id = VLV_DISP_PW_DPIO_CMN_BC,
3553                 {
3554                         .bxt.phy = DPIO_PHY0,
3555                 },
3556         },
3557 };
3558
3559 static const struct i915_power_well_desc glk_power_wells[] = {
3560         {
3561                 .name = "always-on",
3562                 .always_on = true,
3563                 .domains = POWER_DOMAIN_MASK,
3564                 .ops = &i9xx_always_on_power_well_ops,
3565                 .id = DISP_PW_ID_NONE,
3566         },
3567         {
3568                 .name = "power well 1",
3569                 /* Handled by the DMC firmware */
3570                 .always_on = true,
3571                 .domains = 0,
3572                 .ops = &hsw_power_well_ops,
3573                 .id = SKL_DISP_PW_1,
3574                 {
3575                         .hsw.regs = &hsw_power_well_regs,
3576                         .hsw.idx = SKL_PW_CTL_IDX_PW_1,
3577                         .hsw.has_fuses = true,
3578                 },
3579         },
3580         {
3581                 .name = "DC off",
3582                 .domains = GLK_DISPLAY_DC_OFF_POWER_DOMAINS,
3583                 .ops = &gen9_dc_off_power_well_ops,
3584                 .id = SKL_DISP_DC_OFF,
3585         },
3586         {
3587                 .name = "power well 2",
3588                 .domains = GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS,
3589                 .ops = &hsw_power_well_ops,
3590                 .id = SKL_DISP_PW_2,
3591                 {
3592                         .hsw.regs = &hsw_power_well_regs,
3593                         .hsw.idx = SKL_PW_CTL_IDX_PW_2,
3594                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
3595                         .hsw.has_vga = true,
3596                         .hsw.has_fuses = true,
3597                 },
3598         },
3599         {
3600                 .name = "dpio-common-a",
3601                 .domains = GLK_DPIO_CMN_A_POWER_DOMAINS,
3602                 .ops = &bxt_dpio_cmn_power_well_ops,
3603                 .id = BXT_DISP_PW_DPIO_CMN_A,
3604                 {
3605                         .bxt.phy = DPIO_PHY1,
3606                 },
3607         },
3608         {
3609                 .name = "dpio-common-b",
3610                 .domains = GLK_DPIO_CMN_B_POWER_DOMAINS,
3611                 .ops = &bxt_dpio_cmn_power_well_ops,
3612                 .id = VLV_DISP_PW_DPIO_CMN_BC,
3613                 {
3614                         .bxt.phy = DPIO_PHY0,
3615                 },
3616         },
3617         {
3618                 .name = "dpio-common-c",
3619                 .domains = GLK_DPIO_CMN_C_POWER_DOMAINS,
3620                 .ops = &bxt_dpio_cmn_power_well_ops,
3621                 .id = GLK_DISP_PW_DPIO_CMN_C,
3622                 {
3623                         .bxt.phy = DPIO_PHY2,
3624                 },
3625         },
3626         {
3627                 .name = "AUX A",
3628                 .domains = GLK_DISPLAY_AUX_A_POWER_DOMAINS,
3629                 .ops = &hsw_power_well_ops,
3630                 .id = DISP_PW_ID_NONE,
3631                 {
3632                         .hsw.regs = &hsw_power_well_regs,
3633                         .hsw.idx = GLK_PW_CTL_IDX_AUX_A,
3634                 },
3635         },
3636         {
3637                 .name = "AUX B",
3638                 .domains = GLK_DISPLAY_AUX_B_POWER_DOMAINS,
3639                 .ops = &hsw_power_well_ops,
3640                 .id = DISP_PW_ID_NONE,
3641                 {
3642                         .hsw.regs = &hsw_power_well_regs,
3643                         .hsw.idx = GLK_PW_CTL_IDX_AUX_B,
3644                 },
3645         },
3646         {
3647                 .name = "AUX C",
3648                 .domains = GLK_DISPLAY_AUX_C_POWER_DOMAINS,
3649                 .ops = &hsw_power_well_ops,
3650                 .id = DISP_PW_ID_NONE,
3651                 {
3652                         .hsw.regs = &hsw_power_well_regs,
3653                         .hsw.idx = GLK_PW_CTL_IDX_AUX_C,
3654                 },
3655         },
3656         {
3657                 .name = "DDI A IO power well",
3658                 .domains = GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS,
3659                 .ops = &hsw_power_well_ops,
3660                 .id = DISP_PW_ID_NONE,
3661                 {
3662                         .hsw.regs = &hsw_power_well_regs,
3663                         .hsw.idx = GLK_PW_CTL_IDX_DDI_A,
3664                 },
3665         },
3666         {
3667                 .name = "DDI B IO power well",
3668                 .domains = GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS,
3669                 .ops = &hsw_power_well_ops,
3670                 .id = DISP_PW_ID_NONE,
3671                 {
3672                         .hsw.regs = &hsw_power_well_regs,
3673                         .hsw.idx = SKL_PW_CTL_IDX_DDI_B,
3674                 },
3675         },
3676         {
3677                 .name = "DDI C IO power well",
3678                 .domains = GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS,
3679                 .ops = &hsw_power_well_ops,
3680                 .id = DISP_PW_ID_NONE,
3681                 {
3682                         .hsw.regs = &hsw_power_well_regs,
3683                         .hsw.idx = SKL_PW_CTL_IDX_DDI_C,
3684                 },
3685         },
3686 };
3687
3688 static const struct i915_power_well_desc cnl_power_wells[] = {
3689         {
3690                 .name = "always-on",
3691                 .always_on = true,
3692                 .domains = POWER_DOMAIN_MASK,
3693                 .ops = &i9xx_always_on_power_well_ops,
3694                 .id = DISP_PW_ID_NONE,
3695         },
3696         {
3697                 .name = "power well 1",
3698                 /* Handled by the DMC firmware */
3699                 .always_on = true,
3700                 .domains = 0,
3701                 .ops = &hsw_power_well_ops,
3702                 .id = SKL_DISP_PW_1,
3703                 {
3704                         .hsw.regs = &hsw_power_well_regs,
3705                         .hsw.idx = SKL_PW_CTL_IDX_PW_1,
3706                         .hsw.has_fuses = true,
3707                 },
3708         },
3709         {
3710                 .name = "AUX A",
3711                 .domains = CNL_DISPLAY_AUX_A_POWER_DOMAINS,
3712                 .ops = &hsw_power_well_ops,
3713                 .id = DISP_PW_ID_NONE,
3714                 {
3715                         .hsw.regs = &hsw_power_well_regs,
3716                         .hsw.idx = GLK_PW_CTL_IDX_AUX_A,
3717                 },
3718         },
3719         {
3720                 .name = "AUX B",
3721                 .domains = CNL_DISPLAY_AUX_B_POWER_DOMAINS,
3722                 .ops = &hsw_power_well_ops,
3723                 .id = DISP_PW_ID_NONE,
3724                 {
3725                         .hsw.regs = &hsw_power_well_regs,
3726                         .hsw.idx = GLK_PW_CTL_IDX_AUX_B,
3727                 },
3728         },
3729         {
3730                 .name = "AUX C",
3731                 .domains = CNL_DISPLAY_AUX_C_POWER_DOMAINS,
3732                 .ops = &hsw_power_well_ops,
3733                 .id = DISP_PW_ID_NONE,
3734                 {
3735                         .hsw.regs = &hsw_power_well_regs,
3736                         .hsw.idx = GLK_PW_CTL_IDX_AUX_C,
3737                 },
3738         },
3739         {
3740                 .name = "AUX D",
3741                 .domains = CNL_DISPLAY_AUX_D_POWER_DOMAINS,
3742                 .ops = &hsw_power_well_ops,
3743                 .id = DISP_PW_ID_NONE,
3744                 {
3745                         .hsw.regs = &hsw_power_well_regs,
3746                         .hsw.idx = CNL_PW_CTL_IDX_AUX_D,
3747                 },
3748         },
3749         {
3750                 .name = "DC off",
3751                 .domains = CNL_DISPLAY_DC_OFF_POWER_DOMAINS,
3752                 .ops = &gen9_dc_off_power_well_ops,
3753                 .id = SKL_DISP_DC_OFF,
3754         },
3755         {
3756                 .name = "power well 2",
3757                 .domains = CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
3758                 .ops = &hsw_power_well_ops,
3759                 .id = SKL_DISP_PW_2,
3760                 {
3761                         .hsw.regs = &hsw_power_well_regs,
3762                         .hsw.idx = SKL_PW_CTL_IDX_PW_2,
3763                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
3764                         .hsw.has_vga = true,
3765                         .hsw.has_fuses = true,
3766                 },
3767         },
3768         {
3769                 .name = "DDI A IO power well",
3770                 .domains = CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS,
3771                 .ops = &hsw_power_well_ops,
3772                 .id = DISP_PW_ID_NONE,
3773                 {
3774                         .hsw.regs = &hsw_power_well_regs,
3775                         .hsw.idx = GLK_PW_CTL_IDX_DDI_A,
3776                 },
3777         },
3778         {
3779                 .name = "DDI B IO power well",
3780                 .domains = CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS,
3781                 .ops = &hsw_power_well_ops,
3782                 .id = DISP_PW_ID_NONE,
3783                 {
3784                         .hsw.regs = &hsw_power_well_regs,
3785                         .hsw.idx = SKL_PW_CTL_IDX_DDI_B,
3786                 },
3787         },
3788         {
3789                 .name = "DDI C IO power well",
3790                 .domains = CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS,
3791                 .ops = &hsw_power_well_ops,
3792                 .id = DISP_PW_ID_NONE,
3793                 {
3794                         .hsw.regs = &hsw_power_well_regs,
3795                         .hsw.idx = SKL_PW_CTL_IDX_DDI_C,
3796                 },
3797         },
3798         {
3799                 .name = "DDI D IO power well",
3800                 .domains = CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS,
3801                 .ops = &hsw_power_well_ops,
3802                 .id = DISP_PW_ID_NONE,
3803                 {
3804                         .hsw.regs = &hsw_power_well_regs,
3805                         .hsw.idx = SKL_PW_CTL_IDX_DDI_D,
3806                 },
3807         },
3808         {
3809                 .name = "DDI F IO power well",
3810                 .domains = CNL_DISPLAY_DDI_F_IO_POWER_DOMAINS,
3811                 .ops = &hsw_power_well_ops,
3812                 .id = CNL_DISP_PW_DDI_F_IO,
3813                 {
3814                         .hsw.regs = &hsw_power_well_regs,
3815                         .hsw.idx = CNL_PW_CTL_IDX_DDI_F,
3816                 },
3817         },
3818         {
3819                 .name = "AUX F",
3820                 .domains = CNL_DISPLAY_AUX_F_POWER_DOMAINS,
3821                 .ops = &hsw_power_well_ops,
3822                 .id = CNL_DISP_PW_DDI_F_AUX,
3823                 {
3824                         .hsw.regs = &hsw_power_well_regs,
3825                         .hsw.idx = CNL_PW_CTL_IDX_AUX_F,
3826                 },
3827         },
3828 };
3829
3830 static const struct i915_power_well_ops icl_aux_power_well_ops = {
3831         .sync_hw = hsw_power_well_sync_hw,
3832         .enable = icl_aux_power_well_enable,
3833         .disable = icl_aux_power_well_disable,
3834         .is_enabled = hsw_power_well_enabled,
3835 };
3836
3837 static const struct i915_power_well_regs icl_aux_power_well_regs = {
3838         .bios   = ICL_PWR_WELL_CTL_AUX1,
3839         .driver = ICL_PWR_WELL_CTL_AUX2,
3840         .debug  = ICL_PWR_WELL_CTL_AUX4,
3841 };
3842
3843 static const struct i915_power_well_regs icl_ddi_power_well_regs = {
3844         .bios   = ICL_PWR_WELL_CTL_DDI1,
3845         .driver = ICL_PWR_WELL_CTL_DDI2,
3846         .debug  = ICL_PWR_WELL_CTL_DDI4,
3847 };
3848
3849 static const struct i915_power_well_desc icl_power_wells[] = {
3850         {
3851                 .name = "always-on",
3852                 .always_on = true,
3853                 .domains = POWER_DOMAIN_MASK,
3854                 .ops = &i9xx_always_on_power_well_ops,
3855                 .id = DISP_PW_ID_NONE,
3856         },
3857         {
3858                 .name = "power well 1",
3859                 /* Handled by the DMC firmware */
3860                 .always_on = true,
3861                 .domains = 0,
3862                 .ops = &hsw_power_well_ops,
3863                 .id = SKL_DISP_PW_1,
3864                 {
3865                         .hsw.regs = &hsw_power_well_regs,
3866                         .hsw.idx = ICL_PW_CTL_IDX_PW_1,
3867                         .hsw.has_fuses = true,
3868                 },
3869         },
3870         {
3871                 .name = "DC off",
3872                 .domains = ICL_DISPLAY_DC_OFF_POWER_DOMAINS,
3873                 .ops = &gen9_dc_off_power_well_ops,
3874                 .id = SKL_DISP_DC_OFF,
3875         },
3876         {
3877                 .name = "power well 2",
3878                 .domains = ICL_PW_2_POWER_DOMAINS,
3879                 .ops = &hsw_power_well_ops,
3880                 .id = SKL_DISP_PW_2,
3881                 {
3882                         .hsw.regs = &hsw_power_well_regs,
3883                         .hsw.idx = ICL_PW_CTL_IDX_PW_2,
3884                         .hsw.has_fuses = true,
3885                 },
3886         },
3887         {
3888                 .name = "power well 3",
3889                 .domains = ICL_PW_3_POWER_DOMAINS,
3890                 .ops = &hsw_power_well_ops,
3891                 .id = ICL_DISP_PW_3,
3892                 {
3893                         .hsw.regs = &hsw_power_well_regs,
3894                         .hsw.idx = ICL_PW_CTL_IDX_PW_3,
3895                         .hsw.irq_pipe_mask = BIT(PIPE_B),
3896                         .hsw.has_vga = true,
3897                         .hsw.has_fuses = true,
3898                 },
3899         },
3900         {
3901                 .name = "DDI A IO",
3902                 .domains = ICL_DDI_IO_A_POWER_DOMAINS,
3903                 .ops = &hsw_power_well_ops,
3904                 .id = DISP_PW_ID_NONE,
3905                 {
3906                         .hsw.regs = &icl_ddi_power_well_regs,
3907                         .hsw.idx = ICL_PW_CTL_IDX_DDI_A,
3908                 },
3909         },
3910         {
3911                 .name = "DDI B IO",
3912                 .domains = ICL_DDI_IO_B_POWER_DOMAINS,
3913                 .ops = &hsw_power_well_ops,
3914                 .id = DISP_PW_ID_NONE,
3915                 {
3916                         .hsw.regs = &icl_ddi_power_well_regs,
3917                         .hsw.idx = ICL_PW_CTL_IDX_DDI_B,
3918                 },
3919         },
3920         {
3921                 .name = "DDI C IO",
3922                 .domains = ICL_DDI_IO_C_POWER_DOMAINS,
3923                 .ops = &hsw_power_well_ops,
3924                 .id = DISP_PW_ID_NONE,
3925                 {
3926                         .hsw.regs = &icl_ddi_power_well_regs,
3927                         .hsw.idx = ICL_PW_CTL_IDX_DDI_C,
3928                 },
3929         },
3930         {
3931                 .name = "DDI D IO",
3932                 .domains = ICL_DDI_IO_D_POWER_DOMAINS,
3933                 .ops = &hsw_power_well_ops,
3934                 .id = DISP_PW_ID_NONE,
3935                 {
3936                         .hsw.regs = &icl_ddi_power_well_regs,
3937                         .hsw.idx = ICL_PW_CTL_IDX_DDI_D,
3938                 },
3939         },
3940         {
3941                 .name = "DDI E IO",
3942                 .domains = ICL_DDI_IO_E_POWER_DOMAINS,
3943                 .ops = &hsw_power_well_ops,
3944                 .id = DISP_PW_ID_NONE,
3945                 {
3946                         .hsw.regs = &icl_ddi_power_well_regs,
3947                         .hsw.idx = ICL_PW_CTL_IDX_DDI_E,
3948                 },
3949         },
3950         {
3951                 .name = "DDI F IO",
3952                 .domains = ICL_DDI_IO_F_POWER_DOMAINS,
3953                 .ops = &hsw_power_well_ops,
3954                 .id = DISP_PW_ID_NONE,
3955                 {
3956                         .hsw.regs = &icl_ddi_power_well_regs,
3957                         .hsw.idx = ICL_PW_CTL_IDX_DDI_F,
3958                 },
3959         },
3960         {
3961                 .name = "AUX A",
3962                 .domains = ICL_AUX_A_IO_POWER_DOMAINS,
3963                 .ops = &icl_aux_power_well_ops,
3964                 .id = DISP_PW_ID_NONE,
3965                 {
3966                         .hsw.regs = &icl_aux_power_well_regs,
3967                         .hsw.idx = ICL_PW_CTL_IDX_AUX_A,
3968                 },
3969         },
3970         {
3971                 .name = "AUX B",
3972                 .domains = ICL_AUX_B_IO_POWER_DOMAINS,
3973                 .ops = &icl_aux_power_well_ops,
3974                 .id = DISP_PW_ID_NONE,
3975                 {
3976                         .hsw.regs = &icl_aux_power_well_regs,
3977                         .hsw.idx = ICL_PW_CTL_IDX_AUX_B,
3978                 },
3979         },
3980         {
3981                 .name = "AUX C TC1",
3982                 .domains = ICL_AUX_C_TC1_IO_POWER_DOMAINS,
3983                 .ops = &icl_aux_power_well_ops,
3984                 .id = DISP_PW_ID_NONE,
3985                 {
3986                         .hsw.regs = &icl_aux_power_well_regs,
3987                         .hsw.idx = ICL_PW_CTL_IDX_AUX_C,
3988                         .hsw.is_tc_tbt = false,
3989                 },
3990         },
3991         {
3992                 .name = "AUX D TC2",
3993                 .domains = ICL_AUX_D_TC2_IO_POWER_DOMAINS,
3994                 .ops = &icl_aux_power_well_ops,
3995                 .id = DISP_PW_ID_NONE,
3996                 {
3997                         .hsw.regs = &icl_aux_power_well_regs,
3998                         .hsw.idx = ICL_PW_CTL_IDX_AUX_D,
3999                         .hsw.is_tc_tbt = false,
4000                 },
4001         },
4002         {
4003                 .name = "AUX E TC3",
4004                 .domains = ICL_AUX_E_TC3_IO_POWER_DOMAINS,
4005                 .ops = &icl_aux_power_well_ops,
4006                 .id = DISP_PW_ID_NONE,
4007                 {
4008                         .hsw.regs = &icl_aux_power_well_regs,
4009                         .hsw.idx = ICL_PW_CTL_IDX_AUX_E,
4010                         .hsw.is_tc_tbt = false,
4011                 },
4012         },
4013         {
4014                 .name = "AUX F TC4",
4015                 .domains = ICL_AUX_F_TC4_IO_POWER_DOMAINS,
4016                 .ops = &icl_aux_power_well_ops,
4017                 .id = DISP_PW_ID_NONE,
4018                 {
4019                         .hsw.regs = &icl_aux_power_well_regs,
4020                         .hsw.idx = ICL_PW_CTL_IDX_AUX_F,
4021                         .hsw.is_tc_tbt = false,
4022                 },
4023         },
4024         {
4025                 .name = "AUX C TBT1",
4026                 .domains = ICL_AUX_C_TBT1_IO_POWER_DOMAINS,
4027                 .ops = &icl_aux_power_well_ops,
4028                 .id = DISP_PW_ID_NONE,
4029                 {
4030                         .hsw.regs = &icl_aux_power_well_regs,
4031                         .hsw.idx = ICL_PW_CTL_IDX_AUX_TBT1,
4032                         .hsw.is_tc_tbt = true,
4033                 },
4034         },
4035         {
4036                 .name = "AUX D TBT2",
4037                 .domains = ICL_AUX_D_TBT2_IO_POWER_DOMAINS,
4038                 .ops = &icl_aux_power_well_ops,
4039                 .id = DISP_PW_ID_NONE,
4040                 {
4041                         .hsw.regs = &icl_aux_power_well_regs,
4042                         .hsw.idx = ICL_PW_CTL_IDX_AUX_TBT2,
4043                         .hsw.is_tc_tbt = true,
4044                 },
4045         },
4046         {
4047                 .name = "AUX E TBT3",
4048                 .domains = ICL_AUX_E_TBT3_IO_POWER_DOMAINS,
4049                 .ops = &icl_aux_power_well_ops,
4050                 .id = DISP_PW_ID_NONE,
4051                 {
4052                         .hsw.regs = &icl_aux_power_well_regs,
4053                         .hsw.idx = ICL_PW_CTL_IDX_AUX_TBT3,
4054                         .hsw.is_tc_tbt = true,
4055                 },
4056         },
4057         {
4058                 .name = "AUX F TBT4",
4059                 .domains = ICL_AUX_F_TBT4_IO_POWER_DOMAINS,
4060                 .ops = &icl_aux_power_well_ops,
4061                 .id = DISP_PW_ID_NONE,
4062                 {
4063                         .hsw.regs = &icl_aux_power_well_regs,
4064                         .hsw.idx = ICL_PW_CTL_IDX_AUX_TBT4,
4065                         .hsw.is_tc_tbt = true,
4066                 },
4067         },
4068         {
4069                 .name = "power well 4",
4070                 .domains = ICL_PW_4_POWER_DOMAINS,
4071                 .ops = &hsw_power_well_ops,
4072                 .id = DISP_PW_ID_NONE,
4073                 {
4074                         .hsw.regs = &hsw_power_well_regs,
4075                         .hsw.idx = ICL_PW_CTL_IDX_PW_4,
4076                         .hsw.has_fuses = true,
4077                         .hsw.irq_pipe_mask = BIT(PIPE_C),
4078                 },
4079         },
4080 };
4081
4082 static void
4083 tgl_tc_cold_request(struct drm_i915_private *i915, bool block)
4084 {
4085         u8 tries = 0;
4086         int ret;
4087
4088         while (1) {
4089                 u32 low_val;
4090                 u32 high_val = 0;
4091
4092                 if (block)
4093                         low_val = TGL_PCODE_EXIT_TCCOLD_DATA_L_BLOCK_REQ;
4094                 else
4095                         low_val = TGL_PCODE_EXIT_TCCOLD_DATA_L_UNBLOCK_REQ;
4096
4097                 /*
4098                  * Spec states that we should timeout the request after 200us
4099                  * but the function below will timeout after 500us
4100                  */
4101                 ret = sandybridge_pcode_read(i915, TGL_PCODE_TCCOLD, &low_val,
4102                                              &high_val);
4103                 if (ret == 0) {
4104                         if (block &&
4105                             (low_val & TGL_PCODE_EXIT_TCCOLD_DATA_L_EXIT_FAILED))
4106                                 ret = -EIO;
4107                         else
4108                                 break;
4109                 }
4110
4111                 if (++tries == 3)
4112                         break;
4113
4114                 msleep(1);
4115         }
4116
4117         if (ret)
4118                 drm_err(&i915->drm, "TC cold %sblock failed\n",
4119                         block ? "" : "un");
4120         else
4121                 drm_dbg_kms(&i915->drm, "TC cold %sblock succeeded\n",
4122                             block ? "" : "un");
4123 }
4124
4125 static void
4126 tgl_tc_cold_off_power_well_enable(struct drm_i915_private *i915,
4127                                   struct i915_power_well *power_well)
4128 {
4129         tgl_tc_cold_request(i915, true);
4130 }
4131
4132 static void
4133 tgl_tc_cold_off_power_well_disable(struct drm_i915_private *i915,
4134                                    struct i915_power_well *power_well)
4135 {
4136         tgl_tc_cold_request(i915, false);
4137 }
4138
4139 static void
4140 tgl_tc_cold_off_power_well_sync_hw(struct drm_i915_private *i915,
4141                                    struct i915_power_well *power_well)
4142 {
4143         if (power_well->count > 0)
4144                 tgl_tc_cold_off_power_well_enable(i915, power_well);
4145         else
4146                 tgl_tc_cold_off_power_well_disable(i915, power_well);
4147 }
4148
4149 static bool
4150 tgl_tc_cold_off_power_well_is_enabled(struct drm_i915_private *dev_priv,
4151                                       struct i915_power_well *power_well)
4152 {
4153         /*
4154          * Not the correctly implementation but there is no way to just read it
4155          * from PCODE, so returning count to avoid state mismatch errors
4156          */
4157         return power_well->count;
4158 }
4159
4160 static const struct i915_power_well_ops tgl_tc_cold_off_ops = {
4161         .sync_hw = tgl_tc_cold_off_power_well_sync_hw,
4162         .enable = tgl_tc_cold_off_power_well_enable,
4163         .disable = tgl_tc_cold_off_power_well_disable,
4164         .is_enabled = tgl_tc_cold_off_power_well_is_enabled,
4165 };
4166
4167 static const struct i915_power_well_desc tgl_power_wells[] = {
4168         {
4169                 .name = "always-on",
4170                 .always_on = true,
4171                 .domains = POWER_DOMAIN_MASK,
4172                 .ops = &i9xx_always_on_power_well_ops,
4173                 .id = DISP_PW_ID_NONE,
4174         },
4175         {
4176                 .name = "power well 1",
4177                 /* Handled by the DMC firmware */
4178                 .always_on = true,
4179                 .domains = 0,
4180                 .ops = &hsw_power_well_ops,
4181                 .id = SKL_DISP_PW_1,
4182                 {
4183                         .hsw.regs = &hsw_power_well_regs,
4184                         .hsw.idx = ICL_PW_CTL_IDX_PW_1,
4185                         .hsw.has_fuses = true,
4186                 },
4187         },
4188         {
4189                 .name = "DC off",
4190                 .domains = TGL_DISPLAY_DC_OFF_POWER_DOMAINS,
4191                 .ops = &gen9_dc_off_power_well_ops,
4192                 .id = SKL_DISP_DC_OFF,
4193         },
4194         {
4195                 .name = "power well 2",
4196                 .domains = TGL_PW_2_POWER_DOMAINS,
4197                 .ops = &hsw_power_well_ops,
4198                 .id = SKL_DISP_PW_2,
4199                 {
4200                         .hsw.regs = &hsw_power_well_regs,
4201                         .hsw.idx = ICL_PW_CTL_IDX_PW_2,
4202                         .hsw.has_fuses = true,
4203                 },
4204         },
4205         {
4206                 .name = "power well 3",
4207                 .domains = TGL_PW_3_POWER_DOMAINS,
4208                 .ops = &hsw_power_well_ops,
4209                 .id = ICL_DISP_PW_3,
4210                 {
4211                         .hsw.regs = &hsw_power_well_regs,
4212                         .hsw.idx = ICL_PW_CTL_IDX_PW_3,
4213                         .hsw.irq_pipe_mask = BIT(PIPE_B),
4214                         .hsw.has_vga = true,
4215                         .hsw.has_fuses = true,
4216                 },
4217         },
4218         {
4219                 .name = "DDI A IO",
4220                 .domains = ICL_DDI_IO_A_POWER_DOMAINS,
4221                 .ops = &hsw_power_well_ops,
4222                 .id = DISP_PW_ID_NONE,
4223                 {
4224                         .hsw.regs = &icl_ddi_power_well_regs,
4225                         .hsw.idx = ICL_PW_CTL_IDX_DDI_A,
4226                 }
4227         },
4228         {
4229                 .name = "DDI B IO",
4230                 .domains = ICL_DDI_IO_B_POWER_DOMAINS,
4231                 .ops = &hsw_power_well_ops,
4232                 .id = DISP_PW_ID_NONE,
4233                 {
4234                         .hsw.regs = &icl_ddi_power_well_regs,
4235                         .hsw.idx = ICL_PW_CTL_IDX_DDI_B,
4236                 }
4237         },
4238         {
4239                 .name = "DDI C IO",
4240                 .domains = ICL_DDI_IO_C_POWER_DOMAINS,
4241                 .ops = &hsw_power_well_ops,
4242                 .id = DISP_PW_ID_NONE,
4243                 {
4244                         .hsw.regs = &icl_ddi_power_well_regs,
4245                         .hsw.idx = ICL_PW_CTL_IDX_DDI_C,
4246                 }
4247         },
4248         {
4249                 .name = "DDI IO TC1",
4250                 .domains = TGL_DDI_IO_TC1_POWER_DOMAINS,
4251                 .ops = &hsw_power_well_ops,
4252                 .id = DISP_PW_ID_NONE,
4253                 {
4254                         .hsw.regs = &icl_ddi_power_well_regs,
4255                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC1,
4256                 },
4257         },
4258         {
4259                 .name = "DDI IO TC2",
4260                 .domains = TGL_DDI_IO_TC2_POWER_DOMAINS,
4261                 .ops = &hsw_power_well_ops,
4262                 .id = DISP_PW_ID_NONE,
4263                 {
4264                         .hsw.regs = &icl_ddi_power_well_regs,
4265                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC2,
4266                 },
4267         },
4268         {
4269                 .name = "DDI IO TC3",
4270                 .domains = TGL_DDI_IO_TC3_POWER_DOMAINS,
4271                 .ops = &hsw_power_well_ops,
4272                 .id = DISP_PW_ID_NONE,
4273                 {
4274                         .hsw.regs = &icl_ddi_power_well_regs,
4275                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC3,
4276                 },
4277         },
4278         {
4279                 .name = "DDI IO TC4",
4280                 .domains = TGL_DDI_IO_TC4_POWER_DOMAINS,
4281                 .ops = &hsw_power_well_ops,
4282                 .id = DISP_PW_ID_NONE,
4283                 {
4284                         .hsw.regs = &icl_ddi_power_well_regs,
4285                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC4,
4286                 },
4287         },
4288         {
4289                 .name = "DDI IO TC5",
4290                 .domains = TGL_DDI_IO_TC5_POWER_DOMAINS,
4291                 .ops = &hsw_power_well_ops,
4292                 .id = DISP_PW_ID_NONE,
4293                 {
4294                         .hsw.regs = &icl_ddi_power_well_regs,
4295                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC5,
4296                 },
4297         },
4298         {
4299                 .name = "DDI IO TC6",
4300                 .domains = TGL_DDI_IO_TC6_POWER_DOMAINS,
4301                 .ops = &hsw_power_well_ops,
4302                 .id = DISP_PW_ID_NONE,
4303                 {
4304                         .hsw.regs = &icl_ddi_power_well_regs,
4305                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC6,
4306                 },
4307         },
4308         {
4309                 .name = "TC cold off",
4310                 .domains = TGL_TC_COLD_OFF_POWER_DOMAINS,
4311                 .ops = &tgl_tc_cold_off_ops,
4312                 .id = TGL_DISP_PW_TC_COLD_OFF,
4313         },
4314         {
4315                 .name = "AUX A",
4316                 .domains = TGL_AUX_A_IO_POWER_DOMAINS,
4317                 .ops = &icl_aux_power_well_ops,
4318                 .id = DISP_PW_ID_NONE,
4319                 {
4320                         .hsw.regs = &icl_aux_power_well_regs,
4321                         .hsw.idx = ICL_PW_CTL_IDX_AUX_A,
4322                 },
4323         },
4324         {
4325                 .name = "AUX B",
4326                 .domains = TGL_AUX_B_IO_POWER_DOMAINS,
4327                 .ops = &icl_aux_power_well_ops,
4328                 .id = DISP_PW_ID_NONE,
4329                 {
4330                         .hsw.regs = &icl_aux_power_well_regs,
4331                         .hsw.idx = ICL_PW_CTL_IDX_AUX_B,
4332                 },
4333         },
4334         {
4335                 .name = "AUX C",
4336                 .domains = TGL_AUX_C_IO_POWER_DOMAINS,
4337                 .ops = &icl_aux_power_well_ops,
4338                 .id = DISP_PW_ID_NONE,
4339                 {
4340                         .hsw.regs = &icl_aux_power_well_regs,
4341                         .hsw.idx = ICL_PW_CTL_IDX_AUX_C,
4342                 },
4343         },
4344         {
4345                 .name = "AUX USBC1",
4346                 .domains = TGL_AUX_IO_USBC1_POWER_DOMAINS,
4347                 .ops = &icl_aux_power_well_ops,
4348                 .id = DISP_PW_ID_NONE,
4349                 {
4350                         .hsw.regs = &icl_aux_power_well_regs,
4351                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC1,
4352                         .hsw.is_tc_tbt = false,
4353                 },
4354         },
4355         {
4356                 .name = "AUX USBC2",
4357                 .domains = TGL_AUX_IO_USBC2_POWER_DOMAINS,
4358                 .ops = &icl_aux_power_well_ops,
4359                 .id = DISP_PW_ID_NONE,
4360                 {
4361                         .hsw.regs = &icl_aux_power_well_regs,
4362                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC2,
4363                         .hsw.is_tc_tbt = false,
4364                 },
4365         },
4366         {
4367                 .name = "AUX USBC3",
4368                 .domains = TGL_AUX_IO_USBC3_POWER_DOMAINS,
4369                 .ops = &icl_aux_power_well_ops,
4370                 .id = DISP_PW_ID_NONE,
4371                 {
4372                         .hsw.regs = &icl_aux_power_well_regs,
4373                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC3,
4374                         .hsw.is_tc_tbt = false,
4375                 },
4376         },
4377         {
4378                 .name = "AUX USBC4",
4379                 .domains = TGL_AUX_IO_USBC4_POWER_DOMAINS,
4380                 .ops = &icl_aux_power_well_ops,
4381                 .id = DISP_PW_ID_NONE,
4382                 {
4383                         .hsw.regs = &icl_aux_power_well_regs,
4384                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC4,
4385                         .hsw.is_tc_tbt = false,
4386                 },
4387         },
4388         {
4389                 .name = "AUX USBC5",
4390                 .domains = TGL_AUX_IO_USBC5_POWER_DOMAINS,
4391                 .ops = &icl_aux_power_well_ops,
4392                 .id = DISP_PW_ID_NONE,
4393                 {
4394                         .hsw.regs = &icl_aux_power_well_regs,
4395                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC5,
4396                         .hsw.is_tc_tbt = false,
4397                 },
4398         },
4399         {
4400                 .name = "AUX USBC6",
4401                 .domains = TGL_AUX_IO_USBC6_POWER_DOMAINS,
4402                 .ops = &icl_aux_power_well_ops,
4403                 .id = DISP_PW_ID_NONE,
4404                 {
4405                         .hsw.regs = &icl_aux_power_well_regs,
4406                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC6,
4407                         .hsw.is_tc_tbt = false,
4408                 },
4409         },
4410         {
4411                 .name = "AUX TBT1",
4412                 .domains = TGL_AUX_IO_TBT1_POWER_DOMAINS,
4413                 .ops = &icl_aux_power_well_ops,
4414                 .id = DISP_PW_ID_NONE,
4415                 {
4416                         .hsw.regs = &icl_aux_power_well_regs,
4417                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT1,
4418                         .hsw.is_tc_tbt = true,
4419                 },
4420         },
4421         {
4422                 .name = "AUX TBT2",
4423                 .domains = TGL_AUX_IO_TBT2_POWER_DOMAINS,
4424                 .ops = &icl_aux_power_well_ops,
4425                 .id = DISP_PW_ID_NONE,
4426                 {
4427                         .hsw.regs = &icl_aux_power_well_regs,
4428                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT2,
4429                         .hsw.is_tc_tbt = true,
4430                 },
4431         },
4432         {
4433                 .name = "AUX TBT3",
4434                 .domains = TGL_AUX_IO_TBT3_POWER_DOMAINS,
4435                 .ops = &icl_aux_power_well_ops,
4436                 .id = DISP_PW_ID_NONE,
4437                 {
4438                         .hsw.regs = &icl_aux_power_well_regs,
4439                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT3,
4440                         .hsw.is_tc_tbt = true,
4441                 },
4442         },
4443         {
4444                 .name = "AUX TBT4",
4445                 .domains = TGL_AUX_IO_TBT4_POWER_DOMAINS,
4446                 .ops = &icl_aux_power_well_ops,
4447                 .id = DISP_PW_ID_NONE,
4448                 {
4449                         .hsw.regs = &icl_aux_power_well_regs,
4450                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT4,
4451                         .hsw.is_tc_tbt = true,
4452                 },
4453         },
4454         {
4455                 .name = "AUX TBT5",
4456                 .domains = TGL_AUX_IO_TBT5_POWER_DOMAINS,
4457                 .ops = &icl_aux_power_well_ops,
4458                 .id = DISP_PW_ID_NONE,
4459                 {
4460                         .hsw.regs = &icl_aux_power_well_regs,
4461                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT5,
4462                         .hsw.is_tc_tbt = true,
4463                 },
4464         },
4465         {
4466                 .name = "AUX TBT6",
4467                 .domains = TGL_AUX_IO_TBT6_POWER_DOMAINS,
4468                 .ops = &icl_aux_power_well_ops,
4469                 .id = DISP_PW_ID_NONE,
4470                 {
4471                         .hsw.regs = &icl_aux_power_well_regs,
4472                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT6,
4473                         .hsw.is_tc_tbt = true,
4474                 },
4475         },
4476         {
4477                 .name = "power well 4",
4478                 .domains = TGL_PW_4_POWER_DOMAINS,
4479                 .ops = &hsw_power_well_ops,
4480                 .id = DISP_PW_ID_NONE,
4481                 {
4482                         .hsw.regs = &hsw_power_well_regs,
4483                         .hsw.idx = ICL_PW_CTL_IDX_PW_4,
4484                         .hsw.has_fuses = true,
4485                         .hsw.irq_pipe_mask = BIT(PIPE_C),
4486                 }
4487         },
4488         {
4489                 .name = "power well 5",
4490                 .domains = TGL_PW_5_POWER_DOMAINS,
4491                 .ops = &hsw_power_well_ops,
4492                 .id = DISP_PW_ID_NONE,
4493                 {
4494                         .hsw.regs = &hsw_power_well_regs,
4495                         .hsw.idx = TGL_PW_CTL_IDX_PW_5,
4496                         .hsw.has_fuses = true,
4497                         .hsw.irq_pipe_mask = BIT(PIPE_D),
4498                 },
4499         },
4500 };
4501
4502 static const struct i915_power_well_desc rkl_power_wells[] = {
4503         {
4504                 .name = "always-on",
4505                 .always_on = true,
4506                 .domains = POWER_DOMAIN_MASK,
4507                 .ops = &i9xx_always_on_power_well_ops,
4508                 .id = DISP_PW_ID_NONE,
4509         },
4510         {
4511                 .name = "power well 1",
4512                 /* Handled by the DMC firmware */
4513                 .always_on = true,
4514                 .domains = 0,
4515                 .ops = &hsw_power_well_ops,
4516                 .id = SKL_DISP_PW_1,
4517                 {
4518                         .hsw.regs = &hsw_power_well_regs,
4519                         .hsw.idx = ICL_PW_CTL_IDX_PW_1,
4520                         .hsw.has_fuses = true,
4521                 },
4522         },
4523         {
4524                 .name = "DC off",
4525                 .domains = RKL_DISPLAY_DC_OFF_POWER_DOMAINS,
4526                 .ops = &gen9_dc_off_power_well_ops,
4527                 .id = SKL_DISP_DC_OFF,
4528         },
4529         {
4530                 .name = "power well 3",
4531                 .domains = RKL_PW_3_POWER_DOMAINS,
4532                 .ops = &hsw_power_well_ops,
4533                 .id = ICL_DISP_PW_3,
4534                 {
4535                         .hsw.regs = &hsw_power_well_regs,
4536                         .hsw.idx = ICL_PW_CTL_IDX_PW_3,
4537                         .hsw.irq_pipe_mask = BIT(PIPE_B),
4538                         .hsw.has_vga = true,
4539                         .hsw.has_fuses = true,
4540                 },
4541         },
4542         {
4543                 .name = "power well 4",
4544                 .domains = RKL_PW_4_POWER_DOMAINS,
4545                 .ops = &hsw_power_well_ops,
4546                 .id = DISP_PW_ID_NONE,
4547                 {
4548                         .hsw.regs = &hsw_power_well_regs,
4549                         .hsw.idx = ICL_PW_CTL_IDX_PW_4,
4550                         .hsw.has_fuses = true,
4551                         .hsw.irq_pipe_mask = BIT(PIPE_C),
4552                 }
4553         },
4554         {
4555                 .name = "DDI A IO",
4556                 .domains = ICL_DDI_IO_A_POWER_DOMAINS,
4557                 .ops = &hsw_power_well_ops,
4558                 .id = DISP_PW_ID_NONE,
4559                 {
4560                         .hsw.regs = &icl_ddi_power_well_regs,
4561                         .hsw.idx = ICL_PW_CTL_IDX_DDI_A,
4562                 }
4563         },
4564         {
4565                 .name = "DDI B IO",
4566                 .domains = ICL_DDI_IO_B_POWER_DOMAINS,
4567                 .ops = &hsw_power_well_ops,
4568                 .id = DISP_PW_ID_NONE,
4569                 {
4570                         .hsw.regs = &icl_ddi_power_well_regs,
4571                         .hsw.idx = ICL_PW_CTL_IDX_DDI_B,
4572                 }
4573         },
4574         {
4575                 .name = "DDI IO TC1",
4576                 .domains = TGL_DDI_IO_TC1_POWER_DOMAINS,
4577                 .ops = &hsw_power_well_ops,
4578                 .id = DISP_PW_ID_NONE,
4579                 {
4580                         .hsw.regs = &icl_ddi_power_well_regs,
4581                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC1,
4582                 },
4583         },
4584         {
4585                 .name = "DDI IO TC2",
4586                 .domains = TGL_DDI_IO_TC2_POWER_DOMAINS,
4587                 .ops = &hsw_power_well_ops,
4588                 .id = DISP_PW_ID_NONE,
4589                 {
4590                         .hsw.regs = &icl_ddi_power_well_regs,
4591                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC2,
4592                 },
4593         },
4594         {
4595                 .name = "AUX A",
4596                 .domains = ICL_AUX_A_IO_POWER_DOMAINS,
4597                 .ops = &icl_aux_power_well_ops,
4598                 .id = DISP_PW_ID_NONE,
4599                 {
4600                         .hsw.regs = &icl_aux_power_well_regs,
4601                         .hsw.idx = ICL_PW_CTL_IDX_AUX_A,
4602                 },
4603         },
4604         {
4605                 .name = "AUX B",
4606                 .domains = ICL_AUX_B_IO_POWER_DOMAINS,
4607                 .ops = &icl_aux_power_well_ops,
4608                 .id = DISP_PW_ID_NONE,
4609                 {
4610                         .hsw.regs = &icl_aux_power_well_regs,
4611                         .hsw.idx = ICL_PW_CTL_IDX_AUX_B,
4612                 },
4613         },
4614         {
4615                 .name = "AUX USBC1",
4616                 .domains = TGL_AUX_IO_USBC1_POWER_DOMAINS,
4617                 .ops = &icl_aux_power_well_ops,
4618                 .id = DISP_PW_ID_NONE,
4619                 {
4620                         .hsw.regs = &icl_aux_power_well_regs,
4621                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC1,
4622                 },
4623         },
4624         {
4625                 .name = "AUX USBC2",
4626                 .domains = TGL_AUX_IO_USBC2_POWER_DOMAINS,
4627                 .ops = &icl_aux_power_well_ops,
4628                 .id = DISP_PW_ID_NONE,
4629                 {
4630                         .hsw.regs = &icl_aux_power_well_regs,
4631                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC2,
4632                 },
4633         },
4634 };
4635
4636 static const struct i915_power_well_desc xelpd_power_wells[] = {
4637         {
4638                 .name = "always-on",
4639                 .always_on = true,
4640                 .domains = POWER_DOMAIN_MASK,
4641                 .ops = &i9xx_always_on_power_well_ops,
4642                 .id = DISP_PW_ID_NONE,
4643         },
4644         {
4645                 .name = "power well 1",
4646                 /* Handled by the DMC firmware */
4647                 .always_on = true,
4648                 .domains = 0,
4649                 .ops = &hsw_power_well_ops,
4650                 .id = SKL_DISP_PW_1,
4651                 {
4652                         .hsw.regs = &hsw_power_well_regs,
4653                         .hsw.idx = ICL_PW_CTL_IDX_PW_1,
4654                         .hsw.has_fuses = true,
4655                 },
4656         },
4657         {
4658                 .name = "DC off",
4659                 .domains = XELPD_DISPLAY_DC_OFF_POWER_DOMAINS,
4660                 .ops = &gen9_dc_off_power_well_ops,
4661                 .id = SKL_DISP_DC_OFF,
4662         },
4663         {
4664                 .name = "power well 2",
4665                 .domains = XELPD_PW_2_POWER_DOMAINS,
4666                 .ops = &hsw_power_well_ops,
4667                 .id = SKL_DISP_PW_2,
4668                 {
4669                         .hsw.regs = &hsw_power_well_regs,
4670                         .hsw.idx = ICL_PW_CTL_IDX_PW_2,
4671                         .hsw.has_vga = true,
4672                         .hsw.has_fuses = true,
4673                 },
4674         },
4675         {
4676                 .name = "power well A",
4677                 .domains = XELPD_PW_A_POWER_DOMAINS,
4678                 .ops = &hsw_power_well_ops,
4679                 .id = DISP_PW_ID_NONE,
4680                 {
4681                         .hsw.regs = &hsw_power_well_regs,
4682                         .hsw.idx = XELPD_PW_CTL_IDX_PW_A,
4683                         .hsw.irq_pipe_mask = BIT(PIPE_A),
4684                         .hsw.has_fuses = true,
4685                 },
4686         },
4687         {
4688                 .name = "power well B",
4689                 .domains = XELPD_PW_B_POWER_DOMAINS,
4690                 .ops = &hsw_power_well_ops,
4691                 .id = DISP_PW_ID_NONE,
4692                 {
4693                         .hsw.regs = &hsw_power_well_regs,
4694                         .hsw.idx = XELPD_PW_CTL_IDX_PW_B,
4695                         .hsw.irq_pipe_mask = BIT(PIPE_B),
4696                         .hsw.has_fuses = true,
4697                 },
4698         },
4699         {
4700                 .name = "power well C",
4701                 .domains = XELPD_PW_C_POWER_DOMAINS,
4702                 .ops = &hsw_power_well_ops,
4703                 .id = DISP_PW_ID_NONE,
4704                 {
4705                         .hsw.regs = &hsw_power_well_regs,
4706                         .hsw.idx = XELPD_PW_CTL_IDX_PW_C,
4707                         .hsw.irq_pipe_mask = BIT(PIPE_C),
4708                         .hsw.has_fuses = true,
4709                 },
4710         },
4711         {
4712                 .name = "power well D",
4713                 .domains = XELPD_PW_D_POWER_DOMAINS,
4714                 .ops = &hsw_power_well_ops,
4715                 .id = DISP_PW_ID_NONE,
4716                 {
4717                         .hsw.regs = &hsw_power_well_regs,
4718                         .hsw.idx = XELPD_PW_CTL_IDX_PW_D,
4719                         .hsw.irq_pipe_mask = BIT(PIPE_D),
4720                         .hsw.has_fuses = true,
4721                 },
4722         },
4723         {
4724                 .name = "DDI A IO",
4725                 .domains = ICL_DDI_IO_A_POWER_DOMAINS,
4726                 .ops = &hsw_power_well_ops,
4727                 .id = DISP_PW_ID_NONE,
4728                 {
4729                         .hsw.regs = &icl_ddi_power_well_regs,
4730                         .hsw.idx = ICL_PW_CTL_IDX_DDI_A,
4731                 }
4732         },
4733         {
4734                 .name = "DDI B IO",
4735                 .domains = ICL_DDI_IO_B_POWER_DOMAINS,
4736                 .ops = &hsw_power_well_ops,
4737                 .id = DISP_PW_ID_NONE,
4738                 {
4739                         .hsw.regs = &icl_ddi_power_well_regs,
4740                         .hsw.idx = ICL_PW_CTL_IDX_DDI_B,
4741                 }
4742         },
4743         {
4744                 .name = "DDI C IO",
4745                 .domains = ICL_DDI_IO_C_POWER_DOMAINS,
4746                 .ops = &hsw_power_well_ops,
4747                 .id = DISP_PW_ID_NONE,
4748                 {
4749                         .hsw.regs = &icl_ddi_power_well_regs,
4750                         .hsw.idx = ICL_PW_CTL_IDX_DDI_C,
4751                 }
4752         },
4753         {
4754                 .name = "DDI IO D_XELPD",
4755                 .domains = XELPD_DDI_IO_D_XELPD_POWER_DOMAINS,
4756                 .ops = &hsw_power_well_ops,
4757                 .id = DISP_PW_ID_NONE,
4758                 {
4759                         .hsw.regs = &icl_ddi_power_well_regs,
4760                         .hsw.idx = XELPD_PW_CTL_IDX_DDI_D,
4761                 }
4762         },
4763         {
4764                 .name = "DDI IO E_XELPD",
4765                 .domains = XELPD_DDI_IO_E_XELPD_POWER_DOMAINS,
4766                 .ops = &hsw_power_well_ops,
4767                 .id = DISP_PW_ID_NONE,
4768                 {
4769                         .hsw.regs = &icl_ddi_power_well_regs,
4770                         .hsw.idx = XELPD_PW_CTL_IDX_DDI_E,
4771                 }
4772         },
4773         {
4774                 .name = "DDI IO TC1",
4775                 .domains = XELPD_DDI_IO_TC1_POWER_DOMAINS,
4776                 .ops = &hsw_power_well_ops,
4777                 .id = DISP_PW_ID_NONE,
4778                 {
4779                         .hsw.regs = &icl_ddi_power_well_regs,
4780                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC1,
4781                 }
4782         },
4783         {
4784                 .name = "DDI IO TC2",
4785                 .domains = XELPD_DDI_IO_TC2_POWER_DOMAINS,
4786                 .ops = &hsw_power_well_ops,
4787                 .id = DISP_PW_ID_NONE,
4788                 {
4789                         .hsw.regs = &icl_ddi_power_well_regs,
4790                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC2,
4791                 }
4792         },
4793         {
4794                 .name = "DDI IO TC3",
4795                 .domains = XELPD_DDI_IO_TC3_POWER_DOMAINS,
4796                 .ops = &hsw_power_well_ops,
4797                 .id = DISP_PW_ID_NONE,
4798                 {
4799                         .hsw.regs = &icl_ddi_power_well_regs,
4800                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC3,
4801                 }
4802         },
4803         {
4804                 .name = "DDI IO TC4",
4805                 .domains = XELPD_DDI_IO_TC4_POWER_DOMAINS,
4806                 .ops = &hsw_power_well_ops,
4807                 .id = DISP_PW_ID_NONE,
4808                 {
4809                         .hsw.regs = &icl_ddi_power_well_regs,
4810                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC4,
4811                 }
4812         },
4813         {
4814                 .name = "AUX A",
4815                 .domains = ICL_AUX_A_IO_POWER_DOMAINS,
4816                 .ops = &icl_aux_power_well_ops,
4817                 .id = DISP_PW_ID_NONE,
4818                 {
4819                         .hsw.regs = &icl_aux_power_well_regs,
4820                         .hsw.idx = ICL_PW_CTL_IDX_AUX_A,
4821                 },
4822         },
4823         {
4824                 .name = "AUX B",
4825                 .domains = ICL_AUX_B_IO_POWER_DOMAINS,
4826                 .ops = &icl_aux_power_well_ops,
4827                 .id = DISP_PW_ID_NONE,
4828                 {
4829                         .hsw.regs = &icl_aux_power_well_regs,
4830                         .hsw.idx = ICL_PW_CTL_IDX_AUX_B,
4831                 },
4832         },
4833         {
4834                 .name = "AUX C",
4835                 .domains = TGL_AUX_C_IO_POWER_DOMAINS,
4836                 .ops = &icl_aux_power_well_ops,
4837                 .id = DISP_PW_ID_NONE,
4838                 {
4839                         .hsw.regs = &icl_aux_power_well_regs,
4840                         .hsw.idx = ICL_PW_CTL_IDX_AUX_C,
4841                 },
4842         },
4843         {
4844                 .name = "AUX D_XELPD",
4845                 .domains = XELPD_AUX_IO_D_XELPD_POWER_DOMAINS,
4846                 .ops = &icl_aux_power_well_ops,
4847                 .id = DISP_PW_ID_NONE,
4848                 {
4849                         .hsw.regs = &icl_aux_power_well_regs,
4850                         .hsw.idx = XELPD_PW_CTL_IDX_AUX_D,
4851                 },
4852         },
4853         {
4854                 .name = "AUX E_XELPD",
4855                 .domains = XELPD_AUX_IO_E_XELPD_POWER_DOMAINS,
4856                 .ops = &icl_aux_power_well_ops,
4857                 .id = DISP_PW_ID_NONE,
4858                 {
4859                         .hsw.regs = &icl_aux_power_well_regs,
4860                         .hsw.idx = XELPD_PW_CTL_IDX_AUX_E,
4861                 },
4862         },
4863         {
4864                 .name = "AUX USBC1",
4865                 .domains = XELPD_AUX_IO_USBC1_POWER_DOMAINS,
4866                 .ops = &icl_aux_power_well_ops,
4867                 .id = DISP_PW_ID_NONE,
4868                 {
4869                         .hsw.regs = &icl_aux_power_well_regs,
4870                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC1,
4871                 },
4872         },
4873         {
4874                 .name = "AUX USBC2",
4875                 .domains = XELPD_AUX_IO_USBC2_POWER_DOMAINS,
4876                 .ops = &icl_aux_power_well_ops,
4877                 .id = DISP_PW_ID_NONE,
4878                 {
4879                         .hsw.regs = &icl_aux_power_well_regs,
4880                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC2,
4881                 },
4882         },
4883         {
4884                 .name = "AUX USBC3",
4885                 .domains = XELPD_AUX_IO_USBC3_POWER_DOMAINS,
4886                 .ops = &icl_aux_power_well_ops,
4887                 .id = DISP_PW_ID_NONE,
4888                 {
4889                         .hsw.regs = &icl_aux_power_well_regs,
4890                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC3,
4891                 },
4892         },
4893         {
4894                 .name = "AUX USBC4",
4895                 .domains = XELPD_AUX_IO_USBC4_POWER_DOMAINS,
4896                 .ops = &icl_aux_power_well_ops,
4897                 .id = DISP_PW_ID_NONE,
4898                 {
4899                         .hsw.regs = &icl_aux_power_well_regs,
4900                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC4,
4901                 },
4902         },
4903         {
4904                 .name = "AUX TBT1",
4905                 .domains = XELPD_AUX_IO_TBT1_POWER_DOMAINS,
4906                 .ops = &icl_aux_power_well_ops,
4907                 .id = DISP_PW_ID_NONE,
4908                 {
4909                         .hsw.regs = &icl_aux_power_well_regs,
4910                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT1,
4911                         .hsw.is_tc_tbt = true,
4912                 },
4913         },
4914         {
4915                 .name = "AUX TBT2",
4916                 .domains = XELPD_AUX_IO_TBT2_POWER_DOMAINS,
4917                 .ops = &icl_aux_power_well_ops,
4918                 .id = DISP_PW_ID_NONE,
4919                 {
4920                         .hsw.regs = &icl_aux_power_well_regs,
4921                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT2,
4922                         .hsw.is_tc_tbt = true,
4923                 },
4924         },
4925         {
4926                 .name = "AUX TBT3",
4927                 .domains = XELPD_AUX_IO_TBT3_POWER_DOMAINS,
4928                 .ops = &icl_aux_power_well_ops,
4929                 .id = DISP_PW_ID_NONE,
4930                 {
4931                         .hsw.regs = &icl_aux_power_well_regs,
4932                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT3,
4933                         .hsw.is_tc_tbt = true,
4934                 },
4935         },
4936         {
4937                 .name = "AUX TBT4",
4938                 .domains = XELPD_AUX_IO_TBT4_POWER_DOMAINS,
4939                 .ops = &icl_aux_power_well_ops,
4940                 .id = DISP_PW_ID_NONE,
4941                 {
4942                         .hsw.regs = &icl_aux_power_well_regs,
4943                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT4,
4944                         .hsw.is_tc_tbt = true,
4945                 },
4946         },
4947 };
4948
4949 static int
4950 sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
4951                                    int disable_power_well)
4952 {
4953         if (disable_power_well >= 0)
4954                 return !!disable_power_well;
4955
4956         return 1;
4957 }
4958
4959 static u32 get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
4960                                int enable_dc)
4961 {
4962         u32 mask;
4963         int requested_dc;
4964         int max_dc;
4965
4966         if (!HAS_DISPLAY(dev_priv))
4967                 return 0;
4968
4969         if (IS_DG1(dev_priv))
4970                 max_dc = 3;
4971         else if (DISPLAY_VER(dev_priv) >= 12)
4972                 max_dc = 4;
4973         else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
4974                 max_dc = 1;
4975         else if (DISPLAY_VER(dev_priv) >= 9)
4976                 max_dc = 2;
4977         else
4978                 max_dc = 0;
4979
4980         /*
4981          * DC9 has a separate HW flow from the rest of the DC states,
4982          * not depending on the DMC firmware. It's needed by system
4983          * suspend/resume, so allow it unconditionally.
4984          */
4985         mask = IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv) ||
4986                 DISPLAY_VER(dev_priv) >= 11 ?
4987                DC_STATE_EN_DC9 : 0;
4988
4989         if (!dev_priv->params.disable_power_well)
4990                 max_dc = 0;
4991
4992         if (enable_dc >= 0 && enable_dc <= max_dc) {
4993                 requested_dc = enable_dc;
4994         } else if (enable_dc == -1) {
4995                 requested_dc = max_dc;
4996         } else if (enable_dc > max_dc && enable_dc <= 4) {
4997                 drm_dbg_kms(&dev_priv->drm,
4998                             "Adjusting requested max DC state (%d->%d)\n",
4999                             enable_dc, max_dc);
5000                 requested_dc = max_dc;
5001         } else {
5002                 drm_err(&dev_priv->drm,
5003                         "Unexpected value for enable_dc (%d)\n", enable_dc);
5004                 requested_dc = max_dc;
5005         }
5006
5007         switch (requested_dc) {
5008         case 4:
5009                 mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC6;
5010                 break;
5011         case 3:
5012                 mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC5;
5013                 break;
5014         case 2:
5015                 mask |= DC_STATE_EN_UPTO_DC6;
5016                 break;
5017         case 1:
5018                 mask |= DC_STATE_EN_UPTO_DC5;
5019                 break;
5020         }
5021
5022         drm_dbg_kms(&dev_priv->drm, "Allowed DC state mask %02x\n", mask);
5023
5024         return mask;
5025 }
5026
5027 static int
5028 __set_power_wells(struct i915_power_domains *power_domains,
5029                   const struct i915_power_well_desc *power_well_descs,
5030                   int power_well_descs_sz, u64 skip_mask)
5031 {
5032         struct drm_i915_private *i915 = container_of(power_domains,
5033                                                      struct drm_i915_private,
5034                                                      power_domains);
5035         u64 power_well_ids = 0;
5036         int power_well_count = 0;
5037         int i, plt_idx = 0;
5038
5039         for (i = 0; i < power_well_descs_sz; i++)
5040                 if (!(BIT_ULL(power_well_descs[i].id) & skip_mask))
5041                         power_well_count++;
5042
5043         power_domains->power_well_count = power_well_count;
5044         power_domains->power_wells =
5045                                 kcalloc(power_well_count,
5046                                         sizeof(*power_domains->power_wells),
5047                                         GFP_KERNEL);
5048         if (!power_domains->power_wells)
5049                 return -ENOMEM;
5050
5051         for (i = 0; i < power_well_descs_sz; i++) {
5052                 enum i915_power_well_id id = power_well_descs[i].id;
5053
5054                 if (BIT_ULL(id) & skip_mask)
5055                         continue;
5056
5057                 power_domains->power_wells[plt_idx++].desc =
5058                         &power_well_descs[i];
5059
5060                 if (id == DISP_PW_ID_NONE)
5061                         continue;
5062
5063                 drm_WARN_ON(&i915->drm, id >= sizeof(power_well_ids) * 8);
5064                 drm_WARN_ON(&i915->drm, power_well_ids & BIT_ULL(id));
5065                 power_well_ids |= BIT_ULL(id);
5066         }
5067
5068         return 0;
5069 }
5070
5071 #define set_power_wells_mask(power_domains, __power_well_descs, skip_mask) \
5072         __set_power_wells(power_domains, __power_well_descs, \
5073                           ARRAY_SIZE(__power_well_descs), skip_mask)
5074
5075 #define set_power_wells(power_domains, __power_well_descs) \
5076         set_power_wells_mask(power_domains, __power_well_descs, 0)
5077
5078 /**
5079  * intel_power_domains_init - initializes the power domain structures
5080  * @dev_priv: i915 device instance
5081  *
5082  * Initializes the power domain structures for @dev_priv depending upon the
5083  * supported platform.
5084  */
5085 int intel_power_domains_init(struct drm_i915_private *dev_priv)
5086 {
5087         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5088         int err;
5089
5090         dev_priv->params.disable_power_well =
5091                 sanitize_disable_power_well_option(dev_priv,
5092                                                    dev_priv->params.disable_power_well);
5093         dev_priv->dmc.allowed_dc_mask =
5094                 get_allowed_dc_mask(dev_priv, dev_priv->params.enable_dc);
5095
5096         dev_priv->dmc.target_dc_state =
5097                 sanitize_target_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
5098
5099         BUILD_BUG_ON(POWER_DOMAIN_NUM > 64);
5100
5101         mutex_init(&power_domains->lock);
5102
5103         INIT_DELAYED_WORK(&power_domains->async_put_work,
5104                           intel_display_power_put_async_work);
5105
5106         /*
5107          * The enabling order will be from lower to higher indexed wells,
5108          * the disabling order is reversed.
5109          */
5110         if (!HAS_DISPLAY(dev_priv)) {
5111                 power_domains->power_well_count = 0;
5112                 err = 0;
5113         } else if (DISPLAY_VER(dev_priv) >= 13) {
5114                 err = set_power_wells(power_domains, xelpd_power_wells);
5115         } else if (IS_ALDERLAKE_S(dev_priv) || IS_DG1(dev_priv)) {
5116                 err = set_power_wells_mask(power_domains, tgl_power_wells,
5117                                            BIT_ULL(TGL_DISP_PW_TC_COLD_OFF));
5118         } else if (IS_ROCKETLAKE(dev_priv)) {
5119                 err = set_power_wells(power_domains, rkl_power_wells);
5120         } else if (DISPLAY_VER(dev_priv) == 12) {
5121                 err = set_power_wells(power_domains, tgl_power_wells);
5122         } else if (DISPLAY_VER(dev_priv) == 11) {
5123                 err = set_power_wells(power_domains, icl_power_wells);
5124         } else if (IS_CNL_WITH_PORT_F(dev_priv)) {
5125                 err = set_power_wells(power_domains, cnl_power_wells);
5126         } else if (IS_CANNONLAKE(dev_priv)) {
5127                 err = set_power_wells_mask(power_domains, cnl_power_wells,
5128                                            BIT_ULL(CNL_DISP_PW_DDI_F_IO) |
5129                                            BIT_ULL(CNL_DISP_PW_DDI_F_AUX));
5130         } else if (IS_GEMINILAKE(dev_priv)) {
5131                 err = set_power_wells(power_domains, glk_power_wells);
5132         } else if (IS_BROXTON(dev_priv)) {
5133                 err = set_power_wells(power_domains, bxt_power_wells);
5134         } else if (DISPLAY_VER(dev_priv) == 9) {
5135                 err = set_power_wells(power_domains, skl_power_wells);
5136         } else if (IS_CHERRYVIEW(dev_priv)) {
5137                 err = set_power_wells(power_domains, chv_power_wells);
5138         } else if (IS_BROADWELL(dev_priv)) {
5139                 err = set_power_wells(power_domains, bdw_power_wells);
5140         } else if (IS_HASWELL(dev_priv)) {
5141                 err = set_power_wells(power_domains, hsw_power_wells);
5142         } else if (IS_VALLEYVIEW(dev_priv)) {
5143                 err = set_power_wells(power_domains, vlv_power_wells);
5144         } else if (IS_I830(dev_priv)) {
5145                 err = set_power_wells(power_domains, i830_power_wells);
5146         } else {
5147                 err = set_power_wells(power_domains, i9xx_always_on_power_well);
5148         }
5149
5150         return err;
5151 }
5152
5153 /**
5154  * intel_power_domains_cleanup - clean up power domains resources
5155  * @dev_priv: i915 device instance
5156  *
5157  * Release any resources acquired by intel_power_domains_init()
5158  */
5159 void intel_power_domains_cleanup(struct drm_i915_private *dev_priv)
5160 {
5161         kfree(dev_priv->power_domains.power_wells);
5162 }
5163
5164 static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
5165 {
5166         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5167         struct i915_power_well *power_well;
5168
5169         mutex_lock(&power_domains->lock);
5170         for_each_power_well(dev_priv, power_well) {
5171                 power_well->desc->ops->sync_hw(dev_priv, power_well);
5172                 power_well->hw_enabled =
5173                         power_well->desc->ops->is_enabled(dev_priv, power_well);
5174         }
5175         mutex_unlock(&power_domains->lock);
5176 }
5177
5178 static void gen9_dbuf_slice_set(struct drm_i915_private *dev_priv,
5179                                 enum dbuf_slice slice, bool enable)
5180 {
5181         i915_reg_t reg = DBUF_CTL_S(slice);
5182         bool state;
5183
5184         intel_de_rmw(dev_priv, reg, DBUF_POWER_REQUEST,
5185                      enable ? DBUF_POWER_REQUEST : 0);
5186         intel_de_posting_read(dev_priv, reg);
5187         udelay(10);
5188
5189         state = intel_de_read(dev_priv, reg) & DBUF_POWER_STATE;
5190         drm_WARN(&dev_priv->drm, enable != state,
5191                  "DBuf slice %d power %s timeout!\n",
5192                  slice, enabledisable(enable));
5193 }
5194
5195 void gen9_dbuf_slices_update(struct drm_i915_private *dev_priv,
5196                              u8 req_slices)
5197 {
5198         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5199         u8 slice_mask = INTEL_INFO(dev_priv)->dbuf.slice_mask;
5200         enum dbuf_slice slice;
5201
5202         drm_WARN(&dev_priv->drm, req_slices & ~slice_mask,
5203                  "Invalid set of dbuf slices (0x%x) requested (total dbuf slices 0x%x)\n",
5204                  req_slices, slice_mask);
5205
5206         drm_dbg_kms(&dev_priv->drm, "Updating dbuf slices to 0x%x\n",
5207                     req_slices);
5208
5209         /*
5210          * Might be running this in parallel to gen9_dc_off_power_well_enable
5211          * being called from intel_dp_detect for instance,
5212          * which causes assertion triggered by race condition,
5213          * as gen9_assert_dbuf_enabled might preempt this when registers
5214          * were already updated, while dev_priv was not.
5215          */
5216         mutex_lock(&power_domains->lock);
5217
5218         for_each_dbuf_slice(dev_priv, slice)
5219                 gen9_dbuf_slice_set(dev_priv, slice, req_slices & BIT(slice));
5220
5221         dev_priv->dbuf.enabled_slices = req_slices;
5222
5223         mutex_unlock(&power_domains->lock);
5224 }
5225
5226 static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
5227 {
5228         dev_priv->dbuf.enabled_slices =
5229                 intel_enabled_dbuf_slices_mask(dev_priv);
5230
5231         /*
5232          * Just power up at least 1 slice, we will
5233          * figure out later which slices we have and what we need.
5234          */
5235         gen9_dbuf_slices_update(dev_priv, BIT(DBUF_S1) |
5236                                 dev_priv->dbuf.enabled_slices);
5237 }
5238
5239 static void gen9_dbuf_disable(struct drm_i915_private *dev_priv)
5240 {
5241         gen9_dbuf_slices_update(dev_priv, 0);
5242 }
5243
5244 static void gen12_dbuf_slices_config(struct drm_i915_private *dev_priv)
5245 {
5246         enum dbuf_slice slice;
5247
5248         for_each_dbuf_slice(dev_priv, slice)
5249                 intel_de_rmw(dev_priv, DBUF_CTL_S(slice),
5250                              DBUF_TRACKER_STATE_SERVICE_MASK,
5251                              DBUF_TRACKER_STATE_SERVICE(8));
5252 }
5253
5254 static void icl_mbus_init(struct drm_i915_private *dev_priv)
5255 {
5256         unsigned long abox_regs = INTEL_INFO(dev_priv)->abox_mask;
5257         u32 mask, val, i;
5258
5259         mask = MBUS_ABOX_BT_CREDIT_POOL1_MASK |
5260                 MBUS_ABOX_BT_CREDIT_POOL2_MASK |
5261                 MBUS_ABOX_B_CREDIT_MASK |
5262                 MBUS_ABOX_BW_CREDIT_MASK;
5263         val = MBUS_ABOX_BT_CREDIT_POOL1(16) |
5264                 MBUS_ABOX_BT_CREDIT_POOL2(16) |
5265                 MBUS_ABOX_B_CREDIT(1) |
5266                 MBUS_ABOX_BW_CREDIT(1);
5267
5268         /*
5269          * gen12 platforms that use abox1 and abox2 for pixel data reads still
5270          * expect us to program the abox_ctl0 register as well, even though
5271          * we don't have to program other instance-0 registers like BW_BUDDY.
5272          */
5273         if (DISPLAY_VER(dev_priv) == 12)
5274                 abox_regs |= BIT(0);
5275
5276         for_each_set_bit(i, &abox_regs, sizeof(abox_regs))
5277                 intel_de_rmw(dev_priv, MBUS_ABOX_CTL(i), mask, val);
5278 }
5279
5280 static void hsw_assert_cdclk(struct drm_i915_private *dev_priv)
5281 {
5282         u32 val = intel_de_read(dev_priv, LCPLL_CTL);
5283
5284         /*
5285          * The LCPLL register should be turned on by the BIOS. For now
5286          * let's just check its state and print errors in case
5287          * something is wrong.  Don't even try to turn it on.
5288          */
5289
5290         if (val & LCPLL_CD_SOURCE_FCLK)
5291                 drm_err(&dev_priv->drm, "CDCLK source is not LCPLL\n");
5292
5293         if (val & LCPLL_PLL_DISABLE)
5294                 drm_err(&dev_priv->drm, "LCPLL is disabled\n");
5295
5296         if ((val & LCPLL_REF_MASK) != LCPLL_REF_NON_SSC)
5297                 drm_err(&dev_priv->drm, "LCPLL not using non-SSC reference\n");
5298 }
5299
5300 static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
5301 {
5302         struct drm_device *dev = &dev_priv->drm;
5303         struct intel_crtc *crtc;
5304
5305         for_each_intel_crtc(dev, crtc)
5306                 I915_STATE_WARN(crtc->active, "CRTC for pipe %c enabled\n",
5307                                 pipe_name(crtc->pipe));
5308
5309         I915_STATE_WARN(intel_de_read(dev_priv, HSW_PWR_WELL_CTL2),
5310                         "Display power well on\n");
5311         I915_STATE_WARN(intel_de_read(dev_priv, SPLL_CTL) & SPLL_PLL_ENABLE,
5312                         "SPLL enabled\n");
5313         I915_STATE_WARN(intel_de_read(dev_priv, WRPLL_CTL(0)) & WRPLL_PLL_ENABLE,
5314                         "WRPLL1 enabled\n");
5315         I915_STATE_WARN(intel_de_read(dev_priv, WRPLL_CTL(1)) & WRPLL_PLL_ENABLE,
5316                         "WRPLL2 enabled\n");
5317         I915_STATE_WARN(intel_de_read(dev_priv, PP_STATUS(0)) & PP_ON,
5318                         "Panel power on\n");
5319         I915_STATE_WARN(intel_de_read(dev_priv, BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
5320                         "CPU PWM1 enabled\n");
5321         if (IS_HASWELL(dev_priv))
5322                 I915_STATE_WARN(intel_de_read(dev_priv, HSW_BLC_PWM2_CTL) & BLM_PWM_ENABLE,
5323                                 "CPU PWM2 enabled\n");
5324         I915_STATE_WARN(intel_de_read(dev_priv, BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE,
5325                         "PCH PWM1 enabled\n");
5326         I915_STATE_WARN(intel_de_read(dev_priv, UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
5327                         "Utility pin enabled\n");
5328         I915_STATE_WARN(intel_de_read(dev_priv, PCH_GTC_CTL) & PCH_GTC_ENABLE,
5329                         "PCH GTC enabled\n");
5330
5331         /*
5332          * In theory we can still leave IRQs enabled, as long as only the HPD
5333          * interrupts remain enabled. We used to check for that, but since it's
5334          * gen-specific and since we only disable LCPLL after we fully disable
5335          * the interrupts, the check below should be enough.
5336          */
5337         I915_STATE_WARN(intel_irqs_enabled(dev_priv), "IRQs enabled\n");
5338 }
5339
5340 static u32 hsw_read_dcomp(struct drm_i915_private *dev_priv)
5341 {
5342         if (IS_HASWELL(dev_priv))
5343                 return intel_de_read(dev_priv, D_COMP_HSW);
5344         else
5345                 return intel_de_read(dev_priv, D_COMP_BDW);
5346 }
5347
5348 static void hsw_write_dcomp(struct drm_i915_private *dev_priv, u32 val)
5349 {
5350         if (IS_HASWELL(dev_priv)) {
5351                 if (sandybridge_pcode_write(dev_priv,
5352                                             GEN6_PCODE_WRITE_D_COMP, val))
5353                         drm_dbg_kms(&dev_priv->drm,
5354                                     "Failed to write to D_COMP\n");
5355         } else {
5356                 intel_de_write(dev_priv, D_COMP_BDW, val);
5357                 intel_de_posting_read(dev_priv, D_COMP_BDW);
5358         }
5359 }
5360
5361 /*
5362  * This function implements pieces of two sequences from BSpec:
5363  * - Sequence for display software to disable LCPLL
5364  * - Sequence for display software to allow package C8+
5365  * The steps implemented here are just the steps that actually touch the LCPLL
5366  * register. Callers should take care of disabling all the display engine
5367  * functions, doing the mode unset, fixing interrupts, etc.
5368  */
5369 static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
5370                               bool switch_to_fclk, bool allow_power_down)
5371 {
5372         u32 val;
5373
5374         assert_can_disable_lcpll(dev_priv);
5375
5376         val = intel_de_read(dev_priv, LCPLL_CTL);
5377
5378         if (switch_to_fclk) {
5379                 val |= LCPLL_CD_SOURCE_FCLK;
5380                 intel_de_write(dev_priv, LCPLL_CTL, val);
5381
5382                 if (wait_for_us(intel_de_read(dev_priv, LCPLL_CTL) &
5383                                 LCPLL_CD_SOURCE_FCLK_DONE, 1))
5384                         drm_err(&dev_priv->drm, "Switching to FCLK failed\n");
5385
5386                 val = intel_de_read(dev_priv, LCPLL_CTL);
5387         }
5388
5389         val |= LCPLL_PLL_DISABLE;
5390         intel_de_write(dev_priv, LCPLL_CTL, val);
5391         intel_de_posting_read(dev_priv, LCPLL_CTL);
5392
5393         if (intel_de_wait_for_clear(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 1))
5394                 drm_err(&dev_priv->drm, "LCPLL still locked\n");
5395
5396         val = hsw_read_dcomp(dev_priv);
5397         val |= D_COMP_COMP_DISABLE;
5398         hsw_write_dcomp(dev_priv, val);
5399         ndelay(100);
5400
5401         if (wait_for((hsw_read_dcomp(dev_priv) &
5402                       D_COMP_RCOMP_IN_PROGRESS) == 0, 1))
5403                 drm_err(&dev_priv->drm, "D_COMP RCOMP still in progress\n");
5404
5405         if (allow_power_down) {
5406                 val = intel_de_read(dev_priv, LCPLL_CTL);
5407                 val |= LCPLL_POWER_DOWN_ALLOW;
5408                 intel_de_write(dev_priv, LCPLL_CTL, val);
5409                 intel_de_posting_read(dev_priv, LCPLL_CTL);
5410         }
5411 }
5412
5413 /*
5414  * Fully restores LCPLL, disallowing power down and switching back to LCPLL
5415  * source.
5416  */
5417 static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
5418 {
5419         u32 val;
5420
5421         val = intel_de_read(dev_priv, LCPLL_CTL);
5422
5423         if ((val & (LCPLL_PLL_LOCK | LCPLL_PLL_DISABLE | LCPLL_CD_SOURCE_FCLK |
5424                     LCPLL_POWER_DOWN_ALLOW)) == LCPLL_PLL_LOCK)
5425                 return;
5426
5427         /*
5428          * Make sure we're not on PC8 state before disabling PC8, otherwise
5429          * we'll hang the machine. To prevent PC8 state, just enable force_wake.
5430          */
5431         intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
5432
5433         if (val & LCPLL_POWER_DOWN_ALLOW) {
5434                 val &= ~LCPLL_POWER_DOWN_ALLOW;
5435                 intel_de_write(dev_priv, LCPLL_CTL, val);
5436                 intel_de_posting_read(dev_priv, LCPLL_CTL);
5437         }
5438
5439         val = hsw_read_dcomp(dev_priv);
5440         val |= D_COMP_COMP_FORCE;
5441         val &= ~D_COMP_COMP_DISABLE;
5442         hsw_write_dcomp(dev_priv, val);
5443
5444         val = intel_de_read(dev_priv, LCPLL_CTL);
5445         val &= ~LCPLL_PLL_DISABLE;
5446         intel_de_write(dev_priv, LCPLL_CTL, val);
5447
5448         if (intel_de_wait_for_set(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 5))
5449                 drm_err(&dev_priv->drm, "LCPLL not locked yet\n");
5450
5451         if (val & LCPLL_CD_SOURCE_FCLK) {
5452                 val = intel_de_read(dev_priv, LCPLL_CTL);
5453                 val &= ~LCPLL_CD_SOURCE_FCLK;
5454                 intel_de_write(dev_priv, LCPLL_CTL, val);
5455
5456                 if (wait_for_us((intel_de_read(dev_priv, LCPLL_CTL) &
5457                                  LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
5458                         drm_err(&dev_priv->drm,
5459                                 "Switching back to LCPLL failed\n");
5460         }
5461
5462         intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
5463
5464         intel_update_cdclk(dev_priv);
5465         intel_dump_cdclk_config(&dev_priv->cdclk.hw, "Current CDCLK");
5466 }
5467
5468 /*
5469  * Package states C8 and deeper are really deep PC states that can only be
5470  * reached when all the devices on the system allow it, so even if the graphics
5471  * device allows PC8+, it doesn't mean the system will actually get to these
5472  * states. Our driver only allows PC8+ when going into runtime PM.
5473  *
5474  * The requirements for PC8+ are that all the outputs are disabled, the power
5475  * well is disabled and most interrupts are disabled, and these are also
5476  * requirements for runtime PM. When these conditions are met, we manually do
5477  * the other conditions: disable the interrupts, clocks and switch LCPLL refclk
5478  * to Fclk. If we're in PC8+ and we get an non-hotplug interrupt, we can hard
5479  * hang the machine.
5480  *
5481  * When we really reach PC8 or deeper states (not just when we allow it) we lose
5482  * the state of some registers, so when we come back from PC8+ we need to
5483  * restore this state. We don't get into PC8+ if we're not in RC6, so we don't
5484  * need to take care of the registers kept by RC6. Notice that this happens even
5485  * if we don't put the device in PCI D3 state (which is what currently happens
5486  * because of the runtime PM support).
5487  *
5488  * For more, read "Display Sequences for Package C8" on the hardware
5489  * documentation.
5490  */
5491 static void hsw_enable_pc8(struct drm_i915_private *dev_priv)
5492 {
5493         u32 val;
5494
5495         drm_dbg_kms(&dev_priv->drm, "Enabling package C8+\n");
5496
5497         if (HAS_PCH_LPT_LP(dev_priv)) {
5498                 val = intel_de_read(dev_priv, SOUTH_DSPCLK_GATE_D);
5499                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
5500                 intel_de_write(dev_priv, SOUTH_DSPCLK_GATE_D, val);
5501         }
5502
5503         lpt_disable_clkout_dp(dev_priv);
5504         hsw_disable_lcpll(dev_priv, true, true);
5505 }
5506
5507 static void hsw_disable_pc8(struct drm_i915_private *dev_priv)
5508 {
5509         u32 val;
5510
5511         drm_dbg_kms(&dev_priv->drm, "Disabling package C8+\n");
5512
5513         hsw_restore_lcpll(dev_priv);
5514         intel_init_pch_refclk(dev_priv);
5515
5516         if (HAS_PCH_LPT_LP(dev_priv)) {
5517                 val = intel_de_read(dev_priv, SOUTH_DSPCLK_GATE_D);
5518                 val |= PCH_LP_PARTITION_LEVEL_DISABLE;
5519                 intel_de_write(dev_priv, SOUTH_DSPCLK_GATE_D, val);
5520         }
5521 }
5522
5523 static void intel_pch_reset_handshake(struct drm_i915_private *dev_priv,
5524                                       bool enable)
5525 {
5526         i915_reg_t reg;
5527         u32 reset_bits, val;
5528
5529         if (IS_IVYBRIDGE(dev_priv)) {
5530                 reg = GEN7_MSG_CTL;
5531                 reset_bits = WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK;
5532         } else {
5533                 reg = HSW_NDE_RSTWRN_OPT;
5534                 reset_bits = RESET_PCH_HANDSHAKE_ENABLE;
5535         }
5536
5537         val = intel_de_read(dev_priv, reg);
5538
5539         if (enable)
5540                 val |= reset_bits;
5541         else
5542                 val &= ~reset_bits;
5543
5544         intel_de_write(dev_priv, reg, val);
5545 }
5546
5547 static void skl_display_core_init(struct drm_i915_private *dev_priv,
5548                                   bool resume)
5549 {
5550         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5551         struct i915_power_well *well;
5552
5553         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
5554
5555         /* enable PCH reset handshake */
5556         intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
5557
5558         if (!HAS_DISPLAY(dev_priv))
5559                 return;
5560
5561         /* enable PG1 and Misc I/O */
5562         mutex_lock(&power_domains->lock);
5563
5564         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
5565         intel_power_well_enable(dev_priv, well);
5566
5567         well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
5568         intel_power_well_enable(dev_priv, well);
5569
5570         mutex_unlock(&power_domains->lock);
5571
5572         intel_cdclk_init_hw(dev_priv);
5573
5574         gen9_dbuf_enable(dev_priv);
5575
5576         if (resume && dev_priv->dmc.dmc_payload)
5577                 intel_dmc_load_program(dev_priv);
5578 }
5579
5580 static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
5581 {
5582         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5583         struct i915_power_well *well;
5584
5585         if (!HAS_DISPLAY(dev_priv))
5586                 return;
5587
5588         gen9_disable_dc_states(dev_priv);
5589
5590         gen9_dbuf_disable(dev_priv);
5591
5592         intel_cdclk_uninit_hw(dev_priv);
5593
5594         /* The spec doesn't call for removing the reset handshake flag */
5595         /* disable PG1 and Misc I/O */
5596
5597         mutex_lock(&power_domains->lock);
5598
5599         /*
5600          * BSpec says to keep the MISC IO power well enabled here, only
5601          * remove our request for power well 1.
5602          * Note that even though the driver's request is removed power well 1
5603          * may stay enabled after this due to DMC's own request on it.
5604          */
5605         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
5606         intel_power_well_disable(dev_priv, well);
5607
5608         mutex_unlock(&power_domains->lock);
5609
5610         usleep_range(10, 30);           /* 10 us delay per Bspec */
5611 }
5612
5613 static void bxt_display_core_init(struct drm_i915_private *dev_priv, bool resume)
5614 {
5615         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5616         struct i915_power_well *well;
5617
5618         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
5619
5620         /*
5621          * NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
5622          * or else the reset will hang because there is no PCH to respond.
5623          * Move the handshake programming to initialization sequence.
5624          * Previously was left up to BIOS.
5625          */
5626         intel_pch_reset_handshake(dev_priv, false);
5627
5628         if (!HAS_DISPLAY(dev_priv))
5629                 return;
5630
5631         /* Enable PG1 */
5632         mutex_lock(&power_domains->lock);
5633
5634         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
5635         intel_power_well_enable(dev_priv, well);
5636
5637         mutex_unlock(&power_domains->lock);
5638
5639         intel_cdclk_init_hw(dev_priv);
5640
5641         gen9_dbuf_enable(dev_priv);
5642
5643         if (resume && dev_priv->dmc.dmc_payload)
5644                 intel_dmc_load_program(dev_priv);
5645 }
5646
5647 static void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
5648 {
5649         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5650         struct i915_power_well *well;
5651
5652         if (!HAS_DISPLAY(dev_priv))
5653                 return;
5654
5655         gen9_disable_dc_states(dev_priv);
5656
5657         gen9_dbuf_disable(dev_priv);
5658
5659         intel_cdclk_uninit_hw(dev_priv);
5660
5661         /* The spec doesn't call for removing the reset handshake flag */
5662
5663         /*
5664          * Disable PW1 (PG1).
5665          * Note that even though the driver's request is removed power well 1
5666          * may stay enabled after this due to DMC's own request on it.
5667          */
5668         mutex_lock(&power_domains->lock);
5669
5670         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
5671         intel_power_well_disable(dev_priv, well);
5672
5673         mutex_unlock(&power_domains->lock);
5674
5675         usleep_range(10, 30);           /* 10 us delay per Bspec */
5676 }
5677
5678 static void cnl_display_core_init(struct drm_i915_private *dev_priv, bool resume)
5679 {
5680         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5681         struct i915_power_well *well;
5682
5683         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
5684
5685         /* 1. Enable PCH Reset Handshake */
5686         intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
5687
5688         if (!HAS_DISPLAY(dev_priv))
5689                 return;
5690
5691         /* 2-3. */
5692         intel_combo_phy_init(dev_priv);
5693
5694         /*
5695          * 4. Enable Power Well 1 (PG1).
5696          *    The AUX IO power wells will be enabled on demand.
5697          */
5698         mutex_lock(&power_domains->lock);
5699         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
5700         intel_power_well_enable(dev_priv, well);
5701         mutex_unlock(&power_domains->lock);
5702
5703         /* 5. Enable CD clock */
5704         intel_cdclk_init_hw(dev_priv);
5705
5706         /* 6. Enable DBUF */
5707         gen9_dbuf_enable(dev_priv);
5708
5709         if (resume && dev_priv->dmc.dmc_payload)
5710                 intel_dmc_load_program(dev_priv);
5711 }
5712
5713 static void cnl_display_core_uninit(struct drm_i915_private *dev_priv)
5714 {
5715         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5716         struct i915_power_well *well;
5717
5718         if (!HAS_DISPLAY(dev_priv))
5719                 return;
5720
5721         gen9_disable_dc_states(dev_priv);
5722
5723         /* 1. Disable all display engine functions -> aready done */
5724
5725         /* 2. Disable DBUF */
5726         gen9_dbuf_disable(dev_priv);
5727
5728         /* 3. Disable CD clock */
5729         intel_cdclk_uninit_hw(dev_priv);
5730
5731         /*
5732          * 4. Disable Power Well 1 (PG1).
5733          *    The AUX IO power wells are toggled on demand, so they are already
5734          *    disabled at this point.
5735          */
5736         mutex_lock(&power_domains->lock);
5737         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
5738         intel_power_well_disable(dev_priv, well);
5739         mutex_unlock(&power_domains->lock);
5740
5741         usleep_range(10, 30);           /* 10 us delay per Bspec */
5742
5743         /* 5. */
5744         intel_combo_phy_uninit(dev_priv);
5745 }
5746
5747 struct buddy_page_mask {
5748         u32 page_mask;
5749         u8 type;
5750         u8 num_channels;
5751 };
5752
5753 static const struct buddy_page_mask tgl_buddy_page_masks[] = {
5754         { .num_channels = 1, .type = INTEL_DRAM_DDR4,   .page_mask = 0xF },
5755         { .num_channels = 1, .type = INTEL_DRAM_DDR5,   .page_mask = 0xF },
5756         { .num_channels = 2, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x1C },
5757         { .num_channels = 2, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x1C },
5758         { .num_channels = 2, .type = INTEL_DRAM_DDR4,   .page_mask = 0x1F },
5759         { .num_channels = 2, .type = INTEL_DRAM_DDR5,   .page_mask = 0x1E },
5760         { .num_channels = 4, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x38 },
5761         { .num_channels = 4, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x38 },
5762         {}
5763 };
5764
5765 static const struct buddy_page_mask wa_1409767108_buddy_page_masks[] = {
5766         { .num_channels = 1, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x1 },
5767         { .num_channels = 1, .type = INTEL_DRAM_DDR4,   .page_mask = 0x1 },
5768         { .num_channels = 1, .type = INTEL_DRAM_DDR5,   .page_mask = 0x1 },
5769         { .num_channels = 1, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x1 },
5770         { .num_channels = 2, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x3 },
5771         { .num_channels = 2, .type = INTEL_DRAM_DDR4,   .page_mask = 0x3 },
5772         { .num_channels = 2, .type = INTEL_DRAM_DDR5,   .page_mask = 0x3 },
5773         { .num_channels = 2, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x3 },
5774         {}
5775 };
5776
5777 static void tgl_bw_buddy_init(struct drm_i915_private *dev_priv)
5778 {
5779         enum intel_dram_type type = dev_priv->dram_info.type;
5780         u8 num_channels = dev_priv->dram_info.num_channels;
5781         const struct buddy_page_mask *table;
5782         unsigned long abox_mask = INTEL_INFO(dev_priv)->abox_mask;
5783         int config, i;
5784
5785         if (IS_ALDERLAKE_S(dev_priv) ||
5786             IS_DG1_REVID(dev_priv, DG1_REVID_A0, DG1_REVID_A0) ||
5787             IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
5788                 /* Wa_1409767108:tgl,dg1,adl-s */
5789                 table = wa_1409767108_buddy_page_masks;
5790         else
5791                 table = tgl_buddy_page_masks;
5792
5793         for (config = 0; table[config].page_mask != 0; config++)
5794                 if (table[config].num_channels == num_channels &&
5795                     table[config].type == type)
5796                         break;
5797
5798         if (table[config].page_mask == 0) {
5799                 drm_dbg(&dev_priv->drm,
5800                         "Unknown memory configuration; disabling address buddy logic.\n");
5801                 for_each_set_bit(i, &abox_mask, sizeof(abox_mask))
5802                         intel_de_write(dev_priv, BW_BUDDY_CTL(i),
5803                                        BW_BUDDY_DISABLE);
5804         } else {
5805                 for_each_set_bit(i, &abox_mask, sizeof(abox_mask)) {
5806                         intel_de_write(dev_priv, BW_BUDDY_PAGE_MASK(i),
5807                                        table[config].page_mask);
5808
5809                         /* Wa_22010178259:tgl,rkl */
5810                         intel_de_rmw(dev_priv, BW_BUDDY_CTL(i),
5811                                      BW_BUDDY_TLB_REQ_TIMER_MASK,
5812                                      BW_BUDDY_TLB_REQ_TIMER(0x8));
5813                 }
5814         }
5815 }
5816
5817 static void icl_display_core_init(struct drm_i915_private *dev_priv,
5818                                   bool resume)
5819 {
5820         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5821         struct i915_power_well *well;
5822         u32 val;
5823
5824         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
5825
5826         /* Wa_14011294188:ehl,jsl,tgl,rkl,adl-s */
5827         if (INTEL_PCH_TYPE(dev_priv) >= PCH_JSP &&
5828             INTEL_PCH_TYPE(dev_priv) < PCH_DG1)
5829                 intel_de_rmw(dev_priv, SOUTH_DSPCLK_GATE_D, 0,
5830                              PCH_DPMGUNIT_CLOCK_GATE_DISABLE);
5831
5832         /* 1. Enable PCH reset handshake. */
5833         intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
5834
5835         if (!HAS_DISPLAY(dev_priv))
5836                 return;
5837
5838         /* 2. Initialize all combo phys */
5839         intel_combo_phy_init(dev_priv);
5840
5841         /*
5842          * 3. Enable Power Well 1 (PG1).
5843          *    The AUX IO power wells will be enabled on demand.
5844          */
5845         mutex_lock(&power_domains->lock);
5846         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
5847         intel_power_well_enable(dev_priv, well);
5848         mutex_unlock(&power_domains->lock);
5849
5850         /* 4. Enable CDCLK. */
5851         intel_cdclk_init_hw(dev_priv);
5852
5853         if (DISPLAY_VER(dev_priv) >= 12)
5854                 gen12_dbuf_slices_config(dev_priv);
5855
5856         /* 5. Enable DBUF. */
5857         gen9_dbuf_enable(dev_priv);
5858
5859         /* 6. Setup MBUS. */
5860         icl_mbus_init(dev_priv);
5861
5862         /* 7. Program arbiter BW_BUDDY registers */
5863         if (DISPLAY_VER(dev_priv) >= 12)
5864                 tgl_bw_buddy_init(dev_priv);
5865
5866         if (resume && dev_priv->dmc.dmc_payload)
5867                 intel_dmc_load_program(dev_priv);
5868
5869         /* Wa_14011508470 */
5870         if (DISPLAY_VER(dev_priv) == 12) {
5871                 val = DCPR_CLEAR_MEMSTAT_DIS | DCPR_SEND_RESP_IMM |
5872                       DCPR_MASK_LPMODE | DCPR_MASK_MAXLATENCY_MEMUP_CLR;
5873                 intel_uncore_rmw(&dev_priv->uncore, GEN11_CHICKEN_DCPR_2, 0, val);
5874         }
5875
5876         /* Wa_14011503030:xelpd */
5877         if (DISPLAY_VER(dev_priv) >= 13)
5878                 intel_de_write(dev_priv, XELPD_DISPLAY_ERR_FATAL_MASK, ~0);
5879 }
5880
5881 static void icl_display_core_uninit(struct drm_i915_private *dev_priv)
5882 {
5883         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5884         struct i915_power_well *well;
5885
5886         if (!HAS_DISPLAY(dev_priv))
5887                 return;
5888
5889         gen9_disable_dc_states(dev_priv);
5890
5891         /* 1. Disable all display engine functions -> aready done */
5892
5893         /* 2. Disable DBUF */
5894         gen9_dbuf_disable(dev_priv);
5895
5896         /* 3. Disable CD clock */
5897         intel_cdclk_uninit_hw(dev_priv);
5898
5899         /*
5900          * 4. Disable Power Well 1 (PG1).
5901          *    The AUX IO power wells are toggled on demand, so they are already
5902          *    disabled at this point.
5903          */
5904         mutex_lock(&power_domains->lock);
5905         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
5906         intel_power_well_disable(dev_priv, well);
5907         mutex_unlock(&power_domains->lock);
5908
5909         /* 5. */
5910         intel_combo_phy_uninit(dev_priv);
5911 }
5912
5913 static void chv_phy_control_init(struct drm_i915_private *dev_priv)
5914 {
5915         struct i915_power_well *cmn_bc =
5916                 lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
5917         struct i915_power_well *cmn_d =
5918                 lookup_power_well(dev_priv, CHV_DISP_PW_DPIO_CMN_D);
5919
5920         /*
5921          * DISPLAY_PHY_CONTROL can get corrupted if read. As a
5922          * workaround never ever read DISPLAY_PHY_CONTROL, and
5923          * instead maintain a shadow copy ourselves. Use the actual
5924          * power well state and lane status to reconstruct the
5925          * expected initial value.
5926          */
5927         dev_priv->chv_phy_control =
5928                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
5929                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
5930                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) |
5931                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH1) |
5932                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY1, DPIO_CH0);
5933
5934         /*
5935          * If all lanes are disabled we leave the override disabled
5936          * with all power down bits cleared to match the state we
5937          * would use after disabling the port. Otherwise enable the
5938          * override and set the lane powerdown bits accding to the
5939          * current lane status.
5940          */
5941         if (cmn_bc->desc->ops->is_enabled(dev_priv, cmn_bc)) {
5942                 u32 status = intel_de_read(dev_priv, DPLL(PIPE_A));
5943                 unsigned int mask;
5944
5945                 mask = status & DPLL_PORTB_READY_MASK;
5946                 if (mask == 0xf)
5947                         mask = 0x0;
5948                 else
5949                         dev_priv->chv_phy_control |=
5950                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0);
5951
5952                 dev_priv->chv_phy_control |=
5953                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0);
5954
5955                 mask = (status & DPLL_PORTC_READY_MASK) >> 4;
5956                 if (mask == 0xf)
5957                         mask = 0x0;
5958                 else
5959                         dev_priv->chv_phy_control |=
5960                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1);
5961
5962                 dev_priv->chv_phy_control |=
5963                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1);
5964
5965                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
5966
5967                 dev_priv->chv_phy_assert[DPIO_PHY0] = false;
5968         } else {
5969                 dev_priv->chv_phy_assert[DPIO_PHY0] = true;
5970         }
5971
5972         if (cmn_d->desc->ops->is_enabled(dev_priv, cmn_d)) {
5973                 u32 status = intel_de_read(dev_priv, DPIO_PHY_STATUS);
5974                 unsigned int mask;
5975
5976                 mask = status & DPLL_PORTD_READY_MASK;
5977
5978                 if (mask == 0xf)
5979                         mask = 0x0;
5980                 else
5981                         dev_priv->chv_phy_control |=
5982                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0);
5983
5984                 dev_priv->chv_phy_control |=
5985                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0);
5986
5987                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
5988
5989                 dev_priv->chv_phy_assert[DPIO_PHY1] = false;
5990         } else {
5991                 dev_priv->chv_phy_assert[DPIO_PHY1] = true;
5992         }
5993
5994         drm_dbg_kms(&dev_priv->drm, "Initial PHY_CONTROL=0x%08x\n",
5995                     dev_priv->chv_phy_control);
5996
5997         /* Defer application of initial phy_control to enabling the powerwell */
5998 }
5999
6000 static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
6001 {
6002         struct i915_power_well *cmn =
6003                 lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
6004         struct i915_power_well *disp2d =
6005                 lookup_power_well(dev_priv, VLV_DISP_PW_DISP2D);
6006
6007         /* If the display might be already active skip this */
6008         if (cmn->desc->ops->is_enabled(dev_priv, cmn) &&
6009             disp2d->desc->ops->is_enabled(dev_priv, disp2d) &&
6010             intel_de_read(dev_priv, DPIO_CTL) & DPIO_CMNRST)
6011                 return;
6012
6013         drm_dbg_kms(&dev_priv->drm, "toggling display PHY side reset\n");
6014
6015         /* cmnlane needs DPLL registers */
6016         disp2d->desc->ops->enable(dev_priv, disp2d);
6017
6018         /*
6019          * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
6020          * Need to assert and de-assert PHY SB reset by gating the
6021          * common lane power, then un-gating it.
6022          * Simply ungating isn't enough to reset the PHY enough to get
6023          * ports and lanes running.
6024          */
6025         cmn->desc->ops->disable(dev_priv, cmn);
6026 }
6027
6028 static bool vlv_punit_is_power_gated(struct drm_i915_private *dev_priv, u32 reg0)
6029 {
6030         bool ret;
6031
6032         vlv_punit_get(dev_priv);
6033         ret = (vlv_punit_read(dev_priv, reg0) & SSPM0_SSC_MASK) == SSPM0_SSC_PWR_GATE;
6034         vlv_punit_put(dev_priv);
6035
6036         return ret;
6037 }
6038
6039 static void assert_ved_power_gated(struct drm_i915_private *dev_priv)
6040 {
6041         drm_WARN(&dev_priv->drm,
6042                  !vlv_punit_is_power_gated(dev_priv, PUNIT_REG_VEDSSPM0),
6043                  "VED not power gated\n");
6044 }
6045
6046 static void assert_isp_power_gated(struct drm_i915_private *dev_priv)
6047 {
6048         static const struct pci_device_id isp_ids[] = {
6049                 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x0f38)},
6050                 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x22b8)},
6051                 {}
6052         };
6053
6054         drm_WARN(&dev_priv->drm, !pci_dev_present(isp_ids) &&
6055                  !vlv_punit_is_power_gated(dev_priv, PUNIT_REG_ISPSSPM0),
6056                  "ISP not power gated\n");
6057 }
6058
6059 static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv);
6060
6061 /**
6062  * intel_power_domains_init_hw - initialize hardware power domain state
6063  * @i915: i915 device instance
6064  * @resume: Called from resume code paths or not
6065  *
6066  * This function initializes the hardware power domain state and enables all
6067  * power wells belonging to the INIT power domain. Power wells in other
6068  * domains (and not in the INIT domain) are referenced or disabled by
6069  * intel_modeset_readout_hw_state(). After that the reference count of each
6070  * power well must match its HW enabled state, see
6071  * intel_power_domains_verify_state().
6072  *
6073  * It will return with power domains disabled (to be enabled later by
6074  * intel_power_domains_enable()) and must be paired with
6075  * intel_power_domains_driver_remove().
6076  */
6077 void intel_power_domains_init_hw(struct drm_i915_private *i915, bool resume)
6078 {
6079         struct i915_power_domains *power_domains = &i915->power_domains;
6080
6081         power_domains->initializing = true;
6082
6083         if (DISPLAY_VER(i915) >= 11) {
6084                 icl_display_core_init(i915, resume);
6085         } else if (IS_CANNONLAKE(i915)) {
6086                 cnl_display_core_init(i915, resume);
6087         } else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
6088                 bxt_display_core_init(i915, resume);
6089         } else if (DISPLAY_VER(i915) == 9) {
6090                 skl_display_core_init(i915, resume);
6091         } else if (IS_CHERRYVIEW(i915)) {
6092                 mutex_lock(&power_domains->lock);
6093                 chv_phy_control_init(i915);
6094                 mutex_unlock(&power_domains->lock);
6095                 assert_isp_power_gated(i915);
6096         } else if (IS_VALLEYVIEW(i915)) {
6097                 mutex_lock(&power_domains->lock);
6098                 vlv_cmnlane_wa(i915);
6099                 mutex_unlock(&power_domains->lock);
6100                 assert_ved_power_gated(i915);
6101                 assert_isp_power_gated(i915);
6102         } else if (IS_BROADWELL(i915) || IS_HASWELL(i915)) {
6103                 hsw_assert_cdclk(i915);
6104                 intel_pch_reset_handshake(i915, !HAS_PCH_NOP(i915));
6105         } else if (IS_IVYBRIDGE(i915)) {
6106                 intel_pch_reset_handshake(i915, !HAS_PCH_NOP(i915));
6107         }
6108
6109         /*
6110          * Keep all power wells enabled for any dependent HW access during
6111          * initialization and to make sure we keep BIOS enabled display HW
6112          * resources powered until display HW readout is complete. We drop
6113          * this reference in intel_power_domains_enable().
6114          */
6115         drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
6116         power_domains->init_wakeref =
6117                 intel_display_power_get(i915, POWER_DOMAIN_INIT);
6118
6119         /* Disable power support if the user asked so. */
6120         if (!i915->params.disable_power_well) {
6121                 drm_WARN_ON(&i915->drm, power_domains->disable_wakeref);
6122                 i915->power_domains.disable_wakeref = intel_display_power_get(i915,
6123                                                                               POWER_DOMAIN_INIT);
6124         }
6125         intel_power_domains_sync_hw(i915);
6126
6127         power_domains->initializing = false;
6128 }
6129
6130 /**
6131  * intel_power_domains_driver_remove - deinitialize hw power domain state
6132  * @i915: i915 device instance
6133  *
6134  * De-initializes the display power domain HW state. It also ensures that the
6135  * device stays powered up so that the driver can be reloaded.
6136  *
6137  * It must be called with power domains already disabled (after a call to
6138  * intel_power_domains_disable()) and must be paired with
6139  * intel_power_domains_init_hw().
6140  */
6141 void intel_power_domains_driver_remove(struct drm_i915_private *i915)
6142 {
6143         intel_wakeref_t wakeref __maybe_unused =
6144                 fetch_and_zero(&i915->power_domains.init_wakeref);
6145
6146         /* Remove the refcount we took to keep power well support disabled. */
6147         if (!i915->params.disable_power_well)
6148                 intel_display_power_put(i915, POWER_DOMAIN_INIT,
6149                                         fetch_and_zero(&i915->power_domains.disable_wakeref));
6150
6151         intel_display_power_flush_work_sync(i915);
6152
6153         intel_power_domains_verify_state(i915);
6154
6155         /* Keep the power well enabled, but cancel its rpm wakeref. */
6156         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
6157 }
6158
6159 /**
6160  * intel_power_domains_enable - enable toggling of display power wells
6161  * @i915: i915 device instance
6162  *
6163  * Enable the ondemand enabling/disabling of the display power wells. Note that
6164  * power wells not belonging to POWER_DOMAIN_INIT are allowed to be toggled
6165  * only at specific points of the display modeset sequence, thus they are not
6166  * affected by the intel_power_domains_enable()/disable() calls. The purpose
6167  * of these function is to keep the rest of power wells enabled until the end
6168  * of display HW readout (which will acquire the power references reflecting
6169  * the current HW state).
6170  */
6171 void intel_power_domains_enable(struct drm_i915_private *i915)
6172 {
6173         intel_wakeref_t wakeref __maybe_unused =
6174                 fetch_and_zero(&i915->power_domains.init_wakeref);
6175
6176         intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref);
6177         intel_power_domains_verify_state(i915);
6178 }
6179
6180 /**
6181  * intel_power_domains_disable - disable toggling of display power wells
6182  * @i915: i915 device instance
6183  *
6184  * Disable the ondemand enabling/disabling of the display power wells. See
6185  * intel_power_domains_enable() for which power wells this call controls.
6186  */
6187 void intel_power_domains_disable(struct drm_i915_private *i915)
6188 {
6189         struct i915_power_domains *power_domains = &i915->power_domains;
6190
6191         drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
6192         power_domains->init_wakeref =
6193                 intel_display_power_get(i915, POWER_DOMAIN_INIT);
6194
6195         intel_power_domains_verify_state(i915);
6196 }
6197
6198 /**
6199  * intel_power_domains_suspend - suspend power domain state
6200  * @i915: i915 device instance
6201  * @suspend_mode: specifies the target suspend state (idle, mem, hibernation)
6202  *
6203  * This function prepares the hardware power domain state before entering
6204  * system suspend.
6205  *
6206  * It must be called with power domains already disabled (after a call to
6207  * intel_power_domains_disable()) and paired with intel_power_domains_resume().
6208  */
6209 void intel_power_domains_suspend(struct drm_i915_private *i915,
6210                                  enum i915_drm_suspend_mode suspend_mode)
6211 {
6212         struct i915_power_domains *power_domains = &i915->power_domains;
6213         intel_wakeref_t wakeref __maybe_unused =
6214                 fetch_and_zero(&power_domains->init_wakeref);
6215
6216         intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref);
6217
6218         /*
6219          * In case of suspend-to-idle (aka S0ix) on a DMC platform without DC9
6220          * support don't manually deinit the power domains. This also means the
6221          * DMC firmware will stay active, it will power down any HW
6222          * resources as required and also enable deeper system power states
6223          * that would be blocked if the firmware was inactive.
6224          */
6225         if (!(i915->dmc.allowed_dc_mask & DC_STATE_EN_DC9) &&
6226             suspend_mode == I915_DRM_SUSPEND_IDLE &&
6227             i915->dmc.dmc_payload) {
6228                 intel_display_power_flush_work(i915);
6229                 intel_power_domains_verify_state(i915);
6230                 return;
6231         }
6232
6233         /*
6234          * Even if power well support was disabled we still want to disable
6235          * power wells if power domains must be deinitialized for suspend.
6236          */
6237         if (!i915->params.disable_power_well)
6238                 intel_display_power_put(i915, POWER_DOMAIN_INIT,
6239                                         fetch_and_zero(&i915->power_domains.disable_wakeref));
6240
6241         intel_display_power_flush_work(i915);
6242         intel_power_domains_verify_state(i915);
6243
6244         if (DISPLAY_VER(i915) >= 11)
6245                 icl_display_core_uninit(i915);
6246         else if (IS_CANNONLAKE(i915))
6247                 cnl_display_core_uninit(i915);
6248         else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915))
6249                 bxt_display_core_uninit(i915);
6250         else if (DISPLAY_VER(i915) == 9)
6251                 skl_display_core_uninit(i915);
6252
6253         power_domains->display_core_suspended = true;
6254 }
6255
6256 /**
6257  * intel_power_domains_resume - resume power domain state
6258  * @i915: i915 device instance
6259  *
6260  * This function resume the hardware power domain state during system resume.
6261  *
6262  * It will return with power domain support disabled (to be enabled later by
6263  * intel_power_domains_enable()) and must be paired with
6264  * intel_power_domains_suspend().
6265  */
6266 void intel_power_domains_resume(struct drm_i915_private *i915)
6267 {
6268         struct i915_power_domains *power_domains = &i915->power_domains;
6269
6270         if (power_domains->display_core_suspended) {
6271                 intel_power_domains_init_hw(i915, true);
6272                 power_domains->display_core_suspended = false;
6273         } else {
6274                 drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
6275                 power_domains->init_wakeref =
6276                         intel_display_power_get(i915, POWER_DOMAIN_INIT);
6277         }
6278
6279         intel_power_domains_verify_state(i915);
6280 }
6281
6282 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
6283
6284 static void intel_power_domains_dump_info(struct drm_i915_private *i915)
6285 {
6286         struct i915_power_domains *power_domains = &i915->power_domains;
6287         struct i915_power_well *power_well;
6288
6289         for_each_power_well(i915, power_well) {
6290                 enum intel_display_power_domain domain;
6291
6292                 drm_dbg(&i915->drm, "%-25s %d\n",
6293                         power_well->desc->name, power_well->count);
6294
6295                 for_each_power_domain(domain, power_well->desc->domains)
6296                         drm_dbg(&i915->drm, "  %-23s %d\n",
6297                                 intel_display_power_domain_str(domain),
6298                                 power_domains->domain_use_count[domain]);
6299         }
6300 }
6301
6302 /**
6303  * intel_power_domains_verify_state - verify the HW/SW state for all power wells
6304  * @i915: i915 device instance
6305  *
6306  * Verify if the reference count of each power well matches its HW enabled
6307  * state and the total refcount of the domains it belongs to. This must be
6308  * called after modeset HW state sanitization, which is responsible for
6309  * acquiring reference counts for any power wells in use and disabling the
6310  * ones left on by BIOS but not required by any active output.
6311  */
6312 static void intel_power_domains_verify_state(struct drm_i915_private *i915)
6313 {
6314         struct i915_power_domains *power_domains = &i915->power_domains;
6315         struct i915_power_well *power_well;
6316         bool dump_domain_info;
6317
6318         mutex_lock(&power_domains->lock);
6319
6320         verify_async_put_domains_state(power_domains);
6321
6322         dump_domain_info = false;
6323         for_each_power_well(i915, power_well) {
6324                 enum intel_display_power_domain domain;
6325                 int domains_count;
6326                 bool enabled;
6327
6328                 enabled = power_well->desc->ops->is_enabled(i915, power_well);
6329                 if ((power_well->count || power_well->desc->always_on) !=
6330                     enabled)
6331                         drm_err(&i915->drm,
6332                                 "power well %s state mismatch (refcount %d/enabled %d)",
6333                                 power_well->desc->name,
6334                                 power_well->count, enabled);
6335
6336                 domains_count = 0;
6337                 for_each_power_domain(domain, power_well->desc->domains)
6338                         domains_count += power_domains->domain_use_count[domain];
6339
6340                 if (power_well->count != domains_count) {
6341                         drm_err(&i915->drm,
6342                                 "power well %s refcount/domain refcount mismatch "
6343                                 "(refcount %d/domains refcount %d)\n",
6344                                 power_well->desc->name, power_well->count,
6345                                 domains_count);
6346                         dump_domain_info = true;
6347                 }
6348         }
6349
6350         if (dump_domain_info) {
6351                 static bool dumped;
6352
6353                 if (!dumped) {
6354                         intel_power_domains_dump_info(i915);
6355                         dumped = true;
6356                 }
6357         }
6358
6359         mutex_unlock(&power_domains->lock);
6360 }
6361
6362 #else
6363
6364 static void intel_power_domains_verify_state(struct drm_i915_private *i915)
6365 {
6366 }
6367
6368 #endif
6369
6370 void intel_display_power_suspend_late(struct drm_i915_private *i915)
6371 {
6372         if (DISPLAY_VER(i915) >= 11 || IS_GEMINILAKE(i915) ||
6373             IS_BROXTON(i915)) {
6374                 bxt_enable_dc9(i915);
6375                 /* Tweaked Wa_14010685332:icp,jsp,mcc */
6376                 if (INTEL_PCH_TYPE(i915) >= PCH_ICP && INTEL_PCH_TYPE(i915) <= PCH_MCC)
6377                         intel_de_rmw(i915, SOUTH_CHICKEN1,
6378                                      SBCLK_RUN_REFCLK_DIS, SBCLK_RUN_REFCLK_DIS);
6379         } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
6380                 hsw_enable_pc8(i915);
6381         }
6382 }
6383
6384 void intel_display_power_resume_early(struct drm_i915_private *i915)
6385 {
6386         if (DISPLAY_VER(i915) >= 11 || IS_GEMINILAKE(i915) ||
6387             IS_BROXTON(i915)) {
6388                 gen9_sanitize_dc_state(i915);
6389                 bxt_disable_dc9(i915);
6390                 /* Tweaked Wa_14010685332:icp,jsp,mcc */
6391                 if (INTEL_PCH_TYPE(i915) >= PCH_ICP && INTEL_PCH_TYPE(i915) <= PCH_MCC)
6392                         intel_de_rmw(i915, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, 0);
6393
6394         } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
6395                 hsw_disable_pc8(i915);
6396         }
6397 }
6398
6399 void intel_display_power_suspend(struct drm_i915_private *i915)
6400 {
6401         if (DISPLAY_VER(i915) >= 11) {
6402                 icl_display_core_uninit(i915);
6403                 bxt_enable_dc9(i915);
6404         } else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
6405                 bxt_display_core_uninit(i915);
6406                 bxt_enable_dc9(i915);
6407         } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
6408                 hsw_enable_pc8(i915);
6409         }
6410 }
6411
6412 void intel_display_power_resume(struct drm_i915_private *i915)
6413 {
6414         if (DISPLAY_VER(i915) >= 11) {
6415                 bxt_disable_dc9(i915);
6416                 icl_display_core_init(i915, true);
6417                 if (i915->dmc.dmc_payload) {
6418                         if (i915->dmc.allowed_dc_mask &
6419                             DC_STATE_EN_UPTO_DC6)
6420                                 skl_enable_dc6(i915);
6421                         else if (i915->dmc.allowed_dc_mask &
6422                                  DC_STATE_EN_UPTO_DC5)
6423                                 gen9_enable_dc5(i915);
6424                 }
6425         } else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
6426                 bxt_disable_dc9(i915);
6427                 bxt_display_core_init(i915, true);
6428                 if (i915->dmc.dmc_payload &&
6429                     (i915->dmc.allowed_dc_mask & DC_STATE_EN_UPTO_DC5))
6430                         gen9_enable_dc5(i915);
6431         } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
6432                 hsw_disable_pc8(i915);
6433         }
6434 }