drivers/gpu/drm/i915/display/intel_display_power.c

   1 /* SPDX-License-Identifier: MIT */
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include "display/intel_crt.h"
   7
   8 #include "i915_drv.h"
   9 #include "i915_irq.h"
  10 #include "intel_cdclk.h"
  11 #include "intel_combo_phy.h"
  12 #include "intel_display_power.h"
  13 #include "intel_de.h"
  14 #include "intel_display_types.h"
  15 #include "intel_dmc.h"
  16 #include "intel_dpio_phy.h"
  17 #include "intel_hotplug.h"
  18 #include "intel_pm.h"
  19 #include "intel_pps.h"
  20 #include "intel_sideband.h"
  21 #include "intel_tc.h"
  22 #include "intel_vga.h"
  23
  24 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
  25                                          enum i915_power_well_id power_well_id);
  26
  27 const char *
  28 intel_display_power_domain_str(enum intel_display_power_domain domain)
  29 {
  30         switch (domain) {
  31         case POWER_DOMAIN_DISPLAY_CORE:
  32                 return "DISPLAY_CORE";
  33         case POWER_DOMAIN_PIPE_A:
  34                 return "PIPE_A";
  35         case POWER_DOMAIN_PIPE_B:
  36                 return "PIPE_B";
  37         case POWER_DOMAIN_PIPE_C:
  38                 return "PIPE_C";
  39         case POWER_DOMAIN_PIPE_D:
  40                 return "PIPE_D";
  41         case POWER_DOMAIN_PIPE_A_PANEL_FITTER:
  42                 return "PIPE_A_PANEL_FITTER";
  43         case POWER_DOMAIN_PIPE_B_PANEL_FITTER:
  44                 return "PIPE_B_PANEL_FITTER";
  45         case POWER_DOMAIN_PIPE_C_PANEL_FITTER:
  46                 return "PIPE_C_PANEL_FITTER";
  47         case POWER_DOMAIN_PIPE_D_PANEL_FITTER:
  48                 return "PIPE_D_PANEL_FITTER";
  49         case POWER_DOMAIN_TRANSCODER_A:
  50                 return "TRANSCODER_A";
  51         case POWER_DOMAIN_TRANSCODER_B:
  52                 return "TRANSCODER_B";
  53         case POWER_DOMAIN_TRANSCODER_C:
  54                 return "TRANSCODER_C";
  55         case POWER_DOMAIN_TRANSCODER_D:
  56                 return "TRANSCODER_D";
  57         case POWER_DOMAIN_TRANSCODER_EDP:
  58                 return "TRANSCODER_EDP";
  59         case POWER_DOMAIN_TRANSCODER_VDSC_PW2:
  60                 return "TRANSCODER_VDSC_PW2";
  61         case POWER_DOMAIN_TRANSCODER_DSI_A:
  62                 return "TRANSCODER_DSI_A";
  63         case POWER_DOMAIN_TRANSCODER_DSI_C:
  64                 return "TRANSCODER_DSI_C";
  65         case POWER_DOMAIN_PORT_DDI_A_LANES:
  66                 return "PORT_DDI_A_LANES";
  67         case POWER_DOMAIN_PORT_DDI_B_LANES:
  68                 return "PORT_DDI_B_LANES";
  69         case POWER_DOMAIN_PORT_DDI_C_LANES:
  70                 return "PORT_DDI_C_LANES";
  71         case POWER_DOMAIN_PORT_DDI_D_LANES:
  72                 return "PORT_DDI_D_LANES";
  73         case POWER_DOMAIN_PORT_DDI_E_LANES:
  74                 return "PORT_DDI_E_LANES";
  75         case POWER_DOMAIN_PORT_DDI_F_LANES:
  76                 return "PORT_DDI_F_LANES";
  77         case POWER_DOMAIN_PORT_DDI_G_LANES:
  78                 return "PORT_DDI_G_LANES";
  79         case POWER_DOMAIN_PORT_DDI_H_LANES:
  80                 return "PORT_DDI_H_LANES";
  81         case POWER_DOMAIN_PORT_DDI_I_LANES:
  82                 return "PORT_DDI_I_LANES";
  83         case POWER_DOMAIN_PORT_DDI_A_IO:
  84                 return "PORT_DDI_A_IO";
  85         case POWER_DOMAIN_PORT_DDI_B_IO:
  86                 return "PORT_DDI_B_IO";
  87         case POWER_DOMAIN_PORT_DDI_C_IO:
  88                 return "PORT_DDI_C_IO";
  89         case POWER_DOMAIN_PORT_DDI_D_IO:
  90                 return "PORT_DDI_D_IO";
  91         case POWER_DOMAIN_PORT_DDI_E_IO:
  92                 return "PORT_DDI_E_IO";
  93         case POWER_DOMAIN_PORT_DDI_F_IO:
  94                 return "PORT_DDI_F_IO";
  95         case POWER_DOMAIN_PORT_DDI_G_IO:
  96                 return "PORT_DDI_G_IO";
  97         case POWER_DOMAIN_PORT_DDI_H_IO:
  98                 return "PORT_DDI_H_IO";
  99         case POWER_DOMAIN_PORT_DDI_I_IO:
 100                 return "PORT_DDI_I_IO";
 101         case POWER_DOMAIN_PORT_DSI:
 102                 return "PORT_DSI";
 103         case POWER_DOMAIN_PORT_CRT:
 104                 return "PORT_CRT";
 105         case POWER_DOMAIN_PORT_OTHER:
 106                 return "PORT_OTHER";
 107         case POWER_DOMAIN_VGA:
 108                 return "VGA";
 109         case POWER_DOMAIN_AUDIO:
 110                 return "AUDIO";
 111         case POWER_DOMAIN_AUX_A:
 112                 return "AUX_A";
 113         case POWER_DOMAIN_AUX_B:
 114                 return "AUX_B";
 115         case POWER_DOMAIN_AUX_C:
 116                 return "AUX_C";
 117         case POWER_DOMAIN_AUX_D:
 118                 return "AUX_D";
 119         case POWER_DOMAIN_AUX_E:
 120                 return "AUX_E";
 121         case POWER_DOMAIN_AUX_F:
 122                 return "AUX_F";
 123         case POWER_DOMAIN_AUX_G:
 124                 return "AUX_G";
 125         case POWER_DOMAIN_AUX_H:
 126                 return "AUX_H";
 127         case POWER_DOMAIN_AUX_I:
 128                 return "AUX_I";
 129         case POWER_DOMAIN_AUX_IO_A:
 130                 return "AUX_IO_A";
 131         case POWER_DOMAIN_AUX_C_TBT:
 132                 return "AUX_C_TBT";
 133         case POWER_DOMAIN_AUX_D_TBT:
 134                 return "AUX_D_TBT";
 135         case POWER_DOMAIN_AUX_E_TBT:
 136                 return "AUX_E_TBT";
 137         case POWER_DOMAIN_AUX_F_TBT:
 138                 return "AUX_F_TBT";
 139         case POWER_DOMAIN_AUX_G_TBT:
 140                 return "AUX_G_TBT";
 141         case POWER_DOMAIN_AUX_H_TBT:
 142                 return "AUX_H_TBT";
 143         case POWER_DOMAIN_AUX_I_TBT:
 144                 return "AUX_I_TBT";
 145         case POWER_DOMAIN_GMBUS:
 146                 return "GMBUS";
 147         case POWER_DOMAIN_INIT:
 148                 return "INIT";
 149         case POWER_DOMAIN_MODESET:
 150                 return "MODESET";
 151         case POWER_DOMAIN_GT_IRQ:
 152                 return "GT_IRQ";
 153         case POWER_DOMAIN_DPLL_DC_OFF:
 154                 return "DPLL_DC_OFF";
 155         case POWER_DOMAIN_TC_COLD_OFF:
 156                 return "TC_COLD_OFF";
 157         default:
 158                 MISSING_CASE(domain);
 159                 return "?";
 160         }
 161 }
 162
 163 static void intel_power_well_enable(struct drm_i915_private *dev_priv,
 164                                     struct i915_power_well *power_well)
 165 {
 166         drm_dbg_kms(&dev_priv->drm, "enabling %s\n", power_well->desc->name);
 167         power_well->desc->ops->enable(dev_priv, power_well);
 168         power_well->hw_enabled = true;
 169 }
 170
 171 static void intel_power_well_disable(struct drm_i915_private *dev_priv,
 172                                      struct i915_power_well *power_well)
 173 {
 174         drm_dbg_kms(&dev_priv->drm, "disabling %s\n", power_well->desc->name);
 175         power_well->hw_enabled = false;
 176         power_well->desc->ops->disable(dev_priv, power_well);
 177 }
 178
 179 static void intel_power_well_get(struct drm_i915_private *dev_priv,
 180                                  struct i915_power_well *power_well)
 181 {
 182         if (!power_well->count++)
 183                 intel_power_well_enable(dev_priv, power_well);
 184 }
 185
 186 static void intel_power_well_put(struct drm_i915_private *dev_priv,
 187                                  struct i915_power_well *power_well)
 188 {
 189         drm_WARN(&dev_priv->drm, !power_well->count,
 190                  "Use count on power well %s is already zero",
 191                  power_well->desc->name);
 192
 193         if (!--power_well->count)
 194                 intel_power_well_disable(dev_priv, power_well);
 195 }
 196
 197 /**
 198  * __intel_display_power_is_enabled - unlocked check for a power domain
 199  * @dev_priv: i915 device instance
 200  * @domain: power domain to check
 201  *
 202  * This is the unlocked version of intel_display_power_is_enabled() and should
 203  * only be used from error capture and recovery code where deadlocks are
 204  * possible.
 205  *
 206  * Returns:
 207  * True when the power domain is enabled, false otherwise.
 208  */
 209 bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 210                                       enum intel_display_power_domain domain)
 211 {
 212         struct i915_power_well *power_well;
 213         bool is_enabled;
 214
 215         if (dev_priv->runtime_pm.suspended)
 216                 return false;
 217
 218         is_enabled = true;
 219
 220         for_each_power_domain_well_reverse(dev_priv, power_well, BIT_ULL(domain)) {
 221                 if (power_well->desc->always_on)
 222                         continue;
 223
 224                 if (!power_well->hw_enabled) {
 225                         is_enabled = false;
 226                         break;
 227                 }
 228         }
 229
 230         return is_enabled;
 231 }
 232
 233 /**
 234  * intel_display_power_is_enabled - check for a power domain
 235  * @dev_priv: i915 device instance
 236  * @domain: power domain to check
 237  *
 238  * This function can be used to check the hw power domain state. It is mostly
 239  * used in hardware state readout functions. Everywhere else code should rely
 240  * upon explicit power domain reference counting to ensure that the hardware
 241  * block is powered up before accessing it.
 242  *
 243  * Callers must hold the relevant modesetting locks to ensure that concurrent
 244  * threads can't disable the power well while the caller tries to read a few
 245  * registers.
 246  *
 247  * Returns:
 248  * True when the power domain is enabled, false otherwise.
 249  */
 250 bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 251                                     enum intel_display_power_domain domain)
 252 {
 253         struct i915_power_domains *power_domains;
 254         bool ret;
 255
 256         power_domains = &dev_priv->power_domains;
 257
 258         mutex_lock(&power_domains->lock);
 259         ret = __intel_display_power_is_enabled(dev_priv, domain);
 260         mutex_unlock(&power_domains->lock);
 261
 262         return ret;
 263 }
 264
 265 /*
 266  * Starting with Haswell, we have a "Power Down Well" that can be turned off
 267  * when not needed anymore. We have 4 registers that can request the power well
 268  * to be enabled, and it will only be disabled if none of the registers is
 269  * requesting it to be enabled.
 270  */
 271 static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv,
 272                                        u8 irq_pipe_mask, bool has_vga)
 273 {
 274         if (has_vga)
 275                 intel_vga_reset_io_mem(dev_priv);
 276
 277         if (irq_pipe_mask)
 278                 gen8_irq_power_well_post_enable(dev_priv, irq_pipe_mask);
 279 }
 280
 281 static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv,
 282                                        u8 irq_pipe_mask)
 283 {
 284         if (irq_pipe_mask)
 285                 gen8_irq_power_well_pre_disable(dev_priv, irq_pipe_mask);
 286 }
 287
 288 #define ICL_AUX_PW_TO_CH(pw_idx)        \
 289         ((pw_idx) - ICL_PW_CTL_IDX_AUX_A + AUX_CH_A)
 290
 291 #define ICL_TBT_AUX_PW_TO_CH(pw_idx)    \
 292         ((pw_idx) - ICL_PW_CTL_IDX_AUX_TBT1 + AUX_CH_C)
 293
 294 static enum aux_ch icl_aux_pw_to_ch(const struct i915_power_well *power_well)
 295 {
 296         int pw_idx = power_well->desc->hsw.idx;
 297
 298         return power_well->desc->hsw.is_tc_tbt ? ICL_TBT_AUX_PW_TO_CH(pw_idx) :
 299                                                  ICL_AUX_PW_TO_CH(pw_idx);
 300 }
 301
 302 static struct intel_digital_port *
 303 aux_ch_to_digital_port(struct drm_i915_private *dev_priv,
 304                        enum aux_ch aux_ch)
 305 {
 306         struct intel_digital_port *dig_port = NULL;
 307         struct intel_encoder *encoder;
 308
 309         for_each_intel_encoder(&dev_priv->drm, encoder) {
 310                 /* We'll check the MST primary port */
 311                 if (encoder->type == INTEL_OUTPUT_DP_MST)
 312                         continue;
 313
 314                 dig_port = enc_to_dig_port(encoder);
 315                 if (!dig_port)
 316                         continue;
 317
 318                 if (dig_port->aux_ch != aux_ch) {
 319                         dig_port = NULL;
 320                         continue;
 321                 }
 322
 323                 break;
 324         }
 325
 326         return dig_port;
 327 }
 328
 329 static enum phy icl_aux_pw_to_phy(struct drm_i915_private *i915,
 330                                   const struct i915_power_well *power_well)
 331 {
 332         enum aux_ch aux_ch = icl_aux_pw_to_ch(power_well);
 333         struct intel_digital_port *dig_port = aux_ch_to_digital_port(i915, aux_ch);
 334
 335         return intel_port_to_phy(i915, dig_port->base.port);
 336 }
 337
 338 static void hsw_wait_for_power_well_enable(struct drm_i915_private *dev_priv,
 339                                            struct i915_power_well *power_well,
 340                                            bool timeout_expected)
 341 {
 342         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 343         int pw_idx = power_well->desc->hsw.idx;
 344
 345         /* Timeout for PW1:10 us, AUX:not specified, other PWs:20 us. */
 346         if (intel_de_wait_for_set(dev_priv, regs->driver,
 347                                   HSW_PWR_WELL_CTL_STATE(pw_idx), 1)) {
 348                 drm_dbg_kms(&dev_priv->drm, "%s power well enable timeout\n",
 349                             power_well->desc->name);
 350
 351                 drm_WARN_ON(&dev_priv->drm, !timeout_expected);
 352
 353         }
 354 }
 355
 356 static u32 hsw_power_well_requesters(struct drm_i915_private *dev_priv,
 357                                      const struct i915_power_well_regs *regs,
 358                                      int pw_idx)
 359 {
 360         u32 req_mask = HSW_PWR_WELL_CTL_REQ(pw_idx);
 361         u32 ret;
 362
 363         ret = intel_de_read(dev_priv, regs->bios) & req_mask ? 1 : 0;
 364         ret |= intel_de_read(dev_priv, regs->driver) & req_mask ? 2 : 0;
 365         if (regs->kvmr.reg)
 366                 ret |= intel_de_read(dev_priv, regs->kvmr) & req_mask ? 4 : 0;
 367         ret |= intel_de_read(dev_priv, regs->debug) & req_mask ? 8 : 0;
 368
 369         return ret;
 370 }
 371
 372 static void hsw_wait_for_power_well_disable(struct drm_i915_private *dev_priv,
 373                                             struct i915_power_well *power_well)
 374 {
 375         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 376         int pw_idx = power_well->desc->hsw.idx;
 377         bool disabled;
 378         u32 reqs;
 379
 380         /*
 381          * Bspec doesn't require waiting for PWs to get disabled, but still do
 382          * this for paranoia. The known cases where a PW will be forced on:
 383          * - a KVMR request on any power well via the KVMR request register
 384          * - a DMC request on PW1 and MISC_IO power wells via the BIOS and
 385          *   DEBUG request registers
 386          * Skip the wait in case any of the request bits are set and print a
 387          * diagnostic message.
 388          */
 389         wait_for((disabled = !(intel_de_read(dev_priv, regs->driver) &
 390                                HSW_PWR_WELL_CTL_STATE(pw_idx))) ||
 391                  (reqs = hsw_power_well_requesters(dev_priv, regs, pw_idx)), 1);
 392         if (disabled)
 393                 return;
 394
 395         drm_dbg_kms(&dev_priv->drm,
 396                     "%s forced on (bios:%d driver:%d kvmr:%d debug:%d)\n",
 397                     power_well->desc->name,
 398                     !!(reqs & 1), !!(reqs & 2), !!(reqs & 4), !!(reqs & 8));
 399 }
 400
 401 static void gen9_wait_for_power_well_fuses(struct drm_i915_private *dev_priv,
 402                                            enum skl_power_gate pg)
 403 {
 404         /* Timeout 5us for PG#0, for other PGs 1us */
 405         drm_WARN_ON(&dev_priv->drm,
 406                     intel_de_wait_for_set(dev_priv, SKL_FUSE_STATUS,
 407                                           SKL_FUSE_PG_DIST_STATUS(pg), 1));
 408 }
 409
 410 static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
 411                                   struct i915_power_well *power_well)
 412 {
 413         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 414         int pw_idx = power_well->desc->hsw.idx;
 415         u32 val;
 416
 417         if (power_well->desc->hsw.has_fuses) {
 418                 enum skl_power_gate pg;
 419
 420                 pg = DISPLAY_VER(dev_priv) >= 11 ? ICL_PW_CTL_IDX_TO_PG(pw_idx) :
 421                                                  SKL_PW_CTL_IDX_TO_PG(pw_idx);
 422                 /*
 423                  * For PW1 we have to wait both for the PW0/PG0 fuse state
 424                  * before enabling the power well and PW1/PG1's own fuse
 425                  * state after the enabling. For all other power wells with
 426                  * fuses we only have to wait for that PW/PG's fuse state
 427                  * after the enabling.
 428                  */
 429                 if (pg == SKL_PG1)
 430                         gen9_wait_for_power_well_fuses(dev_priv, SKL_PG0);
 431         }
 432
 433         val = intel_de_read(dev_priv, regs->driver);
 434         intel_de_write(dev_priv, regs->driver,
 435                        val | HSW_PWR_WELL_CTL_REQ(pw_idx));
 436
 437         hsw_wait_for_power_well_enable(dev_priv, power_well, false);
 438
 439         /* Display WA #1178: cnl */
 440         if (IS_CANNONLAKE(dev_priv) &&
 441             pw_idx >= GLK_PW_CTL_IDX_AUX_B &&
 442             pw_idx <= CNL_PW_CTL_IDX_AUX_F) {
 443                 u32 val;
 444
 445                 val = intel_de_read(dev_priv, CNL_AUX_ANAOVRD1(pw_idx));
 446                 val |= CNL_AUX_ANAOVRD1_ENABLE | CNL_AUX_ANAOVRD1_LDO_BYPASS;
 447                 intel_de_write(dev_priv, CNL_AUX_ANAOVRD1(pw_idx), val);
 448         }
 449
 450         if (power_well->desc->hsw.has_fuses) {
 451                 enum skl_power_gate pg;
 452
 453                 pg = DISPLAY_VER(dev_priv) >= 11 ? ICL_PW_CTL_IDX_TO_PG(pw_idx) :
 454                                                  SKL_PW_CTL_IDX_TO_PG(pw_idx);
 455                 gen9_wait_for_power_well_fuses(dev_priv, pg);
 456         }
 457
 458         hsw_power_well_post_enable(dev_priv,
 459                                    power_well->desc->hsw.irq_pipe_mask,
 460                                    power_well->desc->hsw.has_vga);
 461 }
 462
 463 static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
 464                                    struct i915_power_well *power_well)
 465 {
 466         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 467         int pw_idx = power_well->desc->hsw.idx;
 468         u32 val;
 469
 470         hsw_power_well_pre_disable(dev_priv,
 471                                    power_well->desc->hsw.irq_pipe_mask);
 472
 473         val = intel_de_read(dev_priv, regs->driver);
 474         intel_de_write(dev_priv, regs->driver,
 475                        val & ~HSW_PWR_WELL_CTL_REQ(pw_idx));
 476         hsw_wait_for_power_well_disable(dev_priv, power_well);
 477 }
 478
 479 static void
 480 icl_combo_phy_aux_power_well_enable(struct drm_i915_private *dev_priv,
 481                                     struct i915_power_well *power_well)
 482 {
 483         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 484         int pw_idx = power_well->desc->hsw.idx;
 485         enum phy phy = icl_aux_pw_to_phy(dev_priv, power_well);
 486         u32 val;
 487
 488         drm_WARN_ON(&dev_priv->drm, !IS_ICELAKE(dev_priv));
 489
 490         val = intel_de_read(dev_priv, regs->driver);
 491         intel_de_write(dev_priv, regs->driver,
 492                        val | HSW_PWR_WELL_CTL_REQ(pw_idx));
 493
 494         if (DISPLAY_VER(dev_priv) < 12) {
 495                 val = intel_de_read(dev_priv, ICL_PORT_CL_DW12(phy));
 496                 intel_de_write(dev_priv, ICL_PORT_CL_DW12(phy),
 497                                val | ICL_LANE_ENABLE_AUX);
 498         }
 499
 500         hsw_wait_for_power_well_enable(dev_priv, power_well, false);
 501
 502         /* Display WA #1178: icl */
 503         if (pw_idx >= ICL_PW_CTL_IDX_AUX_A && pw_idx <= ICL_PW_CTL_IDX_AUX_B &&
 504             !intel_bios_is_port_edp(dev_priv, (enum port)phy)) {
 505                 val = intel_de_read(dev_priv, ICL_AUX_ANAOVRD1(pw_idx));
 506                 val |= ICL_AUX_ANAOVRD1_ENABLE | ICL_AUX_ANAOVRD1_LDO_BYPASS;
 507                 intel_de_write(dev_priv, ICL_AUX_ANAOVRD1(pw_idx), val);
 508         }
 509 }
 510
 511 static void
 512 icl_combo_phy_aux_power_well_disable(struct drm_i915_private *dev_priv,
 513                                      struct i915_power_well *power_well)
 514 {
 515         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 516         int pw_idx = power_well->desc->hsw.idx;
 517         enum phy phy = icl_aux_pw_to_phy(dev_priv, power_well);
 518         u32 val;
 519
 520         drm_WARN_ON(&dev_priv->drm, !IS_ICELAKE(dev_priv));
 521
 522         val = intel_de_read(dev_priv, ICL_PORT_CL_DW12(phy));
 523         intel_de_write(dev_priv, ICL_PORT_CL_DW12(phy),
 524                        val & ~ICL_LANE_ENABLE_AUX);
 525
 526         val = intel_de_read(dev_priv, regs->driver);
 527         intel_de_write(dev_priv, regs->driver,
 528                        val & ~HSW_PWR_WELL_CTL_REQ(pw_idx));
 529
 530         hsw_wait_for_power_well_disable(dev_priv, power_well);
 531 }
 532
 533 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
 534
 535 static u64 async_put_domains_mask(struct i915_power_domains *power_domains);
 536
 537 static int power_well_async_ref_count(struct drm_i915_private *dev_priv,
 538                                       struct i915_power_well *power_well)
 539 {
 540         int refs = hweight64(power_well->desc->domains &
 541                              async_put_domains_mask(&dev_priv->power_domains));
 542
 543         drm_WARN_ON(&dev_priv->drm, refs > power_well->count);
 544
 545         return refs;
 546 }
 547
 548 static void icl_tc_port_assert_ref_held(struct drm_i915_private *dev_priv,
 549                                         struct i915_power_well *power_well,
 550                                         struct intel_digital_port *dig_port)
 551 {
 552         /* Bypass the check if all references are released asynchronously */
 553         if (power_well_async_ref_count(dev_priv, power_well) ==
 554             power_well->count)
 555                 return;
 556
 557         if (drm_WARN_ON(&dev_priv->drm, !dig_port))
 558                 return;
 559
 560         if (DISPLAY_VER(dev_priv) == 11 && dig_port->tc_legacy_port)
 561                 return;
 562
 563         drm_WARN_ON(&dev_priv->drm, !intel_tc_port_ref_held(dig_port));
 564 }
 565
 566 #else
 567
 568 static void icl_tc_port_assert_ref_held(struct drm_i915_private *dev_priv,
 569                                         struct i915_power_well *power_well,
 570                                         struct intel_digital_port *dig_port)
 571 {
 572 }
 573
 574 #endif
 575
 576 #define TGL_AUX_PW_TO_TC_PORT(pw_idx)   ((pw_idx) - TGL_PW_CTL_IDX_AUX_TC1)
 577
 578 static void icl_tc_cold_exit(struct drm_i915_private *i915)
 579 {
 580         int ret, tries = 0;
 581
 582         while (1) {
 583                 ret = sandybridge_pcode_write_timeout(i915,
 584                                                       ICL_PCODE_EXIT_TCCOLD,
 585                                                       0, 250, 1);
 586                 if (ret != -EAGAIN || ++tries == 3)
 587                         break;
 588                 msleep(1);
 589         }
 590
 591         /* Spec states that TC cold exit can take up to 1ms to complete */
 592         if (!ret)
 593                 msleep(1);
 594
 595         /* TODO: turn failure into a error as soon i915 CI updates ICL IFWI */
 596         drm_dbg_kms(&i915->drm, "TC cold block %s\n", ret ? "failed" :
 597                     "succeeded");
 598 }
 599
 600 static void
 601 icl_tc_phy_aux_power_well_enable(struct drm_i915_private *dev_priv,
 602                                  struct i915_power_well *power_well)
 603 {
 604         enum aux_ch aux_ch = icl_aux_pw_to_ch(power_well);
 605         struct intel_digital_port *dig_port = aux_ch_to_digital_port(dev_priv, aux_ch);
 606         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 607         bool is_tbt = power_well->desc->hsw.is_tc_tbt;
 608         bool timeout_expected;
 609         u32 val;
 610
 611         icl_tc_port_assert_ref_held(dev_priv, power_well, dig_port);
 612
 613         val = intel_de_read(dev_priv, DP_AUX_CH_CTL(aux_ch));
 614         val &= ~DP_AUX_CH_CTL_TBT_IO;
 615         if (is_tbt)
 616                 val |= DP_AUX_CH_CTL_TBT_IO;
 617         intel_de_write(dev_priv, DP_AUX_CH_CTL(aux_ch), val);
 618
 619         val = intel_de_read(dev_priv, regs->driver);
 620         intel_de_write(dev_priv, regs->driver,
 621                        val | HSW_PWR_WELL_CTL_REQ(power_well->desc->hsw.idx));
 622
 623         /*
 624          * An AUX timeout is expected if the TBT DP tunnel is down,
 625          * or need to enable AUX on a legacy TypeC port as part of the TC-cold
 626          * exit sequence.
 627          */
 628         timeout_expected = is_tbt || intel_tc_cold_requires_aux_pw(dig_port);
 629         if (DISPLAY_VER(dev_priv) == 11 && dig_port->tc_legacy_port)
 630                 icl_tc_cold_exit(dev_priv);
 631
 632         hsw_wait_for_power_well_enable(dev_priv, power_well, timeout_expected);
 633
 634         if (DISPLAY_VER(dev_priv) >= 12 && !is_tbt) {
 635                 enum tc_port tc_port;
 636
 637                 tc_port = TGL_AUX_PW_TO_TC_PORT(power_well->desc->hsw.idx);
 638                 intel_de_write(dev_priv, HIP_INDEX_REG(tc_port),
 639                                HIP_INDEX_VAL(tc_port, 0x2));
 640
 641                 if (intel_de_wait_for_set(dev_priv, DKL_CMN_UC_DW_27(tc_port),
 642                                           DKL_CMN_UC_DW27_UC_HEALTH, 1))
 643                         drm_warn(&dev_priv->drm,
 644                                  "Timeout waiting TC uC health\n");
 645         }
 646 }
 647
 648 static void
 649 icl_tc_phy_aux_power_well_disable(struct drm_i915_private *dev_priv,
 650                                   struct i915_power_well *power_well)
 651 {
 652         enum aux_ch aux_ch = icl_aux_pw_to_ch(power_well);
 653         struct intel_digital_port *dig_port = aux_ch_to_digital_port(dev_priv, aux_ch);
 654
 655         icl_tc_port_assert_ref_held(dev_priv, power_well, dig_port);
 656
 657         hsw_power_well_disable(dev_priv, power_well);
 658 }
 659
 660 static void
 661 icl_aux_power_well_enable(struct drm_i915_private *dev_priv,
 662                           struct i915_power_well *power_well)
 663 {
 664         enum phy phy = icl_aux_pw_to_phy(dev_priv, power_well);
 665
 666         if (intel_phy_is_tc(dev_priv, phy))
 667                 return icl_tc_phy_aux_power_well_enable(dev_priv, power_well);
 668         else if (IS_ICELAKE(dev_priv))
 669                 return icl_combo_phy_aux_power_well_enable(dev_priv,
 670                                                            power_well);
 671         else
 672                 return hsw_power_well_enable(dev_priv, power_well);
 673 }
 674
 675 static void
 676 icl_aux_power_well_disable(struct drm_i915_private *dev_priv,
 677                            struct i915_power_well *power_well)
 678 {
 679         enum phy phy = icl_aux_pw_to_phy(dev_priv, power_well);
 680
 681         if (intel_phy_is_tc(dev_priv, phy))
 682                 return icl_tc_phy_aux_power_well_disable(dev_priv, power_well);
 683         else if (IS_ICELAKE(dev_priv))
 684                 return icl_combo_phy_aux_power_well_disable(dev_priv,
 685                                                             power_well);
 686         else
 687                 return hsw_power_well_disable(dev_priv, power_well);
 688 }
 689
 690 /*
 691  * We should only use the power well if we explicitly asked the hardware to
 692  * enable it, so check if it's enabled and also check if we've requested it to
 693  * be enabled.
 694  */
 695 static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
 696                                    struct i915_power_well *power_well)
 697 {
 698         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
 699         enum i915_power_well_id id = power_well->desc->id;
 700         int pw_idx = power_well->desc->hsw.idx;
 701         u32 mask = HSW_PWR_WELL_CTL_REQ(pw_idx) |
 702                    HSW_PWR_WELL_CTL_STATE(pw_idx);
 703         u32 val;
 704
 705         val = intel_de_read(dev_priv, regs->driver);
 706
 707         /*
 708          * On GEN9 big core due to a DMC bug the driver's request bits for PW1
 709          * and the MISC_IO PW will be not restored, so check instead for the
 710          * BIOS's own request bits, which are forced-on for these power wells
 711          * when exiting DC5/6.
 712          */
 713         if (DISPLAY_VER(dev_priv) == 9 && !IS_BROXTON(dev_priv) &&
 714             (id == SKL_DISP_PW_1 || id == SKL_DISP_PW_MISC_IO))
 715                 val |= intel_de_read(dev_priv, regs->bios);
 716
 717         return (val & mask) == mask;
 718 }
 719
 720 static void assert_can_enable_dc9(struct drm_i915_private *dev_priv)
 721 {
 722         drm_WARN_ONCE(&dev_priv->drm,
 723                       (intel_de_read(dev_priv, DC_STATE_EN) & DC_STATE_EN_DC9),
 724                       "DC9 already programmed to be enabled.\n");
 725         drm_WARN_ONCE(&dev_priv->drm,
 726                       intel_de_read(dev_priv, DC_STATE_EN) &
 727                       DC_STATE_EN_UPTO_DC5,
 728                       "DC5 still not disabled to enable DC9.\n");
 729         drm_WARN_ONCE(&dev_priv->drm,
 730                       intel_de_read(dev_priv, HSW_PWR_WELL_CTL2) &
 731                       HSW_PWR_WELL_CTL_REQ(SKL_PW_CTL_IDX_PW_2),
 732                       "Power well 2 on.\n");
 733         drm_WARN_ONCE(&dev_priv->drm, intel_irqs_enabled(dev_priv),
 734                       "Interrupts not disabled yet.\n");
 735
 736          /*
 737           * TODO: check for the following to verify the conditions to enter DC9
 738           * state are satisfied:
 739           * 1] Check relevant display engine registers to verify if mode set
 740           * disable sequence was followed.
 741           * 2] Check if display uninitialize sequence is initialized.
 742           */
 743 }
 744
 745 static void assert_can_disable_dc9(struct drm_i915_private *dev_priv)
 746 {
 747         drm_WARN_ONCE(&dev_priv->drm, intel_irqs_enabled(dev_priv),
 748                       "Interrupts not disabled yet.\n");
 749         drm_WARN_ONCE(&dev_priv->drm,
 750                       intel_de_read(dev_priv, DC_STATE_EN) &
 751                       DC_STATE_EN_UPTO_DC5,
 752                       "DC5 still not disabled.\n");
 753
 754          /*
 755           * TODO: check for the following to verify DC9 state was indeed
 756           * entered before programming to disable it:
 757           * 1] Check relevant display engine registers to verify if mode
 758           *  set disable sequence was followed.
 759           * 2] Check if display uninitialize sequence is initialized.
 760           */
 761 }
 762
 763 static void gen9_write_dc_state(struct drm_i915_private *dev_priv,
 764                                 u32 state)
 765 {
 766         int rewrites = 0;
 767         int rereads = 0;
 768         u32 v;
 769
 770         intel_de_write(dev_priv, DC_STATE_EN, state);
 771
 772         /* It has been observed that disabling the dc6 state sometimes
 773          * doesn't stick and dmc keeps returning old value. Make sure
 774          * the write really sticks enough times and also force rewrite until
 775          * we are confident that state is exactly what we want.
 776          */
 777         do  {
 778                 v = intel_de_read(dev_priv, DC_STATE_EN);
 779
 780                 if (v != state) {
 781                         intel_de_write(dev_priv, DC_STATE_EN, state);
 782                         rewrites++;
 783                         rereads = 0;
 784                 } else if (rereads++ > 5) {
 785                         break;
 786                 }
 787
 788         } while (rewrites < 100);
 789
 790         if (v != state)
 791                 drm_err(&dev_priv->drm,
 792                         "Writing dc state to 0x%x failed, now 0x%x\n",
 793                         state, v);
 794
 795         /* Most of the times we need one retry, avoid spam */
 796         if (rewrites > 1)
 797                 drm_dbg_kms(&dev_priv->drm,
 798                             "Rewrote dc state to 0x%x %d times\n",
 799                             state, rewrites);
 800 }
 801
 802 static u32 gen9_dc_mask(struct drm_i915_private *dev_priv)
 803 {
 804         u32 mask;
 805
 806         mask = DC_STATE_EN_UPTO_DC5;
 807
 808         if (DISPLAY_VER(dev_priv) >= 12)
 809                 mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC6
 810                                           | DC_STATE_EN_DC9;
 811         else if (DISPLAY_VER(dev_priv) == 11)
 812                 mask |= DC_STATE_EN_UPTO_DC6 | DC_STATE_EN_DC9;
 813         else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
 814                 mask |= DC_STATE_EN_DC9;
 815         else
 816                 mask |= DC_STATE_EN_UPTO_DC6;
 817
 818         return mask;
 819 }
 820
 821 static void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv)
 822 {
 823         u32 val;
 824
 825         if (!HAS_DISPLAY(dev_priv))
 826                 return;
 827
 828         val = intel_de_read(dev_priv, DC_STATE_EN) & gen9_dc_mask(dev_priv);
 829
 830         drm_dbg_kms(&dev_priv->drm,
 831                     "Resetting DC state tracking from %02x to %02x\n",
 832                     dev_priv->dmc.dc_state, val);
 833         dev_priv->dmc.dc_state = val;
 834 }
 835
 836 /**
 837  * gen9_set_dc_state - set target display C power state
 838  * @dev_priv: i915 device instance
 839  * @state: target DC power state
 840  * - DC_STATE_DISABLE
 841  * - DC_STATE_EN_UPTO_DC5
 842  * - DC_STATE_EN_UPTO_DC6
 843  * - DC_STATE_EN_DC9
 844  *
 845  * Signal to DMC firmware/HW the target DC power state passed in @state.
 846  * DMC/HW can turn off individual display clocks and power rails when entering
 847  * a deeper DC power state (higher in number) and turns these back when exiting
 848  * that state to a shallower power state (lower in number). The HW will decide
 849  * when to actually enter a given state on an on-demand basis, for instance
 850  * depending on the active state of display pipes. The state of display
 851  * registers backed by affected power rails are saved/restored as needed.
 852  *
 853  * Based on the above enabling a deeper DC power state is asynchronous wrt.
 854  * enabling it. Disabling a deeper power state is synchronous: for instance
 855  * setting %DC_STATE_DISABLE won't complete until all HW resources are turned
 856  * back on and register state is restored. This is guaranteed by the MMIO write
 857  * to DC_STATE_EN blocking until the state is restored.
 858  */
 859 static void gen9_set_dc_state(struct drm_i915_private *dev_priv, u32 state)
 860 {
 861         u32 val;
 862         u32 mask;
 863
 864         if (!HAS_DISPLAY(dev_priv))
 865                 return;
 866
 867         if (drm_WARN_ON_ONCE(&dev_priv->drm,
 868                              state & ~dev_priv->dmc.allowed_dc_mask))
 869                 state &= dev_priv->dmc.allowed_dc_mask;
 870
 871         val = intel_de_read(dev_priv, DC_STATE_EN);
 872         mask = gen9_dc_mask(dev_priv);
 873         drm_dbg_kms(&dev_priv->drm, "Setting DC state from %02x to %02x\n",
 874                     val & mask, state);
 875
 876         /* Check if DMC is ignoring our DC state requests */
 877         if ((val & mask) != dev_priv->dmc.dc_state)
 878                 drm_err(&dev_priv->drm, "DC state mismatch (0x%x -> 0x%x)\n",
 879                         dev_priv->dmc.dc_state, val & mask);
 880
 881         val &= ~mask;
 882         val |= state;
 883
 884         gen9_write_dc_state(dev_priv, val);
 885
 886         dev_priv->dmc.dc_state = val & mask;
 887 }
 888
 889 static u32
 890 sanitize_target_dc_state(struct drm_i915_private *dev_priv,
 891                          u32 target_dc_state)
 892 {
 893         u32 states[] = {
 894                 DC_STATE_EN_UPTO_DC6,
 895                 DC_STATE_EN_UPTO_DC5,
 896                 DC_STATE_EN_DC3CO,
 897                 DC_STATE_DISABLE,
 898         };
 899         int i;
 900
 901         for (i = 0; i < ARRAY_SIZE(states) - 1; i++) {
 902                 if (target_dc_state != states[i])
 903                         continue;
 904
 905                 if (dev_priv->dmc.allowed_dc_mask & target_dc_state)
 906                         break;
 907
 908                 target_dc_state = states[i + 1];
 909         }
 910
 911         return target_dc_state;
 912 }
 913
 914 static void tgl_enable_dc3co(struct drm_i915_private *dev_priv)
 915 {
 916         drm_dbg_kms(&dev_priv->drm, "Enabling DC3CO\n");
 917         gen9_set_dc_state(dev_priv, DC_STATE_EN_DC3CO);
 918 }
 919
 920 static void tgl_disable_dc3co(struct drm_i915_private *dev_priv)
 921 {
 922         u32 val;
 923
 924         drm_dbg_kms(&dev_priv->drm, "Disabling DC3CO\n");
 925         val = intel_de_read(dev_priv, DC_STATE_EN);
 926         val &= ~DC_STATE_DC3CO_STATUS;
 927         intel_de_write(dev_priv, DC_STATE_EN, val);
 928         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 929         /*
 930          * Delay of 200us DC3CO Exit time B.Spec 49196
 931          */
 932         usleep_range(200, 210);
 933 }
 934
 935 static void bxt_enable_dc9(struct drm_i915_private *dev_priv)
 936 {
 937         assert_can_enable_dc9(dev_priv);
 938
 939         drm_dbg_kms(&dev_priv->drm, "Enabling DC9\n");
 940         /*
 941          * Power sequencer reset is not needed on
 942          * platforms with South Display Engine on PCH,
 943          * because PPS registers are always on.
 944          */
 945         if (!HAS_PCH_SPLIT(dev_priv))
 946                 intel_pps_reset_all(dev_priv);
 947         gen9_set_dc_state(dev_priv, DC_STATE_EN_DC9);
 948 }
 949
 950 static void bxt_disable_dc9(struct drm_i915_private *dev_priv)
 951 {
 952         assert_can_disable_dc9(dev_priv);
 953
 954         drm_dbg_kms(&dev_priv->drm, "Disabling DC9\n");
 955
 956         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 957
 958         intel_pps_unlock_regs_wa(dev_priv);
 959 }
 960
 961 static void assert_dmc_loaded(struct drm_i915_private *dev_priv)
 962 {
 963         drm_WARN_ONCE(&dev_priv->drm,
 964                       !intel_de_read(dev_priv, DMC_PROGRAM(0)),
 965                       "DMC program storage start is NULL\n");
 966         drm_WARN_ONCE(&dev_priv->drm, !intel_de_read(dev_priv, DMC_SSP_BASE),
 967                       "DMC SSP Base Not fine\n");
 968         drm_WARN_ONCE(&dev_priv->drm, !intel_de_read(dev_priv, DMC_HTP_SKL),
 969                       "DMC HTP Not fine\n");
 970 }
 971
 972 static struct i915_power_well *
 973 lookup_power_well(struct drm_i915_private *dev_priv,
 974                   enum i915_power_well_id power_well_id)
 975 {
 976         struct i915_power_well *power_well;
 977
 978         for_each_power_well(dev_priv, power_well)
 979                 if (power_well->desc->id == power_well_id)
 980                         return power_well;
 981
 982         /*
 983          * It's not feasible to add error checking code to the callers since
 984          * this condition really shouldn't happen and it doesn't even make sense
 985          * to abort things like display initialization sequences. Just return
 986          * the first power well and hope the WARN gets reported so we can fix
 987          * our driver.
 988          */
 989         drm_WARN(&dev_priv->drm, 1,
 990                  "Power well %d not defined for this platform\n",
 991                  power_well_id);
 992         return &dev_priv->power_domains.power_wells[0];
 993 }
 994
 995 /**
 996  * intel_display_power_set_target_dc_state - Set target dc state.
 997  * @dev_priv: i915 device
 998  * @state: state which needs to be set as target_dc_state.
 999  *
1000  * This function set the "DC off" power well target_dc_state,
1001  * based upon this target_dc_stste, "DC off" power well will
1002  * enable desired DC state.
1003  */
1004 void intel_display_power_set_target_dc_state(struct drm_i915_private *dev_priv,
1005                                              u32 state)
1006 {
1007         struct i915_power_well *power_well;
1008         bool dc_off_enabled;
1009         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1010
1011         mutex_lock(&power_domains->lock);
1012         power_well = lookup_power_well(dev_priv, SKL_DISP_DC_OFF);
1013
1014         if (drm_WARN_ON(&dev_priv->drm, !power_well))
1015                 goto unlock;
1016
1017         state = sanitize_target_dc_state(dev_priv, state);
1018
1019         if (state == dev_priv->dmc.target_dc_state)
1020                 goto unlock;
1021
1022         dc_off_enabled = power_well->desc->ops->is_enabled(dev_priv,
1023                                                            power_well);
1024         /*
1025          * If DC off power well is disabled, need to enable and disable the
1026          * DC off power well to effect target DC state.
1027          */
1028         if (!dc_off_enabled)
1029                 power_well->desc->ops->enable(dev_priv, power_well);
1030
1031         dev_priv->dmc.target_dc_state = state;
1032
1033         if (!dc_off_enabled)
1034                 power_well->desc->ops->disable(dev_priv, power_well);
1035
1036 unlock:
1037         mutex_unlock(&power_domains->lock);
1038 }
1039
1040 static void assert_can_enable_dc5(struct drm_i915_private *dev_priv)
1041 {
1042         enum i915_power_well_id high_pg;
1043
1044         /* Power wells at this level and above must be disabled for DC5 entry */
1045         if (DISPLAY_VER(dev_priv) == 12)
1046                 high_pg = ICL_DISP_PW_3;
1047         else
1048                 high_pg = SKL_DISP_PW_2;
1049
1050         drm_WARN_ONCE(&dev_priv->drm,
1051                       intel_display_power_well_is_enabled(dev_priv, high_pg),
1052                       "Power wells above platform's DC5 limit still enabled.\n");
1053
1054         drm_WARN_ONCE(&dev_priv->drm,
1055                       (intel_de_read(dev_priv, DC_STATE_EN) &
1056                        DC_STATE_EN_UPTO_DC5),
1057                       "DC5 already programmed to be enabled.\n");
1058         assert_rpm_wakelock_held(&dev_priv->runtime_pm);
1059
1060         assert_dmc_loaded(dev_priv);
1061 }
1062
1063 static void gen9_enable_dc5(struct drm_i915_private *dev_priv)
1064 {
1065         assert_can_enable_dc5(dev_priv);
1066
1067         drm_dbg_kms(&dev_priv->drm, "Enabling DC5\n");
1068
1069         /* Wa Display #1183: skl,kbl,cfl */
1070         if (DISPLAY_VER(dev_priv) == 9 && !IS_BROXTON(dev_priv))
1071                 intel_de_write(dev_priv, GEN8_CHICKEN_DCPR_1,
1072                                intel_de_read(dev_priv, GEN8_CHICKEN_DCPR_1) | SKL_SELECT_ALTERNATE_DC_EXIT);
1073
1074         gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5);
1075 }
1076
1077 static void assert_can_enable_dc6(struct drm_i915_private *dev_priv)
1078 {
1079         drm_WARN_ONCE(&dev_priv->drm,
1080                       intel_de_read(dev_priv, UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
1081                       "Backlight is not disabled.\n");
1082         drm_WARN_ONCE(&dev_priv->drm,
1083                       (intel_de_read(dev_priv, DC_STATE_EN) &
1084                        DC_STATE_EN_UPTO_DC6),
1085                       "DC6 already programmed to be enabled.\n");
1086
1087         assert_dmc_loaded(dev_priv);
1088 }
1089
1090 static void skl_enable_dc6(struct drm_i915_private *dev_priv)
1091 {
1092         assert_can_enable_dc6(dev_priv);
1093
1094         drm_dbg_kms(&dev_priv->drm, "Enabling DC6\n");
1095
1096         /* Wa Display #1183: skl,kbl,cfl */
1097         if (DISPLAY_VER(dev_priv) == 9 && !IS_BROXTON(dev_priv))
1098                 intel_de_write(dev_priv, GEN8_CHICKEN_DCPR_1,
1099                                intel_de_read(dev_priv, GEN8_CHICKEN_DCPR_1) | SKL_SELECT_ALTERNATE_DC_EXIT);
1100
1101         gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
1102 }
1103
1104 static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
1105                                    struct i915_power_well *power_well)
1106 {
1107         const struct i915_power_well_regs *regs = power_well->desc->hsw.regs;
1108         int pw_idx = power_well->desc->hsw.idx;
1109         u32 mask = HSW_PWR_WELL_CTL_REQ(pw_idx);
1110         u32 bios_req = intel_de_read(dev_priv, regs->bios);
1111
1112         /* Take over the request bit if set by BIOS. */
1113         if (bios_req & mask) {
1114                 u32 drv_req = intel_de_read(dev_priv, regs->driver);
1115
1116                 if (!(drv_req & mask))
1117                         intel_de_write(dev_priv, regs->driver, drv_req | mask);
1118                 intel_de_write(dev_priv, regs->bios, bios_req & ~mask);
1119         }
1120 }
1121
1122 static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1123                                            struct i915_power_well *power_well)
1124 {
1125         bxt_ddi_phy_init(dev_priv, power_well->desc->bxt.phy);
1126 }
1127
1128 static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1129                                             struct i915_power_well *power_well)
1130 {
1131         bxt_ddi_phy_uninit(dev_priv, power_well->desc->bxt.phy);
1132 }
1133
1134 static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv,
1135                                             struct i915_power_well *power_well)
1136 {
1137         return bxt_ddi_phy_is_enabled(dev_priv, power_well->desc->bxt.phy);
1138 }
1139
1140 static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv)
1141 {
1142         struct i915_power_well *power_well;
1143
1144         power_well = lookup_power_well(dev_priv, BXT_DISP_PW_DPIO_CMN_A);
1145         if (power_well->count > 0)
1146                 bxt_ddi_phy_verify_state(dev_priv, power_well->desc->bxt.phy);
1147
1148         power_well = lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
1149         if (power_well->count > 0)
1150                 bxt_ddi_phy_verify_state(dev_priv, power_well->desc->bxt.phy);
1151
1152         if (IS_GEMINILAKE(dev_priv)) {
1153                 power_well = lookup_power_well(dev_priv,
1154                                                GLK_DISP_PW_DPIO_CMN_C);
1155                 if (power_well->count > 0)
1156                         bxt_ddi_phy_verify_state(dev_priv,
1157                                                  power_well->desc->bxt.phy);
1158         }
1159 }
1160
1161 static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv,
1162                                            struct i915_power_well *power_well)
1163 {
1164         return ((intel_de_read(dev_priv, DC_STATE_EN) & DC_STATE_EN_DC3CO) == 0 &&
1165                 (intel_de_read(dev_priv, DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0);
1166 }
1167
1168 static void gen9_assert_dbuf_enabled(struct drm_i915_private *dev_priv)
1169 {
1170         u8 hw_enabled_dbuf_slices = intel_enabled_dbuf_slices_mask(dev_priv);
1171         u8 enabled_dbuf_slices = dev_priv->dbuf.enabled_slices;
1172
1173         drm_WARN(&dev_priv->drm,
1174                  hw_enabled_dbuf_slices != enabled_dbuf_slices,
1175                  "Unexpected DBuf power power state (0x%08x, expected 0x%08x)\n",
1176                  hw_enabled_dbuf_slices,
1177                  enabled_dbuf_slices);
1178 }
1179
1180 static void gen9_disable_dc_states(struct drm_i915_private *dev_priv)
1181 {
1182         struct intel_cdclk_config cdclk_config = {};
1183
1184         if (dev_priv->dmc.target_dc_state == DC_STATE_EN_DC3CO) {
1185                 tgl_disable_dc3co(dev_priv);
1186                 return;
1187         }
1188
1189         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
1190
1191         if (!HAS_DISPLAY(dev_priv))
1192                 return;
1193
1194         dev_priv->display.get_cdclk(dev_priv, &cdclk_config);
1195         /* Can't read out voltage_level so can't use intel_cdclk_changed() */
1196         drm_WARN_ON(&dev_priv->drm,
1197                     intel_cdclk_needs_modeset(&dev_priv->cdclk.hw,
1198                                               &cdclk_config));
1199
1200         gen9_assert_dbuf_enabled(dev_priv);
1201
1202         if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
1203                 bxt_verify_ddi_phy_power_wells(dev_priv);
1204
1205         if (DISPLAY_VER(dev_priv) >= 11)
1206                 /*
1207                  * DMC retains HW context only for port A, the other combo
1208                  * PHY's HW context for port B is lost after DC transitions,
1209                  * so we need to restore it manually.
1210                  */
1211                 intel_combo_phy_init(dev_priv);
1212 }
1213
1214 static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
1215                                           struct i915_power_well *power_well)
1216 {
1217         gen9_disable_dc_states(dev_priv);
1218 }
1219
1220 static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv,
1221                                            struct i915_power_well *power_well)
1222 {
1223         if (!intel_dmc_has_payload(dev_priv))
1224                 return;
1225
1226         switch (dev_priv->dmc.target_dc_state) {
1227         case DC_STATE_EN_DC3CO:
1228                 tgl_enable_dc3co(dev_priv);
1229                 break;
1230         case DC_STATE_EN_UPTO_DC6:
1231                 skl_enable_dc6(dev_priv);
1232                 break;
1233         case DC_STATE_EN_UPTO_DC5:
1234                 gen9_enable_dc5(dev_priv);
1235                 break;
1236         }
1237 }
1238
1239 static void i9xx_power_well_sync_hw_noop(struct drm_i915_private *dev_priv,
1240                                          struct i915_power_well *power_well)
1241 {
1242 }
1243
1244 static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
1245                                            struct i915_power_well *power_well)
1246 {
1247 }
1248
1249 static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
1250                                              struct i915_power_well *power_well)
1251 {
1252         return true;
1253 }
1254
1255 static void i830_pipes_power_well_enable(struct drm_i915_private *dev_priv,
1256                                          struct i915_power_well *power_well)
1257 {
1258         if ((intel_de_read(dev_priv, PIPECONF(PIPE_A)) & PIPECONF_ENABLE) == 0)
1259                 i830_enable_pipe(dev_priv, PIPE_A);
1260         if ((intel_de_read(dev_priv, PIPECONF(PIPE_B)) & PIPECONF_ENABLE) == 0)
1261                 i830_enable_pipe(dev_priv, PIPE_B);
1262 }
1263
1264 static void i830_pipes_power_well_disable(struct drm_i915_private *dev_priv,
1265                                           struct i915_power_well *power_well)
1266 {
1267         i830_disable_pipe(dev_priv, PIPE_B);
1268         i830_disable_pipe(dev_priv, PIPE_A);
1269 }
1270
1271 static bool i830_pipes_power_well_enabled(struct drm_i915_private *dev_priv,
1272                                           struct i915_power_well *power_well)
1273 {
1274         return intel_de_read(dev_priv, PIPECONF(PIPE_A)) & PIPECONF_ENABLE &&
1275                 intel_de_read(dev_priv, PIPECONF(PIPE_B)) & PIPECONF_ENABLE;
1276 }
1277
1278 static void i830_pipes_power_well_sync_hw(struct drm_i915_private *dev_priv,
1279                                           struct i915_power_well *power_well)
1280 {
1281         if (power_well->count > 0)
1282                 i830_pipes_power_well_enable(dev_priv, power_well);
1283         else
1284                 i830_pipes_power_well_disable(dev_priv, power_well);
1285 }
1286
1287 static void vlv_set_power_well(struct drm_i915_private *dev_priv,
1288                                struct i915_power_well *power_well, bool enable)
1289 {
1290         int pw_idx = power_well->desc->vlv.idx;
1291         u32 mask;
1292         u32 state;
1293         u32 ctrl;
1294
1295         mask = PUNIT_PWRGT_MASK(pw_idx);
1296         state = enable ? PUNIT_PWRGT_PWR_ON(pw_idx) :
1297                          PUNIT_PWRGT_PWR_GATE(pw_idx);
1298
1299         vlv_punit_get(dev_priv);
1300
1301 #define COND \
1302         ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)
1303
1304         if (COND)
1305                 goto out;
1306
1307         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
1308         ctrl &= ~mask;
1309         ctrl |= state;
1310         vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);
1311
1312         if (wait_for(COND, 100))
1313                 drm_err(&dev_priv->drm,
1314                         "timeout setting power well state %08x (%08x)\n",
1315                         state,
1316                         vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));
1317
1318 #undef COND
1319
1320 out:
1321         vlv_punit_put(dev_priv);
1322 }
1323
1324 static void vlv_power_well_enable(struct drm_i915_private *dev_priv,
1325                                   struct i915_power_well *power_well)
1326 {
1327         vlv_set_power_well(dev_priv, power_well, true);
1328 }
1329
1330 static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
1331                                    struct i915_power_well *power_well)
1332 {
1333         vlv_set_power_well(dev_priv, power_well, false);
1334 }
1335
1336 static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
1337                                    struct i915_power_well *power_well)
1338 {
1339         int pw_idx = power_well->desc->vlv.idx;
1340         bool enabled = false;
1341         u32 mask;
1342         u32 state;
1343         u32 ctrl;
1344
1345         mask = PUNIT_PWRGT_MASK(pw_idx);
1346         ctrl = PUNIT_PWRGT_PWR_ON(pw_idx);
1347
1348         vlv_punit_get(dev_priv);
1349
1350         state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
1351         /*
1352          * We only ever set the power-on and power-gate states, anything
1353          * else is unexpected.
1354          */
1355         drm_WARN_ON(&dev_priv->drm, state != PUNIT_PWRGT_PWR_ON(pw_idx) &&
1356                     state != PUNIT_PWRGT_PWR_GATE(pw_idx));
1357         if (state == ctrl)
1358                 enabled = true;
1359
1360         /*
1361          * A transient state at this point would mean some unexpected party
1362          * is poking at the power controls too.
1363          */
1364         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
1365         drm_WARN_ON(&dev_priv->drm, ctrl != state);
1366
1367         vlv_punit_put(dev_priv);
1368
1369         return enabled;
1370 }
1371
1372 static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
1373 {
1374         u32 val;
1375
1376         /*
1377          * On driver load, a pipe may be active and driving a DSI display.
1378          * Preserve DPOUNIT_CLOCK_GATE_DISABLE to avoid the pipe getting stuck
1379          * (and never recovering) in this case. intel_dsi_post_disable() will
1380          * clear it when we turn off the display.
1381          */
1382         val = intel_de_read(dev_priv, DSPCLK_GATE_D);
1383         val &= DPOUNIT_CLOCK_GATE_DISABLE;
1384         val |= VRHUNIT_CLOCK_GATE_DISABLE;
1385         intel_de_write(dev_priv, DSPCLK_GATE_D, val);
1386
1387         /*
1388          * Disable trickle feed and enable pnd deadline calculation
1389          */
1390         intel_de_write(dev_priv, MI_ARB_VLV,
1391                        MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
1392         intel_de_write(dev_priv, CBR1_VLV, 0);
1393
1394         drm_WARN_ON(&dev_priv->drm, RUNTIME_INFO(dev_priv)->rawclk_freq == 0);
1395         intel_de_write(dev_priv, RAWCLK_FREQ_VLV,
1396                        DIV_ROUND_CLOSEST(RUNTIME_INFO(dev_priv)->rawclk_freq,
1397                                          1000));
1398 }
1399
1400 static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
1401 {
1402         struct intel_encoder *encoder;
1403         enum pipe pipe;
1404
1405         /*
1406          * Enable the CRI clock source so we can get at the
1407          * display and the reference clock for VGA
1408          * hotplug / manual detection. Supposedly DSI also
1409          * needs the ref clock up and running.
1410          *
1411          * CHV DPLL B/C have some issues if VGA mode is enabled.
1412          */
1413         for_each_pipe(dev_priv, pipe) {
1414                 u32 val = intel_de_read(dev_priv, DPLL(pipe));
1415
1416                 val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1417                 if (pipe != PIPE_A)
1418                         val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1419
1420                 intel_de_write(dev_priv, DPLL(pipe), val);
1421         }
1422
1423         vlv_init_display_clock_gating(dev_priv);
1424
1425         spin_lock_irq(&dev_priv->irq_lock);
1426         valleyview_enable_display_irqs(dev_priv);
1427         spin_unlock_irq(&dev_priv->irq_lock);
1428
1429         /*
1430          * During driver initialization/resume we can avoid restoring the
1431          * part of the HW/SW state that will be inited anyway explicitly.
1432          */
1433         if (dev_priv->power_domains.initializing)
1434                 return;
1435
1436         intel_hpd_init(dev_priv);
1437         intel_hpd_poll_disable(dev_priv);
1438
1439         /* Re-enable the ADPA, if we have one */
1440         for_each_intel_encoder(&dev_priv->drm, encoder) {
1441                 if (encoder->type == INTEL_OUTPUT_ANALOG)
1442                         intel_crt_reset(&encoder->base);
1443         }
1444
1445         intel_vga_redisable_power_on(dev_priv);
1446
1447         intel_pps_unlock_regs_wa(dev_priv);
1448 }
1449
1450 static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv)
1451 {
1452         spin_lock_irq(&dev_priv->irq_lock);
1453         valleyview_disable_display_irqs(dev_priv);
1454         spin_unlock_irq(&dev_priv->irq_lock);
1455
1456         /* make sure we're done processing display irqs */
1457         intel_synchronize_irq(dev_priv);
1458
1459         intel_pps_reset_all(dev_priv);
1460
1461         /* Prevent us from re-enabling polling on accident in late suspend */
1462         if (!dev_priv->drm.dev->power.is_suspended)
1463                 intel_hpd_poll_enable(dev_priv);
1464 }
1465
1466 static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
1467                                           struct i915_power_well *power_well)
1468 {
1469         vlv_set_power_well(dev_priv, power_well, true);
1470
1471         vlv_display_power_well_init(dev_priv);
1472 }
1473
1474 static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
1475                                            struct i915_power_well *power_well)
1476 {
1477         vlv_display_power_well_deinit(dev_priv);
1478
1479         vlv_set_power_well(dev_priv, power_well, false);
1480 }
1481
1482 static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1483                                            struct i915_power_well *power_well)
1484 {
1485         /* since ref/cri clock was enabled */
1486         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1487
1488         vlv_set_power_well(dev_priv, power_well, true);
1489
1490         /*
1491          * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx -
1492          *  6.  De-assert cmn_reset/side_reset. Same as VLV X0.
1493          *   a. GUnit 0x2110 bit[0] set to 1 (def 0)
1494          *   b. The other bits such as sfr settings / modesel may all
1495          *      be set to 0.
1496          *
1497          * This should only be done on init and resume from S3 with
1498          * both PLLs disabled, or we risk losing DPIO and PLL
1499          * synchronization.
1500          */
1501         intel_de_write(dev_priv, DPIO_CTL,
1502                        intel_de_read(dev_priv, DPIO_CTL) | DPIO_CMNRST);
1503 }
1504
1505 static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1506                                             struct i915_power_well *power_well)
1507 {
1508         enum pipe pipe;
1509
1510         for_each_pipe(dev_priv, pipe)
1511                 assert_pll_disabled(dev_priv, pipe);
1512
1513         /* Assert common reset */
1514         intel_de_write(dev_priv, DPIO_CTL,
1515                        intel_de_read(dev_priv, DPIO_CTL) & ~DPIO_CMNRST);
1516
1517         vlv_set_power_well(dev_priv, power_well, false);
1518 }
1519
1520 #define POWER_DOMAIN_MASK (GENMASK_ULL(POWER_DOMAIN_NUM - 1, 0))
1521
1522 #define BITS_SET(val, bits) (((val) & (bits)) == (bits))
1523
1524 static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
1525 {
1526         struct i915_power_well *cmn_bc =
1527                 lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
1528         struct i915_power_well *cmn_d =
1529                 lookup_power_well(dev_priv, CHV_DISP_PW_DPIO_CMN_D);
1530         u32 phy_control = dev_priv->chv_phy_control;
1531         u32 phy_status = 0;
1532         u32 phy_status_mask = 0xffffffff;
1533
1534         /*
1535          * The BIOS can leave the PHY is some weird state
1536          * where it doesn't fully power down some parts.
1537          * Disable the asserts until the PHY has been fully
1538          * reset (ie. the power well has been disabled at
1539          * least once).
1540          */
1541         if (!dev_priv->chv_phy_assert[DPIO_PHY0])
1542                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0) |
1543                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0) |
1544                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1) |
1545                                      PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1) |
1546                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0) |
1547                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1));
1548
1549         if (!dev_priv->chv_phy_assert[DPIO_PHY1])
1550                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0) |
1551                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0) |
1552                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1));
1553
1554         if (cmn_bc->desc->ops->is_enabled(dev_priv, cmn_bc)) {
1555                 phy_status |= PHY_POWERGOOD(DPIO_PHY0);
1556
1557                 /* this assumes override is only used to enable lanes */
1558                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0)) == 0)
1559                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0);
1560
1561                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1)) == 0)
1562                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1);
1563
1564                 /* CL1 is on whenever anything is on in either channel */
1565                 if (BITS_SET(phy_control,
1566                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0) |
1567                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)))
1568                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0);
1569
1570                 /*
1571                  * The DPLLB check accounts for the pipe B + port A usage
1572                  * with CL2 powered up but all the lanes in the second channel
1573                  * powered down.
1574                  */
1575                 if (BITS_SET(phy_control,
1576                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)) &&
1577                     (intel_de_read(dev_priv, DPLL(PIPE_B)) & DPLL_VCO_ENABLE) == 0)
1578                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1);
1579
1580                 if (BITS_SET(phy_control,
1581                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH0)))
1582                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0);
1583                 if (BITS_SET(phy_control,
1584                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH0)))
1585                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1);
1586
1587                 if (BITS_SET(phy_control,
1588                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH1)))
1589                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0);
1590                 if (BITS_SET(phy_control,
1591                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH1)))
1592                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1);
1593         }
1594
1595         if (cmn_d->desc->ops->is_enabled(dev_priv, cmn_d)) {
1596                 phy_status |= PHY_POWERGOOD(DPIO_PHY1);
1597
1598                 /* this assumes override is only used to enable lanes */
1599                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0)) == 0)
1600                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0);
1601
1602                 if (BITS_SET(phy_control,
1603                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0)))
1604                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0);
1605
1606                 if (BITS_SET(phy_control,
1607                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY1, DPIO_CH0)))
1608                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0);
1609                 if (BITS_SET(phy_control,
1610                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY1, DPIO_CH0)))
1611                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1);
1612         }
1613
1614         phy_status &= phy_status_mask;
1615
1616         /*
1617          * The PHY may be busy with some initial calibration and whatnot,
1618          * so the power state can take a while to actually change.
1619          */
1620         if (intel_de_wait_for_register(dev_priv, DISPLAY_PHY_STATUS,
1621                                        phy_status_mask, phy_status, 10))
1622                 drm_err(&dev_priv->drm,
1623                         "Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n",
1624                         intel_de_read(dev_priv, DISPLAY_PHY_STATUS) & phy_status_mask,
1625                         phy_status, dev_priv->chv_phy_control);
1626 }
1627
1628 #undef BITS_SET
1629
1630 static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1631                                            struct i915_power_well *power_well)
1632 {
1633         enum dpio_phy phy;
1634         enum pipe pipe;
1635         u32 tmp;
1636
1637         drm_WARN_ON_ONCE(&dev_priv->drm,
1638                          power_well->desc->id != VLV_DISP_PW_DPIO_CMN_BC &&
1639                          power_well->desc->id != CHV_DISP_PW_DPIO_CMN_D);
1640
1641         if (power_well->desc->id == VLV_DISP_PW_DPIO_CMN_BC) {
1642                 pipe = PIPE_A;
1643                 phy = DPIO_PHY0;
1644         } else {
1645                 pipe = PIPE_C;
1646                 phy = DPIO_PHY1;
1647         }
1648
1649         /* since ref/cri clock was enabled */
1650         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1651         vlv_set_power_well(dev_priv, power_well, true);
1652
1653         /* Poll for phypwrgood signal */
1654         if (intel_de_wait_for_set(dev_priv, DISPLAY_PHY_STATUS,
1655                                   PHY_POWERGOOD(phy), 1))
1656                 drm_err(&dev_priv->drm, "Display PHY %d is not power up\n",
1657                         phy);
1658
1659         vlv_dpio_get(dev_priv);
1660
1661         /* Enable dynamic power down */
1662         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW28);
1663         tmp |= DPIO_DYNPWRDOWNEN_CH0 | DPIO_CL1POWERDOWNEN |
1664                 DPIO_SUS_CLK_CONFIG_GATE_CLKREQ;
1665         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW28, tmp);
1666
1667         if (power_well->desc->id == VLV_DISP_PW_DPIO_CMN_BC) {
1668                 tmp = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW6_CH1);
1669                 tmp |= DPIO_DYNPWRDOWNEN_CH1;
1670                 vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW6_CH1, tmp);
1671         } else {
1672                 /*
1673                  * Force the non-existing CL2 off. BXT does this
1674                  * too, so maybe it saves some power even though
1675                  * CL2 doesn't exist?
1676                  */
1677                 tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30);
1678                 tmp |= DPIO_CL2_LDOFUSE_PWRENB;
1679                 vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, tmp);
1680         }
1681
1682         vlv_dpio_put(dev_priv);
1683
1684         dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy);
1685         intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1686                        dev_priv->chv_phy_control);
1687
1688         drm_dbg_kms(&dev_priv->drm,
1689                     "Enabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1690                     phy, dev_priv->chv_phy_control);
1691
1692         assert_chv_phy_status(dev_priv);
1693 }
1694
1695 static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1696                                             struct i915_power_well *power_well)
1697 {
1698         enum dpio_phy phy;
1699
1700         drm_WARN_ON_ONCE(&dev_priv->drm,
1701                          power_well->desc->id != VLV_DISP_PW_DPIO_CMN_BC &&
1702                          power_well->desc->id != CHV_DISP_PW_DPIO_CMN_D);
1703
1704         if (power_well->desc->id == VLV_DISP_PW_DPIO_CMN_BC) {
1705                 phy = DPIO_PHY0;
1706                 assert_pll_disabled(dev_priv, PIPE_A);
1707                 assert_pll_disabled(dev_priv, PIPE_B);
1708         } else {
1709                 phy = DPIO_PHY1;
1710                 assert_pll_disabled(dev_priv, PIPE_C);
1711         }
1712
1713         dev_priv->chv_phy_control &= ~PHY_COM_LANE_RESET_DEASSERT(phy);
1714         intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1715                        dev_priv->chv_phy_control);
1716
1717         vlv_set_power_well(dev_priv, power_well, false);
1718
1719         drm_dbg_kms(&dev_priv->drm,
1720                     "Disabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1721                     phy, dev_priv->chv_phy_control);
1722
1723         /* PHY is fully reset now, so we can enable the PHY state asserts */
1724         dev_priv->chv_phy_assert[phy] = true;
1725
1726         assert_chv_phy_status(dev_priv);
1727 }
1728
1729 static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1730                                      enum dpio_channel ch, bool override, unsigned int mask)
1731 {
1732         enum pipe pipe = phy == DPIO_PHY0 ? PIPE_A : PIPE_C;
1733         u32 reg, val, expected, actual;
1734
1735         /*
1736          * The BIOS can leave the PHY is some weird state
1737          * where it doesn't fully power down some parts.
1738          * Disable the asserts until the PHY has been fully
1739          * reset (ie. the power well has been disabled at
1740          * least once).
1741          */
1742         if (!dev_priv->chv_phy_assert[phy])
1743                 return;
1744
1745         if (ch == DPIO_CH0)
1746                 reg = _CHV_CMN_DW0_CH0;
1747         else
1748                 reg = _CHV_CMN_DW6_CH1;
1749
1750         vlv_dpio_get(dev_priv);
1751         val = vlv_dpio_read(dev_priv, pipe, reg);
1752         vlv_dpio_put(dev_priv);
1753
1754         /*
1755          * This assumes !override is only used when the port is disabled.
1756          * All lanes should power down even without the override when
1757          * the port is disabled.
1758          */
1759         if (!override || mask == 0xf) {
1760                 expected = DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1761                 /*
1762                  * If CH1 common lane is not active anymore
1763                  * (eg. for pipe B DPLL) the entire channel will
1764                  * shut down, which causes the common lane registers
1765                  * to read as 0. That means we can't actually check
1766                  * the lane power down status bits, but as the entire
1767                  * register reads as 0 it's a good indication that the
1768                  * channel is indeed entirely powered down.
1769                  */
1770                 if (ch == DPIO_CH1 && val == 0)
1771                         expected = 0;
1772         } else if (mask != 0x0) {
1773                 expected = DPIO_ANYDL_POWERDOWN;
1774         } else {
1775                 expected = 0;
1776         }
1777
1778         if (ch == DPIO_CH0)
1779                 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH0;
1780         else
1781                 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH1;
1782         actual &= DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1783
1784         drm_WARN(&dev_priv->drm, actual != expected,
1785                  "Unexpected DPIO lane power down: all %d, any %d. Expected: all %d, any %d. (0x%x = 0x%08x)\n",
1786                  !!(actual & DPIO_ALLDL_POWERDOWN),
1787                  !!(actual & DPIO_ANYDL_POWERDOWN),
1788                  !!(expected & DPIO_ALLDL_POWERDOWN),
1789                  !!(expected & DPIO_ANYDL_POWERDOWN),
1790                  reg, val);
1791 }
1792
1793 bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1794                           enum dpio_channel ch, bool override)
1795 {
1796         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1797         bool was_override;
1798
1799         mutex_lock(&power_domains->lock);
1800
1801         was_override = dev_priv->chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1802
1803         if (override == was_override)
1804                 goto out;
1805
1806         if (override)
1807                 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1808         else
1809                 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1810
1811         intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1812                        dev_priv->chv_phy_control);
1813
1814         drm_dbg_kms(&dev_priv->drm,
1815                     "Power gating DPIO PHY%d CH%d (DPIO_PHY_CONTROL=0x%08x)\n",
1816                     phy, ch, dev_priv->chv_phy_control);
1817
1818         assert_chv_phy_status(dev_priv);
1819
1820 out:
1821         mutex_unlock(&power_domains->lock);
1822
1823         return was_override;
1824 }
1825
1826 void chv_phy_powergate_lanes(struct intel_encoder *encoder,
1827                              bool override, unsigned int mask)
1828 {
1829         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
1830         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1831         enum dpio_phy phy = vlv_dig_port_to_phy(enc_to_dig_port(encoder));
1832         enum dpio_channel ch = vlv_dig_port_to_channel(enc_to_dig_port(encoder));
1833
1834         mutex_lock(&power_domains->lock);
1835
1836         dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch);
1837         dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch);
1838
1839         if (override)
1840                 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1841         else
1842                 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1843
1844         intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1845                        dev_priv->chv_phy_control);
1846
1847         drm_dbg_kms(&dev_priv->drm,
1848                     "Power gating DPIO PHY%d CH%d lanes 0x%x (PHY_CONTROL=0x%08x)\n",
1849                     phy, ch, mask, dev_priv->chv_phy_control);
1850
1851         assert_chv_phy_status(dev_priv);
1852
1853         assert_chv_phy_powergate(dev_priv, phy, ch, override, mask);
1854
1855         mutex_unlock(&power_domains->lock);
1856 }
1857
1858 static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
1859                                         struct i915_power_well *power_well)
1860 {
1861         enum pipe pipe = PIPE_A;
1862         bool enabled;
1863         u32 state, ctrl;
1864
1865         vlv_punit_get(dev_priv);
1866
1867         state = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSS_MASK(pipe);
1868         /*
1869          * We only ever set the power-on and power-gate states, anything
1870          * else is unexpected.
1871          */
1872         drm_WARN_ON(&dev_priv->drm, state != DP_SSS_PWR_ON(pipe) &&
1873                     state != DP_SSS_PWR_GATE(pipe));
1874         enabled = state == DP_SSS_PWR_ON(pipe);
1875
1876         /*
1877          * A transient state at this point would mean some unexpected party
1878          * is poking at the power controls too.
1879          */
1880         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSC_MASK(pipe);
1881         drm_WARN_ON(&dev_priv->drm, ctrl << 16 != state);
1882
1883         vlv_punit_put(dev_priv);
1884
1885         return enabled;
1886 }
1887
1888 static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
1889                                     struct i915_power_well *power_well,
1890                                     bool enable)
1891 {
1892         enum pipe pipe = PIPE_A;
1893         u32 state;
1894         u32 ctrl;
1895
1896         state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe);
1897
1898         vlv_punit_get(dev_priv);
1899
1900 #define COND \
1901         ((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSS_MASK(pipe)) == state)
1902
1903         if (COND)
1904                 goto out;
1905
1906         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
1907         ctrl &= ~DP_SSC_MASK(pipe);
1908         ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
1909         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, ctrl);
1910
1911         if (wait_for(COND, 100))
1912                 drm_err(&dev_priv->drm,
1913                         "timeout setting power well state %08x (%08x)\n",
1914                         state,
1915                         vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM));
1916
1917 #undef COND
1918
1919 out:
1920         vlv_punit_put(dev_priv);
1921 }
1922
1923 static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv,
1924                                         struct i915_power_well *power_well)
1925 {
1926         intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
1927                        dev_priv->chv_phy_control);
1928 }
1929
1930 static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
1931                                        struct i915_power_well *power_well)
1932 {
1933         chv_set_pipe_power_well(dev_priv, power_well, true);
1934
1935         vlv_display_power_well_init(dev_priv);
1936 }
1937
1938 static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv,
1939                                         struct i915_power_well *power_well)
1940 {
1941         vlv_display_power_well_deinit(dev_priv);
1942
1943         chv_set_pipe_power_well(dev_priv, power_well, false);
1944 }
1945
1946 static u64 __async_put_domains_mask(struct i915_power_domains *power_domains)
1947 {
1948         return power_domains->async_put_domains[0] |
1949                power_domains->async_put_domains[1];
1950 }
1951
1952 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
1953
1954 static bool
1955 assert_async_put_domain_masks_disjoint(struct i915_power_domains *power_domains)
1956 {
1957         struct drm_i915_private *i915 = container_of(power_domains,
1958                                                      struct drm_i915_private,
1959                                                      power_domains);
1960         return !drm_WARN_ON(&i915->drm, power_domains->async_put_domains[0] &
1961                             power_domains->async_put_domains[1]);
1962 }
1963
1964 static bool
1965 __async_put_domains_state_ok(struct i915_power_domains *power_domains)
1966 {
1967         struct drm_i915_private *i915 = container_of(power_domains,
1968                                                      struct drm_i915_private,
1969                                                      power_domains);
1970         enum intel_display_power_domain domain;
1971         bool err = false;
1972
1973         err |= !assert_async_put_domain_masks_disjoint(power_domains);
1974         err |= drm_WARN_ON(&i915->drm, !!power_domains->async_put_wakeref !=
1975                            !!__async_put_domains_mask(power_domains));
1976
1977         for_each_power_domain(domain, __async_put_domains_mask(power_domains))
1978                 err |= drm_WARN_ON(&i915->drm,
1979                                    power_domains->domain_use_count[domain] != 1);
1980
1981         return !err;
1982 }
1983
1984 static void print_power_domains(struct i915_power_domains *power_domains,
1985                                 const char *prefix, u64 mask)
1986 {
1987         struct drm_i915_private *i915 = container_of(power_domains,
1988                                                      struct drm_i915_private,
1989                                                      power_domains);
1990         enum intel_display_power_domain domain;
1991
1992         drm_dbg(&i915->drm, "%s (%lu):\n", prefix, hweight64(mask));
1993         for_each_power_domain(domain, mask)
1994                 drm_dbg(&i915->drm, "%s use_count %d\n",
1995                         intel_display_power_domain_str(domain),
1996                         power_domains->domain_use_count[domain]);
1997 }
1998
1999 static void
2000 print_async_put_domains_state(struct i915_power_domains *power_domains)
2001 {
2002         struct drm_i915_private *i915 = container_of(power_domains,
2003                                                      struct drm_i915_private,
2004                                                      power_domains);
2005
2006         drm_dbg(&i915->drm, "async_put_wakeref %u\n",
2007                 power_domains->async_put_wakeref);
2008
2009         print_power_domains(power_domains, "async_put_domains[0]",
2010                             power_domains->async_put_domains[0]);
2011         print_power_domains(power_domains, "async_put_domains[1]",
2012                             power_domains->async_put_domains[1]);
2013 }
2014
2015 static void
2016 verify_async_put_domains_state(struct i915_power_domains *power_domains)
2017 {
2018         if (!__async_put_domains_state_ok(power_domains))
2019                 print_async_put_domains_state(power_domains);
2020 }
2021
2022 #else
2023
2024 static void
2025 assert_async_put_domain_masks_disjoint(struct i915_power_domains *power_domains)
2026 {
2027 }
2028
2029 static void
2030 verify_async_put_domains_state(struct i915_power_domains *power_domains)
2031 {
2032 }
2033
2034 #endif /* CONFIG_DRM_I915_DEBUG_RUNTIME_PM */
2035
2036 static u64 async_put_domains_mask(struct i915_power_domains *power_domains)
2037 {
2038         assert_async_put_domain_masks_disjoint(power_domains);
2039
2040         return __async_put_domains_mask(power_domains);
2041 }
2042
2043 static void
2044 async_put_domains_clear_domain(struct i915_power_domains *power_domains,
2045                                enum intel_display_power_domain domain)
2046 {
2047         assert_async_put_domain_masks_disjoint(power_domains);
2048
2049         power_domains->async_put_domains[0] &= ~BIT_ULL(domain);
2050         power_domains->async_put_domains[1] &= ~BIT_ULL(domain);
2051 }
2052
2053 static bool
2054 intel_display_power_grab_async_put_ref(struct drm_i915_private *dev_priv,
2055                                        enum intel_display_power_domain domain)
2056 {
2057         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2058         bool ret = false;
2059
2060         if (!(async_put_domains_mask(power_domains) & BIT_ULL(domain)))
2061                 goto out_verify;
2062
2063         async_put_domains_clear_domain(power_domains, domain);
2064
2065         ret = true;
2066
2067         if (async_put_domains_mask(power_domains))
2068                 goto out_verify;
2069
2070         cancel_delayed_work(&power_domains->async_put_work);
2071         intel_runtime_pm_put_raw(&dev_priv->runtime_pm,
2072                                  fetch_and_zero(&power_domains->async_put_wakeref));
2073 out_verify:
2074         verify_async_put_domains_state(power_domains);
2075
2076         return ret;
2077 }
2078
2079 static void
2080 __intel_display_power_get_domain(struct drm_i915_private *dev_priv,
2081                                  enum intel_display_power_domain domain)
2082 {
2083         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2084         struct i915_power_well *power_well;
2085
2086         if (intel_display_power_grab_async_put_ref(dev_priv, domain))
2087                 return;
2088
2089         for_each_power_domain_well(dev_priv, power_well, BIT_ULL(domain))
2090                 intel_power_well_get(dev_priv, power_well);
2091
2092         power_domains->domain_use_count[domain]++;
2093 }
2094
2095 /**
2096  * intel_display_power_get - grab a power domain reference
2097  * @dev_priv: i915 device instance
2098  * @domain: power domain to reference
2099  *
2100  * This function grabs a power domain reference for @domain and ensures that the
2101  * power domain and all its parents are powered up. Therefore users should only
2102  * grab a reference to the innermost power domain they need.
2103  *
2104  * Any power domain reference obtained by this function must have a symmetric
2105  * call to intel_display_power_put() to release the reference again.
2106  */
2107 intel_wakeref_t intel_display_power_get(struct drm_i915_private *dev_priv,
2108                                         enum intel_display_power_domain domain)
2109 {
2110         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2111         intel_wakeref_t wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
2112
2113         mutex_lock(&power_domains->lock);
2114         __intel_display_power_get_domain(dev_priv, domain);
2115         mutex_unlock(&power_domains->lock);
2116
2117         return wakeref;
2118 }
2119
2120 /**
2121  * intel_display_power_get_if_enabled - grab a reference for an enabled display power domain
2122  * @dev_priv: i915 device instance
2123  * @domain: power domain to reference
2124  *
2125  * This function grabs a power domain reference for @domain and ensures that the
2126  * power domain and all its parents are powered up. Therefore users should only
2127  * grab a reference to the innermost power domain they need.
2128  *
2129  * Any power domain reference obtained by this function must have a symmetric
2130  * call to intel_display_power_put() to release the reference again.
2131  */
2132 intel_wakeref_t
2133 intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv,
2134                                    enum intel_display_power_domain domain)
2135 {
2136         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2137         intel_wakeref_t wakeref;
2138         bool is_enabled;
2139
2140         wakeref = intel_runtime_pm_get_if_in_use(&dev_priv->runtime_pm);
2141         if (!wakeref)
2142                 return false;
2143
2144         mutex_lock(&power_domains->lock);
2145
2146         if (__intel_display_power_is_enabled(dev_priv, domain)) {
2147                 __intel_display_power_get_domain(dev_priv, domain);
2148                 is_enabled = true;
2149         } else {
2150                 is_enabled = false;
2151         }
2152
2153         mutex_unlock(&power_domains->lock);
2154
2155         if (!is_enabled) {
2156                 intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
2157                 wakeref = 0;
2158         }
2159
2160         return wakeref;
2161 }
2162
2163 static void
2164 __intel_display_power_put_domain(struct drm_i915_private *dev_priv,
2165                                  enum intel_display_power_domain domain)
2166 {
2167         struct i915_power_domains *power_domains;
2168         struct i915_power_well *power_well;
2169         const char *name = intel_display_power_domain_str(domain);
2170
2171         power_domains = &dev_priv->power_domains;
2172
2173         drm_WARN(&dev_priv->drm, !power_domains->domain_use_count[domain],
2174                  "Use count on domain %s is already zero\n",
2175                  name);
2176         drm_WARN(&dev_priv->drm,
2177                  async_put_domains_mask(power_domains) & BIT_ULL(domain),
2178                  "Async disabling of domain %s is pending\n",
2179                  name);
2180
2181         power_domains->domain_use_count[domain]--;
2182
2183         for_each_power_domain_well_reverse(dev_priv, power_well, BIT_ULL(domain))
2184                 intel_power_well_put(dev_priv, power_well);
2185 }
2186
2187 static void __intel_display_power_put(struct drm_i915_private *dev_priv,
2188                                       enum intel_display_power_domain domain)
2189 {
2190         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2191
2192         mutex_lock(&power_domains->lock);
2193         __intel_display_power_put_domain(dev_priv, domain);
2194         mutex_unlock(&power_domains->lock);
2195 }
2196
2197 static void
2198 queue_async_put_domains_work(struct i915_power_domains *power_domains,
2199                              intel_wakeref_t wakeref)
2200 {
2201         struct drm_i915_private *i915 = container_of(power_domains,
2202                                                      struct drm_i915_private,
2203                                                      power_domains);
2204         drm_WARN_ON(&i915->drm, power_domains->async_put_wakeref);
2205         power_domains->async_put_wakeref = wakeref;
2206         drm_WARN_ON(&i915->drm, !queue_delayed_work(system_unbound_wq,
2207                                                     &power_domains->async_put_work,
2208                                                     msecs_to_jiffies(100)));
2209 }
2210
2211 static void
2212 release_async_put_domains(struct i915_power_domains *power_domains, u64 mask)
2213 {
2214         struct drm_i915_private *dev_priv =
2215                 container_of(power_domains, struct drm_i915_private,
2216                              power_domains);
2217         struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
2218         enum intel_display_power_domain domain;
2219         intel_wakeref_t wakeref;
2220
2221         /*
2222          * The caller must hold already raw wakeref, upgrade that to a proper
2223          * wakeref to make the state checker happy about the HW access during
2224          * power well disabling.
2225          */
2226         assert_rpm_raw_wakeref_held(rpm);
2227         wakeref = intel_runtime_pm_get(rpm);
2228
2229         for_each_power_domain(domain, mask) {
2230                 /* Clear before put, so put's sanity check is happy. */
2231                 async_put_domains_clear_domain(power_domains, domain);
2232                 __intel_display_power_put_domain(dev_priv, domain);
2233         }
2234
2235         intel_runtime_pm_put(rpm, wakeref);
2236 }
2237
2238 static void
2239 intel_display_power_put_async_work(struct work_struct *work)
2240 {
2241         struct drm_i915_private *dev_priv =
2242                 container_of(work, struct drm_i915_private,
2243                              power_domains.async_put_work.work);
2244         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2245         struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
2246         intel_wakeref_t new_work_wakeref = intel_runtime_pm_get_raw(rpm);
2247         intel_wakeref_t old_work_wakeref = 0;
2248
2249         mutex_lock(&power_domains->lock);
2250
2251         /*
2252          * Bail out if all the domain refs pending to be released were grabbed
2253          * by subsequent gets or a flush_work.
2254          */
2255         old_work_wakeref = fetch_and_zero(&power_domains->async_put_wakeref);
2256         if (!old_work_wakeref)
2257                 goto out_verify;
2258
2259         release_async_put_domains(power_domains,
2260                                   power_domains->async_put_domains[0]);
2261
2262         /* Requeue the work if more domains were async put meanwhile. */
2263         if (power_domains->async_put_domains[1]) {
2264                 power_domains->async_put_domains[0] =
2265                         fetch_and_zero(&power_domains->async_put_domains[1]);
2266                 queue_async_put_domains_work(power_domains,
2267                                              fetch_and_zero(&new_work_wakeref));
2268         } else {
2269                 /*
2270                  * Cancel the work that got queued after this one got dequeued,
2271                  * since here we released the corresponding async-put reference.
2272                  */
2273                 cancel_delayed_work(&power_domains->async_put_work);
2274         }
2275
2276 out_verify:
2277         verify_async_put_domains_state(power_domains);
2278
2279         mutex_unlock(&power_domains->lock);
2280
2281         if (old_work_wakeref)
2282                 intel_runtime_pm_put_raw(rpm, old_work_wakeref);
2283         if (new_work_wakeref)
2284                 intel_runtime_pm_put_raw(rpm, new_work_wakeref);
2285 }
2286
2287 /**
2288  * intel_display_power_put_async - release a power domain reference asynchronously
2289  * @i915: i915 device instance
2290  * @domain: power domain to reference
2291  * @wakeref: wakeref acquired for the reference that is being released
2292  *
2293  * This function drops the power domain reference obtained by
2294  * intel_display_power_get*() and schedules a work to power down the
2295  * corresponding hardware block if this is the last reference.
2296  */
2297 void __intel_display_power_put_async(struct drm_i915_private *i915,
2298                                      enum intel_display_power_domain domain,
2299                                      intel_wakeref_t wakeref)
2300 {
2301         struct i915_power_domains *power_domains = &i915->power_domains;
2302         struct intel_runtime_pm *rpm = &i915->runtime_pm;
2303         intel_wakeref_t work_wakeref = intel_runtime_pm_get_raw(rpm);
2304
2305         mutex_lock(&power_domains->lock);
2306
2307         if (power_domains->domain_use_count[domain] > 1) {
2308                 __intel_display_power_put_domain(i915, domain);
2309
2310                 goto out_verify;
2311         }
2312
2313         drm_WARN_ON(&i915->drm, power_domains->domain_use_count[domain] != 1);
2314
2315         /* Let a pending work requeue itself or queue a new one. */
2316         if (power_domains->async_put_wakeref) {
2317                 power_domains->async_put_domains[1] |= BIT_ULL(domain);
2318         } else {
2319                 power_domains->async_put_domains[0] |= BIT_ULL(domain);
2320                 queue_async_put_domains_work(power_domains,
2321                                              fetch_and_zero(&work_wakeref));
2322         }
2323
2324 out_verify:
2325         verify_async_put_domains_state(power_domains);
2326
2327         mutex_unlock(&power_domains->lock);
2328
2329         if (work_wakeref)
2330                 intel_runtime_pm_put_raw(rpm, work_wakeref);
2331
2332         intel_runtime_pm_put(rpm, wakeref);
2333 }
2334
2335 /**
2336  * intel_display_power_flush_work - flushes the async display power disabling work
2337  * @i915: i915 device instance
2338  *
2339  * Flushes any pending work that was scheduled by a preceding
2340  * intel_display_power_put_async() call, completing the disabling of the
2341  * corresponding power domains.
2342  *
2343  * Note that the work handler function may still be running after this
2344  * function returns; to ensure that the work handler isn't running use
2345  * intel_display_power_flush_work_sync() instead.
2346  */
2347 void intel_display_power_flush_work(struct drm_i915_private *i915)
2348 {
2349         struct i915_power_domains *power_domains = &i915->power_domains;
2350         intel_wakeref_t work_wakeref;
2351
2352         mutex_lock(&power_domains->lock);
2353
2354         work_wakeref = fetch_and_zero(&power_domains->async_put_wakeref);
2355         if (!work_wakeref)
2356                 goto out_verify;
2357
2358         release_async_put_domains(power_domains,
2359                                   async_put_domains_mask(power_domains));
2360         cancel_delayed_work(&power_domains->async_put_work);
2361
2362 out_verify:
2363         verify_async_put_domains_state(power_domains);
2364
2365         mutex_unlock(&power_domains->lock);
2366
2367         if (work_wakeref)
2368                 intel_runtime_pm_put_raw(&i915->runtime_pm, work_wakeref);
2369 }
2370
2371 /**
2372  * intel_display_power_flush_work_sync - flushes and syncs the async display power disabling work
2373  * @i915: i915 device instance
2374  *
2375  * Like intel_display_power_flush_work(), but also ensure that the work
2376  * handler function is not running any more when this function returns.
2377  */
2378 static void
2379 intel_display_power_flush_work_sync(struct drm_i915_private *i915)
2380 {
2381         struct i915_power_domains *power_domains = &i915->power_domains;
2382
2383         intel_display_power_flush_work(i915);
2384         cancel_delayed_work_sync(&power_domains->async_put_work);
2385
2386         verify_async_put_domains_state(power_domains);
2387
2388         drm_WARN_ON(&i915->drm, power_domains->async_put_wakeref);
2389 }
2390
2391 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
2392 /**
2393  * intel_display_power_put - release a power domain reference
2394  * @dev_priv: i915 device instance
2395  * @domain: power domain to reference
2396  * @wakeref: wakeref acquired for the reference that is being released
2397  *
2398  * This function drops the power domain reference obtained by
2399  * intel_display_power_get() and might power down the corresponding hardware
2400  * block right away if this is the last reference.
2401  */
2402 void intel_display_power_put(struct drm_i915_private *dev_priv,
2403                              enum intel_display_power_domain domain,
2404                              intel_wakeref_t wakeref)
2405 {
2406         __intel_display_power_put(dev_priv, domain);
2407         intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
2408 }
2409 #else
2410 /**
2411  * intel_display_power_put_unchecked - release an unchecked power domain reference
2412  * @dev_priv: i915 device instance
2413  * @domain: power domain to reference
2414  *
2415  * This function drops the power domain reference obtained by
2416  * intel_display_power_get() and might power down the corresponding hardware
2417  * block right away if this is the last reference.
2418  *
2419  * This function is only for the power domain code's internal use to suppress wakeref
2420  * tracking when the correspondig debug kconfig option is disabled, should not
2421  * be used otherwise.
2422  */
2423 void intel_display_power_put_unchecked(struct drm_i915_private *dev_priv,
2424                                        enum intel_display_power_domain domain)
2425 {
2426         __intel_display_power_put(dev_priv, domain);
2427         intel_runtime_pm_put_unchecked(&dev_priv->runtime_pm);
2428 }
2429 #endif
2430
2431 void
2432 intel_display_power_get_in_set(struct drm_i915_private *i915,
2433                                struct intel_display_power_domain_set *power_domain_set,
2434                                enum intel_display_power_domain domain)
2435 {
2436         intel_wakeref_t __maybe_unused wf;
2437
2438         drm_WARN_ON(&i915->drm, power_domain_set->mask & BIT_ULL(domain));
2439
2440         wf = intel_display_power_get(i915, domain);
2441 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
2442         power_domain_set->wakerefs[domain] = wf;
2443 #endif
2444         power_domain_set->mask |= BIT_ULL(domain);
2445 }
2446
2447 bool
2448 intel_display_power_get_in_set_if_enabled(struct drm_i915_private *i915,
2449                                           struct intel_display_power_domain_set *power_domain_set,
2450                                           enum intel_display_power_domain domain)
2451 {
2452         intel_wakeref_t wf;
2453
2454         drm_WARN_ON(&i915->drm, power_domain_set->mask & BIT_ULL(domain));
2455
2456         wf = intel_display_power_get_if_enabled(i915, domain);
2457         if (!wf)
2458                 return false;
2459
2460 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
2461         power_domain_set->wakerefs[domain] = wf;
2462 #endif
2463         power_domain_set->mask |= BIT_ULL(domain);
2464
2465         return true;
2466 }
2467
2468 void
2469 intel_display_power_put_mask_in_set(struct drm_i915_private *i915,
2470                                     struct intel_display_power_domain_set *power_domain_set,
2471                                     u64 mask)
2472 {
2473         enum intel_display_power_domain domain;
2474
2475         drm_WARN_ON(&i915->drm, mask & ~power_domain_set->mask);
2476
2477         for_each_power_domain(domain, mask) {
2478                 intel_wakeref_t __maybe_unused wf = -1;
2479
2480 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
2481                 wf = fetch_and_zero(&power_domain_set->wakerefs[domain]);
2482 #endif
2483                 intel_display_power_put(i915, domain, wf);
2484                 power_domain_set->mask &= ~BIT_ULL(domain);
2485         }
2486 }
2487
2488 #define I830_PIPES_POWER_DOMAINS (              \
2489         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
2490         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
2491         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
2492         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
2493         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
2494         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
2495         BIT_ULL(POWER_DOMAIN_INIT))
2496
2497 #define VLV_DISPLAY_POWER_DOMAINS (             \
2498         BIT_ULL(POWER_DOMAIN_DISPLAY_CORE) |    \
2499         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
2500         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
2501         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
2502         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
2503         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
2504         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
2505         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2506         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2507         BIT_ULL(POWER_DOMAIN_PORT_DSI) |                \
2508         BIT_ULL(POWER_DOMAIN_PORT_CRT) |                \
2509         BIT_ULL(POWER_DOMAIN_VGA) |                     \
2510         BIT_ULL(POWER_DOMAIN_AUDIO) |           \
2511         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2512         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2513         BIT_ULL(POWER_DOMAIN_GMBUS) |           \
2514         BIT_ULL(POWER_DOMAIN_INIT))
2515
2516 #define VLV_DPIO_CMN_BC_POWER_DOMAINS (         \
2517         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2518         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2519         BIT_ULL(POWER_DOMAIN_PORT_CRT) |                \
2520         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2521         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2522         BIT_ULL(POWER_DOMAIN_INIT))
2523
2524 #define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS (  \
2525         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2526         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2527         BIT_ULL(POWER_DOMAIN_INIT))
2528
2529 #define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS (  \
2530         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2531         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2532         BIT_ULL(POWER_DOMAIN_INIT))
2533
2534 #define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS (  \
2535         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2536         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2537         BIT_ULL(POWER_DOMAIN_INIT))
2538
2539 #define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS (  \
2540         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2541         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2542         BIT_ULL(POWER_DOMAIN_INIT))
2543
2544 #define CHV_DISPLAY_POWER_DOMAINS (             \
2545         BIT_ULL(POWER_DOMAIN_DISPLAY_CORE) |    \
2546         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
2547         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
2548         BIT_ULL(POWER_DOMAIN_PIPE_C) |          \
2549         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
2550         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
2551         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
2552         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
2553         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
2554         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |    \
2555         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2556         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2557         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
2558         BIT_ULL(POWER_DOMAIN_PORT_DSI) |                \
2559         BIT_ULL(POWER_DOMAIN_VGA) |                     \
2560         BIT_ULL(POWER_DOMAIN_AUDIO) |           \
2561         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2562         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2563         BIT_ULL(POWER_DOMAIN_AUX_D) |           \
2564         BIT_ULL(POWER_DOMAIN_GMBUS) |           \
2565         BIT_ULL(POWER_DOMAIN_INIT))
2566
2567 #define CHV_DPIO_CMN_BC_POWER_DOMAINS (         \
2568         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2569         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2570         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2571         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2572         BIT_ULL(POWER_DOMAIN_INIT))
2573
2574 #define CHV_DPIO_CMN_D_POWER_DOMAINS (          \
2575         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
2576         BIT_ULL(POWER_DOMAIN_AUX_D) |           \
2577         BIT_ULL(POWER_DOMAIN_INIT))
2578
2579 #define HSW_DISPLAY_POWER_DOMAINS (                     \
2580         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2581         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2582         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |             \
2583         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
2584         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
2585         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2586         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2587         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2588         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2589         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2590         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
2591         BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */    \
2592         BIT_ULL(POWER_DOMAIN_VGA) |                             \
2593         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2594         BIT_ULL(POWER_DOMAIN_INIT))
2595
2596 #define BDW_DISPLAY_POWER_DOMAINS (                     \
2597         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2598         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2599         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
2600         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
2601         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2602         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2603         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2604         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2605         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2606         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
2607         BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */    \
2608         BIT_ULL(POWER_DOMAIN_VGA) |                             \
2609         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2610         BIT_ULL(POWER_DOMAIN_INIT))
2611
2612 #define SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
2613         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2614         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2615         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2616         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2617         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2618         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
2619         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
2620         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2621         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2622         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
2623         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |                \
2624         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
2625         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2626         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
2627         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2628         BIT_ULL(POWER_DOMAIN_VGA) |                             \
2629         BIT_ULL(POWER_DOMAIN_INIT))
2630 #define SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS (          \
2631         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |           \
2632         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO) |           \
2633         BIT_ULL(POWER_DOMAIN_INIT))
2634 #define SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS (            \
2635         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |           \
2636         BIT_ULL(POWER_DOMAIN_INIT))
2637 #define SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS (            \
2638         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |           \
2639         BIT_ULL(POWER_DOMAIN_INIT))
2640 #define SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS (            \
2641         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |           \
2642         BIT_ULL(POWER_DOMAIN_INIT))
2643 #define SKL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
2644         SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
2645         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
2646         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
2647         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2648         BIT_ULL(POWER_DOMAIN_INIT))
2649
2650 #define BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
2651         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2652         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2653         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2654         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2655         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2656         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
2657         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
2658         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2659         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2660         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
2661         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2662         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2663         BIT_ULL(POWER_DOMAIN_VGA) |                             \
2664         BIT_ULL(POWER_DOMAIN_INIT))
2665 #define BXT_DISPLAY_DC_OFF_POWER_DOMAINS (              \
2666         BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
2667         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
2668         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
2669         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2670         BIT_ULL(POWER_DOMAIN_GMBUS) |                   \
2671         BIT_ULL(POWER_DOMAIN_INIT))
2672 #define BXT_DPIO_CMN_A_POWER_DOMAINS (                  \
2673         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
2674         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2675         BIT_ULL(POWER_DOMAIN_INIT))
2676 #define BXT_DPIO_CMN_BC_POWER_DOMAINS (                 \
2677         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2678         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2679         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
2680         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2681         BIT_ULL(POWER_DOMAIN_INIT))
2682
2683 #define GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
2684         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2685         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2686         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2687         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2688         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2689         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
2690         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
2691         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2692         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2693         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
2694         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2695         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2696         BIT_ULL(POWER_DOMAIN_VGA) |                             \
2697         BIT_ULL(POWER_DOMAIN_INIT))
2698 #define GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS (            \
2699         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO))
2700 #define GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS (            \
2701         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO))
2702 #define GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS (            \
2703         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO))
2704 #define GLK_DPIO_CMN_A_POWER_DOMAINS (                  \
2705         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
2706         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2707         BIT_ULL(POWER_DOMAIN_INIT))
2708 #define GLK_DPIO_CMN_B_POWER_DOMAINS (                  \
2709         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2710         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
2711         BIT_ULL(POWER_DOMAIN_INIT))
2712 #define GLK_DPIO_CMN_C_POWER_DOMAINS (                  \
2713         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2714         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2715         BIT_ULL(POWER_DOMAIN_INIT))
2716 #define GLK_DISPLAY_AUX_A_POWER_DOMAINS (               \
2717         BIT_ULL(POWER_DOMAIN_AUX_A) |           \
2718         BIT_ULL(POWER_DOMAIN_AUX_IO_A) |                \
2719         BIT_ULL(POWER_DOMAIN_INIT))
2720 #define GLK_DISPLAY_AUX_B_POWER_DOMAINS (               \
2721         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
2722         BIT_ULL(POWER_DOMAIN_INIT))
2723 #define GLK_DISPLAY_AUX_C_POWER_DOMAINS (               \
2724         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
2725         BIT_ULL(POWER_DOMAIN_INIT))
2726 #define GLK_DISPLAY_DC_OFF_POWER_DOMAINS (              \
2727         GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
2728         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
2729         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
2730         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2731         BIT_ULL(POWER_DOMAIN_GMBUS) |                   \
2732         BIT_ULL(POWER_DOMAIN_INIT))
2733
2734 #define CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
2735         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2736         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2737         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2738         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2739         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2740         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
2741         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
2742         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
2743         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
2744         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
2745         BIT_ULL(POWER_DOMAIN_PORT_DDI_F_LANES) |                \
2746         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
2747         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2748         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
2749         BIT_ULL(POWER_DOMAIN_AUX_F) |                   \
2750         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2751         BIT_ULL(POWER_DOMAIN_VGA) |                             \
2752         BIT_ULL(POWER_DOMAIN_INIT))
2753 #define CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS (            \
2754         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |           \
2755         BIT_ULL(POWER_DOMAIN_INIT))
2756 #define CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS (            \
2757         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |           \
2758         BIT_ULL(POWER_DOMAIN_INIT))
2759 #define CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS (            \
2760         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |           \
2761         BIT_ULL(POWER_DOMAIN_INIT))
2762 #define CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS (            \
2763         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |           \
2764         BIT_ULL(POWER_DOMAIN_INIT))
2765 #define CNL_DISPLAY_AUX_A_POWER_DOMAINS (               \
2766         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2767         BIT_ULL(POWER_DOMAIN_AUX_IO_A) |                \
2768         BIT_ULL(POWER_DOMAIN_INIT))
2769 #define CNL_DISPLAY_AUX_B_POWER_DOMAINS (               \
2770         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
2771         BIT_ULL(POWER_DOMAIN_INIT))
2772 #define CNL_DISPLAY_AUX_C_POWER_DOMAINS (               \
2773         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2774         BIT_ULL(POWER_DOMAIN_INIT))
2775 #define CNL_DISPLAY_AUX_D_POWER_DOMAINS (               \
2776         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
2777         BIT_ULL(POWER_DOMAIN_INIT))
2778 #define CNL_DISPLAY_AUX_F_POWER_DOMAINS (               \
2779         BIT_ULL(POWER_DOMAIN_AUX_F) |                   \
2780         BIT_ULL(POWER_DOMAIN_INIT))
2781 #define CNL_DISPLAY_DDI_F_IO_POWER_DOMAINS (            \
2782         BIT_ULL(POWER_DOMAIN_PORT_DDI_F_IO) |           \
2783         BIT_ULL(POWER_DOMAIN_INIT))
2784 #define CNL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
2785         CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
2786         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
2787         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
2788         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2789         BIT_ULL(POWER_DOMAIN_INIT))
2790
2791 /*
2792  * ICL PW_0/PG_0 domains (HW/DMC control):
2793  * - PCI
2794  * - clocks except port PLL
2795  * - central power except FBC
2796  * - shared functions except pipe interrupts, pipe MBUS, DBUF registers
2797  * ICL PW_1/PG_1 domains (HW/DMC control):
2798  * - DBUF function
2799  * - PIPE_A and its planes, except VGA
2800  * - transcoder EDP + PSR
2801  * - transcoder DSI
2802  * - DDI_A
2803  * - FBC
2804  */
2805 #define ICL_PW_4_POWER_DOMAINS (                        \
2806         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2807         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
2808         BIT_ULL(POWER_DOMAIN_INIT))
2809         /* VDSC/joining */
2810 #define ICL_PW_3_POWER_DOMAINS (                        \
2811         ICL_PW_4_POWER_DOMAINS |                        \
2812         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2813         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
2814         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2815         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2816         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
2817         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
2818         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
2819         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
2820         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |        \
2821         BIT_ULL(POWER_DOMAIN_PORT_DDI_F_LANES) |        \
2822         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
2823         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2824         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
2825         BIT_ULL(POWER_DOMAIN_AUX_E) |                   \
2826         BIT_ULL(POWER_DOMAIN_AUX_F) |                   \
2827         BIT_ULL(POWER_DOMAIN_AUX_C_TBT) |               \
2828         BIT_ULL(POWER_DOMAIN_AUX_D_TBT) |               \
2829         BIT_ULL(POWER_DOMAIN_AUX_E_TBT) |               \
2830         BIT_ULL(POWER_DOMAIN_AUX_F_TBT) |               \
2831         BIT_ULL(POWER_DOMAIN_VGA) |                     \
2832         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2833         BIT_ULL(POWER_DOMAIN_INIT))
2834         /*
2835          * - transcoder WD
2836          * - KVMR (HW control)
2837          */
2838 #define ICL_PW_2_POWER_DOMAINS (                        \
2839         ICL_PW_3_POWER_DOMAINS |                        \
2840         BIT_ULL(POWER_DOMAIN_TRANSCODER_VDSC_PW2) |             \
2841         BIT_ULL(POWER_DOMAIN_INIT))
2842         /*
2843          * - KVMR (HW control)
2844          */
2845 #define ICL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
2846         ICL_PW_2_POWER_DOMAINS |                        \
2847         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
2848         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2849         BIT_ULL(POWER_DOMAIN_DPLL_DC_OFF) |                     \
2850         BIT_ULL(POWER_DOMAIN_INIT))
2851
2852 #define ICL_DDI_IO_A_POWER_DOMAINS (                    \
2853         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO))
2854 #define ICL_DDI_IO_B_POWER_DOMAINS (                    \
2855         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO))
2856 #define ICL_DDI_IO_C_POWER_DOMAINS (                    \
2857         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO))
2858 #define ICL_DDI_IO_D_POWER_DOMAINS (                    \
2859         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO))
2860 #define ICL_DDI_IO_E_POWER_DOMAINS (                    \
2861         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO))
2862 #define ICL_DDI_IO_F_POWER_DOMAINS (                    \
2863         BIT_ULL(POWER_DOMAIN_PORT_DDI_F_IO))
2864
2865 #define ICL_AUX_A_IO_POWER_DOMAINS (                    \
2866         BIT_ULL(POWER_DOMAIN_AUX_IO_A) |                \
2867         BIT_ULL(POWER_DOMAIN_AUX_A))
2868 #define ICL_AUX_B_IO_POWER_DOMAINS (                    \
2869         BIT_ULL(POWER_DOMAIN_AUX_B))
2870 #define ICL_AUX_C_TC1_IO_POWER_DOMAINS (                \
2871         BIT_ULL(POWER_DOMAIN_AUX_C))
2872 #define ICL_AUX_D_TC2_IO_POWER_DOMAINS (                \
2873         BIT_ULL(POWER_DOMAIN_AUX_D))
2874 #define ICL_AUX_E_TC3_IO_POWER_DOMAINS (                \
2875         BIT_ULL(POWER_DOMAIN_AUX_E))
2876 #define ICL_AUX_F_TC4_IO_POWER_DOMAINS (                \
2877         BIT_ULL(POWER_DOMAIN_AUX_F))
2878 #define ICL_AUX_C_TBT1_IO_POWER_DOMAINS (               \
2879         BIT_ULL(POWER_DOMAIN_AUX_C_TBT))
2880 #define ICL_AUX_D_TBT2_IO_POWER_DOMAINS (               \
2881         BIT_ULL(POWER_DOMAIN_AUX_D_TBT))
2882 #define ICL_AUX_E_TBT3_IO_POWER_DOMAINS (               \
2883         BIT_ULL(POWER_DOMAIN_AUX_E_TBT))
2884 #define ICL_AUX_F_TBT4_IO_POWER_DOMAINS (               \
2885         BIT_ULL(POWER_DOMAIN_AUX_F_TBT))
2886
2887 #define TGL_PW_5_POWER_DOMAINS (                        \
2888         BIT_ULL(POWER_DOMAIN_PIPE_D) |                  \
2889         BIT_ULL(POWER_DOMAIN_TRANSCODER_D) |            \
2890         BIT_ULL(POWER_DOMAIN_PIPE_D_PANEL_FITTER) |     \
2891         BIT_ULL(POWER_DOMAIN_INIT))
2892
2893 #define TGL_PW_4_POWER_DOMAINS (                        \
2894         TGL_PW_5_POWER_DOMAINS |                        \
2895         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2896         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2897         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
2898         BIT_ULL(POWER_DOMAIN_INIT))
2899
2900 #define TGL_PW_3_POWER_DOMAINS (                        \
2901         TGL_PW_4_POWER_DOMAINS |                        \
2902         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2903         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2904         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
2905         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC1) |      \
2906         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC2) |      \
2907         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC3) |      \
2908         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC4) |      \
2909         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC5) |      \
2910         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC6) |      \
2911         BIT_ULL(POWER_DOMAIN_AUX_USBC1) |               \
2912         BIT_ULL(POWER_DOMAIN_AUX_USBC2) |               \
2913         BIT_ULL(POWER_DOMAIN_AUX_USBC3) |               \
2914         BIT_ULL(POWER_DOMAIN_AUX_USBC4) |               \
2915         BIT_ULL(POWER_DOMAIN_AUX_USBC5) |               \
2916         BIT_ULL(POWER_DOMAIN_AUX_USBC6) |               \
2917         BIT_ULL(POWER_DOMAIN_AUX_TBT1) |                \
2918         BIT_ULL(POWER_DOMAIN_AUX_TBT2) |                \
2919         BIT_ULL(POWER_DOMAIN_AUX_TBT3) |                \
2920         BIT_ULL(POWER_DOMAIN_AUX_TBT4) |                \
2921         BIT_ULL(POWER_DOMAIN_AUX_TBT5) |                \
2922         BIT_ULL(POWER_DOMAIN_AUX_TBT6) |                \
2923         BIT_ULL(POWER_DOMAIN_VGA) |                     \
2924         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2925         BIT_ULL(POWER_DOMAIN_INIT))
2926
2927 #define TGL_PW_2_POWER_DOMAINS (                        \
2928         TGL_PW_3_POWER_DOMAINS |                        \
2929         BIT_ULL(POWER_DOMAIN_TRANSCODER_VDSC_PW2) |     \
2930         BIT_ULL(POWER_DOMAIN_INIT))
2931
2932 #define TGL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
2933         TGL_PW_3_POWER_DOMAINS |                        \
2934         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
2935         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
2936         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
2937         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
2938         BIT_ULL(POWER_DOMAIN_INIT))
2939
2940 #define TGL_DDI_IO_TC1_POWER_DOMAINS    BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_TC1)
2941 #define TGL_DDI_IO_TC2_POWER_DOMAINS    BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_TC2)
2942 #define TGL_DDI_IO_TC3_POWER_DOMAINS    BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_TC3)
2943 #define TGL_DDI_IO_TC4_POWER_DOMAINS    BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_TC4)
2944 #define TGL_DDI_IO_TC5_POWER_DOMAINS    BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_TC5)
2945 #define TGL_DDI_IO_TC6_POWER_DOMAINS    BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_TC6)
2946
2947 #define TGL_AUX_A_IO_POWER_DOMAINS (            \
2948         BIT_ULL(POWER_DOMAIN_AUX_IO_A) |        \
2949         BIT_ULL(POWER_DOMAIN_AUX_A))
2950 #define TGL_AUX_B_IO_POWER_DOMAINS (            \
2951         BIT_ULL(POWER_DOMAIN_AUX_B))
2952 #define TGL_AUX_C_IO_POWER_DOMAINS (            \
2953         BIT_ULL(POWER_DOMAIN_AUX_C))
2954
2955 #define TGL_AUX_IO_USBC1_POWER_DOMAINS  BIT_ULL(POWER_DOMAIN_AUX_USBC1)
2956 #define TGL_AUX_IO_USBC2_POWER_DOMAINS  BIT_ULL(POWER_DOMAIN_AUX_USBC2)
2957 #define TGL_AUX_IO_USBC3_POWER_DOMAINS  BIT_ULL(POWER_DOMAIN_AUX_USBC3)
2958 #define TGL_AUX_IO_USBC4_POWER_DOMAINS  BIT_ULL(POWER_DOMAIN_AUX_USBC4)
2959 #define TGL_AUX_IO_USBC5_POWER_DOMAINS  BIT_ULL(POWER_DOMAIN_AUX_USBC5)
2960 #define TGL_AUX_IO_USBC6_POWER_DOMAINS  BIT_ULL(POWER_DOMAIN_AUX_USBC6)
2961
2962 #define TGL_AUX_IO_TBT1_POWER_DOMAINS   BIT_ULL(POWER_DOMAIN_AUX_TBT1)
2963 #define TGL_AUX_IO_TBT2_POWER_DOMAINS   BIT_ULL(POWER_DOMAIN_AUX_TBT2)
2964 #define TGL_AUX_IO_TBT3_POWER_DOMAINS   BIT_ULL(POWER_DOMAIN_AUX_TBT3)
2965 #define TGL_AUX_IO_TBT4_POWER_DOMAINS   BIT_ULL(POWER_DOMAIN_AUX_TBT4)
2966 #define TGL_AUX_IO_TBT5_POWER_DOMAINS   BIT_ULL(POWER_DOMAIN_AUX_TBT5)
2967 #define TGL_AUX_IO_TBT6_POWER_DOMAINS   BIT_ULL(POWER_DOMAIN_AUX_TBT6)
2968
2969 #define TGL_TC_COLD_OFF_POWER_DOMAINS (         \
2970         BIT_ULL(POWER_DOMAIN_AUX_USBC1) |       \
2971         BIT_ULL(POWER_DOMAIN_AUX_USBC2) |       \
2972         BIT_ULL(POWER_DOMAIN_AUX_USBC3) |       \
2973         BIT_ULL(POWER_DOMAIN_AUX_USBC4) |       \
2974         BIT_ULL(POWER_DOMAIN_AUX_USBC5) |       \
2975         BIT_ULL(POWER_DOMAIN_AUX_USBC6) |       \
2976         BIT_ULL(POWER_DOMAIN_AUX_TBT1) |        \
2977         BIT_ULL(POWER_DOMAIN_AUX_TBT2) |        \
2978         BIT_ULL(POWER_DOMAIN_AUX_TBT3) |        \
2979         BIT_ULL(POWER_DOMAIN_AUX_TBT4) |        \
2980         BIT_ULL(POWER_DOMAIN_AUX_TBT5) |        \
2981         BIT_ULL(POWER_DOMAIN_AUX_TBT6) |        \
2982         BIT_ULL(POWER_DOMAIN_TC_COLD_OFF))
2983
2984 #define RKL_PW_4_POWER_DOMAINS (                        \
2985         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
2986         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
2987         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
2988         BIT_ULL(POWER_DOMAIN_INIT))
2989
2990 #define RKL_PW_3_POWER_DOMAINS (                        \
2991         RKL_PW_4_POWER_DOMAINS |                        \
2992         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
2993         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
2994         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
2995         BIT_ULL(POWER_DOMAIN_VGA) |                     \
2996         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
2997         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC1) |      \
2998         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC2) |      \
2999         BIT_ULL(POWER_DOMAIN_AUX_USBC1) |               \
3000         BIT_ULL(POWER_DOMAIN_AUX_USBC2) |               \
3001         BIT_ULL(POWER_DOMAIN_INIT))
3002
3003 /*
3004  * There is no PW_2/PG_2 on RKL.
3005  *
3006  * RKL PW_1/PG_1 domains (under HW/DMC control):
3007  * - DBUF function (note: registers are in PW0)
3008  * - PIPE_A and its planes and VDSC/joining, except VGA
3009  * - transcoder A
3010  * - DDI_A and DDI_B
3011  * - FBC
3012  *
3013  * RKL PW_0/PG_0 domains (under HW/DMC control):
3014  * - PCI
3015  * - clocks except port PLL
3016  * - shared functions:
3017  *     * interrupts except pipe interrupts
3018  *     * MBus except PIPE_MBUS_DBOX_CTL
3019  *     * DBUF registers
3020  * - central power except FBC
3021  * - top-level GTC (DDI-level GTC is in the well associated with the DDI)
3022  */
3023
3024 #define RKL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
3025         RKL_PW_3_POWER_DOMAINS |                        \
3026         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
3027         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
3028         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
3029         BIT_ULL(POWER_DOMAIN_INIT))
3030
3031 /*
3032  * XE_LPD Power Domains
3033  *
3034  * Previous platforms required that PG(n-1) be enabled before PG(n).  That
3035  * dependency chain turns into a dependency tree on XE_LPD:
3036  *
3037  *       PG0
3038  *        |
3039  *     --PG1--
3040  *    /       \
3041  *  PGA     --PG2--
3042  *         /   |   \
3043  *       PGB  PGC  PGD
3044  *
3045  * Power wells must be enabled from top to bottom and disabled from bottom
3046  * to top.  This allows pipes to be power gated independently.
3047  */
3048
3049 #define XELPD_PW_D_POWER_DOMAINS (                      \
3050         BIT_ULL(POWER_DOMAIN_PIPE_D) |                  \
3051         BIT_ULL(POWER_DOMAIN_PIPE_D_PANEL_FITTER) |     \
3052         BIT_ULL(POWER_DOMAIN_TRANSCODER_D) |            \
3053         BIT_ULL(POWER_DOMAIN_INIT))
3054
3055 #define XELPD_PW_C_POWER_DOMAINS (                      \
3056         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
3057         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
3058         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
3059         BIT_ULL(POWER_DOMAIN_INIT))
3060
3061 #define XELPD_PW_B_POWER_DOMAINS (                      \
3062         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
3063         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
3064         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
3065         BIT_ULL(POWER_DOMAIN_INIT))
3066
3067 #define XELPD_PW_A_POWER_DOMAINS (                      \
3068         BIT_ULL(POWER_DOMAIN_PIPE_A) |                  \
3069         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
3070         BIT_ULL(POWER_DOMAIN_INIT))
3071
3072 #define XELPD_PW_2_POWER_DOMAINS (                      \
3073         XELPD_PW_B_POWER_DOMAINS |                      \
3074         XELPD_PW_C_POWER_DOMAINS |                      \
3075         XELPD_PW_D_POWER_DOMAINS |                      \
3076         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
3077         BIT_ULL(POWER_DOMAIN_VGA) |                     \
3078         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
3079         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_D_XELPD) |  \
3080         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_E_XELPD) |  \
3081         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC1) |      \
3082         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC2) |      \
3083         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC3) |      \
3084         BIT_ULL(POWER_DOMAIN_PORT_DDI_LANES_TC4) |      \
3085         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
3086         BIT_ULL(POWER_DOMAIN_AUX_D_XELPD) |             \
3087         BIT_ULL(POWER_DOMAIN_AUX_E_XELPD) |             \
3088         BIT_ULL(POWER_DOMAIN_AUX_USBC1) |                       \
3089         BIT_ULL(POWER_DOMAIN_AUX_USBC2) |                       \
3090         BIT_ULL(POWER_DOMAIN_AUX_USBC3) |                       \
3091         BIT_ULL(POWER_DOMAIN_AUX_USBC4) |                       \
3092         BIT_ULL(POWER_DOMAIN_AUX_TBT1) |                        \
3093         BIT_ULL(POWER_DOMAIN_AUX_TBT2) |                        \
3094         BIT_ULL(POWER_DOMAIN_AUX_TBT3) |                        \
3095         BIT_ULL(POWER_DOMAIN_AUX_TBT4) |                        \
3096         BIT_ULL(POWER_DOMAIN_INIT))
3097
3098 /*
3099  * XELPD PW_1/PG_1 domains (under HW/DMC control):
3100  *  - DBUF function (registers are in PW0)
3101  *  - Transcoder A
3102  *  - DDI_A and DDI_B
3103  *
3104  * XELPD PW_0/PW_1 domains (under HW/DMC control):
3105  *  - PCI
3106  *  - Clocks except port PLL
3107  *  - Shared functions:
3108  *     * interrupts except pipe interrupts
3109  *     * MBus except PIPE_MBUS_DBOX_CTL
3110  *     * DBUF registers
3111  *  - Central power except FBC
3112  *  - Top-level GTC (DDI-level GTC is in the well associated with the DDI)
3113  */
3114
3115 #define XELPD_DISPLAY_DC_OFF_POWER_DOMAINS (            \
3116         XELPD_PW_2_POWER_DOMAINS |                      \
3117         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
3118         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
3119         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
3120         BIT_ULL(POWER_DOMAIN_INIT))
3121
3122 #define XELPD_AUX_IO_D_XELPD_POWER_DOMAINS      BIT_ULL(POWER_DOMAIN_AUX_D_XELPD)
3123 #define XELPD_AUX_IO_E_XELPD_POWER_DOMAINS      BIT_ULL(POWER_DOMAIN_AUX_E_XELPD)
3124 #define XELPD_AUX_IO_USBC1_POWER_DOMAINS        BIT_ULL(POWER_DOMAIN_AUX_USBC1)
3125 #define XELPD_AUX_IO_USBC2_POWER_DOMAINS        BIT_ULL(POWER_DOMAIN_AUX_USBC2)
3126 #define XELPD_AUX_IO_USBC3_POWER_DOMAINS        BIT_ULL(POWER_DOMAIN_AUX_USBC3)
3127 #define XELPD_AUX_IO_USBC4_POWER_DOMAINS        BIT_ULL(POWER_DOMAIN_AUX_USBC4)
3128
3129 #define XELPD_AUX_IO_TBT1_POWER_DOMAINS         BIT_ULL(POWER_DOMAIN_AUX_TBT1)
3130 #define XELPD_AUX_IO_TBT2_POWER_DOMAINS         BIT_ULL(POWER_DOMAIN_AUX_TBT2)
3131 #define XELPD_AUX_IO_TBT3_POWER_DOMAINS         BIT_ULL(POWER_DOMAIN_AUX_TBT3)
3132 #define XELPD_AUX_IO_TBT4_POWER_DOMAINS         BIT_ULL(POWER_DOMAIN_AUX_TBT4)
3133
3134 #define XELPD_DDI_IO_D_XELPD_POWER_DOMAINS      BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_D_XELPD)
3135 #define XELPD_DDI_IO_E_XELPD_POWER_DOMAINS      BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_E_XELPD)
3136 #define XELPD_DDI_IO_TC1_POWER_DOMAINS          BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_TC1)
3137 #define XELPD_DDI_IO_TC2_POWER_DOMAINS          BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_TC2)
3138 #define XELPD_DDI_IO_TC3_POWER_DOMAINS          BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_TC3)
3139 #define XELPD_DDI_IO_TC4_POWER_DOMAINS          BIT_ULL(POWER_DOMAIN_PORT_DDI_IO_TC4)
3140
3141 static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
3142         .sync_hw = i9xx_power_well_sync_hw_noop,
3143         .enable = i9xx_always_on_power_well_noop,
3144         .disable = i9xx_always_on_power_well_noop,
3145         .is_enabled = i9xx_always_on_power_well_enabled,
3146 };
3147
3148 static const struct i915_power_well_ops chv_pipe_power_well_ops = {
3149         .sync_hw = chv_pipe_power_well_sync_hw,
3150         .enable = chv_pipe_power_well_enable,
3151         .disable = chv_pipe_power_well_disable,
3152         .is_enabled = chv_pipe_power_well_enabled,
3153 };
3154
3155 static const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = {
3156         .sync_hw = i9xx_power_well_sync_hw_noop,
3157         .enable = chv_dpio_cmn_power_well_enable,
3158         .disable = chv_dpio_cmn_power_well_disable,
3159         .is_enabled = vlv_power_well_enabled,
3160 };
3161
3162 static const struct i915_power_well_desc i9xx_always_on_power_well[] = {
3163         {
3164                 .name = "always-on",
3165                 .always_on = true,
3166                 .domains = POWER_DOMAIN_MASK,
3167                 .ops = &i9xx_always_on_power_well_ops,
3168                 .id = DISP_PW_ID_NONE,
3169         },
3170 };
3171
3172 static const struct i915_power_well_ops i830_pipes_power_well_ops = {
3173         .sync_hw = i830_pipes_power_well_sync_hw,
3174         .enable = i830_pipes_power_well_enable,
3175         .disable = i830_pipes_power_well_disable,
3176         .is_enabled = i830_pipes_power_well_enabled,
3177 };
3178
3179 static const struct i915_power_well_desc i830_power_wells[] = {
3180         {
3181                 .name = "always-on",
3182                 .always_on = true,
3183                 .domains = POWER_DOMAIN_MASK,
3184                 .ops = &i9xx_always_on_power_well_ops,
3185                 .id = DISP_PW_ID_NONE,
3186         },
3187         {
3188                 .name = "pipes",
3189                 .domains = I830_PIPES_POWER_DOMAINS,
3190                 .ops = &i830_pipes_power_well_ops,
3191                 .id = DISP_PW_ID_NONE,
3192         },
3193 };
3194
3195 static const struct i915_power_well_ops hsw_power_well_ops = {
3196         .sync_hw = hsw_power_well_sync_hw,
3197         .enable = hsw_power_well_enable,
3198         .disable = hsw_power_well_disable,
3199         .is_enabled = hsw_power_well_enabled,
3200 };
3201
3202 static const struct i915_power_well_ops gen9_dc_off_power_well_ops = {
3203         .sync_hw = i9xx_power_well_sync_hw_noop,
3204         .enable = gen9_dc_off_power_well_enable,
3205         .disable = gen9_dc_off_power_well_disable,
3206         .is_enabled = gen9_dc_off_power_well_enabled,
3207 };
3208
3209 static const struct i915_power_well_ops bxt_dpio_cmn_power_well_ops = {
3210         .sync_hw = i9xx_power_well_sync_hw_noop,
3211         .enable = bxt_dpio_cmn_power_well_enable,
3212         .disable = bxt_dpio_cmn_power_well_disable,
3213         .is_enabled = bxt_dpio_cmn_power_well_enabled,
3214 };
3215
3216 static const struct i915_power_well_regs hsw_power_well_regs = {
3217         .bios   = HSW_PWR_WELL_CTL1,
3218         .driver = HSW_PWR_WELL_CTL2,
3219         .kvmr   = HSW_PWR_WELL_CTL3,
3220         .debug  = HSW_PWR_WELL_CTL4,
3221 };
3222
3223 static const struct i915_power_well_desc hsw_power_wells[] = {
3224         {
3225                 .name = "always-on",
3226                 .always_on = true,
3227                 .domains = POWER_DOMAIN_MASK,
3228                 .ops = &i9xx_always_on_power_well_ops,
3229                 .id = DISP_PW_ID_NONE,
3230         },
3231         {
3232                 .name = "display",
3233                 .domains = HSW_DISPLAY_POWER_DOMAINS,
3234                 .ops = &hsw_power_well_ops,
3235                 .id = HSW_DISP_PW_GLOBAL,
3236                 {
3237                         .hsw.regs = &hsw_power_well_regs,
3238                         .hsw.idx = HSW_PW_CTL_IDX_GLOBAL,
3239                         .hsw.has_vga = true,
3240                 },
3241         },
3242 };
3243
3244 static const struct i915_power_well_desc bdw_power_wells[] = {
3245         {
3246                 .name = "always-on",
3247                 .always_on = true,
3248                 .domains = POWER_DOMAIN_MASK,
3249                 .ops = &i9xx_always_on_power_well_ops,
3250                 .id = DISP_PW_ID_NONE,
3251         },
3252         {
3253                 .name = "display",
3254                 .domains = BDW_DISPLAY_POWER_DOMAINS,
3255                 .ops = &hsw_power_well_ops,
3256                 .id = HSW_DISP_PW_GLOBAL,
3257                 {
3258                         .hsw.regs = &hsw_power_well_regs,
3259                         .hsw.idx = HSW_PW_CTL_IDX_GLOBAL,
3260                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
3261                         .hsw.has_vga = true,
3262                 },
3263         },
3264 };
3265
3266 static const struct i915_power_well_ops vlv_display_power_well_ops = {
3267         .sync_hw = i9xx_power_well_sync_hw_noop,
3268         .enable = vlv_display_power_well_enable,
3269         .disable = vlv_display_power_well_disable,
3270         .is_enabled = vlv_power_well_enabled,
3271 };
3272
3273 static const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = {
3274         .sync_hw = i9xx_power_well_sync_hw_noop,
3275         .enable = vlv_dpio_cmn_power_well_enable,
3276         .disable = vlv_dpio_cmn_power_well_disable,
3277         .is_enabled = vlv_power_well_enabled,
3278 };
3279
3280 static const struct i915_power_well_ops vlv_dpio_power_well_ops = {
3281         .sync_hw = i9xx_power_well_sync_hw_noop,
3282         .enable = vlv_power_well_enable,
3283         .disable = vlv_power_well_disable,
3284         .is_enabled = vlv_power_well_enabled,
3285 };
3286
3287 static const struct i915_power_well_desc vlv_power_wells[] = {
3288         {
3289                 .name = "always-on",
3290                 .always_on = true,
3291                 .domains = POWER_DOMAIN_MASK,
3292                 .ops = &i9xx_always_on_power_well_ops,
3293                 .id = DISP_PW_ID_NONE,
3294         },
3295         {
3296                 .name = "display",
3297                 .domains = VLV_DISPLAY_POWER_DOMAINS,
3298                 .ops = &vlv_display_power_well_ops,
3299                 .id = VLV_DISP_PW_DISP2D,
3300                 {
3301                         .vlv.idx = PUNIT_PWGT_IDX_DISP2D,
3302                 },
3303         },
3304         {
3305                 .name = "dpio-tx-b-01",
3306                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
3307                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
3308                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
3309                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
3310                 .ops = &vlv_dpio_power_well_ops,
3311                 .id = DISP_PW_ID_NONE,
3312                 {
3313                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_TX_B_LANES_01,
3314                 },
3315         },
3316         {
3317                 .name = "dpio-tx-b-23",
3318                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
3319                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
3320                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
3321                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
3322                 .ops = &vlv_dpio_power_well_ops,
3323                 .id = DISP_PW_ID_NONE,
3324                 {
3325                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_TX_B_LANES_23,
3326                 },
3327         },
3328         {
3329                 .name = "dpio-tx-c-01",
3330                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
3331                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
3332                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
3333                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
3334                 .ops = &vlv_dpio_power_well_ops,
3335                 .id = DISP_PW_ID_NONE,
3336                 {
3337                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_TX_C_LANES_01,
3338                 },
3339         },
3340         {
3341                 .name = "dpio-tx-c-23",
3342                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
3343                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
3344                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
3345                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
3346                 .ops = &vlv_dpio_power_well_ops,
3347                 .id = DISP_PW_ID_NONE,
3348                 {
3349                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_TX_C_LANES_23,
3350                 },
3351         },
3352         {
3353                 .name = "dpio-common",
3354                 .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS,
3355                 .ops = &vlv_dpio_cmn_power_well_ops,
3356                 .id = VLV_DISP_PW_DPIO_CMN_BC,
3357                 {
3358                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_CMN_BC,
3359                 },
3360         },
3361 };
3362
3363 static const struct i915_power_well_desc chv_power_wells[] = {
3364         {
3365                 .name = "always-on",
3366                 .always_on = true,
3367                 .domains = POWER_DOMAIN_MASK,
3368                 .ops = &i9xx_always_on_power_well_ops,
3369                 .id = DISP_PW_ID_NONE,
3370         },
3371         {
3372                 .name = "display",
3373                 /*
3374                  * Pipe A power well is the new disp2d well. Pipe B and C
3375                  * power wells don't actually exist. Pipe A power well is
3376                  * required for any pipe to work.
3377                  */
3378                 .domains = CHV_DISPLAY_POWER_DOMAINS,
3379                 .ops = &chv_pipe_power_well_ops,
3380                 .id = DISP_PW_ID_NONE,
3381         },
3382         {
3383                 .name = "dpio-common-bc",
3384                 .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS,
3385                 .ops = &chv_dpio_cmn_power_well_ops,
3386                 .id = VLV_DISP_PW_DPIO_CMN_BC,
3387                 {
3388                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_CMN_BC,
3389                 },
3390         },
3391         {
3392                 .name = "dpio-common-d",
3393                 .domains = CHV_DPIO_CMN_D_POWER_DOMAINS,
3394                 .ops = &chv_dpio_cmn_power_well_ops,
3395                 .id = CHV_DISP_PW_DPIO_CMN_D,
3396                 {
3397                         .vlv.idx = PUNIT_PWGT_IDX_DPIO_CMN_D,
3398                 },
3399         },
3400 };
3401
3402 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
3403                                          enum i915_power_well_id power_well_id)
3404 {
3405         struct i915_power_well *power_well;
3406         bool ret;
3407
3408         power_well = lookup_power_well(dev_priv, power_well_id);
3409         ret = power_well->desc->ops->is_enabled(dev_priv, power_well);
3410
3411         return ret;
3412 }
3413
3414 static const struct i915_power_well_desc skl_power_wells[] = {
3415         {
3416                 .name = "always-on",
3417                 .always_on = true,
3418                 .domains = POWER_DOMAIN_MASK,
3419                 .ops = &i9xx_always_on_power_well_ops,
3420                 .id = DISP_PW_ID_NONE,
3421         },
3422         {
3423                 .name = "power well 1",
3424                 /* Handled by the DMC firmware */
3425                 .always_on = true,
3426                 .domains = 0,
3427                 .ops = &hsw_power_well_ops,
3428                 .id = SKL_DISP_PW_1,
3429                 {
3430                         .hsw.regs = &hsw_power_well_regs,
3431                         .hsw.idx = SKL_PW_CTL_IDX_PW_1,
3432                         .hsw.has_fuses = true,
3433                 },
3434         },
3435         {
3436                 .name = "MISC IO power well",
3437                 /* Handled by the DMC firmware */
3438                 .always_on = true,
3439                 .domains = 0,
3440                 .ops = &hsw_power_well_ops,
3441                 .id = SKL_DISP_PW_MISC_IO,
3442                 {
3443                         .hsw.regs = &hsw_power_well_regs,
3444                         .hsw.idx = SKL_PW_CTL_IDX_MISC_IO,
3445                 },
3446         },
3447         {
3448                 .name = "DC off",
3449                 .domains = SKL_DISPLAY_DC_OFF_POWER_DOMAINS,
3450                 .ops = &gen9_dc_off_power_well_ops,
3451                 .id = SKL_DISP_DC_OFF,
3452         },
3453         {
3454                 .name = "power well 2",
3455                 .domains = SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
3456                 .ops = &hsw_power_well_ops,
3457                 .id = SKL_DISP_PW_2,
3458                 {
3459                         .hsw.regs = &hsw_power_well_regs,
3460                         .hsw.idx = SKL_PW_CTL_IDX_PW_2,
3461                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
3462                         .hsw.has_vga = true,
3463                         .hsw.has_fuses = true,
3464                 },
3465         },
3466         {
3467                 .name = "DDI A/E IO power well",
3468                 .domains = SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS,
3469                 .ops = &hsw_power_well_ops,
3470                 .id = DISP_PW_ID_NONE,
3471                 {
3472                         .hsw.regs = &hsw_power_well_regs,
3473                         .hsw.idx = SKL_PW_CTL_IDX_DDI_A_E,
3474                 },
3475         },
3476         {
3477                 .name = "DDI B IO power well",
3478                 .domains = SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS,
3479                 .ops = &hsw_power_well_ops,
3480                 .id = DISP_PW_ID_NONE,
3481                 {
3482                         .hsw.regs = &hsw_power_well_regs,
3483                         .hsw.idx = SKL_PW_CTL_IDX_DDI_B,
3484                 },
3485         },
3486         {
3487                 .name = "DDI C IO power well",
3488                 .domains = SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS,
3489                 .ops = &hsw_power_well_ops,
3490                 .id = DISP_PW_ID_NONE,
3491                 {
3492                         .hsw.regs = &hsw_power_well_regs,
3493                         .hsw.idx = SKL_PW_CTL_IDX_DDI_C,
3494                 },
3495         },
3496         {
3497                 .name = "DDI D IO power well",
3498                 .domains = SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS,
3499                 .ops = &hsw_power_well_ops,
3500                 .id = DISP_PW_ID_NONE,
3501                 {
3502                         .hsw.regs = &hsw_power_well_regs,
3503                         .hsw.idx = SKL_PW_CTL_IDX_DDI_D,
3504                 },
3505         },
3506 };
3507
3508 static const struct i915_power_well_desc bxt_power_wells[] = {
3509         {
3510                 .name = "always-on",
3511                 .always_on = true,
3512                 .domains = POWER_DOMAIN_MASK,
3513                 .ops = &i9xx_always_on_power_well_ops,
3514                 .id = DISP_PW_ID_NONE,
3515         },
3516         {
3517                 .name = "power well 1",
3518                 /* Handled by the DMC firmware */
3519                 .always_on = true,
3520                 .domains = 0,
3521                 .ops = &hsw_power_well_ops,
3522                 .id = SKL_DISP_PW_1,
3523                 {
3524                         .hsw.regs = &hsw_power_well_regs,
3525                         .hsw.idx = SKL_PW_CTL_IDX_PW_1,
3526                         .hsw.has_fuses = true,
3527                 },
3528         },
3529         {
3530                 .name = "DC off",
3531                 .domains = BXT_DISPLAY_DC_OFF_POWER_DOMAINS,
3532                 .ops = &gen9_dc_off_power_well_ops,
3533                 .id = SKL_DISP_DC_OFF,
3534         },
3535         {
3536                 .name = "power well 2",
3537                 .domains = BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS,
3538                 .ops = &hsw_power_well_ops,
3539                 .id = SKL_DISP_PW_2,
3540                 {
3541                         .hsw.regs = &hsw_power_well_regs,
3542                         .hsw.idx = SKL_PW_CTL_IDX_PW_2,
3543                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
3544                         .hsw.has_vga = true,
3545                         .hsw.has_fuses = true,
3546                 },
3547         },
3548         {
3549                 .name = "dpio-common-a",
3550                 .domains = BXT_DPIO_CMN_A_POWER_DOMAINS,
3551                 .ops = &bxt_dpio_cmn_power_well_ops,
3552                 .id = BXT_DISP_PW_DPIO_CMN_A,
3553                 {
3554                         .bxt.phy = DPIO_PHY1,
3555                 },
3556         },
3557         {
3558                 .name = "dpio-common-bc",
3559                 .domains = BXT_DPIO_CMN_BC_POWER_DOMAINS,
3560                 .ops = &bxt_dpio_cmn_power_well_ops,
3561                 .id = VLV_DISP_PW_DPIO_CMN_BC,
3562                 {
3563                         .bxt.phy = DPIO_PHY0,
3564                 },
3565         },
3566 };
3567
3568 static const struct i915_power_well_desc glk_power_wells[] = {
3569         {
3570                 .name = "always-on",
3571                 .always_on = true,
3572                 .domains = POWER_DOMAIN_MASK,
3573                 .ops = &i9xx_always_on_power_well_ops,
3574                 .id = DISP_PW_ID_NONE,
3575         },
3576         {
3577                 .name = "power well 1",
3578                 /* Handled by the DMC firmware */
3579                 .always_on = true,
3580                 .domains = 0,
3581                 .ops = &hsw_power_well_ops,
3582                 .id = SKL_DISP_PW_1,
3583                 {
3584                         .hsw.regs = &hsw_power_well_regs,
3585                         .hsw.idx = SKL_PW_CTL_IDX_PW_1,
3586                         .hsw.has_fuses = true,
3587                 },
3588         },
3589         {
3590                 .name = "DC off",
3591                 .domains = GLK_DISPLAY_DC_OFF_POWER_DOMAINS,
3592                 .ops = &gen9_dc_off_power_well_ops,
3593                 .id = SKL_DISP_DC_OFF,
3594         },
3595         {
3596                 .name = "power well 2",
3597                 .domains = GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS,
3598                 .ops = &hsw_power_well_ops,
3599                 .id = SKL_DISP_PW_2,
3600                 {
3601                         .hsw.regs = &hsw_power_well_regs,
3602                         .hsw.idx = SKL_PW_CTL_IDX_PW_2,
3603                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
3604                         .hsw.has_vga = true,
3605                         .hsw.has_fuses = true,
3606                 },
3607         },
3608         {
3609                 .name = "dpio-common-a",
3610                 .domains = GLK_DPIO_CMN_A_POWER_DOMAINS,
3611                 .ops = &bxt_dpio_cmn_power_well_ops,
3612                 .id = BXT_DISP_PW_DPIO_CMN_A,
3613                 {
3614                         .bxt.phy = DPIO_PHY1,
3615                 },
3616         },
3617         {
3618                 .name = "dpio-common-b",
3619                 .domains = GLK_DPIO_CMN_B_POWER_DOMAINS,
3620                 .ops = &bxt_dpio_cmn_power_well_ops,
3621                 .id = VLV_DISP_PW_DPIO_CMN_BC,
3622                 {
3623                         .bxt.phy = DPIO_PHY0,
3624                 },
3625         },
3626         {
3627                 .name = "dpio-common-c",
3628                 .domains = GLK_DPIO_CMN_C_POWER_DOMAINS,
3629                 .ops = &bxt_dpio_cmn_power_well_ops,
3630                 .id = GLK_DISP_PW_DPIO_CMN_C,
3631                 {
3632                         .bxt.phy = DPIO_PHY2,
3633                 },
3634         },
3635         {
3636                 .name = "AUX A",
3637                 .domains = GLK_DISPLAY_AUX_A_POWER_DOMAINS,
3638                 .ops = &hsw_power_well_ops,
3639                 .id = DISP_PW_ID_NONE,
3640                 {
3641                         .hsw.regs = &hsw_power_well_regs,
3642                         .hsw.idx = GLK_PW_CTL_IDX_AUX_A,
3643                 },
3644         },
3645         {
3646                 .name = "AUX B",
3647                 .domains = GLK_DISPLAY_AUX_B_POWER_DOMAINS,
3648                 .ops = &hsw_power_well_ops,
3649                 .id = DISP_PW_ID_NONE,
3650                 {
3651                         .hsw.regs = &hsw_power_well_regs,
3652                         .hsw.idx = GLK_PW_CTL_IDX_AUX_B,
3653                 },
3654         },
3655         {
3656                 .name = "AUX C",
3657                 .domains = GLK_DISPLAY_AUX_C_POWER_DOMAINS,
3658                 .ops = &hsw_power_well_ops,
3659                 .id = DISP_PW_ID_NONE,
3660                 {
3661                         .hsw.regs = &hsw_power_well_regs,
3662                         .hsw.idx = GLK_PW_CTL_IDX_AUX_C,
3663                 },
3664         },
3665         {
3666                 .name = "DDI A IO power well",
3667                 .domains = GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS,
3668                 .ops = &hsw_power_well_ops,
3669                 .id = DISP_PW_ID_NONE,
3670                 {
3671                         .hsw.regs = &hsw_power_well_regs,
3672                         .hsw.idx = GLK_PW_CTL_IDX_DDI_A,
3673                 },
3674         },
3675         {
3676                 .name = "DDI B IO power well",
3677                 .domains = GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS,
3678                 .ops = &hsw_power_well_ops,
3679                 .id = DISP_PW_ID_NONE,
3680                 {
3681                         .hsw.regs = &hsw_power_well_regs,
3682                         .hsw.idx = SKL_PW_CTL_IDX_DDI_B,
3683                 },
3684         },
3685         {
3686                 .name = "DDI C IO power well",
3687                 .domains = GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS,
3688                 .ops = &hsw_power_well_ops,
3689                 .id = DISP_PW_ID_NONE,
3690                 {
3691                         .hsw.regs = &hsw_power_well_regs,
3692                         .hsw.idx = SKL_PW_CTL_IDX_DDI_C,
3693                 },
3694         },
3695 };
3696
3697 static const struct i915_power_well_desc cnl_power_wells[] = {
3698         {
3699                 .name = "always-on",
3700                 .always_on = true,
3701                 .domains = POWER_DOMAIN_MASK,
3702                 .ops = &i9xx_always_on_power_well_ops,
3703                 .id = DISP_PW_ID_NONE,
3704         },
3705         {
3706                 .name = "power well 1",
3707                 /* Handled by the DMC firmware */
3708                 .always_on = true,
3709                 .domains = 0,
3710                 .ops = &hsw_power_well_ops,
3711                 .id = SKL_DISP_PW_1,
3712                 {
3713                         .hsw.regs = &hsw_power_well_regs,
3714                         .hsw.idx = SKL_PW_CTL_IDX_PW_1,
3715                         .hsw.has_fuses = true,
3716                 },
3717         },
3718         {
3719                 .name = "AUX A",
3720                 .domains = CNL_DISPLAY_AUX_A_POWER_DOMAINS,
3721                 .ops = &hsw_power_well_ops,
3722                 .id = DISP_PW_ID_NONE,
3723                 {
3724                         .hsw.regs = &hsw_power_well_regs,
3725                         .hsw.idx = GLK_PW_CTL_IDX_AUX_A,
3726                 },
3727         },
3728         {
3729                 .name = "AUX B",
3730                 .domains = CNL_DISPLAY_AUX_B_POWER_DOMAINS,
3731                 .ops = &hsw_power_well_ops,
3732                 .id = DISP_PW_ID_NONE,
3733                 {
3734                         .hsw.regs = &hsw_power_well_regs,
3735                         .hsw.idx = GLK_PW_CTL_IDX_AUX_B,
3736                 },
3737         },
3738         {
3739                 .name = "AUX C",
3740                 .domains = CNL_DISPLAY_AUX_C_POWER_DOMAINS,
3741                 .ops = &hsw_power_well_ops,
3742                 .id = DISP_PW_ID_NONE,
3743                 {
3744                         .hsw.regs = &hsw_power_well_regs,
3745                         .hsw.idx = GLK_PW_CTL_IDX_AUX_C,
3746                 },
3747         },
3748         {
3749                 .name = "AUX D",
3750                 .domains = CNL_DISPLAY_AUX_D_POWER_DOMAINS,
3751                 .ops = &hsw_power_well_ops,
3752                 .id = DISP_PW_ID_NONE,
3753                 {
3754                         .hsw.regs = &hsw_power_well_regs,
3755                         .hsw.idx = CNL_PW_CTL_IDX_AUX_D,
3756                 },
3757         },
3758         {
3759                 .name = "DC off",
3760                 .domains = CNL_DISPLAY_DC_OFF_POWER_DOMAINS,
3761                 .ops = &gen9_dc_off_power_well_ops,
3762                 .id = SKL_DISP_DC_OFF,
3763         },
3764         {
3765                 .name = "power well 2",
3766                 .domains = CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
3767                 .ops = &hsw_power_well_ops,
3768                 .id = SKL_DISP_PW_2,
3769                 {
3770                         .hsw.regs = &hsw_power_well_regs,
3771                         .hsw.idx = SKL_PW_CTL_IDX_PW_2,
3772                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
3773                         .hsw.has_vga = true,
3774                         .hsw.has_fuses = true,
3775                 },
3776         },
3777         {
3778                 .name = "DDI A IO power well",
3779                 .domains = CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS,
3780                 .ops = &hsw_power_well_ops,
3781                 .id = DISP_PW_ID_NONE,
3782                 {
3783                         .hsw.regs = &hsw_power_well_regs,
3784                         .hsw.idx = GLK_PW_CTL_IDX_DDI_A,
3785                 },
3786         },
3787         {
3788                 .name = "DDI B IO power well",
3789                 .domains = CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS,
3790                 .ops = &hsw_power_well_ops,
3791                 .id = DISP_PW_ID_NONE,
3792                 {
3793                         .hsw.regs = &hsw_power_well_regs,
3794                         .hsw.idx = SKL_PW_CTL_IDX_DDI_B,
3795                 },
3796         },
3797         {
3798                 .name = "DDI C IO power well",
3799                 .domains = CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS,
3800                 .ops = &hsw_power_well_ops,
3801                 .id = DISP_PW_ID_NONE,
3802                 {
3803                         .hsw.regs = &hsw_power_well_regs,
3804                         .hsw.idx = SKL_PW_CTL_IDX_DDI_C,
3805                 },
3806         },
3807         {
3808                 .name = "DDI D IO power well",
3809                 .domains = CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS,
3810                 .ops = &hsw_power_well_ops,
3811                 .id = DISP_PW_ID_NONE,
3812                 {
3813                         .hsw.regs = &hsw_power_well_regs,
3814                         .hsw.idx = SKL_PW_CTL_IDX_DDI_D,
3815                 },
3816         },
3817         {
3818                 .name = "DDI F IO power well",
3819                 .domains = CNL_DISPLAY_DDI_F_IO_POWER_DOMAINS,
3820                 .ops = &hsw_power_well_ops,
3821                 .id = CNL_DISP_PW_DDI_F_IO,
3822                 {
3823                         .hsw.regs = &hsw_power_well_regs,
3824                         .hsw.idx = CNL_PW_CTL_IDX_DDI_F,
3825                 },
3826         },
3827         {
3828                 .name = "AUX F",
3829                 .domains = CNL_DISPLAY_AUX_F_POWER_DOMAINS,
3830                 .ops = &hsw_power_well_ops,
3831                 .id = CNL_DISP_PW_DDI_F_AUX,
3832                 {
3833                         .hsw.regs = &hsw_power_well_regs,
3834                         .hsw.idx = CNL_PW_CTL_IDX_AUX_F,
3835                 },
3836         },
3837 };
3838
3839 static const struct i915_power_well_ops icl_aux_power_well_ops = {
3840         .sync_hw = hsw_power_well_sync_hw,
3841         .enable = icl_aux_power_well_enable,
3842         .disable = icl_aux_power_well_disable,
3843         .is_enabled = hsw_power_well_enabled,
3844 };
3845
3846 static const struct i915_power_well_regs icl_aux_power_well_regs = {
3847         .bios   = ICL_PWR_WELL_CTL_AUX1,
3848         .driver = ICL_PWR_WELL_CTL_AUX2,
3849         .debug  = ICL_PWR_WELL_CTL_AUX4,
3850 };
3851
3852 static const struct i915_power_well_regs icl_ddi_power_well_regs = {
3853         .bios   = ICL_PWR_WELL_CTL_DDI1,
3854         .driver = ICL_PWR_WELL_CTL_DDI2,
3855         .debug  = ICL_PWR_WELL_CTL_DDI4,
3856 };
3857
3858 static const struct i915_power_well_desc icl_power_wells[] = {
3859         {
3860                 .name = "always-on",
3861                 .always_on = true,
3862                 .domains = POWER_DOMAIN_MASK,
3863                 .ops = &i9xx_always_on_power_well_ops,
3864                 .id = DISP_PW_ID_NONE,
3865         },
3866         {
3867                 .name = "power well 1",
3868                 /* Handled by the DMC firmware */
3869                 .always_on = true,
3870                 .domains = 0,
3871                 .ops = &hsw_power_well_ops,
3872                 .id = SKL_DISP_PW_1,
3873                 {
3874                         .hsw.regs = &hsw_power_well_regs,
3875                         .hsw.idx = ICL_PW_CTL_IDX_PW_1,
3876                         .hsw.has_fuses = true,
3877                 },
3878         },
3879         {
3880                 .name = "DC off",
3881                 .domains = ICL_DISPLAY_DC_OFF_POWER_DOMAINS,
3882                 .ops = &gen9_dc_off_power_well_ops,
3883                 .id = SKL_DISP_DC_OFF,
3884         },
3885         {
3886                 .name = "power well 2",
3887                 .domains = ICL_PW_2_POWER_DOMAINS,
3888                 .ops = &hsw_power_well_ops,
3889                 .id = SKL_DISP_PW_2,
3890                 {
3891                         .hsw.regs = &hsw_power_well_regs,
3892                         .hsw.idx = ICL_PW_CTL_IDX_PW_2,
3893                         .hsw.has_fuses = true,
3894                 },
3895         },
3896         {
3897                 .name = "power well 3",
3898                 .domains = ICL_PW_3_POWER_DOMAINS,
3899                 .ops = &hsw_power_well_ops,
3900                 .id = ICL_DISP_PW_3,
3901                 {
3902                         .hsw.regs = &hsw_power_well_regs,
3903                         .hsw.idx = ICL_PW_CTL_IDX_PW_3,
3904                         .hsw.irq_pipe_mask = BIT(PIPE_B),
3905                         .hsw.has_vga = true,
3906                         .hsw.has_fuses = true,
3907                 },
3908         },
3909         {
3910                 .name = "DDI A IO",
3911                 .domains = ICL_DDI_IO_A_POWER_DOMAINS,
3912                 .ops = &hsw_power_well_ops,
3913                 .id = DISP_PW_ID_NONE,
3914                 {
3915                         .hsw.regs = &icl_ddi_power_well_regs,
3916                         .hsw.idx = ICL_PW_CTL_IDX_DDI_A,
3917                 },
3918         },
3919         {
3920                 .name = "DDI B IO",
3921                 .domains = ICL_DDI_IO_B_POWER_DOMAINS,
3922                 .ops = &hsw_power_well_ops,
3923                 .id = DISP_PW_ID_NONE,
3924                 {
3925                         .hsw.regs = &icl_ddi_power_well_regs,
3926                         .hsw.idx = ICL_PW_CTL_IDX_DDI_B,
3927                 },
3928         },
3929         {
3930                 .name = "DDI C IO",
3931                 .domains = ICL_DDI_IO_C_POWER_DOMAINS,
3932                 .ops = &hsw_power_well_ops,
3933                 .id = DISP_PW_ID_NONE,
3934                 {
3935                         .hsw.regs = &icl_ddi_power_well_regs,
3936                         .hsw.idx = ICL_PW_CTL_IDX_DDI_C,
3937                 },
3938         },
3939         {
3940                 .name = "DDI D IO",
3941                 .domains = ICL_DDI_IO_D_POWER_DOMAINS,
3942                 .ops = &hsw_power_well_ops,
3943                 .id = DISP_PW_ID_NONE,
3944                 {
3945                         .hsw.regs = &icl_ddi_power_well_regs,
3946                         .hsw.idx = ICL_PW_CTL_IDX_DDI_D,
3947                 },
3948         },
3949         {
3950                 .name = "DDI E IO",
3951                 .domains = ICL_DDI_IO_E_POWER_DOMAINS,
3952                 .ops = &hsw_power_well_ops,
3953                 .id = DISP_PW_ID_NONE,
3954                 {
3955                         .hsw.regs = &icl_ddi_power_well_regs,
3956                         .hsw.idx = ICL_PW_CTL_IDX_DDI_E,
3957                 },
3958         },
3959         {
3960                 .name = "DDI F IO",
3961                 .domains = ICL_DDI_IO_F_POWER_DOMAINS,
3962                 .ops = &hsw_power_well_ops,
3963                 .id = DISP_PW_ID_NONE,
3964                 {
3965                         .hsw.regs = &icl_ddi_power_well_regs,
3966                         .hsw.idx = ICL_PW_CTL_IDX_DDI_F,
3967                 },
3968         },
3969         {
3970                 .name = "AUX A",
3971                 .domains = ICL_AUX_A_IO_POWER_DOMAINS,
3972                 .ops = &icl_aux_power_well_ops,
3973                 .id = DISP_PW_ID_NONE,
3974                 {
3975                         .hsw.regs = &icl_aux_power_well_regs,
3976                         .hsw.idx = ICL_PW_CTL_IDX_AUX_A,
3977                 },
3978         },
3979         {
3980                 .name = "AUX B",
3981                 .domains = ICL_AUX_B_IO_POWER_DOMAINS,
3982                 .ops = &icl_aux_power_well_ops,
3983                 .id = DISP_PW_ID_NONE,
3984                 {
3985                         .hsw.regs = &icl_aux_power_well_regs,
3986                         .hsw.idx = ICL_PW_CTL_IDX_AUX_B,
3987                 },
3988         },
3989         {
3990                 .name = "AUX C TC1",
3991                 .domains = ICL_AUX_C_TC1_IO_POWER_DOMAINS,
3992                 .ops = &icl_aux_power_well_ops,
3993                 .id = DISP_PW_ID_NONE,
3994                 {
3995                         .hsw.regs = &icl_aux_power_well_regs,
3996                         .hsw.idx = ICL_PW_CTL_IDX_AUX_C,
3997                         .hsw.is_tc_tbt = false,
3998                 },
3999         },
4000         {
4001                 .name = "AUX D TC2",
4002                 .domains = ICL_AUX_D_TC2_IO_POWER_DOMAINS,
4003                 .ops = &icl_aux_power_well_ops,
4004                 .id = DISP_PW_ID_NONE,
4005                 {
4006                         .hsw.regs = &icl_aux_power_well_regs,
4007                         .hsw.idx = ICL_PW_CTL_IDX_AUX_D,
4008                         .hsw.is_tc_tbt = false,
4009                 },
4010         },
4011         {
4012                 .name = "AUX E TC3",
4013                 .domains = ICL_AUX_E_TC3_IO_POWER_DOMAINS,
4014                 .ops = &icl_aux_power_well_ops,
4015                 .id = DISP_PW_ID_NONE,
4016                 {
4017                         .hsw.regs = &icl_aux_power_well_regs,
4018                         .hsw.idx = ICL_PW_CTL_IDX_AUX_E,
4019                         .hsw.is_tc_tbt = false,
4020                 },
4021         },
4022         {
4023                 .name = "AUX F TC4",
4024                 .domains = ICL_AUX_F_TC4_IO_POWER_DOMAINS,
4025                 .ops = &icl_aux_power_well_ops,
4026                 .id = DISP_PW_ID_NONE,
4027                 {
4028                         .hsw.regs = &icl_aux_power_well_regs,
4029                         .hsw.idx = ICL_PW_CTL_IDX_AUX_F,
4030                         .hsw.is_tc_tbt = false,
4031                 },
4032         },
4033         {
4034                 .name = "AUX C TBT1",
4035                 .domains = ICL_AUX_C_TBT1_IO_POWER_DOMAINS,
4036                 .ops = &icl_aux_power_well_ops,
4037                 .id = DISP_PW_ID_NONE,
4038                 {
4039                         .hsw.regs = &icl_aux_power_well_regs,
4040                         .hsw.idx = ICL_PW_CTL_IDX_AUX_TBT1,
4041                         .hsw.is_tc_tbt = true,
4042                 },
4043         },
4044         {
4045                 .name = "AUX D TBT2",
4046                 .domains = ICL_AUX_D_TBT2_IO_POWER_DOMAINS,
4047                 .ops = &icl_aux_power_well_ops,
4048                 .id = DISP_PW_ID_NONE,
4049                 {
4050                         .hsw.regs = &icl_aux_power_well_regs,
4051                         .hsw.idx = ICL_PW_CTL_IDX_AUX_TBT2,
4052                         .hsw.is_tc_tbt = true,
4053                 },
4054         },
4055         {
4056                 .name = "AUX E TBT3",
4057                 .domains = ICL_AUX_E_TBT3_IO_POWER_DOMAINS,
4058                 .ops = &icl_aux_power_well_ops,
4059                 .id = DISP_PW_ID_NONE,
4060                 {
4061                         .hsw.regs = &icl_aux_power_well_regs,
4062                         .hsw.idx = ICL_PW_CTL_IDX_AUX_TBT3,
4063                         .hsw.is_tc_tbt = true,
4064                 },
4065         },
4066         {
4067                 .name = "AUX F TBT4",
4068                 .domains = ICL_AUX_F_TBT4_IO_POWER_DOMAINS,
4069                 .ops = &icl_aux_power_well_ops,
4070                 .id = DISP_PW_ID_NONE,
4071                 {
4072                         .hsw.regs = &icl_aux_power_well_regs,
4073                         .hsw.idx = ICL_PW_CTL_IDX_AUX_TBT4,
4074                         .hsw.is_tc_tbt = true,
4075                 },
4076         },
4077         {
4078                 .name = "power well 4",
4079                 .domains = ICL_PW_4_POWER_DOMAINS,
4080                 .ops = &hsw_power_well_ops,
4081                 .id = DISP_PW_ID_NONE,
4082                 {
4083                         .hsw.regs = &hsw_power_well_regs,
4084                         .hsw.idx = ICL_PW_CTL_IDX_PW_4,
4085                         .hsw.has_fuses = true,
4086                         .hsw.irq_pipe_mask = BIT(PIPE_C),
4087                 },
4088         },
4089 };
4090
4091 static void
4092 tgl_tc_cold_request(struct drm_i915_private *i915, bool block)
4093 {
4094         u8 tries = 0;
4095         int ret;
4096
4097         while (1) {
4098                 u32 low_val;
4099                 u32 high_val = 0;
4100
4101                 if (block)
4102                         low_val = TGL_PCODE_EXIT_TCCOLD_DATA_L_BLOCK_REQ;
4103                 else
4104                         low_val = TGL_PCODE_EXIT_TCCOLD_DATA_L_UNBLOCK_REQ;
4105
4106                 /*
4107                  * Spec states that we should timeout the request after 200us
4108                  * but the function below will timeout after 500us
4109                  */
4110                 ret = sandybridge_pcode_read(i915, TGL_PCODE_TCCOLD, &low_val,
4111                                              &high_val);
4112                 if (ret == 0) {
4113                         if (block &&
4114                             (low_val & TGL_PCODE_EXIT_TCCOLD_DATA_L_EXIT_FAILED))
4115                                 ret = -EIO;
4116                         else
4117                                 break;
4118                 }
4119
4120                 if (++tries == 3)
4121                         break;
4122
4123                 msleep(1);
4124         }
4125
4126         if (ret)
4127                 drm_err(&i915->drm, "TC cold %sblock failed\n",
4128                         block ? "" : "un");
4129         else
4130                 drm_dbg_kms(&i915->drm, "TC cold %sblock succeeded\n",
4131                             block ? "" : "un");
4132 }
4133
4134 static void
4135 tgl_tc_cold_off_power_well_enable(struct drm_i915_private *i915,
4136                                   struct i915_power_well *power_well)
4137 {
4138         tgl_tc_cold_request(i915, true);
4139 }
4140
4141 static void
4142 tgl_tc_cold_off_power_well_disable(struct drm_i915_private *i915,
4143                                    struct i915_power_well *power_well)
4144 {
4145         tgl_tc_cold_request(i915, false);
4146 }
4147
4148 static void
4149 tgl_tc_cold_off_power_well_sync_hw(struct drm_i915_private *i915,
4150                                    struct i915_power_well *power_well)
4151 {
4152         if (power_well->count > 0)
4153                 tgl_tc_cold_off_power_well_enable(i915, power_well);
4154         else
4155                 tgl_tc_cold_off_power_well_disable(i915, power_well);
4156 }
4157
4158 static bool
4159 tgl_tc_cold_off_power_well_is_enabled(struct drm_i915_private *dev_priv,
4160                                       struct i915_power_well *power_well)
4161 {
4162         /*
4163          * Not the correctly implementation but there is no way to just read it
4164          * from PCODE, so returning count to avoid state mismatch errors
4165          */
4166         return power_well->count;
4167 }
4168
4169 static const struct i915_power_well_ops tgl_tc_cold_off_ops = {
4170         .sync_hw = tgl_tc_cold_off_power_well_sync_hw,
4171         .enable = tgl_tc_cold_off_power_well_enable,
4172         .disable = tgl_tc_cold_off_power_well_disable,
4173         .is_enabled = tgl_tc_cold_off_power_well_is_enabled,
4174 };
4175
4176 static const struct i915_power_well_desc tgl_power_wells[] = {
4177         {
4178                 .name = "always-on",
4179                 .always_on = true,
4180                 .domains = POWER_DOMAIN_MASK,
4181                 .ops = &i9xx_always_on_power_well_ops,
4182                 .id = DISP_PW_ID_NONE,
4183         },
4184         {
4185                 .name = "power well 1",
4186                 /* Handled by the DMC firmware */
4187                 .always_on = true,
4188                 .domains = 0,
4189                 .ops = &hsw_power_well_ops,
4190                 .id = SKL_DISP_PW_1,
4191                 {
4192                         .hsw.regs = &hsw_power_well_regs,
4193                         .hsw.idx = ICL_PW_CTL_IDX_PW_1,
4194                         .hsw.has_fuses = true,
4195                 },
4196         },
4197         {
4198                 .name = "DC off",
4199                 .domains = TGL_DISPLAY_DC_OFF_POWER_DOMAINS,
4200                 .ops = &gen9_dc_off_power_well_ops,
4201                 .id = SKL_DISP_DC_OFF,
4202         },
4203         {
4204                 .name = "power well 2",
4205                 .domains = TGL_PW_2_POWER_DOMAINS,
4206                 .ops = &hsw_power_well_ops,
4207                 .id = SKL_DISP_PW_2,
4208                 {
4209                         .hsw.regs = &hsw_power_well_regs,
4210                         .hsw.idx = ICL_PW_CTL_IDX_PW_2,
4211                         .hsw.has_fuses = true,
4212                 },
4213         },
4214         {
4215                 .name = "power well 3",
4216                 .domains = TGL_PW_3_POWER_DOMAINS,
4217                 .ops = &hsw_power_well_ops,
4218                 .id = ICL_DISP_PW_3,
4219                 {
4220                         .hsw.regs = &hsw_power_well_regs,
4221                         .hsw.idx = ICL_PW_CTL_IDX_PW_3,
4222                         .hsw.irq_pipe_mask = BIT(PIPE_B),
4223                         .hsw.has_vga = true,
4224                         .hsw.has_fuses = true,
4225                 },
4226         },
4227         {
4228                 .name = "DDI A IO",
4229                 .domains = ICL_DDI_IO_A_POWER_DOMAINS,
4230                 .ops = &hsw_power_well_ops,
4231                 .id = DISP_PW_ID_NONE,
4232                 {
4233                         .hsw.regs = &icl_ddi_power_well_regs,
4234                         .hsw.idx = ICL_PW_CTL_IDX_DDI_A,
4235                 }
4236         },
4237         {
4238                 .name = "DDI B IO",
4239                 .domains = ICL_DDI_IO_B_POWER_DOMAINS,
4240                 .ops = &hsw_power_well_ops,
4241                 .id = DISP_PW_ID_NONE,
4242                 {
4243                         .hsw.regs = &icl_ddi_power_well_regs,
4244                         .hsw.idx = ICL_PW_CTL_IDX_DDI_B,
4245                 }
4246         },
4247         {
4248                 .name = "DDI C IO",
4249                 .domains = ICL_DDI_IO_C_POWER_DOMAINS,
4250                 .ops = &hsw_power_well_ops,
4251                 .id = DISP_PW_ID_NONE,
4252                 {
4253                         .hsw.regs = &icl_ddi_power_well_regs,
4254                         .hsw.idx = ICL_PW_CTL_IDX_DDI_C,
4255                 }
4256         },
4257         {
4258                 .name = "DDI IO TC1",
4259                 .domains = TGL_DDI_IO_TC1_POWER_DOMAINS,
4260                 .ops = &hsw_power_well_ops,
4261                 .id = DISP_PW_ID_NONE,
4262                 {
4263                         .hsw.regs = &icl_ddi_power_well_regs,
4264                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC1,
4265                 },
4266         },
4267         {
4268                 .name = "DDI IO TC2",
4269                 .domains = TGL_DDI_IO_TC2_POWER_DOMAINS,
4270                 .ops = &hsw_power_well_ops,
4271                 .id = DISP_PW_ID_NONE,
4272                 {
4273                         .hsw.regs = &icl_ddi_power_well_regs,
4274                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC2,
4275                 },
4276         },
4277         {
4278                 .name = "DDI IO TC3",
4279                 .domains = TGL_DDI_IO_TC3_POWER_DOMAINS,
4280                 .ops = &hsw_power_well_ops,
4281                 .id = DISP_PW_ID_NONE,
4282                 {
4283                         .hsw.regs = &icl_ddi_power_well_regs,
4284                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC3,
4285                 },
4286         },
4287         {
4288                 .name = "DDI IO TC4",
4289                 .domains = TGL_DDI_IO_TC4_POWER_DOMAINS,
4290                 .ops = &hsw_power_well_ops,
4291                 .id = DISP_PW_ID_NONE,
4292                 {
4293                         .hsw.regs = &icl_ddi_power_well_regs,
4294                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC4,
4295                 },
4296         },
4297         {
4298                 .name = "DDI IO TC5",
4299                 .domains = TGL_DDI_IO_TC5_POWER_DOMAINS,
4300                 .ops = &hsw_power_well_ops,
4301                 .id = DISP_PW_ID_NONE,
4302                 {
4303                         .hsw.regs = &icl_ddi_power_well_regs,
4304                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC5,
4305                 },
4306         },
4307         {
4308                 .name = "DDI IO TC6",
4309                 .domains = TGL_DDI_IO_TC6_POWER_DOMAINS,
4310                 .ops = &hsw_power_well_ops,
4311                 .id = DISP_PW_ID_NONE,
4312                 {
4313                         .hsw.regs = &icl_ddi_power_well_regs,
4314                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC6,
4315                 },
4316         },
4317         {
4318                 .name = "TC cold off",
4319                 .domains = TGL_TC_COLD_OFF_POWER_DOMAINS,
4320                 .ops = &tgl_tc_cold_off_ops,
4321                 .id = TGL_DISP_PW_TC_COLD_OFF,
4322         },
4323         {
4324                 .name = "AUX A",
4325                 .domains = TGL_AUX_A_IO_POWER_DOMAINS,
4326                 .ops = &icl_aux_power_well_ops,
4327                 .id = DISP_PW_ID_NONE,
4328                 {
4329                         .hsw.regs = &icl_aux_power_well_regs,
4330                         .hsw.idx = ICL_PW_CTL_IDX_AUX_A,
4331                 },
4332         },
4333         {
4334                 .name = "AUX B",
4335                 .domains = TGL_AUX_B_IO_POWER_DOMAINS,
4336                 .ops = &icl_aux_power_well_ops,
4337                 .id = DISP_PW_ID_NONE,
4338                 {
4339                         .hsw.regs = &icl_aux_power_well_regs,
4340                         .hsw.idx = ICL_PW_CTL_IDX_AUX_B,
4341                 },
4342         },
4343         {
4344                 .name = "AUX C",
4345                 .domains = TGL_AUX_C_IO_POWER_DOMAINS,
4346                 .ops = &icl_aux_power_well_ops,
4347                 .id = DISP_PW_ID_NONE,
4348                 {
4349                         .hsw.regs = &icl_aux_power_well_regs,
4350                         .hsw.idx = ICL_PW_CTL_IDX_AUX_C,
4351                 },
4352         },
4353         {
4354                 .name = "AUX USBC1",
4355                 .domains = TGL_AUX_IO_USBC1_POWER_DOMAINS,
4356                 .ops = &icl_aux_power_well_ops,
4357                 .id = DISP_PW_ID_NONE,
4358                 {
4359                         .hsw.regs = &icl_aux_power_well_regs,
4360                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC1,
4361                         .hsw.is_tc_tbt = false,
4362                 },
4363         },
4364         {
4365                 .name = "AUX USBC2",
4366                 .domains = TGL_AUX_IO_USBC2_POWER_DOMAINS,
4367                 .ops = &icl_aux_power_well_ops,
4368                 .id = DISP_PW_ID_NONE,
4369                 {
4370                         .hsw.regs = &icl_aux_power_well_regs,
4371                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC2,
4372                         .hsw.is_tc_tbt = false,
4373                 },
4374         },
4375         {
4376                 .name = "AUX USBC3",
4377                 .domains = TGL_AUX_IO_USBC3_POWER_DOMAINS,
4378                 .ops = &icl_aux_power_well_ops,
4379                 .id = DISP_PW_ID_NONE,
4380                 {
4381                         .hsw.regs = &icl_aux_power_well_regs,
4382                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC3,
4383                         .hsw.is_tc_tbt = false,
4384                 },
4385         },
4386         {
4387                 .name = "AUX USBC4",
4388                 .domains = TGL_AUX_IO_USBC4_POWER_DOMAINS,
4389                 .ops = &icl_aux_power_well_ops,
4390                 .id = DISP_PW_ID_NONE,
4391                 {
4392                         .hsw.regs = &icl_aux_power_well_regs,
4393                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC4,
4394                         .hsw.is_tc_tbt = false,
4395                 },
4396         },
4397         {
4398                 .name = "AUX USBC5",
4399                 .domains = TGL_AUX_IO_USBC5_POWER_DOMAINS,
4400                 .ops = &icl_aux_power_well_ops,
4401                 .id = DISP_PW_ID_NONE,
4402                 {
4403                         .hsw.regs = &icl_aux_power_well_regs,
4404                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC5,
4405                         .hsw.is_tc_tbt = false,
4406                 },
4407         },
4408         {
4409                 .name = "AUX USBC6",
4410                 .domains = TGL_AUX_IO_USBC6_POWER_DOMAINS,
4411                 .ops = &icl_aux_power_well_ops,
4412                 .id = DISP_PW_ID_NONE,
4413                 {
4414                         .hsw.regs = &icl_aux_power_well_regs,
4415                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC6,
4416                         .hsw.is_tc_tbt = false,
4417                 },
4418         },
4419         {
4420                 .name = "AUX TBT1",
4421                 .domains = TGL_AUX_IO_TBT1_POWER_DOMAINS,
4422                 .ops = &icl_aux_power_well_ops,
4423                 .id = DISP_PW_ID_NONE,
4424                 {
4425                         .hsw.regs = &icl_aux_power_well_regs,
4426                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT1,
4427                         .hsw.is_tc_tbt = true,
4428                 },
4429         },
4430         {
4431                 .name = "AUX TBT2",
4432                 .domains = TGL_AUX_IO_TBT2_POWER_DOMAINS,
4433                 .ops = &icl_aux_power_well_ops,
4434                 .id = DISP_PW_ID_NONE,
4435                 {
4436                         .hsw.regs = &icl_aux_power_well_regs,
4437                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT2,
4438                         .hsw.is_tc_tbt = true,
4439                 },
4440         },
4441         {
4442                 .name = "AUX TBT3",
4443                 .domains = TGL_AUX_IO_TBT3_POWER_DOMAINS,
4444                 .ops = &icl_aux_power_well_ops,
4445                 .id = DISP_PW_ID_NONE,
4446                 {
4447                         .hsw.regs = &icl_aux_power_well_regs,
4448                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT3,
4449                         .hsw.is_tc_tbt = true,
4450                 },
4451         },
4452         {
4453                 .name = "AUX TBT4",
4454                 .domains = TGL_AUX_IO_TBT4_POWER_DOMAINS,
4455                 .ops = &icl_aux_power_well_ops,
4456                 .id = DISP_PW_ID_NONE,
4457                 {
4458                         .hsw.regs = &icl_aux_power_well_regs,
4459                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT4,
4460                         .hsw.is_tc_tbt = true,
4461                 },
4462         },
4463         {
4464                 .name = "AUX TBT5",
4465                 .domains = TGL_AUX_IO_TBT5_POWER_DOMAINS,
4466                 .ops = &icl_aux_power_well_ops,
4467                 .id = DISP_PW_ID_NONE,
4468                 {
4469                         .hsw.regs = &icl_aux_power_well_regs,
4470                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT5,
4471                         .hsw.is_tc_tbt = true,
4472                 },
4473         },
4474         {
4475                 .name = "AUX TBT6",
4476                 .domains = TGL_AUX_IO_TBT6_POWER_DOMAINS,
4477                 .ops = &icl_aux_power_well_ops,
4478                 .id = DISP_PW_ID_NONE,
4479                 {
4480                         .hsw.regs = &icl_aux_power_well_regs,
4481                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT6,
4482                         .hsw.is_tc_tbt = true,
4483                 },
4484         },
4485         {
4486                 .name = "power well 4",
4487                 .domains = TGL_PW_4_POWER_DOMAINS,
4488                 .ops = &hsw_power_well_ops,
4489                 .id = DISP_PW_ID_NONE,
4490                 {
4491                         .hsw.regs = &hsw_power_well_regs,
4492                         .hsw.idx = ICL_PW_CTL_IDX_PW_4,
4493                         .hsw.has_fuses = true,
4494                         .hsw.irq_pipe_mask = BIT(PIPE_C),
4495                 }
4496         },
4497         {
4498                 .name = "power well 5",
4499                 .domains = TGL_PW_5_POWER_DOMAINS,
4500                 .ops = &hsw_power_well_ops,
4501                 .id = DISP_PW_ID_NONE,
4502                 {
4503                         .hsw.regs = &hsw_power_well_regs,
4504                         .hsw.idx = TGL_PW_CTL_IDX_PW_5,
4505                         .hsw.has_fuses = true,
4506                         .hsw.irq_pipe_mask = BIT(PIPE_D),
4507                 },
4508         },
4509 };
4510
4511 static const struct i915_power_well_desc rkl_power_wells[] = {
4512         {
4513                 .name = "always-on",
4514                 .always_on = true,
4515                 .domains = POWER_DOMAIN_MASK,
4516                 .ops = &i9xx_always_on_power_well_ops,
4517                 .id = DISP_PW_ID_NONE,
4518         },
4519         {
4520                 .name = "power well 1",
4521                 /* Handled by the DMC firmware */
4522                 .always_on = true,
4523                 .domains = 0,
4524                 .ops = &hsw_power_well_ops,
4525                 .id = SKL_DISP_PW_1,
4526                 {
4527                         .hsw.regs = &hsw_power_well_regs,
4528                         .hsw.idx = ICL_PW_CTL_IDX_PW_1,
4529                         .hsw.has_fuses = true,
4530                 },
4531         },
4532         {
4533                 .name = "DC off",
4534                 .domains = RKL_DISPLAY_DC_OFF_POWER_DOMAINS,
4535                 .ops = &gen9_dc_off_power_well_ops,
4536                 .id = SKL_DISP_DC_OFF,
4537         },
4538         {
4539                 .name = "power well 3",
4540                 .domains = RKL_PW_3_POWER_DOMAINS,
4541                 .ops = &hsw_power_well_ops,
4542                 .id = ICL_DISP_PW_3,
4543                 {
4544                         .hsw.regs = &hsw_power_well_regs,
4545                         .hsw.idx = ICL_PW_CTL_IDX_PW_3,
4546                         .hsw.irq_pipe_mask = BIT(PIPE_B),
4547                         .hsw.has_vga = true,
4548                         .hsw.has_fuses = true,
4549                 },
4550         },
4551         {
4552                 .name = "power well 4",
4553                 .domains = RKL_PW_4_POWER_DOMAINS,
4554                 .ops = &hsw_power_well_ops,
4555                 .id = DISP_PW_ID_NONE,
4556                 {
4557                         .hsw.regs = &hsw_power_well_regs,
4558                         .hsw.idx = ICL_PW_CTL_IDX_PW_4,
4559                         .hsw.has_fuses = true,
4560                         .hsw.irq_pipe_mask = BIT(PIPE_C),
4561                 }
4562         },
4563         {
4564                 .name = "DDI A IO",
4565                 .domains = ICL_DDI_IO_A_POWER_DOMAINS,
4566                 .ops = &hsw_power_well_ops,
4567                 .id = DISP_PW_ID_NONE,
4568                 {
4569                         .hsw.regs = &icl_ddi_power_well_regs,
4570                         .hsw.idx = ICL_PW_CTL_IDX_DDI_A,
4571                 }
4572         },
4573         {
4574                 .name = "DDI B IO",
4575                 .domains = ICL_DDI_IO_B_POWER_DOMAINS,
4576                 .ops = &hsw_power_well_ops,
4577                 .id = DISP_PW_ID_NONE,
4578                 {
4579                         .hsw.regs = &icl_ddi_power_well_regs,
4580                         .hsw.idx = ICL_PW_CTL_IDX_DDI_B,
4581                 }
4582         },
4583         {
4584                 .name = "DDI IO TC1",
4585                 .domains = TGL_DDI_IO_TC1_POWER_DOMAINS,
4586                 .ops = &hsw_power_well_ops,
4587                 .id = DISP_PW_ID_NONE,
4588                 {
4589                         .hsw.regs = &icl_ddi_power_well_regs,
4590                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC1,
4591                 },
4592         },
4593         {
4594                 .name = "DDI IO TC2",
4595                 .domains = TGL_DDI_IO_TC2_POWER_DOMAINS,
4596                 .ops = &hsw_power_well_ops,
4597                 .id = DISP_PW_ID_NONE,
4598                 {
4599                         .hsw.regs = &icl_ddi_power_well_regs,
4600                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC2,
4601                 },
4602         },
4603         {
4604                 .name = "AUX A",
4605                 .domains = ICL_AUX_A_IO_POWER_DOMAINS,
4606                 .ops = &icl_aux_power_well_ops,
4607                 .id = DISP_PW_ID_NONE,
4608                 {
4609                         .hsw.regs = &icl_aux_power_well_regs,
4610                         .hsw.idx = ICL_PW_CTL_IDX_AUX_A,
4611                 },
4612         },
4613         {
4614                 .name = "AUX B",
4615                 .domains = ICL_AUX_B_IO_POWER_DOMAINS,
4616                 .ops = &icl_aux_power_well_ops,
4617                 .id = DISP_PW_ID_NONE,
4618                 {
4619                         .hsw.regs = &icl_aux_power_well_regs,
4620                         .hsw.idx = ICL_PW_CTL_IDX_AUX_B,
4621                 },
4622         },
4623         {
4624                 .name = "AUX USBC1",
4625                 .domains = TGL_AUX_IO_USBC1_POWER_DOMAINS,
4626                 .ops = &icl_aux_power_well_ops,
4627                 .id = DISP_PW_ID_NONE,
4628                 {
4629                         .hsw.regs = &icl_aux_power_well_regs,
4630                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC1,
4631                 },
4632         },
4633         {
4634                 .name = "AUX USBC2",
4635                 .domains = TGL_AUX_IO_USBC2_POWER_DOMAINS,
4636                 .ops = &icl_aux_power_well_ops,
4637                 .id = DISP_PW_ID_NONE,
4638                 {
4639                         .hsw.regs = &icl_aux_power_well_regs,
4640                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC2,
4641                 },
4642         },
4643 };
4644
4645 static const struct i915_power_well_desc xelpd_power_wells[] = {
4646         {
4647                 .name = "always-on",
4648                 .always_on = true,
4649                 .domains = POWER_DOMAIN_MASK,
4650                 .ops = &i9xx_always_on_power_well_ops,
4651                 .id = DISP_PW_ID_NONE,
4652         },
4653         {
4654                 .name = "power well 1",
4655                 /* Handled by the DMC firmware */
4656                 .always_on = true,
4657                 .domains = 0,
4658                 .ops = &hsw_power_well_ops,
4659                 .id = SKL_DISP_PW_1,
4660                 {
4661                         .hsw.regs = &hsw_power_well_regs,
4662                         .hsw.idx = ICL_PW_CTL_IDX_PW_1,
4663                         .hsw.has_fuses = true,
4664                 },
4665         },
4666         {
4667                 .name = "DC off",
4668                 .domains = XELPD_DISPLAY_DC_OFF_POWER_DOMAINS,
4669                 .ops = &gen9_dc_off_power_well_ops,
4670                 .id = SKL_DISP_DC_OFF,
4671         },
4672         {
4673                 .name = "power well 2",
4674                 .domains = XELPD_PW_2_POWER_DOMAINS,
4675                 .ops = &hsw_power_well_ops,
4676                 .id = SKL_DISP_PW_2,
4677                 {
4678                         .hsw.regs = &hsw_power_well_regs,
4679                         .hsw.idx = ICL_PW_CTL_IDX_PW_2,
4680                         .hsw.has_vga = true,
4681                         .hsw.has_fuses = true,
4682                 },
4683         },
4684         {
4685                 .name = "power well A",
4686                 .domains = XELPD_PW_A_POWER_DOMAINS,
4687                 .ops = &hsw_power_well_ops,
4688                 .id = DISP_PW_ID_NONE,
4689                 {
4690                         .hsw.regs = &hsw_power_well_regs,
4691                         .hsw.idx = XELPD_PW_CTL_IDX_PW_A,
4692                         .hsw.irq_pipe_mask = BIT(PIPE_A),
4693                         .hsw.has_fuses = true,
4694                 },
4695         },
4696         {
4697                 .name = "power well B",
4698                 .domains = XELPD_PW_B_POWER_DOMAINS,
4699                 .ops = &hsw_power_well_ops,
4700                 .id = DISP_PW_ID_NONE,
4701                 {
4702                         .hsw.regs = &hsw_power_well_regs,
4703                         .hsw.idx = XELPD_PW_CTL_IDX_PW_B,
4704                         .hsw.irq_pipe_mask = BIT(PIPE_B),
4705                         .hsw.has_fuses = true,
4706                 },
4707         },
4708         {
4709                 .name = "power well C",
4710                 .domains = XELPD_PW_C_POWER_DOMAINS,
4711                 .ops = &hsw_power_well_ops,
4712                 .id = DISP_PW_ID_NONE,
4713                 {
4714                         .hsw.regs = &hsw_power_well_regs,
4715                         .hsw.idx = XELPD_PW_CTL_IDX_PW_C,
4716                         .hsw.irq_pipe_mask = BIT(PIPE_C),
4717                         .hsw.has_fuses = true,
4718                 },
4719         },
4720         {
4721                 .name = "power well D",
4722                 .domains = XELPD_PW_D_POWER_DOMAINS,
4723                 .ops = &hsw_power_well_ops,
4724                 .id = DISP_PW_ID_NONE,
4725                 {
4726                         .hsw.regs = &hsw_power_well_regs,
4727                         .hsw.idx = XELPD_PW_CTL_IDX_PW_D,
4728                         .hsw.irq_pipe_mask = BIT(PIPE_D),
4729                         .hsw.has_fuses = true,
4730                 },
4731         },
4732         {
4733                 .name = "DDI A IO",
4734                 .domains = ICL_DDI_IO_A_POWER_DOMAINS,
4735                 .ops = &hsw_power_well_ops,
4736                 .id = DISP_PW_ID_NONE,
4737                 {
4738                         .hsw.regs = &icl_ddi_power_well_regs,
4739                         .hsw.idx = ICL_PW_CTL_IDX_DDI_A,
4740                 }
4741         },
4742         {
4743                 .name = "DDI B IO",
4744                 .domains = ICL_DDI_IO_B_POWER_DOMAINS,
4745                 .ops = &hsw_power_well_ops,
4746                 .id = DISP_PW_ID_NONE,
4747                 {
4748                         .hsw.regs = &icl_ddi_power_well_regs,
4749                         .hsw.idx = ICL_PW_CTL_IDX_DDI_B,
4750                 }
4751         },
4752         {
4753                 .name = "DDI C IO",
4754                 .domains = ICL_DDI_IO_C_POWER_DOMAINS,
4755                 .ops = &hsw_power_well_ops,
4756                 .id = DISP_PW_ID_NONE,
4757                 {
4758                         .hsw.regs = &icl_ddi_power_well_regs,
4759                         .hsw.idx = ICL_PW_CTL_IDX_DDI_C,
4760                 }
4761         },
4762         {
4763                 .name = "DDI IO D_XELPD",
4764                 .domains = XELPD_DDI_IO_D_XELPD_POWER_DOMAINS,
4765                 .ops = &hsw_power_well_ops,
4766                 .id = DISP_PW_ID_NONE,
4767                 {
4768                         .hsw.regs = &icl_ddi_power_well_regs,
4769                         .hsw.idx = XELPD_PW_CTL_IDX_DDI_D,
4770                 }
4771         },
4772         {
4773                 .name = "DDI IO E_XELPD",
4774                 .domains = XELPD_DDI_IO_E_XELPD_POWER_DOMAINS,
4775                 .ops = &hsw_power_well_ops,
4776                 .id = DISP_PW_ID_NONE,
4777                 {
4778                         .hsw.regs = &icl_ddi_power_well_regs,
4779                         .hsw.idx = XELPD_PW_CTL_IDX_DDI_E,
4780                 }
4781         },
4782         {
4783                 .name = "DDI IO TC1",
4784                 .domains = XELPD_DDI_IO_TC1_POWER_DOMAINS,
4785                 .ops = &hsw_power_well_ops,
4786                 .id = DISP_PW_ID_NONE,
4787                 {
4788                         .hsw.regs = &icl_ddi_power_well_regs,
4789                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC1,
4790                 }
4791         },
4792         {
4793                 .name = "DDI IO TC2",
4794                 .domains = XELPD_DDI_IO_TC2_POWER_DOMAINS,
4795                 .ops = &hsw_power_well_ops,
4796                 .id = DISP_PW_ID_NONE,
4797                 {
4798                         .hsw.regs = &icl_ddi_power_well_regs,
4799                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC2,
4800                 }
4801         },
4802         {
4803                 .name = "DDI IO TC3",
4804                 .domains = XELPD_DDI_IO_TC3_POWER_DOMAINS,
4805                 .ops = &hsw_power_well_ops,
4806                 .id = DISP_PW_ID_NONE,
4807                 {
4808                         .hsw.regs = &icl_ddi_power_well_regs,
4809                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC3,
4810                 }
4811         },
4812         {
4813                 .name = "DDI IO TC4",
4814                 .domains = XELPD_DDI_IO_TC4_POWER_DOMAINS,
4815                 .ops = &hsw_power_well_ops,
4816                 .id = DISP_PW_ID_NONE,
4817                 {
4818                         .hsw.regs = &icl_ddi_power_well_regs,
4819                         .hsw.idx = TGL_PW_CTL_IDX_DDI_TC4,
4820                 }
4821         },
4822         {
4823                 .name = "AUX A",
4824                 .domains = ICL_AUX_A_IO_POWER_DOMAINS,
4825                 .ops = &icl_aux_power_well_ops,
4826                 .id = DISP_PW_ID_NONE,
4827                 {
4828                         .hsw.regs = &icl_aux_power_well_regs,
4829                         .hsw.idx = ICL_PW_CTL_IDX_AUX_A,
4830                 },
4831         },
4832         {
4833                 .name = "AUX B",
4834                 .domains = ICL_AUX_B_IO_POWER_DOMAINS,
4835                 .ops = &icl_aux_power_well_ops,
4836                 .id = DISP_PW_ID_NONE,
4837                 {
4838                         .hsw.regs = &icl_aux_power_well_regs,
4839                         .hsw.idx = ICL_PW_CTL_IDX_AUX_B,
4840                 },
4841         },
4842         {
4843                 .name = "AUX C",
4844                 .domains = TGL_AUX_C_IO_POWER_DOMAINS,
4845                 .ops = &icl_aux_power_well_ops,
4846                 .id = DISP_PW_ID_NONE,
4847                 {
4848                         .hsw.regs = &icl_aux_power_well_regs,
4849                         .hsw.idx = ICL_PW_CTL_IDX_AUX_C,
4850                 },
4851         },
4852         {
4853                 .name = "AUX D_XELPD",
4854                 .domains = XELPD_AUX_IO_D_XELPD_POWER_DOMAINS,
4855                 .ops = &icl_aux_power_well_ops,
4856                 .id = DISP_PW_ID_NONE,
4857                 {
4858                         .hsw.regs = &icl_aux_power_well_regs,
4859                         .hsw.idx = XELPD_PW_CTL_IDX_AUX_D,
4860                 },
4861         },
4862         {
4863                 .name = "AUX E_XELPD",
4864                 .domains = XELPD_AUX_IO_E_XELPD_POWER_DOMAINS,
4865                 .ops = &icl_aux_power_well_ops,
4866                 .id = DISP_PW_ID_NONE,
4867                 {
4868                         .hsw.regs = &icl_aux_power_well_regs,
4869                         .hsw.idx = XELPD_PW_CTL_IDX_AUX_E,
4870                 },
4871         },
4872         {
4873                 .name = "AUX USBC1",
4874                 .domains = XELPD_AUX_IO_USBC1_POWER_DOMAINS,
4875                 .ops = &icl_aux_power_well_ops,
4876                 .id = DISP_PW_ID_NONE,
4877                 {
4878                         .hsw.regs = &icl_aux_power_well_regs,
4879                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC1,
4880                 },
4881         },
4882         {
4883                 .name = "AUX USBC2",
4884                 .domains = XELPD_AUX_IO_USBC2_POWER_DOMAINS,
4885                 .ops = &icl_aux_power_well_ops,
4886                 .id = DISP_PW_ID_NONE,
4887                 {
4888                         .hsw.regs = &icl_aux_power_well_regs,
4889                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC2,
4890                 },
4891         },
4892         {
4893                 .name = "AUX USBC3",
4894                 .domains = XELPD_AUX_IO_USBC3_POWER_DOMAINS,
4895                 .ops = &icl_aux_power_well_ops,
4896                 .id = DISP_PW_ID_NONE,
4897                 {
4898                         .hsw.regs = &icl_aux_power_well_regs,
4899                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC3,
4900                 },
4901         },
4902         {
4903                 .name = "AUX USBC4",
4904                 .domains = XELPD_AUX_IO_USBC4_POWER_DOMAINS,
4905                 .ops = &icl_aux_power_well_ops,
4906                 .id = DISP_PW_ID_NONE,
4907                 {
4908                         .hsw.regs = &icl_aux_power_well_regs,
4909                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TC4,
4910                 },
4911         },
4912         {
4913                 .name = "AUX TBT1",
4914                 .domains = XELPD_AUX_IO_TBT1_POWER_DOMAINS,
4915                 .ops = &icl_aux_power_well_ops,
4916                 .id = DISP_PW_ID_NONE,
4917                 {
4918                         .hsw.regs = &icl_aux_power_well_regs,
4919                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT1,
4920                         .hsw.is_tc_tbt = true,
4921                 },
4922         },
4923         {
4924                 .name = "AUX TBT2",
4925                 .domains = XELPD_AUX_IO_TBT2_POWER_DOMAINS,
4926                 .ops = &icl_aux_power_well_ops,
4927                 .id = DISP_PW_ID_NONE,
4928                 {
4929                         .hsw.regs = &icl_aux_power_well_regs,
4930                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT2,
4931                         .hsw.is_tc_tbt = true,
4932                 },
4933         },
4934         {
4935                 .name = "AUX TBT3",
4936                 .domains = XELPD_AUX_IO_TBT3_POWER_DOMAINS,
4937                 .ops = &icl_aux_power_well_ops,
4938                 .id = DISP_PW_ID_NONE,
4939                 {
4940                         .hsw.regs = &icl_aux_power_well_regs,
4941                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT3,
4942                         .hsw.is_tc_tbt = true,
4943                 },
4944         },
4945         {
4946                 .name = "AUX TBT4",
4947                 .domains = XELPD_AUX_IO_TBT4_POWER_DOMAINS,
4948                 .ops = &icl_aux_power_well_ops,
4949                 .id = DISP_PW_ID_NONE,
4950                 {
4951                         .hsw.regs = &icl_aux_power_well_regs,
4952                         .hsw.idx = TGL_PW_CTL_IDX_AUX_TBT4,
4953                         .hsw.is_tc_tbt = true,
4954                 },
4955         },
4956 };
4957
4958 static int
4959 sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
4960                                    int disable_power_well)
4961 {
4962         if (disable_power_well >= 0)
4963                 return !!disable_power_well;
4964
4965         return 1;
4966 }
4967
4968 static u32 get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
4969                                int enable_dc)
4970 {
4971         u32 mask;
4972         int requested_dc;
4973         int max_dc;
4974
4975         if (!HAS_DISPLAY(dev_priv))
4976                 return 0;
4977
4978         if (IS_DG1(dev_priv))
4979                 max_dc = 3;
4980         else if (DISPLAY_VER(dev_priv) >= 12)
4981                 max_dc = 4;
4982         else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
4983                 max_dc = 1;
4984         else if (DISPLAY_VER(dev_priv) >= 9)
4985                 max_dc = 2;
4986         else
4987                 max_dc = 0;
4988
4989         /*
4990          * DC9 has a separate HW flow from the rest of the DC states,
4991          * not depending on the DMC firmware. It's needed by system
4992          * suspend/resume, so allow it unconditionally.
4993          */
4994         mask = IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv) ||
4995                 DISPLAY_VER(dev_priv) >= 11 ?
4996                DC_STATE_EN_DC9 : 0;
4997
4998         if (!dev_priv->params.disable_power_well)
4999                 max_dc = 0;
5000
5001         if (enable_dc >= 0 && enable_dc <= max_dc) {
5002                 requested_dc = enable_dc;
5003         } else if (enable_dc == -1) {
5004                 requested_dc = max_dc;
5005         } else if (enable_dc > max_dc && enable_dc <= 4) {
5006                 drm_dbg_kms(&dev_priv->drm,
5007                             "Adjusting requested max DC state (%d->%d)\n",
5008                             enable_dc, max_dc);
5009                 requested_dc = max_dc;
5010         } else {
5011                 drm_err(&dev_priv->drm,
5012                         "Unexpected value for enable_dc (%d)\n", enable_dc);
5013                 requested_dc = max_dc;
5014         }
5015
5016         switch (requested_dc) {
5017         case 4:
5018                 mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC6;
5019                 break;
5020         case 3:
5021                 mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC5;
5022                 break;
5023         case 2:
5024                 mask |= DC_STATE_EN_UPTO_DC6;
5025                 break;
5026         case 1:
5027                 mask |= DC_STATE_EN_UPTO_DC5;
5028                 break;
5029         }
5030
5031         drm_dbg_kms(&dev_priv->drm, "Allowed DC state mask %02x\n", mask);
5032
5033         return mask;
5034 }
5035
5036 static int
5037 __set_power_wells(struct i915_power_domains *power_domains,
5038                   const struct i915_power_well_desc *power_well_descs,
5039                   int power_well_descs_sz, u64 skip_mask)
5040 {
5041         struct drm_i915_private *i915 = container_of(power_domains,
5042                                                      struct drm_i915_private,
5043                                                      power_domains);
5044         u64 power_well_ids = 0;
5045         int power_well_count = 0;
5046         int i, plt_idx = 0;
5047
5048         for (i = 0; i < power_well_descs_sz; i++)
5049                 if (!(BIT_ULL(power_well_descs[i].id) & skip_mask))
5050                         power_well_count++;
5051
5052         power_domains->power_well_count = power_well_count;
5053         power_domains->power_wells =
5054                                 kcalloc(power_well_count,
5055                                         sizeof(*power_domains->power_wells),
5056                                         GFP_KERNEL);
5057         if (!power_domains->power_wells)
5058                 return -ENOMEM;
5059
5060         for (i = 0; i < power_well_descs_sz; i++) {
5061                 enum i915_power_well_id id = power_well_descs[i].id;
5062
5063                 if (BIT_ULL(id) & skip_mask)
5064                         continue;
5065
5066                 power_domains->power_wells[plt_idx++].desc =
5067                         &power_well_descs[i];
5068
5069                 if (id == DISP_PW_ID_NONE)
5070                         continue;
5071
5072                 drm_WARN_ON(&i915->drm, id >= sizeof(power_well_ids) * 8);
5073                 drm_WARN_ON(&i915->drm, power_well_ids & BIT_ULL(id));
5074                 power_well_ids |= BIT_ULL(id);
5075         }
5076
5077         return 0;
5078 }
5079
5080 #define set_power_wells_mask(power_domains, __power_well_descs, skip_mask) \
5081         __set_power_wells(power_domains, __power_well_descs, \
5082                           ARRAY_SIZE(__power_well_descs), skip_mask)
5083
5084 #define set_power_wells(power_domains, __power_well_descs) \
5085         set_power_wells_mask(power_domains, __power_well_descs, 0)
5086
5087 /**
5088  * intel_power_domains_init - initializes the power domain structures
5089  * @dev_priv: i915 device instance
5090  *
5091  * Initializes the power domain structures for @dev_priv depending upon the
5092  * supported platform.
5093  */
5094 int intel_power_domains_init(struct drm_i915_private *dev_priv)
5095 {
5096         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5097         int err;
5098
5099         dev_priv->params.disable_power_well =
5100                 sanitize_disable_power_well_option(dev_priv,
5101                                                    dev_priv->params.disable_power_well);
5102         dev_priv->dmc.allowed_dc_mask =
5103                 get_allowed_dc_mask(dev_priv, dev_priv->params.enable_dc);
5104
5105         dev_priv->dmc.target_dc_state =
5106                 sanitize_target_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
5107
5108         BUILD_BUG_ON(POWER_DOMAIN_NUM > 64);
5109
5110         mutex_init(&power_domains->lock);
5111
5112         INIT_DELAYED_WORK(&power_domains->async_put_work,
5113                           intel_display_power_put_async_work);
5114
5115         /*
5116          * The enabling order will be from lower to higher indexed wells,
5117          * the disabling order is reversed.
5118          */
5119         if (!HAS_DISPLAY(dev_priv)) {
5120                 power_domains->power_well_count = 0;
5121                 err = 0;
5122         } else if (DISPLAY_VER(dev_priv) >= 13) {
5123                 err = set_power_wells(power_domains, xelpd_power_wells);
5124         } else if (IS_ALDERLAKE_S(dev_priv) || IS_DG1(dev_priv)) {
5125                 err = set_power_wells_mask(power_domains, tgl_power_wells,
5126                                            BIT_ULL(TGL_DISP_PW_TC_COLD_OFF));
5127         } else if (IS_ROCKETLAKE(dev_priv)) {
5128                 err = set_power_wells(power_domains, rkl_power_wells);
5129         } else if (DISPLAY_VER(dev_priv) == 12) {
5130                 err = set_power_wells(power_domains, tgl_power_wells);
5131         } else if (DISPLAY_VER(dev_priv) == 11) {
5132                 err = set_power_wells(power_domains, icl_power_wells);
5133         } else if (IS_CNL_WITH_PORT_F(dev_priv)) {
5134                 err = set_power_wells(power_domains, cnl_power_wells);
5135         } else if (IS_CANNONLAKE(dev_priv)) {
5136                 err = set_power_wells_mask(power_domains, cnl_power_wells,
5137                                            BIT_ULL(CNL_DISP_PW_DDI_F_IO) |
5138                                            BIT_ULL(CNL_DISP_PW_DDI_F_AUX));
5139         } else if (IS_GEMINILAKE(dev_priv)) {
5140                 err = set_power_wells(power_domains, glk_power_wells);
5141         } else if (IS_BROXTON(dev_priv)) {
5142                 err = set_power_wells(power_domains, bxt_power_wells);
5143         } else if (DISPLAY_VER(dev_priv) == 9) {
5144                 err = set_power_wells(power_domains, skl_power_wells);
5145         } else if (IS_CHERRYVIEW(dev_priv)) {
5146                 err = set_power_wells(power_domains, chv_power_wells);
5147         } else if (IS_BROADWELL(dev_priv)) {
5148                 err = set_power_wells(power_domains, bdw_power_wells);
5149         } else if (IS_HASWELL(dev_priv)) {
5150                 err = set_power_wells(power_domains, hsw_power_wells);
5151         } else if (IS_VALLEYVIEW(dev_priv)) {
5152                 err = set_power_wells(power_domains, vlv_power_wells);
5153         } else if (IS_I830(dev_priv)) {
5154                 err = set_power_wells(power_domains, i830_power_wells);
5155         } else {
5156                 err = set_power_wells(power_domains, i9xx_always_on_power_well);
5157         }
5158
5159         return err;
5160 }
5161
5162 /**
5163  * intel_power_domains_cleanup - clean up power domains resources
5164  * @dev_priv: i915 device instance
5165  *
5166  * Release any resources acquired by intel_power_domains_init()
5167  */
5168 void intel_power_domains_cleanup(struct drm_i915_private *dev_priv)
5169 {
5170         kfree(dev_priv->power_domains.power_wells);
5171 }
5172
5173 static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
5174 {
5175         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5176         struct i915_power_well *power_well;
5177
5178         mutex_lock(&power_domains->lock);
5179         for_each_power_well(dev_priv, power_well) {
5180                 power_well->desc->ops->sync_hw(dev_priv, power_well);
5181                 power_well->hw_enabled =
5182                         power_well->desc->ops->is_enabled(dev_priv, power_well);
5183         }
5184         mutex_unlock(&power_domains->lock);
5185 }
5186
5187 static void gen9_dbuf_slice_set(struct drm_i915_private *dev_priv,
5188                                 enum dbuf_slice slice, bool enable)
5189 {
5190         i915_reg_t reg = DBUF_CTL_S(slice);
5191         bool state;
5192
5193         intel_de_rmw(dev_priv, reg, DBUF_POWER_REQUEST,
5194                      enable ? DBUF_POWER_REQUEST : 0);
5195         intel_de_posting_read(dev_priv, reg);
5196         udelay(10);
5197
5198         state = intel_de_read(dev_priv, reg) & DBUF_POWER_STATE;
5199         drm_WARN(&dev_priv->drm, enable != state,
5200                  "DBuf slice %d power %s timeout!\n",
5201                  slice, enabledisable(enable));
5202 }
5203
5204 void gen9_dbuf_slices_update(struct drm_i915_private *dev_priv,
5205                              u8 req_slices)
5206 {
5207         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5208         u8 slice_mask = INTEL_INFO(dev_priv)->dbuf.slice_mask;
5209         enum dbuf_slice slice;
5210
5211         drm_WARN(&dev_priv->drm, req_slices & ~slice_mask,
5212                  "Invalid set of dbuf slices (0x%x) requested (total dbuf slices 0x%x)\n",
5213                  req_slices, slice_mask);
5214
5215         drm_dbg_kms(&dev_priv->drm, "Updating dbuf slices to 0x%x\n",
5216                     req_slices);
5217
5218         /*
5219          * Might be running this in parallel to gen9_dc_off_power_well_enable
5220          * being called from intel_dp_detect for instance,
5221          * which causes assertion triggered by race condition,
5222          * as gen9_assert_dbuf_enabled might preempt this when registers
5223          * were already updated, while dev_priv was not.
5224          */
5225         mutex_lock(&power_domains->lock);
5226
5227         for_each_dbuf_slice(dev_priv, slice)
5228                 gen9_dbuf_slice_set(dev_priv, slice, req_slices & BIT(slice));
5229
5230         dev_priv->dbuf.enabled_slices = req_slices;
5231
5232         mutex_unlock(&power_domains->lock);
5233 }
5234
5235 static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
5236 {
5237         dev_priv->dbuf.enabled_slices =
5238                 intel_enabled_dbuf_slices_mask(dev_priv);
5239
5240         /*
5241          * Just power up at least 1 slice, we will
5242          * figure out later which slices we have and what we need.
5243          */
5244         gen9_dbuf_slices_update(dev_priv, BIT(DBUF_S1) |
5245                                 dev_priv->dbuf.enabled_slices);
5246 }
5247
5248 static void gen9_dbuf_disable(struct drm_i915_private *dev_priv)
5249 {
5250         gen9_dbuf_slices_update(dev_priv, 0);
5251 }
5252
5253 static void gen12_dbuf_slices_config(struct drm_i915_private *dev_priv)
5254 {
5255         enum dbuf_slice slice;
5256
5257         if (IS_ALDERLAKE_P(dev_priv))
5258                 return;
5259
5260         for_each_dbuf_slice(dev_priv, slice)
5261                 intel_de_rmw(dev_priv, DBUF_CTL_S(slice),
5262                              DBUF_TRACKER_STATE_SERVICE_MASK,
5263                              DBUF_TRACKER_STATE_SERVICE(8));
5264 }
5265
5266 static void icl_mbus_init(struct drm_i915_private *dev_priv)
5267 {
5268         unsigned long abox_regs = INTEL_INFO(dev_priv)->abox_mask;
5269         u32 mask, val, i;
5270
5271         if (IS_ALDERLAKE_P(dev_priv))
5272                 return;
5273
5274         mask = MBUS_ABOX_BT_CREDIT_POOL1_MASK |
5275                 MBUS_ABOX_BT_CREDIT_POOL2_MASK |
5276                 MBUS_ABOX_B_CREDIT_MASK |
5277                 MBUS_ABOX_BW_CREDIT_MASK;
5278         val = MBUS_ABOX_BT_CREDIT_POOL1(16) |
5279                 MBUS_ABOX_BT_CREDIT_POOL2(16) |
5280                 MBUS_ABOX_B_CREDIT(1) |
5281                 MBUS_ABOX_BW_CREDIT(1);
5282
5283         /*
5284          * gen12 platforms that use abox1 and abox2 for pixel data reads still
5285          * expect us to program the abox_ctl0 register as well, even though
5286          * we don't have to program other instance-0 registers like BW_BUDDY.
5287          */
5288         if (DISPLAY_VER(dev_priv) == 12)
5289                 abox_regs |= BIT(0);
5290
5291         for_each_set_bit(i, &abox_regs, sizeof(abox_regs))
5292                 intel_de_rmw(dev_priv, MBUS_ABOX_CTL(i), mask, val);
5293 }
5294
5295 static void hsw_assert_cdclk(struct drm_i915_private *dev_priv)
5296 {
5297         u32 val = intel_de_read(dev_priv, LCPLL_CTL);
5298
5299         /*
5300          * The LCPLL register should be turned on by the BIOS. For now
5301          * let's just check its state and print errors in case
5302          * something is wrong.  Don't even try to turn it on.
5303          */
5304
5305         if (val & LCPLL_CD_SOURCE_FCLK)
5306                 drm_err(&dev_priv->drm, "CDCLK source is not LCPLL\n");
5307
5308         if (val & LCPLL_PLL_DISABLE)
5309                 drm_err(&dev_priv->drm, "LCPLL is disabled\n");
5310
5311         if ((val & LCPLL_REF_MASK) != LCPLL_REF_NON_SSC)
5312                 drm_err(&dev_priv->drm, "LCPLL not using non-SSC reference\n");
5313 }
5314
5315 static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
5316 {
5317         struct drm_device *dev = &dev_priv->drm;
5318         struct intel_crtc *crtc;
5319
5320         for_each_intel_crtc(dev, crtc)
5321                 I915_STATE_WARN(crtc->active, "CRTC for pipe %c enabled\n",
5322                                 pipe_name(crtc->pipe));
5323
5324         I915_STATE_WARN(intel_de_read(dev_priv, HSW_PWR_WELL_CTL2),
5325                         "Display power well on\n");
5326         I915_STATE_WARN(intel_de_read(dev_priv, SPLL_CTL) & SPLL_PLL_ENABLE,
5327                         "SPLL enabled\n");
5328         I915_STATE_WARN(intel_de_read(dev_priv, WRPLL_CTL(0)) & WRPLL_PLL_ENABLE,
5329                         "WRPLL1 enabled\n");
5330         I915_STATE_WARN(intel_de_read(dev_priv, WRPLL_CTL(1)) & WRPLL_PLL_ENABLE,
5331                         "WRPLL2 enabled\n");
5332         I915_STATE_WARN(intel_de_read(dev_priv, PP_STATUS(0)) & PP_ON,
5333                         "Panel power on\n");
5334         I915_STATE_WARN(intel_de_read(dev_priv, BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
5335                         "CPU PWM1 enabled\n");
5336         if (IS_HASWELL(dev_priv))
5337                 I915_STATE_WARN(intel_de_read(dev_priv, HSW_BLC_PWM2_CTL) & BLM_PWM_ENABLE,
5338                                 "CPU PWM2 enabled\n");
5339         I915_STATE_WARN(intel_de_read(dev_priv, BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE,
5340                         "PCH PWM1 enabled\n");
5341         I915_STATE_WARN(intel_de_read(dev_priv, UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
5342                         "Utility pin enabled\n");
5343         I915_STATE_WARN(intel_de_read(dev_priv, PCH_GTC_CTL) & PCH_GTC_ENABLE,
5344                         "PCH GTC enabled\n");
5345
5346         /*
5347          * In theory we can still leave IRQs enabled, as long as only the HPD
5348          * interrupts remain enabled. We used to check for that, but since it's
5349          * gen-specific and since we only disable LCPLL after we fully disable
5350          * the interrupts, the check below should be enough.
5351          */
5352         I915_STATE_WARN(intel_irqs_enabled(dev_priv), "IRQs enabled\n");
5353 }
5354
5355 static u32 hsw_read_dcomp(struct drm_i915_private *dev_priv)
5356 {
5357         if (IS_HASWELL(dev_priv))
5358                 return intel_de_read(dev_priv, D_COMP_HSW);
5359         else
5360                 return intel_de_read(dev_priv, D_COMP_BDW);
5361 }
5362
5363 static void hsw_write_dcomp(struct drm_i915_private *dev_priv, u32 val)
5364 {
5365         if (IS_HASWELL(dev_priv)) {
5366                 if (sandybridge_pcode_write(dev_priv,
5367                                             GEN6_PCODE_WRITE_D_COMP, val))
5368                         drm_dbg_kms(&dev_priv->drm,
5369                                     "Failed to write to D_COMP\n");
5370         } else {
5371                 intel_de_write(dev_priv, D_COMP_BDW, val);
5372                 intel_de_posting_read(dev_priv, D_COMP_BDW);
5373         }
5374 }
5375
5376 /*
5377  * This function implements pieces of two sequences from BSpec:
5378  * - Sequence for display software to disable LCPLL
5379  * - Sequence for display software to allow package C8+
5380  * The steps implemented here are just the steps that actually touch the LCPLL
5381  * register. Callers should take care of disabling all the display engine
5382  * functions, doing the mode unset, fixing interrupts, etc.
5383  */
5384 static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
5385                               bool switch_to_fclk, bool allow_power_down)
5386 {
5387         u32 val;
5388
5389         assert_can_disable_lcpll(dev_priv);
5390
5391         val = intel_de_read(dev_priv, LCPLL_CTL);
5392
5393         if (switch_to_fclk) {
5394                 val |= LCPLL_CD_SOURCE_FCLK;
5395                 intel_de_write(dev_priv, LCPLL_CTL, val);
5396
5397                 if (wait_for_us(intel_de_read(dev_priv, LCPLL_CTL) &
5398                                 LCPLL_CD_SOURCE_FCLK_DONE, 1))
5399                         drm_err(&dev_priv->drm, "Switching to FCLK failed\n");
5400
5401                 val = intel_de_read(dev_priv, LCPLL_CTL);
5402         }
5403
5404         val |= LCPLL_PLL_DISABLE;
5405         intel_de_write(dev_priv, LCPLL_CTL, val);
5406         intel_de_posting_read(dev_priv, LCPLL_CTL);
5407
5408         if (intel_de_wait_for_clear(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 1))
5409                 drm_err(&dev_priv->drm, "LCPLL still locked\n");
5410
5411         val = hsw_read_dcomp(dev_priv);
5412         val |= D_COMP_COMP_DISABLE;
5413         hsw_write_dcomp(dev_priv, val);
5414         ndelay(100);
5415
5416         if (wait_for((hsw_read_dcomp(dev_priv) &
5417                       D_COMP_RCOMP_IN_PROGRESS) == 0, 1))
5418                 drm_err(&dev_priv->drm, "D_COMP RCOMP still in progress\n");
5419
5420         if (allow_power_down) {
5421                 val = intel_de_read(dev_priv, LCPLL_CTL);
5422                 val |= LCPLL_POWER_DOWN_ALLOW;
5423                 intel_de_write(dev_priv, LCPLL_CTL, val);
5424                 intel_de_posting_read(dev_priv, LCPLL_CTL);
5425         }
5426 }
5427
5428 /*
5429  * Fully restores LCPLL, disallowing power down and switching back to LCPLL
5430  * source.
5431  */
5432 static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
5433 {
5434         u32 val;
5435
5436         val = intel_de_read(dev_priv, LCPLL_CTL);
5437
5438         if ((val & (LCPLL_PLL_LOCK | LCPLL_PLL_DISABLE | LCPLL_CD_SOURCE_FCLK |
5439                     LCPLL_POWER_DOWN_ALLOW)) == LCPLL_PLL_LOCK)
5440                 return;
5441
5442         /*
5443          * Make sure we're not on PC8 state before disabling PC8, otherwise
5444          * we'll hang the machine. To prevent PC8 state, just enable force_wake.
5445          */
5446         intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
5447
5448         if (val & LCPLL_POWER_DOWN_ALLOW) {
5449                 val &= ~LCPLL_POWER_DOWN_ALLOW;
5450                 intel_de_write(dev_priv, LCPLL_CTL, val);
5451                 intel_de_posting_read(dev_priv, LCPLL_CTL);
5452         }
5453
5454         val = hsw_read_dcomp(dev_priv);
5455         val |= D_COMP_COMP_FORCE;
5456         val &= ~D_COMP_COMP_DISABLE;
5457         hsw_write_dcomp(dev_priv, val);
5458
5459         val = intel_de_read(dev_priv, LCPLL_CTL);
5460         val &= ~LCPLL_PLL_DISABLE;
5461         intel_de_write(dev_priv, LCPLL_CTL, val);
5462
5463         if (intel_de_wait_for_set(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 5))
5464                 drm_err(&dev_priv->drm, "LCPLL not locked yet\n");
5465
5466         if (val & LCPLL_CD_SOURCE_FCLK) {
5467                 val = intel_de_read(dev_priv, LCPLL_CTL);
5468                 val &= ~LCPLL_CD_SOURCE_FCLK;
5469                 intel_de_write(dev_priv, LCPLL_CTL, val);
5470
5471                 if (wait_for_us((intel_de_read(dev_priv, LCPLL_CTL) &
5472                                  LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
5473                         drm_err(&dev_priv->drm,
5474                                 "Switching back to LCPLL failed\n");
5475         }
5476
5477         intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
5478
5479         intel_update_cdclk(dev_priv);
5480         intel_dump_cdclk_config(&dev_priv->cdclk.hw, "Current CDCLK");
5481 }
5482
5483 /*
5484  * Package states C8 and deeper are really deep PC states that can only be
5485  * reached when all the devices on the system allow it, so even if the graphics
5486  * device allows PC8+, it doesn't mean the system will actually get to these
5487  * states. Our driver only allows PC8+ when going into runtime PM.
5488  *
5489  * The requirements for PC8+ are that all the outputs are disabled, the power
5490  * well is disabled and most interrupts are disabled, and these are also
5491  * requirements for runtime PM. When these conditions are met, we manually do
5492  * the other conditions: disable the interrupts, clocks and switch LCPLL refclk
5493  * to Fclk. If we're in PC8+ and we get an non-hotplug interrupt, we can hard
5494  * hang the machine.
5495  *
5496  * When we really reach PC8 or deeper states (not just when we allow it) we lose
5497  * the state of some registers, so when we come back from PC8+ we need to
5498  * restore this state. We don't get into PC8+ if we're not in RC6, so we don't
5499  * need to take care of the registers kept by RC6. Notice that this happens even
5500  * if we don't put the device in PCI D3 state (which is what currently happens
5501  * because of the runtime PM support).
5502  *
5503  * For more, read "Display Sequences for Package C8" on the hardware
5504  * documentation.
5505  */
5506 static void hsw_enable_pc8(struct drm_i915_private *dev_priv)
5507 {
5508         u32 val;
5509
5510         drm_dbg_kms(&dev_priv->drm, "Enabling package C8+\n");
5511
5512         if (HAS_PCH_LPT_LP(dev_priv)) {
5513                 val = intel_de_read(dev_priv, SOUTH_DSPCLK_GATE_D);
5514                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
5515                 intel_de_write(dev_priv, SOUTH_DSPCLK_GATE_D, val);
5516         }
5517
5518         lpt_disable_clkout_dp(dev_priv);
5519         hsw_disable_lcpll(dev_priv, true, true);
5520 }
5521
5522 static void hsw_disable_pc8(struct drm_i915_private *dev_priv)
5523 {
5524         u32 val;
5525
5526         drm_dbg_kms(&dev_priv->drm, "Disabling package C8+\n");
5527
5528         hsw_restore_lcpll(dev_priv);
5529         intel_init_pch_refclk(dev_priv);
5530
5531         if (HAS_PCH_LPT_LP(dev_priv)) {
5532                 val = intel_de_read(dev_priv, SOUTH_DSPCLK_GATE_D);
5533                 val |= PCH_LP_PARTITION_LEVEL_DISABLE;
5534                 intel_de_write(dev_priv, SOUTH_DSPCLK_GATE_D, val);
5535         }
5536 }
5537
5538 static void intel_pch_reset_handshake(struct drm_i915_private *dev_priv,
5539                                       bool enable)
5540 {
5541         i915_reg_t reg;
5542         u32 reset_bits, val;
5543
5544         if (IS_IVYBRIDGE(dev_priv)) {
5545                 reg = GEN7_MSG_CTL;
5546                 reset_bits = WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK;
5547         } else {
5548                 reg = HSW_NDE_RSTWRN_OPT;
5549                 reset_bits = RESET_PCH_HANDSHAKE_ENABLE;
5550         }
5551
5552         val = intel_de_read(dev_priv, reg);
5553
5554         if (enable)
5555                 val |= reset_bits;
5556         else
5557                 val &= ~reset_bits;
5558
5559         intel_de_write(dev_priv, reg, val);
5560 }
5561
5562 static void skl_display_core_init(struct drm_i915_private *dev_priv,
5563                                   bool resume)
5564 {
5565         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5566         struct i915_power_well *well;
5567
5568         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
5569
5570         /* enable PCH reset handshake */
5571         intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
5572
5573         if (!HAS_DISPLAY(dev_priv))
5574                 return;
5575
5576         /* enable PG1 and Misc I/O */
5577         mutex_lock(&power_domains->lock);
5578
5579         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
5580         intel_power_well_enable(dev_priv, well);
5581
5582         well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
5583         intel_power_well_enable(dev_priv, well);
5584
5585         mutex_unlock(&power_domains->lock);
5586
5587         intel_cdclk_init_hw(dev_priv);
5588
5589         gen9_dbuf_enable(dev_priv);
5590
5591         if (resume && intel_dmc_has_payload(dev_priv))
5592                 intel_dmc_load_program(dev_priv);
5593 }
5594
5595 static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
5596 {
5597         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5598         struct i915_power_well *well;
5599
5600         if (!HAS_DISPLAY(dev_priv))
5601                 return;
5602
5603         gen9_disable_dc_states(dev_priv);
5604
5605         gen9_dbuf_disable(dev_priv);
5606
5607         intel_cdclk_uninit_hw(dev_priv);
5608
5609         /* The spec doesn't call for removing the reset handshake flag */
5610         /* disable PG1 and Misc I/O */
5611
5612         mutex_lock(&power_domains->lock);
5613
5614         /*
5615          * BSpec says to keep the MISC IO power well enabled here, only
5616          * remove our request for power well 1.
5617          * Note that even though the driver's request is removed power well 1
5618          * may stay enabled after this due to DMC's own request on it.
5619          */
5620         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
5621         intel_power_well_disable(dev_priv, well);
5622
5623         mutex_unlock(&power_domains->lock);
5624
5625         usleep_range(10, 30);           /* 10 us delay per Bspec */
5626 }
5627
5628 static void bxt_display_core_init(struct drm_i915_private *dev_priv, bool resume)
5629 {
5630         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5631         struct i915_power_well *well;
5632
5633         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
5634
5635         /*
5636          * NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
5637          * or else the reset will hang because there is no PCH to respond.
5638          * Move the handshake programming to initialization sequence.
5639          * Previously was left up to BIOS.
5640          */
5641         intel_pch_reset_handshake(dev_priv, false);
5642
5643         if (!HAS_DISPLAY(dev_priv))
5644                 return;
5645
5646         /* Enable PG1 */
5647         mutex_lock(&power_domains->lock);
5648
5649         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
5650         intel_power_well_enable(dev_priv, well);
5651
5652         mutex_unlock(&power_domains->lock);
5653
5654         intel_cdclk_init_hw(dev_priv);
5655
5656         gen9_dbuf_enable(dev_priv);
5657
5658         if (resume && intel_dmc_has_payload(dev_priv))
5659                 intel_dmc_load_program(dev_priv);
5660 }
5661
5662 static void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
5663 {
5664         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5665         struct i915_power_well *well;
5666
5667         if (!HAS_DISPLAY(dev_priv))
5668                 return;
5669
5670         gen9_disable_dc_states(dev_priv);
5671
5672         gen9_dbuf_disable(dev_priv);
5673
5674         intel_cdclk_uninit_hw(dev_priv);
5675
5676         /* The spec doesn't call for removing the reset handshake flag */
5677
5678         /*
5679          * Disable PW1 (PG1).
5680          * Note that even though the driver's request is removed power well 1
5681          * may stay enabled after this due to DMC's own request on it.
5682          */
5683         mutex_lock(&power_domains->lock);
5684
5685         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
5686         intel_power_well_disable(dev_priv, well);
5687
5688         mutex_unlock(&power_domains->lock);
5689
5690         usleep_range(10, 30);           /* 10 us delay per Bspec */
5691 }
5692
5693 static void cnl_display_core_init(struct drm_i915_private *dev_priv, bool resume)
5694 {
5695         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5696         struct i915_power_well *well;
5697
5698         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
5699
5700         /* 1. Enable PCH Reset Handshake */
5701         intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
5702
5703         if (!HAS_DISPLAY(dev_priv))
5704                 return;
5705
5706         /* 2-3. */
5707         intel_combo_phy_init(dev_priv);
5708
5709         /*
5710          * 4. Enable Power Well 1 (PG1).
5711          *    The AUX IO power wells will be enabled on demand.
5712          */
5713         mutex_lock(&power_domains->lock);
5714         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
5715         intel_power_well_enable(dev_priv, well);
5716         mutex_unlock(&power_domains->lock);
5717
5718         /* 5. Enable CD clock */
5719         intel_cdclk_init_hw(dev_priv);
5720
5721         /* 6. Enable DBUF */
5722         gen9_dbuf_enable(dev_priv);
5723
5724         if (resume && intel_dmc_has_payload(dev_priv))
5725                 intel_dmc_load_program(dev_priv);
5726 }
5727
5728 static void cnl_display_core_uninit(struct drm_i915_private *dev_priv)
5729 {
5730         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5731         struct i915_power_well *well;
5732
5733         if (!HAS_DISPLAY(dev_priv))
5734                 return;
5735
5736         gen9_disable_dc_states(dev_priv);
5737
5738         /* 1. Disable all display engine functions -> aready done */
5739
5740         /* 2. Disable DBUF */
5741         gen9_dbuf_disable(dev_priv);
5742
5743         /* 3. Disable CD clock */
5744         intel_cdclk_uninit_hw(dev_priv);
5745
5746         /*
5747          * 4. Disable Power Well 1 (PG1).
5748          *    The AUX IO power wells are toggled on demand, so they are already
5749          *    disabled at this point.
5750          */
5751         mutex_lock(&power_domains->lock);
5752         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
5753         intel_power_well_disable(dev_priv, well);
5754         mutex_unlock(&power_domains->lock);
5755
5756         usleep_range(10, 30);           /* 10 us delay per Bspec */
5757
5758         /* 5. */
5759         intel_combo_phy_uninit(dev_priv);
5760 }
5761
5762 struct buddy_page_mask {
5763         u32 page_mask;
5764         u8 type;
5765         u8 num_channels;
5766 };
5767
5768 static const struct buddy_page_mask tgl_buddy_page_masks[] = {
5769         { .num_channels = 1, .type = INTEL_DRAM_DDR4,   .page_mask = 0xF },
5770         { .num_channels = 1, .type = INTEL_DRAM_DDR5,   .page_mask = 0xF },
5771         { .num_channels = 2, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x1C },
5772         { .num_channels = 2, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x1C },
5773         { .num_channels = 2, .type = INTEL_DRAM_DDR4,   .page_mask = 0x1F },
5774         { .num_channels = 2, .type = INTEL_DRAM_DDR5,   .page_mask = 0x1E },
5775         { .num_channels = 4, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x38 },
5776         { .num_channels = 4, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x38 },
5777         {}
5778 };
5779
5780 static const struct buddy_page_mask wa_1409767108_buddy_page_masks[] = {
5781         { .num_channels = 1, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x1 },
5782         { .num_channels = 1, .type = INTEL_DRAM_DDR4,   .page_mask = 0x1 },
5783         { .num_channels = 1, .type = INTEL_DRAM_DDR5,   .page_mask = 0x1 },
5784         { .num_channels = 1, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x1 },
5785         { .num_channels = 2, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x3 },
5786         { .num_channels = 2, .type = INTEL_DRAM_DDR4,   .page_mask = 0x3 },
5787         { .num_channels = 2, .type = INTEL_DRAM_DDR5,   .page_mask = 0x3 },
5788         { .num_channels = 2, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x3 },
5789         {}
5790 };
5791
5792 static void tgl_bw_buddy_init(struct drm_i915_private *dev_priv)
5793 {
5794         enum intel_dram_type type = dev_priv->dram_info.type;
5795         u8 num_channels = dev_priv->dram_info.num_channels;
5796         const struct buddy_page_mask *table;
5797         unsigned long abox_mask = INTEL_INFO(dev_priv)->abox_mask;
5798         int config, i;
5799
5800         if (IS_ALDERLAKE_S(dev_priv) ||
5801             IS_DG1_REVID(dev_priv, DG1_REVID_A0, DG1_REVID_A0) ||
5802             IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
5803                 /* Wa_1409767108:tgl,dg1,adl-s */
5804                 table = wa_1409767108_buddy_page_masks;
5805         else
5806                 table = tgl_buddy_page_masks;
5807
5808         for (config = 0; table[config].page_mask != 0; config++)
5809                 if (table[config].num_channels == num_channels &&
5810                     table[config].type == type)
5811                         break;
5812
5813         if (table[config].page_mask == 0) {
5814                 drm_dbg(&dev_priv->drm,
5815                         "Unknown memory configuration; disabling address buddy logic.\n");
5816                 for_each_set_bit(i, &abox_mask, sizeof(abox_mask))
5817                         intel_de_write(dev_priv, BW_BUDDY_CTL(i),
5818                                        BW_BUDDY_DISABLE);
5819         } else {
5820                 for_each_set_bit(i, &abox_mask, sizeof(abox_mask)) {
5821                         intel_de_write(dev_priv, BW_BUDDY_PAGE_MASK(i),
5822                                        table[config].page_mask);
5823
5824                         /* Wa_22010178259:tgl,rkl */
5825                         intel_de_rmw(dev_priv, BW_BUDDY_CTL(i),
5826                                      BW_BUDDY_TLB_REQ_TIMER_MASK,
5827                                      BW_BUDDY_TLB_REQ_TIMER(0x8));
5828                 }
5829         }
5830 }
5831
5832 static void icl_display_core_init(struct drm_i915_private *dev_priv,
5833                                   bool resume)
5834 {
5835         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5836         struct i915_power_well *well;
5837         u32 val;
5838
5839         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
5840
5841         /* Wa_14011294188:ehl,jsl,tgl,rkl,adl-s */
5842         if (INTEL_PCH_TYPE(dev_priv) >= PCH_JSP &&
5843             INTEL_PCH_TYPE(dev_priv) < PCH_DG1)
5844                 intel_de_rmw(dev_priv, SOUTH_DSPCLK_GATE_D, 0,
5845                              PCH_DPMGUNIT_CLOCK_GATE_DISABLE);
5846
5847         /* 1. Enable PCH reset handshake. */
5848         intel_pch_reset_handshake(dev_priv, !HAS_PCH_NOP(dev_priv));
5849
5850         if (!HAS_DISPLAY(dev_priv))
5851                 return;
5852
5853         /* 2. Initialize all combo phys */
5854         intel_combo_phy_init(dev_priv);
5855
5856         /*
5857          * 3. Enable Power Well 1 (PG1).
5858          *    The AUX IO power wells will be enabled on demand.
5859          */
5860         mutex_lock(&power_domains->lock);
5861         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
5862         intel_power_well_enable(dev_priv, well);
5863         mutex_unlock(&power_domains->lock);
5864
5865         /* 4. Enable CDCLK. */
5866         intel_cdclk_init_hw(dev_priv);
5867
5868         if (DISPLAY_VER(dev_priv) >= 12)
5869                 gen12_dbuf_slices_config(dev_priv);
5870
5871         /* 5. Enable DBUF. */
5872         gen9_dbuf_enable(dev_priv);
5873
5874         /* 6. Setup MBUS. */
5875         icl_mbus_init(dev_priv);
5876
5877         /* 7. Program arbiter BW_BUDDY registers */
5878         if (DISPLAY_VER(dev_priv) >= 12)
5879                 tgl_bw_buddy_init(dev_priv);
5880
5881         if (resume && intel_dmc_has_payload(dev_priv))
5882                 intel_dmc_load_program(dev_priv);
5883
5884         /* Wa_14011508470 */
5885         if (DISPLAY_VER(dev_priv) == 12) {
5886                 val = DCPR_CLEAR_MEMSTAT_DIS | DCPR_SEND_RESP_IMM |
5887                       DCPR_MASK_LPMODE | DCPR_MASK_MAXLATENCY_MEMUP_CLR;
5888                 intel_uncore_rmw(&dev_priv->uncore, GEN11_CHICKEN_DCPR_2, 0, val);
5889         }
5890
5891         /* Wa_14011503030:xelpd */
5892         if (DISPLAY_VER(dev_priv) >= 13)
5893                 intel_de_write(dev_priv, XELPD_DISPLAY_ERR_FATAL_MASK, ~0);
5894 }
5895
5896 static void icl_display_core_uninit(struct drm_i915_private *dev_priv)
5897 {
5898         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5899         struct i915_power_well *well;
5900
5901         if (!HAS_DISPLAY(dev_priv))
5902                 return;
5903
5904         gen9_disable_dc_states(dev_priv);
5905
5906         /* 1. Disable all display engine functions -> aready done */
5907
5908         /* 2. Disable DBUF */
5909         gen9_dbuf_disable(dev_priv);
5910
5911         /* 3. Disable CD clock */
5912         intel_cdclk_uninit_hw(dev_priv);
5913
5914         /*
5915          * 4. Disable Power Well 1 (PG1).
5916          *    The AUX IO power wells are toggled on demand, so they are already
5917          *    disabled at this point.
5918          */
5919         mutex_lock(&power_domains->lock);
5920         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
5921         intel_power_well_disable(dev_priv, well);
5922         mutex_unlock(&power_domains->lock);
5923
5924         /* 5. */
5925         intel_combo_phy_uninit(dev_priv);
5926 }
5927
5928 static void chv_phy_control_init(struct drm_i915_private *dev_priv)
5929 {
5930         struct i915_power_well *cmn_bc =
5931                 lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
5932         struct i915_power_well *cmn_d =
5933                 lookup_power_well(dev_priv, CHV_DISP_PW_DPIO_CMN_D);
5934
5935         /*
5936          * DISPLAY_PHY_CONTROL can get corrupted if read. As a
5937          * workaround never ever read DISPLAY_PHY_CONTROL, and
5938          * instead maintain a shadow copy ourselves. Use the actual
5939          * power well state and lane status to reconstruct the
5940          * expected initial value.
5941          */
5942         dev_priv->chv_phy_control =
5943                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
5944                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
5945                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) |
5946                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH1) |
5947                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY1, DPIO_CH0);
5948
5949         /*
5950          * If all lanes are disabled we leave the override disabled
5951          * with all power down bits cleared to match the state we
5952          * would use after disabling the port. Otherwise enable the
5953          * override and set the lane powerdown bits accding to the
5954          * current lane status.
5955          */
5956         if (cmn_bc->desc->ops->is_enabled(dev_priv, cmn_bc)) {
5957                 u32 status = intel_de_read(dev_priv, DPLL(PIPE_A));
5958                 unsigned int mask;
5959
5960                 mask = status & DPLL_PORTB_READY_MASK;
5961                 if (mask == 0xf)
5962                         mask = 0x0;
5963                 else
5964                         dev_priv->chv_phy_control |=
5965                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0);
5966
5967                 dev_priv->chv_phy_control |=
5968                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0);
5969
5970                 mask = (status & DPLL_PORTC_READY_MASK) >> 4;
5971                 if (mask == 0xf)
5972                         mask = 0x0;
5973                 else
5974                         dev_priv->chv_phy_control |=
5975                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1);
5976
5977                 dev_priv->chv_phy_control |=
5978                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1);
5979
5980                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
5981
5982                 dev_priv->chv_phy_assert[DPIO_PHY0] = false;
5983         } else {
5984                 dev_priv->chv_phy_assert[DPIO_PHY0] = true;
5985         }
5986
5987         if (cmn_d->desc->ops->is_enabled(dev_priv, cmn_d)) {
5988                 u32 status = intel_de_read(dev_priv, DPIO_PHY_STATUS);
5989                 unsigned int mask;
5990
5991                 mask = status & DPLL_PORTD_READY_MASK;
5992
5993                 if (mask == 0xf)
5994                         mask = 0x0;
5995                 else
5996                         dev_priv->chv_phy_control |=
5997                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0);
5998
5999                 dev_priv->chv_phy_control |=
6000                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0);
6001
6002                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
6003
6004                 dev_priv->chv_phy_assert[DPIO_PHY1] = false;
6005         } else {
6006                 dev_priv->chv_phy_assert[DPIO_PHY1] = true;
6007         }
6008
6009         drm_dbg_kms(&dev_priv->drm, "Initial PHY_CONTROL=0x%08x\n",
6010                     dev_priv->chv_phy_control);
6011
6012         /* Defer application of initial phy_control to enabling the powerwell */
6013 }
6014
6015 static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
6016 {
6017         struct i915_power_well *cmn =
6018                 lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
6019         struct i915_power_well *disp2d =
6020                 lookup_power_well(dev_priv, VLV_DISP_PW_DISP2D);
6021
6022         /* If the display might be already active skip this */
6023         if (cmn->desc->ops->is_enabled(dev_priv, cmn) &&
6024             disp2d->desc->ops->is_enabled(dev_priv, disp2d) &&
6025             intel_de_read(dev_priv, DPIO_CTL) & DPIO_CMNRST)
6026                 return;
6027
6028         drm_dbg_kms(&dev_priv->drm, "toggling display PHY side reset\n");
6029
6030         /* cmnlane needs DPLL registers */
6031         disp2d->desc->ops->enable(dev_priv, disp2d);
6032
6033         /*
6034          * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
6035          * Need to assert and de-assert PHY SB reset by gating the
6036          * common lane power, then un-gating it.
6037          * Simply ungating isn't enough to reset the PHY enough to get
6038          * ports and lanes running.
6039          */
6040         cmn->desc->ops->disable(dev_priv, cmn);
6041 }
6042
6043 static bool vlv_punit_is_power_gated(struct drm_i915_private *dev_priv, u32 reg0)
6044 {
6045         bool ret;
6046
6047         vlv_punit_get(dev_priv);
6048         ret = (vlv_punit_read(dev_priv, reg0) & SSPM0_SSC_MASK) == SSPM0_SSC_PWR_GATE;
6049         vlv_punit_put(dev_priv);
6050
6051         return ret;
6052 }
6053
6054 static void assert_ved_power_gated(struct drm_i915_private *dev_priv)
6055 {
6056         drm_WARN(&dev_priv->drm,
6057                  !vlv_punit_is_power_gated(dev_priv, PUNIT_REG_VEDSSPM0),
6058                  "VED not power gated\n");
6059 }
6060
6061 static void assert_isp_power_gated(struct drm_i915_private *dev_priv)
6062 {
6063         static const struct pci_device_id isp_ids[] = {
6064                 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x0f38)},
6065                 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x22b8)},
6066                 {}
6067         };
6068
6069         drm_WARN(&dev_priv->drm, !pci_dev_present(isp_ids) &&
6070                  !vlv_punit_is_power_gated(dev_priv, PUNIT_REG_ISPSSPM0),
6071                  "ISP not power gated\n");
6072 }
6073
6074 static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv);
6075
6076 /**
6077  * intel_power_domains_init_hw - initialize hardware power domain state
6078  * @i915: i915 device instance
6079  * @resume: Called from resume code paths or not
6080  *
6081  * This function initializes the hardware power domain state and enables all
6082  * power wells belonging to the INIT power domain. Power wells in other
6083  * domains (and not in the INIT domain) are referenced or disabled by
6084  * intel_modeset_readout_hw_state(). After that the reference count of each
6085  * power well must match its HW enabled state, see
6086  * intel_power_domains_verify_state().
6087  *
6088  * It will return with power domains disabled (to be enabled later by
6089  * intel_power_domains_enable()) and must be paired with
6090  * intel_power_domains_driver_remove().
6091  */
6092 void intel_power_domains_init_hw(struct drm_i915_private *i915, bool resume)
6093 {
6094         struct i915_power_domains *power_domains = &i915->power_domains;
6095
6096         power_domains->initializing = true;
6097
6098         if (DISPLAY_VER(i915) >= 11) {
6099                 icl_display_core_init(i915, resume);
6100         } else if (IS_CANNONLAKE(i915)) {
6101                 cnl_display_core_init(i915, resume);
6102         } else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
6103                 bxt_display_core_init(i915, resume);
6104         } else if (DISPLAY_VER(i915) == 9) {
6105                 skl_display_core_init(i915, resume);
6106         } else if (IS_CHERRYVIEW(i915)) {
6107                 mutex_lock(&power_domains->lock);
6108                 chv_phy_control_init(i915);
6109                 mutex_unlock(&power_domains->lock);
6110                 assert_isp_power_gated(i915);
6111         } else if (IS_VALLEYVIEW(i915)) {
6112                 mutex_lock(&power_domains->lock);
6113                 vlv_cmnlane_wa(i915);
6114                 mutex_unlock(&power_domains->lock);
6115                 assert_ved_power_gated(i915);
6116                 assert_isp_power_gated(i915);
6117         } else if (IS_BROADWELL(i915) || IS_HASWELL(i915)) {
6118                 hsw_assert_cdclk(i915);
6119                 intel_pch_reset_handshake(i915, !HAS_PCH_NOP(i915));
6120         } else if (IS_IVYBRIDGE(i915)) {
6121                 intel_pch_reset_handshake(i915, !HAS_PCH_NOP(i915));
6122         }
6123
6124         /*
6125          * Keep all power wells enabled for any dependent HW access during
6126          * initialization and to make sure we keep BIOS enabled display HW
6127          * resources powered until display HW readout is complete. We drop
6128          * this reference in intel_power_domains_enable().
6129          */
6130         drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
6131         power_domains->init_wakeref =
6132                 intel_display_power_get(i915, POWER_DOMAIN_INIT);
6133
6134         /* Disable power support if the user asked so. */
6135         if (!i915->params.disable_power_well) {
6136                 drm_WARN_ON(&i915->drm, power_domains->disable_wakeref);
6137                 i915->power_domains.disable_wakeref = intel_display_power_get(i915,
6138                                                                               POWER_DOMAIN_INIT);
6139         }
6140         intel_power_domains_sync_hw(i915);
6141
6142         power_domains->initializing = false;
6143 }
6144
6145 /**
6146  * intel_power_domains_driver_remove - deinitialize hw power domain state
6147  * @i915: i915 device instance
6148  *
6149  * De-initializes the display power domain HW state. It also ensures that the
6150  * device stays powered up so that the driver can be reloaded.
6151  *
6152  * It must be called with power domains already disabled (after a call to
6153  * intel_power_domains_disable()) and must be paired with
6154  * intel_power_domains_init_hw().
6155  */
6156 void intel_power_domains_driver_remove(struct drm_i915_private *i915)
6157 {
6158         intel_wakeref_t wakeref __maybe_unused =
6159                 fetch_and_zero(&i915->power_domains.init_wakeref);
6160
6161         /* Remove the refcount we took to keep power well support disabled. */
6162         if (!i915->params.disable_power_well)
6163                 intel_display_power_put(i915, POWER_DOMAIN_INIT,
6164                                         fetch_and_zero(&i915->power_domains.disable_wakeref));
6165
6166         intel_display_power_flush_work_sync(i915);
6167
6168         intel_power_domains_verify_state(i915);
6169
6170         /* Keep the power well enabled, but cancel its rpm wakeref. */
6171         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
6172 }
6173
6174 /**
6175  * intel_power_domains_enable - enable toggling of display power wells
6176  * @i915: i915 device instance
6177  *
6178  * Enable the ondemand enabling/disabling of the display power wells. Note that
6179  * power wells not belonging to POWER_DOMAIN_INIT are allowed to be toggled
6180  * only at specific points of the display modeset sequence, thus they are not
6181  * affected by the intel_power_domains_enable()/disable() calls. The purpose
6182  * of these function is to keep the rest of power wells enabled until the end
6183  * of display HW readout (which will acquire the power references reflecting
6184  * the current HW state).
6185  */
6186 void intel_power_domains_enable(struct drm_i915_private *i915)
6187 {
6188         intel_wakeref_t wakeref __maybe_unused =
6189                 fetch_and_zero(&i915->power_domains.init_wakeref);
6190
6191         intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref);
6192         intel_power_domains_verify_state(i915);
6193 }
6194
6195 /**
6196  * intel_power_domains_disable - disable toggling of display power wells
6197  * @i915: i915 device instance
6198  *
6199  * Disable the ondemand enabling/disabling of the display power wells. See
6200  * intel_power_domains_enable() for which power wells this call controls.
6201  */
6202 void intel_power_domains_disable(struct drm_i915_private *i915)
6203 {
6204         struct i915_power_domains *power_domains = &i915->power_domains;
6205
6206         drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
6207         power_domains->init_wakeref =
6208                 intel_display_power_get(i915, POWER_DOMAIN_INIT);
6209
6210         intel_power_domains_verify_state(i915);
6211 }
6212
6213 /**
6214  * intel_power_domains_suspend - suspend power domain state
6215  * @i915: i915 device instance
6216  * @suspend_mode: specifies the target suspend state (idle, mem, hibernation)
6217  *
6218  * This function prepares the hardware power domain state before entering
6219  * system suspend.
6220  *
6221  * It must be called with power domains already disabled (after a call to
6222  * intel_power_domains_disable()) and paired with intel_power_domains_resume().
6223  */
6224 void intel_power_domains_suspend(struct drm_i915_private *i915,
6225                                  enum i915_drm_suspend_mode suspend_mode)
6226 {
6227         struct i915_power_domains *power_domains = &i915->power_domains;
6228         intel_wakeref_t wakeref __maybe_unused =
6229                 fetch_and_zero(&power_domains->init_wakeref);
6230
6231         intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref);
6232
6233         /*
6234          * In case of suspend-to-idle (aka S0ix) on a DMC platform without DC9
6235          * support don't manually deinit the power domains. This also means the
6236          * DMC firmware will stay active, it will power down any HW
6237          * resources as required and also enable deeper system power states
6238          * that would be blocked if the firmware was inactive.
6239          */
6240         if (!(i915->dmc.allowed_dc_mask & DC_STATE_EN_DC9) &&
6241             suspend_mode == I915_DRM_SUSPEND_IDLE &&
6242             intel_dmc_has_payload(i915)) {
6243                 intel_display_power_flush_work(i915);
6244                 intel_power_domains_verify_state(i915);
6245                 return;
6246         }
6247
6248         /*
6249          * Even if power well support was disabled we still want to disable
6250          * power wells if power domains must be deinitialized for suspend.
6251          */
6252         if (!i915->params.disable_power_well)
6253                 intel_display_power_put(i915, POWER_DOMAIN_INIT,
6254                                         fetch_and_zero(&i915->power_domains.disable_wakeref));
6255
6256         intel_display_power_flush_work(i915);
6257         intel_power_domains_verify_state(i915);
6258
6259         if (DISPLAY_VER(i915) >= 11)
6260                 icl_display_core_uninit(i915);
6261         else if (IS_CANNONLAKE(i915))
6262                 cnl_display_core_uninit(i915);
6263         else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915))
6264                 bxt_display_core_uninit(i915);
6265         else if (DISPLAY_VER(i915) == 9)
6266                 skl_display_core_uninit(i915);
6267
6268         power_domains->display_core_suspended = true;
6269 }
6270
6271 /**
6272  * intel_power_domains_resume - resume power domain state
6273  * @i915: i915 device instance
6274  *
6275  * This function resume the hardware power domain state during system resume.
6276  *
6277  * It will return with power domain support disabled (to be enabled later by
6278  * intel_power_domains_enable()) and must be paired with
6279  * intel_power_domains_suspend().
6280  */
6281 void intel_power_domains_resume(struct drm_i915_private *i915)
6282 {
6283         struct i915_power_domains *power_domains = &i915->power_domains;
6284
6285         if (power_domains->display_core_suspended) {
6286                 intel_power_domains_init_hw(i915, true);
6287                 power_domains->display_core_suspended = false;
6288         } else {
6289                 drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
6290                 power_domains->init_wakeref =
6291                         intel_display_power_get(i915, POWER_DOMAIN_INIT);
6292         }
6293
6294         intel_power_domains_verify_state(i915);
6295 }
6296
6297 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
6298
6299 static void intel_power_domains_dump_info(struct drm_i915_private *i915)
6300 {
6301         struct i915_power_domains *power_domains = &i915->power_domains;
6302         struct i915_power_well *power_well;
6303
6304         for_each_power_well(i915, power_well) {
6305                 enum intel_display_power_domain domain;
6306
6307                 drm_dbg(&i915->drm, "%-25s %d\n",
6308                         power_well->desc->name, power_well->count);
6309
6310                 for_each_power_domain(domain, power_well->desc->domains)
6311                         drm_dbg(&i915->drm, "  %-23s %d\n",
6312                                 intel_display_power_domain_str(domain),
6313                                 power_domains->domain_use_count[domain]);
6314         }
6315 }
6316
6317 /**
6318  * intel_power_domains_verify_state - verify the HW/SW state for all power wells
6319  * @i915: i915 device instance
6320  *
6321  * Verify if the reference count of each power well matches its HW enabled
6322  * state and the total refcount of the domains it belongs to. This must be
6323  * called after modeset HW state sanitization, which is responsible for
6324  * acquiring reference counts for any power wells in use and disabling the
6325  * ones left on by BIOS but not required by any active output.
6326  */
6327 static void intel_power_domains_verify_state(struct drm_i915_private *i915)
6328 {
6329         struct i915_power_domains *power_domains = &i915->power_domains;
6330         struct i915_power_well *power_well;
6331         bool dump_domain_info;
6332
6333         mutex_lock(&power_domains->lock);
6334
6335         verify_async_put_domains_state(power_domains);
6336
6337         dump_domain_info = false;
6338         for_each_power_well(i915, power_well) {
6339                 enum intel_display_power_domain domain;
6340                 int domains_count;
6341                 bool enabled;
6342
6343                 enabled = power_well->desc->ops->is_enabled(i915, power_well);
6344                 if ((power_well->count || power_well->desc->always_on) !=
6345                     enabled)
6346                         drm_err(&i915->drm,
6347                                 "power well %s state mismatch (refcount %d/enabled %d)",
6348                                 power_well->desc->name,
6349                                 power_well->count, enabled);
6350
6351                 domains_count = 0;
6352                 for_each_power_domain(domain, power_well->desc->domains)
6353                         domains_count += power_domains->domain_use_count[domain];
6354
6355                 if (power_well->count != domains_count) {
6356                         drm_err(&i915->drm,
6357                                 "power well %s refcount/domain refcount mismatch "
6358                                 "(refcount %d/domains refcount %d)\n",
6359                                 power_well->desc->name, power_well->count,
6360                                 domains_count);
6361                         dump_domain_info = true;
6362                 }
6363         }
6364
6365         if (dump_domain_info) {
6366                 static bool dumped;
6367
6368                 if (!dumped) {
6369                         intel_power_domains_dump_info(i915);
6370                         dumped = true;
6371                 }
6372         }
6373
6374         mutex_unlock(&power_domains->lock);
6375 }
6376
6377 #else
6378
6379 static void intel_power_domains_verify_state(struct drm_i915_private *i915)
6380 {
6381 }
6382
6383 #endif
6384
6385 void intel_display_power_suspend_late(struct drm_i915_private *i915)
6386 {
6387         if (DISPLAY_VER(i915) >= 11 || IS_GEMINILAKE(i915) ||
6388             IS_BROXTON(i915)) {
6389                 bxt_enable_dc9(i915);
6390                 /* Tweaked Wa_14010685332:icp,jsp,mcc */
6391                 if (INTEL_PCH_TYPE(i915) >= PCH_ICP && INTEL_PCH_TYPE(i915) <= PCH_MCC)
6392                         intel_de_rmw(i915, SOUTH_CHICKEN1,
6393                                      SBCLK_RUN_REFCLK_DIS, SBCLK_RUN_REFCLK_DIS);
6394         } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
6395                 hsw_enable_pc8(i915);
6396         }
6397 }
6398
6399 void intel_display_power_resume_early(struct drm_i915_private *i915)
6400 {
6401         if (DISPLAY_VER(i915) >= 11 || IS_GEMINILAKE(i915) ||
6402             IS_BROXTON(i915)) {
6403                 gen9_sanitize_dc_state(i915);
6404                 bxt_disable_dc9(i915);
6405                 /* Tweaked Wa_14010685332:icp,jsp,mcc */
6406                 if (INTEL_PCH_TYPE(i915) >= PCH_ICP && INTEL_PCH_TYPE(i915) <= PCH_MCC)
6407                         intel_de_rmw(i915, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, 0);
6408
6409         } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
6410                 hsw_disable_pc8(i915);
6411         }
6412 }
6413
6414 void intel_display_power_suspend(struct drm_i915_private *i915)
6415 {
6416         if (DISPLAY_VER(i915) >= 11) {
6417                 icl_display_core_uninit(i915);
6418                 bxt_enable_dc9(i915);
6419         } else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
6420                 bxt_display_core_uninit(i915);
6421                 bxt_enable_dc9(i915);
6422         } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
6423                 hsw_enable_pc8(i915);
6424         }
6425 }
6426
6427 void intel_display_power_resume(struct drm_i915_private *i915)
6428 {
6429         if (DISPLAY_VER(i915) >= 11) {
6430                 bxt_disable_dc9(i915);
6431                 icl_display_core_init(i915, true);
6432                 if (intel_dmc_has_payload(i915)) {
6433                         if (i915->dmc.allowed_dc_mask &
6434                             DC_STATE_EN_UPTO_DC6)
6435                                 skl_enable_dc6(i915);
6436                         else if (i915->dmc.allowed_dc_mask &
6437                                  DC_STATE_EN_UPTO_DC5)
6438                                 gen9_enable_dc5(i915);
6439                 }
6440         } else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
6441                 bxt_disable_dc9(i915);
6442                 bxt_display_core_init(i915, true);
6443                 if (intel_dmc_has_payload(i915) &&
6444                     (i915->dmc.allowed_dc_mask & DC_STATE_EN_UPTO_DC5))
6445                         gen9_enable_dc5(i915);
6446         } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
6447                 hsw_disable_pc8(i915);
6448         }
6449 }