drivers/gpu/drm/i915/i915_irq.c

   1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
   2  */
   3 /*
   4  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  */
  28
  29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  30
  31 #include <linux/circ_buf.h>
  32 #include <linux/slab.h>
  33 #include <linux/sysrq.h>
  34
  35 #include <drm/drm_drv.h>
  36 #include <drm/drm_irq.h>
  37
  38 #include "display/intel_display_types.h"
  39 #include "display/intel_fifo_underrun.h"
  40 #include "display/intel_hotplug.h"
  41 #include "display/intel_lpe_audio.h"
  42 #include "display/intel_psr.h"
  43
  44 #include "gt/intel_breadcrumbs.h"
  45 #include "gt/intel_gt.h"
  46 #include "gt/intel_gt_irq.h"
  47 #include "gt/intel_gt_pm_irq.h"
  48 #include "gt/intel_rps.h"
  49
  50 #include "i915_drv.h"
  51 #include "i915_irq.h"
  52 #include "i915_trace.h"
  53 #include "intel_pm.h"
  54
  55 /**
  56  * DOC: interrupt handling
  57  *
  58  * These functions provide the basic support for enabling and disabling the
  59  * interrupt handling support. There's a lot more functionality in i915_irq.c
  60  * and related files, but that will be described in separate chapters.
  61  */
  62
  63 /*
  64  * Interrupt statistic for PMU. Increments the counter only if the
  65  * interrupt originated from the the GPU so interrupts from a device which
  66  * shares the interrupt line are not accounted.
  67  */
  68 static inline void pmu_irq_stats(struct drm_i915_private *i915,
  69                                  irqreturn_t res)
  70 {
  71         if (unlikely(res != IRQ_HANDLED))
  72                 return;
  73
  74         /*
  75          * A clever compiler translates that into INC. A not so clever one
  76          * should at least prevent store tearing.
  77          */
  78         WRITE_ONCE(i915->pmu.irq_count, i915->pmu.irq_count + 1);
  79 }
  80
  81 typedef bool (*long_pulse_detect_func)(enum hpd_pin pin, u32 val);
  82 typedef u32 (*hotplug_enables_func)(struct drm_i915_private *i915,
  83                                     enum hpd_pin pin);
  84
  85 static const u32 hpd_ilk[HPD_NUM_PINS] = {
  86         [HPD_PORT_A] = DE_DP_A_HOTPLUG,
  87 };
  88
  89 static const u32 hpd_ivb[HPD_NUM_PINS] = {
  90         [HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
  91 };
  92
  93 static const u32 hpd_bdw[HPD_NUM_PINS] = {
  94         [HPD_PORT_A] = GEN8_DE_PORT_HOTPLUG(HPD_PORT_A),
  95 };
  96
  97 static const u32 hpd_ibx[HPD_NUM_PINS] = {
  98         [HPD_CRT] = SDE_CRT_HOTPLUG,
  99         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
 100         [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
 101         [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
 102         [HPD_PORT_D] = SDE_PORTD_HOTPLUG,
 103 };
 104
 105 static const u32 hpd_cpt[HPD_NUM_PINS] = {
 106         [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
 107         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
 108         [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
 109         [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
 110         [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
 111 };
 112
 113 static const u32 hpd_spt[HPD_NUM_PINS] = {
 114         [HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
 115         [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
 116         [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
 117         [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
 118         [HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT,
 119 };
 120
 121 static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
 122         [HPD_CRT] = CRT_HOTPLUG_INT_EN,
 123         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
 124         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
 125         [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
 126         [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
 127         [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN,
 128 };
 129
 130 static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
 131         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
 132         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
 133         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
 134         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
 135         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
 136         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS,
 137 };
 138
 139 static const u32 hpd_status_i915[HPD_NUM_PINS] = {
 140         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
 141         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
 142         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
 143         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
 144         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
 145         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS,
 146 };
 147
 148 static const u32 hpd_bxt[HPD_NUM_PINS] = {
 149         [HPD_PORT_A] = GEN8_DE_PORT_HOTPLUG(HPD_PORT_A),
 150         [HPD_PORT_B] = GEN8_DE_PORT_HOTPLUG(HPD_PORT_B),
 151         [HPD_PORT_C] = GEN8_DE_PORT_HOTPLUG(HPD_PORT_C),
 152 };
 153
 154 static const u32 hpd_gen11[HPD_NUM_PINS] = {
 155         [HPD_PORT_TC1] = GEN11_TC_HOTPLUG(HPD_PORT_TC1) | GEN11_TBT_HOTPLUG(HPD_PORT_TC1),
 156         [HPD_PORT_TC2] = GEN11_TC_HOTPLUG(HPD_PORT_TC2) | GEN11_TBT_HOTPLUG(HPD_PORT_TC2),
 157         [HPD_PORT_TC3] = GEN11_TC_HOTPLUG(HPD_PORT_TC3) | GEN11_TBT_HOTPLUG(HPD_PORT_TC3),
 158         [HPD_PORT_TC4] = GEN11_TC_HOTPLUG(HPD_PORT_TC4) | GEN11_TBT_HOTPLUG(HPD_PORT_TC4),
 159         [HPD_PORT_TC5] = GEN11_TC_HOTPLUG(HPD_PORT_TC5) | GEN11_TBT_HOTPLUG(HPD_PORT_TC5),
 160         [HPD_PORT_TC6] = GEN11_TC_HOTPLUG(HPD_PORT_TC6) | GEN11_TBT_HOTPLUG(HPD_PORT_TC6),
 161 };
 162
 163 static const u32 hpd_icp[HPD_NUM_PINS] = {
 164         [HPD_PORT_A] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_A),
 165         [HPD_PORT_B] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_B),
 166         [HPD_PORT_C] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_C),
 167         [HPD_PORT_TC1] = SDE_TC_HOTPLUG_ICP(HPD_PORT_TC1),
 168         [HPD_PORT_TC2] = SDE_TC_HOTPLUG_ICP(HPD_PORT_TC2),
 169         [HPD_PORT_TC3] = SDE_TC_HOTPLUG_ICP(HPD_PORT_TC3),
 170         [HPD_PORT_TC4] = SDE_TC_HOTPLUG_ICP(HPD_PORT_TC4),
 171         [HPD_PORT_TC5] = SDE_TC_HOTPLUG_ICP(HPD_PORT_TC5),
 172         [HPD_PORT_TC6] = SDE_TC_HOTPLUG_ICP(HPD_PORT_TC6),
 173 };
 174
 175 static const u32 hpd_sde_dg1[HPD_NUM_PINS] = {
 176         [HPD_PORT_A] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_A),
 177         [HPD_PORT_B] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_B),
 178         [HPD_PORT_C] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_C),
 179         [HPD_PORT_D] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_D),
 180 };
 181
 182 static void intel_hpd_init_pins(struct drm_i915_private *dev_priv)
 183 {
 184         struct i915_hotplug *hpd = &dev_priv->hotplug;
 185
 186         if (HAS_GMCH(dev_priv)) {
 187                 if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
 188                     IS_CHERRYVIEW(dev_priv))
 189                         hpd->hpd = hpd_status_g4x;
 190                 else
 191                         hpd->hpd = hpd_status_i915;
 192                 return;
 193         }
 194
 195         if (INTEL_GEN(dev_priv) >= 11)
 196                 hpd->hpd = hpd_gen11;
 197         else if (IS_GEN9_LP(dev_priv))
 198                 hpd->hpd = hpd_bxt;
 199         else if (INTEL_GEN(dev_priv) >= 8)
 200                 hpd->hpd = hpd_bdw;
 201         else if (INTEL_GEN(dev_priv) >= 7)
 202                 hpd->hpd = hpd_ivb;
 203         else
 204                 hpd->hpd = hpd_ilk;
 205
 206         if ((INTEL_PCH_TYPE(dev_priv) < PCH_DG1) &&
 207             (!HAS_PCH_SPLIT(dev_priv) || HAS_PCH_NOP(dev_priv)))
 208                 return;
 209
 210         if (HAS_PCH_DG1(dev_priv))
 211                 hpd->pch_hpd = hpd_sde_dg1;
 212         else if (HAS_PCH_TGP(dev_priv) || HAS_PCH_JSP(dev_priv) ||
 213                  HAS_PCH_ICP(dev_priv) || HAS_PCH_MCC(dev_priv))
 214                 hpd->pch_hpd = hpd_icp;
 215         else if (HAS_PCH_CNP(dev_priv) || HAS_PCH_SPT(dev_priv))
 216                 hpd->pch_hpd = hpd_spt;
 217         else if (HAS_PCH_LPT(dev_priv) || HAS_PCH_CPT(dev_priv))
 218                 hpd->pch_hpd = hpd_cpt;
 219         else if (HAS_PCH_IBX(dev_priv))
 220                 hpd->pch_hpd = hpd_ibx;
 221         else
 222                 MISSING_CASE(INTEL_PCH_TYPE(dev_priv));
 223 }
 224
 225 static void
 226 intel_handle_vblank(struct drm_i915_private *dev_priv, enum pipe pipe)
 227 {
 228         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
 229
 230         drm_crtc_handle_vblank(&crtc->base);
 231 }
 232
 233 void gen3_irq_reset(struct intel_uncore *uncore, i915_reg_t imr,
 234                     i915_reg_t iir, i915_reg_t ier)
 235 {
 236         intel_uncore_write(uncore, imr, 0xffffffff);
 237         intel_uncore_posting_read(uncore, imr);
 238
 239         intel_uncore_write(uncore, ier, 0);
 240
 241         /* IIR can theoretically queue up two events. Be paranoid. */
 242         intel_uncore_write(uncore, iir, 0xffffffff);
 243         intel_uncore_posting_read(uncore, iir);
 244         intel_uncore_write(uncore, iir, 0xffffffff);
 245         intel_uncore_posting_read(uncore, iir);
 246 }
 247
 248 void gen2_irq_reset(struct intel_uncore *uncore)
 249 {
 250         intel_uncore_write16(uncore, GEN2_IMR, 0xffff);
 251         intel_uncore_posting_read16(uncore, GEN2_IMR);
 252
 253         intel_uncore_write16(uncore, GEN2_IER, 0);
 254
 255         /* IIR can theoretically queue up two events. Be paranoid. */
 256         intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
 257         intel_uncore_posting_read16(uncore, GEN2_IIR);
 258         intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
 259         intel_uncore_posting_read16(uncore, GEN2_IIR);
 260 }
 261
 262 /*
 263  * We should clear IMR at preinstall/uninstall, and just check at postinstall.
 264  */
 265 static void gen3_assert_iir_is_zero(struct intel_uncore *uncore, i915_reg_t reg)
 266 {
 267         u32 val = intel_uncore_read(uncore, reg);
 268
 269         if (val == 0)
 270                 return;
 271
 272         drm_WARN(&uncore->i915->drm, 1,
 273                  "Interrupt register 0x%x is not zero: 0x%08x\n",
 274                  i915_mmio_reg_offset(reg), val);
 275         intel_uncore_write(uncore, reg, 0xffffffff);
 276         intel_uncore_posting_read(uncore, reg);
 277         intel_uncore_write(uncore, reg, 0xffffffff);
 278         intel_uncore_posting_read(uncore, reg);
 279 }
 280
 281 static void gen2_assert_iir_is_zero(struct intel_uncore *uncore)
 282 {
 283         u16 val = intel_uncore_read16(uncore, GEN2_IIR);
 284
 285         if (val == 0)
 286                 return;
 287
 288         drm_WARN(&uncore->i915->drm, 1,
 289                  "Interrupt register 0x%x is not zero: 0x%08x\n",
 290                  i915_mmio_reg_offset(GEN2_IIR), val);
 291         intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
 292         intel_uncore_posting_read16(uncore, GEN2_IIR);
 293         intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
 294         intel_uncore_posting_read16(uncore, GEN2_IIR);
 295 }
 296
 297 void gen3_irq_init(struct intel_uncore *uncore,
 298                    i915_reg_t imr, u32 imr_val,
 299                    i915_reg_t ier, u32 ier_val,
 300                    i915_reg_t iir)
 301 {
 302         gen3_assert_iir_is_zero(uncore, iir);
 303
 304         intel_uncore_write(uncore, ier, ier_val);
 305         intel_uncore_write(uncore, imr, imr_val);
 306         intel_uncore_posting_read(uncore, imr);
 307 }
 308
 309 void gen2_irq_init(struct intel_uncore *uncore,
 310                    u32 imr_val, u32 ier_val)
 311 {
 312         gen2_assert_iir_is_zero(uncore);
 313
 314         intel_uncore_write16(uncore, GEN2_IER, ier_val);
 315         intel_uncore_write16(uncore, GEN2_IMR, imr_val);
 316         intel_uncore_posting_read16(uncore, GEN2_IMR);
 317 }
 318
 319 /* For display hotplug interrupt */
 320 static inline void
 321 i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
 322                                      u32 mask,
 323                                      u32 bits)
 324 {
 325         u32 val;
 326
 327         lockdep_assert_held(&dev_priv->irq_lock);
 328         drm_WARN_ON(&dev_priv->drm, bits & ~mask);
 329
 330         val = I915_READ(PORT_HOTPLUG_EN);
 331         val &= ~mask;
 332         val |= bits;
 333         I915_WRITE(PORT_HOTPLUG_EN, val);
 334 }
 335
 336 /**
 337  * i915_hotplug_interrupt_update - update hotplug interrupt enable
 338  * @dev_priv: driver private
 339  * @mask: bits to update
 340  * @bits: bits to enable
 341  * NOTE: the HPD enable bits are modified both inside and outside
 342  * of an interrupt context. To avoid that read-modify-write cycles
 343  * interfer, these bits are protected by a spinlock. Since this
 344  * function is usually not called from a context where the lock is
 345  * held already, this function acquires the lock itself. A non-locking
 346  * version is also available.
 347  */
 348 void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
 349                                    u32 mask,
 350                                    u32 bits)
 351 {
 352         spin_lock_irq(&dev_priv->irq_lock);
 353         i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
 354         spin_unlock_irq(&dev_priv->irq_lock);
 355 }
 356
 357 /**
 358  * ilk_update_display_irq - update DEIMR
 359  * @dev_priv: driver private
 360  * @interrupt_mask: mask of interrupt bits to update
 361  * @enabled_irq_mask: mask of interrupt bits to enable
 362  */
 363 void ilk_update_display_irq(struct drm_i915_private *dev_priv,
 364                             u32 interrupt_mask,
 365                             u32 enabled_irq_mask)
 366 {
 367         u32 new_val;
 368
 369         lockdep_assert_held(&dev_priv->irq_lock);
 370         drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
 371
 372         new_val = dev_priv->irq_mask;
 373         new_val &= ~interrupt_mask;
 374         new_val |= (~enabled_irq_mask & interrupt_mask);
 375
 376         if (new_val != dev_priv->irq_mask &&
 377             !drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv))) {
 378                 dev_priv->irq_mask = new_val;
 379                 I915_WRITE(DEIMR, dev_priv->irq_mask);
 380                 POSTING_READ(DEIMR);
 381         }
 382 }
 383
 384 /**
 385  * bdw_update_port_irq - update DE port interrupt
 386  * @dev_priv: driver private
 387  * @interrupt_mask: mask of interrupt bits to update
 388  * @enabled_irq_mask: mask of interrupt bits to enable
 389  */
 390 static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
 391                                 u32 interrupt_mask,
 392                                 u32 enabled_irq_mask)
 393 {
 394         u32 new_val;
 395         u32 old_val;
 396
 397         lockdep_assert_held(&dev_priv->irq_lock);
 398
 399         drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
 400
 401         if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
 402                 return;
 403
 404         old_val = I915_READ(GEN8_DE_PORT_IMR);
 405
 406         new_val = old_val;
 407         new_val &= ~interrupt_mask;
 408         new_val |= (~enabled_irq_mask & interrupt_mask);
 409
 410         if (new_val != old_val) {
 411                 I915_WRITE(GEN8_DE_PORT_IMR, new_val);
 412                 POSTING_READ(GEN8_DE_PORT_IMR);
 413         }
 414 }
 415
 416 /**
 417  * bdw_update_pipe_irq - update DE pipe interrupt
 418  * @dev_priv: driver private
 419  * @pipe: pipe whose interrupt to update
 420  * @interrupt_mask: mask of interrupt bits to update
 421  * @enabled_irq_mask: mask of interrupt bits to enable
 422  */
 423 void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
 424                          enum pipe pipe,
 425                          u32 interrupt_mask,
 426                          u32 enabled_irq_mask)
 427 {
 428         u32 new_val;
 429
 430         lockdep_assert_held(&dev_priv->irq_lock);
 431
 432         drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
 433
 434         if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
 435                 return;
 436
 437         new_val = dev_priv->de_irq_mask[pipe];
 438         new_val &= ~interrupt_mask;
 439         new_val |= (~enabled_irq_mask & interrupt_mask);
 440
 441         if (new_val != dev_priv->de_irq_mask[pipe]) {
 442                 dev_priv->de_irq_mask[pipe] = new_val;
 443                 I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
 444                 POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
 445         }
 446 }
 447
 448 /**
 449  * ibx_display_interrupt_update - update SDEIMR
 450  * @dev_priv: driver private
 451  * @interrupt_mask: mask of interrupt bits to update
 452  * @enabled_irq_mask: mask of interrupt bits to enable
 453  */
 454 void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
 455                                   u32 interrupt_mask,
 456                                   u32 enabled_irq_mask)
 457 {
 458         u32 sdeimr = I915_READ(SDEIMR);
 459         sdeimr &= ~interrupt_mask;
 460         sdeimr |= (~enabled_irq_mask & interrupt_mask);
 461
 462         drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
 463
 464         lockdep_assert_held(&dev_priv->irq_lock);
 465
 466         if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
 467                 return;
 468
 469         I915_WRITE(SDEIMR, sdeimr);
 470         POSTING_READ(SDEIMR);
 471 }
 472
 473 u32 i915_pipestat_enable_mask(struct drm_i915_private *dev_priv,
 474                               enum pipe pipe)
 475 {
 476         u32 status_mask = dev_priv->pipestat_irq_mask[pipe];
 477         u32 enable_mask = status_mask << 16;
 478
 479         lockdep_assert_held(&dev_priv->irq_lock);
 480
 481         if (INTEL_GEN(dev_priv) < 5)
 482                 goto out;
 483
 484         /*
 485          * On pipe A we don't support the PSR interrupt yet,
 486          * on pipe B and C the same bit MBZ.
 487          */
 488         if (drm_WARN_ON_ONCE(&dev_priv->drm,
 489                              status_mask & PIPE_A_PSR_STATUS_VLV))
 490                 return 0;
 491         /*
 492          * On pipe B and C we don't support the PSR interrupt yet, on pipe
 493          * A the same bit is for perf counters which we don't use either.
 494          */
 495         if (drm_WARN_ON_ONCE(&dev_priv->drm,
 496                              status_mask & PIPE_B_PSR_STATUS_VLV))
 497                 return 0;
 498
 499         enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
 500                          SPRITE0_FLIP_DONE_INT_EN_VLV |
 501                          SPRITE1_FLIP_DONE_INT_EN_VLV);
 502         if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
 503                 enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
 504         if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
 505                 enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
 506
 507 out:
 508         drm_WARN_ONCE(&dev_priv->drm,
 509                       enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
 510                       status_mask & ~PIPESTAT_INT_STATUS_MASK,
 511                       "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
 512                       pipe_name(pipe), enable_mask, status_mask);
 513
 514         return enable_mask;
 515 }
 516
 517 void i915_enable_pipestat(struct drm_i915_private *dev_priv,
 518                           enum pipe pipe, u32 status_mask)
 519 {
 520         i915_reg_t reg = PIPESTAT(pipe);
 521         u32 enable_mask;
 522
 523         drm_WARN_ONCE(&dev_priv->drm, status_mask & ~PIPESTAT_INT_STATUS_MASK,
 524                       "pipe %c: status_mask=0x%x\n",
 525                       pipe_name(pipe), status_mask);
 526
 527         lockdep_assert_held(&dev_priv->irq_lock);
 528         drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv));
 529
 530         if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == status_mask)
 531                 return;
 532
 533         dev_priv->pipestat_irq_mask[pipe] |= status_mask;
 534         enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
 535
 536         I915_WRITE(reg, enable_mask | status_mask);
 537         POSTING_READ(reg);
 538 }
 539
 540 void i915_disable_pipestat(struct drm_i915_private *dev_priv,
 541                            enum pipe pipe, u32 status_mask)
 542 {
 543         i915_reg_t reg = PIPESTAT(pipe);
 544         u32 enable_mask;
 545
 546         drm_WARN_ONCE(&dev_priv->drm, status_mask & ~PIPESTAT_INT_STATUS_MASK,
 547                       "pipe %c: status_mask=0x%x\n",
 548                       pipe_name(pipe), status_mask);
 549
 550         lockdep_assert_held(&dev_priv->irq_lock);
 551         drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv));
 552
 553         if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == 0)
 554                 return;
 555
 556         dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
 557         enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
 558
 559         I915_WRITE(reg, enable_mask | status_mask);
 560         POSTING_READ(reg);
 561 }
 562
 563 static bool i915_has_asle(struct drm_i915_private *dev_priv)
 564 {
 565         if (!dev_priv->opregion.asle)
 566                 return false;
 567
 568         return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
 569 }
 570
 571 /**
 572  * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
 573  * @dev_priv: i915 device private
 574  */
 575 static void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv)
 576 {
 577         if (!i915_has_asle(dev_priv))
 578                 return;
 579
 580         spin_lock_irq(&dev_priv->irq_lock);
 581
 582         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
 583         if (INTEL_GEN(dev_priv) >= 4)
 584                 i915_enable_pipestat(dev_priv, PIPE_A,
 585                                      PIPE_LEGACY_BLC_EVENT_STATUS);
 586
 587         spin_unlock_irq(&dev_priv->irq_lock);
 588 }
 589
 590 /*
 591  * This timing diagram depicts the video signal in and
 592  * around the vertical blanking period.
 593  *
 594  * Assumptions about the fictitious mode used in this example:
 595  *  vblank_start >= 3
 596  *  vsync_start = vblank_start + 1
 597  *  vsync_end = vblank_start + 2
 598  *  vtotal = vblank_start + 3
 599  *
 600  *           start of vblank:
 601  *           latch double buffered registers
 602  *           increment frame counter (ctg+)
 603  *           generate start of vblank interrupt (gen4+)
 604  *           |
 605  *           |          frame start:
 606  *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
 607  *           |          may be shifted forward 1-3 extra lines via PIPECONF
 608  *           |          |
 609  *           |          |  start of vsync:
 610  *           |          |  generate vsync interrupt
 611  *           |          |  |
 612  * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
 613  *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
 614  * ----va---> <-----------------vb--------------------> <--------va-------------
 615  *       |          |       <----vs----->                     |
 616  * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
 617  * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
 618  * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
 619  *       |          |                                         |
 620  *       last visible pixel                                   first visible pixel
 621  *                  |                                         increment frame counter (gen3/4)
 622  *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
 623  *
 624  * x  = horizontal active
 625  * _  = horizontal blanking
 626  * hs = horizontal sync
 627  * va = vertical active
 628  * vb = vertical blanking
 629  * vs = vertical sync
 630  * vbs = vblank_start (number)
 631  *
 632  * Summary:
 633  * - most events happen at the start of horizontal sync
 634  * - frame start happens at the start of horizontal blank, 1-4 lines
 635  *   (depending on PIPECONF settings) after the start of vblank
 636  * - gen3/4 pixel and frame counter are synchronized with the start
 637  *   of horizontal active on the first line of vertical active
 638  */
 639
 640 /* Called from drm generic code, passed a 'crtc', which
 641  * we use as a pipe index
 642  */
 643 u32 i915_get_vblank_counter(struct drm_crtc *crtc)
 644 {
 645         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
 646         struct drm_vblank_crtc *vblank = &dev_priv->drm.vblank[drm_crtc_index(crtc)];
 647         const struct drm_display_mode *mode = &vblank->hwmode;
 648         enum pipe pipe = to_intel_crtc(crtc)->pipe;
 649         i915_reg_t high_frame, low_frame;
 650         u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
 651         unsigned long irqflags;
 652
 653         /*
 654          * On i965gm TV output the frame counter only works up to
 655          * the point when we enable the TV encoder. After that the
 656          * frame counter ceases to work and reads zero. We need a
 657          * vblank wait before enabling the TV encoder and so we
 658          * have to enable vblank interrupts while the frame counter
 659          * is still in a working state. However the core vblank code
 660          * does not like us returning non-zero frame counter values
 661          * when we've told it that we don't have a working frame
 662          * counter. Thus we must stop non-zero values leaking out.
 663          */
 664         if (!vblank->max_vblank_count)
 665                 return 0;
 666
 667         htotal = mode->crtc_htotal;
 668         hsync_start = mode->crtc_hsync_start;
 669         vbl_start = mode->crtc_vblank_start;
 670         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 671                 vbl_start = DIV_ROUND_UP(vbl_start, 2);
 672
 673         /* Convert to pixel count */
 674         vbl_start *= htotal;
 675
 676         /* Start of vblank event occurs at start of hsync */
 677         vbl_start -= htotal - hsync_start;
 678
 679         high_frame = PIPEFRAME(pipe);
 680         low_frame = PIPEFRAMEPIXEL(pipe);
 681
 682         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 683
 684         /*
 685          * High & low register fields aren't synchronized, so make sure
 686          * we get a low value that's stable across two reads of the high
 687          * register.
 688          */
 689         do {
 690                 high1 = intel_de_read_fw(dev_priv, high_frame) & PIPE_FRAME_HIGH_MASK;
 691                 low   = intel_de_read_fw(dev_priv, low_frame);
 692                 high2 = intel_de_read_fw(dev_priv, high_frame) & PIPE_FRAME_HIGH_MASK;
 693         } while (high1 != high2);
 694
 695         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 696
 697         high1 >>= PIPE_FRAME_HIGH_SHIFT;
 698         pixel = low & PIPE_PIXEL_MASK;
 699         low >>= PIPE_FRAME_LOW_SHIFT;
 700
 701         /*
 702          * The frame counter increments at beginning of active.
 703          * Cook up a vblank counter by also checking the pixel
 704          * counter against vblank start.
 705          */
 706         return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
 707 }
 708
 709 u32 g4x_get_vblank_counter(struct drm_crtc *crtc)
 710 {
 711         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
 712         struct drm_vblank_crtc *vblank = &dev_priv->drm.vblank[drm_crtc_index(crtc)];
 713         enum pipe pipe = to_intel_crtc(crtc)->pipe;
 714
 715         if (!vblank->max_vblank_count)
 716                 return 0;
 717
 718         return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
 719 }
 720
 721 /*
 722  * On certain encoders on certain platforms, pipe
 723  * scanline register will not work to get the scanline,
 724  * since the timings are driven from the PORT or issues
 725  * with scanline register updates.
 726  * This function will use Framestamp and current
 727  * timestamp registers to calculate the scanline.
 728  */
 729 static u32 __intel_get_crtc_scanline_from_timestamp(struct intel_crtc *crtc)
 730 {
 731         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 732         struct drm_vblank_crtc *vblank =
 733                 &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)];
 734         const struct drm_display_mode *mode = &vblank->hwmode;
 735         u32 vblank_start = mode->crtc_vblank_start;
 736         u32 vtotal = mode->crtc_vtotal;
 737         u32 htotal = mode->crtc_htotal;
 738         u32 clock = mode->crtc_clock;
 739         u32 scanline, scan_prev_time, scan_curr_time, scan_post_time;
 740
 741         /*
 742          * To avoid the race condition where we might cross into the
 743          * next vblank just between the PIPE_FRMTMSTMP and TIMESTAMP_CTR
 744          * reads. We make sure we read PIPE_FRMTMSTMP and TIMESTAMP_CTR
 745          * during the same frame.
 746          */
 747         do {
 748                 /*
 749                  * This field provides read back of the display
 750                  * pipe frame time stamp. The time stamp value
 751                  * is sampled at every start of vertical blank.
 752                  */
 753                 scan_prev_time = intel_de_read_fw(dev_priv,
 754                                                   PIPE_FRMTMSTMP(crtc->pipe));
 755
 756                 /*
 757                  * The TIMESTAMP_CTR register has the current
 758                  * time stamp value.
 759                  */
 760                 scan_curr_time = intel_de_read_fw(dev_priv, IVB_TIMESTAMP_CTR);
 761
 762                 scan_post_time = intel_de_read_fw(dev_priv,
 763                                                   PIPE_FRMTMSTMP(crtc->pipe));
 764         } while (scan_post_time != scan_prev_time);
 765
 766         scanline = div_u64(mul_u32_u32(scan_curr_time - scan_prev_time,
 767                                         clock), 1000 * htotal);
 768         scanline = min(scanline, vtotal - 1);
 769         scanline = (scanline + vblank_start) % vtotal;
 770
 771         return scanline;
 772 }
 773
 774 /*
 775  * intel_de_read_fw(), only for fast reads of display block, no need for
 776  * forcewake etc.
 777  */
 778 static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
 779 {
 780         struct drm_device *dev = crtc->base.dev;
 781         struct drm_i915_private *dev_priv = to_i915(dev);
 782         const struct drm_display_mode *mode;
 783         struct drm_vblank_crtc *vblank;
 784         enum pipe pipe = crtc->pipe;
 785         int position, vtotal;
 786
 787         if (!crtc->active)
 788                 return -1;
 789
 790         vblank = &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)];
 791         mode = &vblank->hwmode;
 792
 793         if (crtc->mode_flags & I915_MODE_FLAG_GET_SCANLINE_FROM_TIMESTAMP)
 794                 return __intel_get_crtc_scanline_from_timestamp(crtc);
 795
 796         vtotal = mode->crtc_vtotal;
 797         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 798                 vtotal /= 2;
 799
 800         if (IS_GEN(dev_priv, 2))
 801                 position = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
 802         else
 803                 position = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
 804
 805         /*
 806          * On HSW, the DSL reg (0x70000) appears to return 0 if we
 807          * read it just before the start of vblank.  So try it again
 808          * so we don't accidentally end up spanning a vblank frame
 809          * increment, causing the pipe_update_end() code to squak at us.
 810          *
 811          * The nature of this problem means we can't simply check the ISR
 812          * bit and return the vblank start value; nor can we use the scanline
 813          * debug register in the transcoder as it appears to have the same
 814          * problem.  We may need to extend this to include other platforms,
 815          * but so far testing only shows the problem on HSW.
 816          */
 817         if (HAS_DDI(dev_priv) && !position) {
 818                 int i, temp;
 819
 820                 for (i = 0; i < 100; i++) {
 821                         udelay(1);
 822                         temp = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
 823                         if (temp != position) {
 824                                 position = temp;
 825                                 break;
 826                         }
 827                 }
 828         }
 829
 830         /*
 831          * See update_scanline_offset() for the details on the
 832          * scanline_offset adjustment.
 833          */
 834         return (position + crtc->scanline_offset) % vtotal;
 835 }
 836
 837 static bool i915_get_crtc_scanoutpos(struct drm_crtc *_crtc,
 838                                      bool in_vblank_irq,
 839                                      int *vpos, int *hpos,
 840                                      ktime_t *stime, ktime_t *etime,
 841                                      const struct drm_display_mode *mode)
 842 {
 843         struct drm_device *dev = _crtc->dev;
 844         struct drm_i915_private *dev_priv = to_i915(dev);
 845         struct intel_crtc *crtc = to_intel_crtc(_crtc);
 846         enum pipe pipe = crtc->pipe;
 847         int position;
 848         int vbl_start, vbl_end, hsync_start, htotal, vtotal;
 849         unsigned long irqflags;
 850         bool use_scanline_counter = INTEL_GEN(dev_priv) >= 5 ||
 851                 IS_G4X(dev_priv) || IS_GEN(dev_priv, 2) ||
 852                 crtc->mode_flags & I915_MODE_FLAG_USE_SCANLINE_COUNTER;
 853
 854         if (drm_WARN_ON(&dev_priv->drm, !mode->crtc_clock)) {
 855                 drm_dbg(&dev_priv->drm,
 856                         "trying to get scanoutpos for disabled "
 857                         "pipe %c\n", pipe_name(pipe));
 858                 return false;
 859         }
 860
 861         htotal = mode->crtc_htotal;
 862         hsync_start = mode->crtc_hsync_start;
 863         vtotal = mode->crtc_vtotal;
 864         vbl_start = mode->crtc_vblank_start;
 865         vbl_end = mode->crtc_vblank_end;
 866
 867         if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
 868                 vbl_start = DIV_ROUND_UP(vbl_start, 2);
 869                 vbl_end /= 2;
 870                 vtotal /= 2;
 871         }
 872
 873         /*
 874          * Lock uncore.lock, as we will do multiple timing critical raw
 875          * register reads, potentially with preemption disabled, so the
 876          * following code must not block on uncore.lock.
 877          */
 878         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 879
 880         /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
 881
 882         /* Get optional system timestamp before query. */
 883         if (stime)
 884                 *stime = ktime_get();
 885
 886         if (use_scanline_counter) {
 887                 /* No obvious pixelcount register. Only query vertical
 888                  * scanout position from Display scan line register.
 889                  */
 890                 position = __intel_get_crtc_scanline(crtc);
 891         } else {
 892                 /* Have access to pixelcount since start of frame.
 893                  * We can split this into vertical and horizontal
 894                  * scanout position.
 895                  */
 896                 position = (intel_de_read_fw(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
 897
 898                 /* convert to pixel counts */
 899                 vbl_start *= htotal;
 900                 vbl_end *= htotal;
 901                 vtotal *= htotal;
 902
 903                 /*
 904                  * In interlaced modes, the pixel counter counts all pixels,
 905                  * so one field will have htotal more pixels. In order to avoid
 906                  * the reported position from jumping backwards when the pixel
 907                  * counter is beyond the length of the shorter field, just
 908                  * clamp the position the length of the shorter field. This
 909                  * matches how the scanline counter based position works since
 910                  * the scanline counter doesn't count the two half lines.
 911                  */
 912                 if (position >= vtotal)
 913                         position = vtotal - 1;
 914
 915                 /*
 916                  * Start of vblank interrupt is triggered at start of hsync,
 917                  * just prior to the first active line of vblank. However we
 918                  * consider lines to start at the leading edge of horizontal
 919                  * active. So, should we get here before we've crossed into
 920                  * the horizontal active of the first line in vblank, we would
 921                  * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
 922                  * always add htotal-hsync_start to the current pixel position.
 923                  */
 924                 position = (position + htotal - hsync_start) % vtotal;
 925         }
 926
 927         /* Get optional system timestamp after query. */
 928         if (etime)
 929                 *etime = ktime_get();
 930
 931         /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
 932
 933         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 934
 935         /*
 936          * While in vblank, position will be negative
 937          * counting up towards 0 at vbl_end. And outside
 938          * vblank, position will be positive counting
 939          * up since vbl_end.
 940          */
 941         if (position >= vbl_start)
 942                 position -= vbl_end;
 943         else
 944                 position += vtotal - vbl_end;
 945
 946         if (use_scanline_counter) {
 947                 *vpos = position;
 948                 *hpos = 0;
 949         } else {
 950                 *vpos = position / htotal;
 951                 *hpos = position - (*vpos * htotal);
 952         }
 953
 954         return true;
 955 }
 956
 957 bool intel_crtc_get_vblank_timestamp(struct drm_crtc *crtc, int *max_error,
 958                                      ktime_t *vblank_time, bool in_vblank_irq)
 959 {
 960         return drm_crtc_vblank_helper_get_vblank_timestamp_internal(
 961                 crtc, max_error, vblank_time, in_vblank_irq,
 962                 i915_get_crtc_scanoutpos);
 963 }
 964
 965 int intel_get_crtc_scanline(struct intel_crtc *crtc)
 966 {
 967         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 968         unsigned long irqflags;
 969         int position;
 970
 971         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 972         position = __intel_get_crtc_scanline(crtc);
 973         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 974
 975         return position;
 976 }
 977
 978 /**
 979  * ivb_parity_work - Workqueue called when a parity error interrupt
 980  * occurred.
 981  * @work: workqueue struct
 982  *
 983  * Doesn't actually do anything except notify userspace. As a consequence of
 984  * this event, userspace should try to remap the bad rows since statistically
 985  * it is likely the same row is more likely to go bad again.
 986  */
 987 static void ivb_parity_work(struct work_struct *work)
 988 {
 989         struct drm_i915_private *dev_priv =
 990                 container_of(work, typeof(*dev_priv), l3_parity.error_work);
 991         struct intel_gt *gt = &dev_priv->gt;
 992         u32 error_status, row, bank, subbank;
 993         char *parity_event[6];
 994         u32 misccpctl;
 995         u8 slice = 0;
 996
 997         /* We must turn off DOP level clock gating to access the L3 registers.
 998          * In order to prevent a get/put style interface, acquire struct mutex
 999          * any time we access those registers.
1000          */
1001         mutex_lock(&dev_priv->drm.struct_mutex);
1002
1003         /* If we've screwed up tracking, just let the interrupt fire again */
1004         if (drm_WARN_ON(&dev_priv->drm, !dev_priv->l3_parity.which_slice))
1005                 goto out;
1006
1007         misccpctl = I915_READ(GEN7_MISCCPCTL);
1008         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
1009         POSTING_READ(GEN7_MISCCPCTL);
1010
1011         while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1012                 i915_reg_t reg;
1013
1014                 slice--;
1015                 if (drm_WARN_ON_ONCE(&dev_priv->drm,
1016                                      slice >= NUM_L3_SLICES(dev_priv)))
1017                         break;
1018
1019                 dev_priv->l3_parity.which_slice &= ~(1<<slice);
1020
1021                 reg = GEN7_L3CDERRST1(slice);
1022
1023                 error_status = I915_READ(reg);
1024                 row = GEN7_PARITY_ERROR_ROW(error_status);
1025                 bank = GEN7_PARITY_ERROR_BANK(error_status);
1026                 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
1027
1028                 I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
1029                 POSTING_READ(reg);
1030
1031                 parity_event[0] = I915_L3_PARITY_UEVENT "=1";
1032                 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
1033                 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
1034                 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
1035                 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
1036                 parity_event[5] = NULL;
1037
1038                 kobject_uevent_env(&dev_priv->drm.primary->kdev->kobj,
1039                                    KOBJ_CHANGE, parity_event);
1040
1041                 DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
1042                           slice, row, bank, subbank);
1043
1044                 kfree(parity_event[4]);
1045                 kfree(parity_event[3]);
1046                 kfree(parity_event[2]);
1047                 kfree(parity_event[1]);
1048         }
1049
1050         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1051
1052 out:
1053         drm_WARN_ON(&dev_priv->drm, dev_priv->l3_parity.which_slice);
1054         spin_lock_irq(&gt->irq_lock);
1055         gen5_gt_enable_irq(gt, GT_PARITY_ERROR(dev_priv));
1056         spin_unlock_irq(&gt->irq_lock);
1057
1058         mutex_unlock(&dev_priv->drm.struct_mutex);
1059 }
1060
1061 static bool gen11_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1062 {
1063         switch (pin) {
1064         case HPD_PORT_TC1:
1065                 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(HPD_PORT_TC1);
1066         case HPD_PORT_TC2:
1067                 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(HPD_PORT_TC2);
1068         case HPD_PORT_TC3:
1069                 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(HPD_PORT_TC3);
1070         case HPD_PORT_TC4:
1071                 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(HPD_PORT_TC4);
1072         case HPD_PORT_TC5:
1073                 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(HPD_PORT_TC5);
1074         case HPD_PORT_TC6:
1075                 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(HPD_PORT_TC6);
1076         default:
1077                 return false;
1078         }
1079 }
1080
1081 static bool bxt_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1082 {
1083         switch (pin) {
1084         case HPD_PORT_A:
1085                 return val & PORTA_HOTPLUG_LONG_DETECT;
1086         case HPD_PORT_B:
1087                 return val & PORTB_HOTPLUG_LONG_DETECT;
1088         case HPD_PORT_C:
1089                 return val & PORTC_HOTPLUG_LONG_DETECT;
1090         default:
1091                 return false;
1092         }
1093 }
1094
1095 static bool icp_ddi_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1096 {
1097         switch (pin) {
1098         case HPD_PORT_A:
1099                 return val & SHOTPLUG_CTL_DDI_HPD_LONG_DETECT(HPD_PORT_A);
1100         case HPD_PORT_B:
1101                 return val & SHOTPLUG_CTL_DDI_HPD_LONG_DETECT(HPD_PORT_B);
1102         case HPD_PORT_C:
1103                 return val & SHOTPLUG_CTL_DDI_HPD_LONG_DETECT(HPD_PORT_C);
1104         case HPD_PORT_D:
1105                 return val & SHOTPLUG_CTL_DDI_HPD_LONG_DETECT(HPD_PORT_D);
1106         default:
1107                 return false;
1108         }
1109 }
1110
1111 static bool icp_tc_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1112 {
1113         switch (pin) {
1114         case HPD_PORT_TC1:
1115                 return val & ICP_TC_HPD_LONG_DETECT(HPD_PORT_TC1);
1116         case HPD_PORT_TC2:
1117                 return val & ICP_TC_HPD_LONG_DETECT(HPD_PORT_TC2);
1118         case HPD_PORT_TC3:
1119                 return val & ICP_TC_HPD_LONG_DETECT(HPD_PORT_TC3);
1120         case HPD_PORT_TC4:
1121                 return val & ICP_TC_HPD_LONG_DETECT(HPD_PORT_TC4);
1122         case HPD_PORT_TC5:
1123                 return val & ICP_TC_HPD_LONG_DETECT(HPD_PORT_TC5);
1124         case HPD_PORT_TC6:
1125                 return val & ICP_TC_HPD_LONG_DETECT(HPD_PORT_TC6);
1126         default:
1127                 return false;
1128         }
1129 }
1130
1131 static bool spt_port_hotplug2_long_detect(enum hpd_pin pin, u32 val)
1132 {
1133         switch (pin) {
1134         case HPD_PORT_E:
1135                 return val & PORTE_HOTPLUG_LONG_DETECT;
1136         default:
1137                 return false;
1138         }
1139 }
1140
1141 static bool spt_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1142 {
1143         switch (pin) {
1144         case HPD_PORT_A:
1145                 return val & PORTA_HOTPLUG_LONG_DETECT;
1146         case HPD_PORT_B:
1147                 return val & PORTB_HOTPLUG_LONG_DETECT;
1148         case HPD_PORT_C:
1149                 return val & PORTC_HOTPLUG_LONG_DETECT;
1150         case HPD_PORT_D:
1151                 return val & PORTD_HOTPLUG_LONG_DETECT;
1152         default:
1153                 return false;
1154         }
1155 }
1156
1157 static bool ilk_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1158 {
1159         switch (pin) {
1160         case HPD_PORT_A:
1161                 return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
1162         default:
1163                 return false;
1164         }
1165 }
1166
1167 static bool pch_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1168 {
1169         switch (pin) {
1170         case HPD_PORT_B:
1171                 return val & PORTB_HOTPLUG_LONG_DETECT;
1172         case HPD_PORT_C:
1173                 return val & PORTC_HOTPLUG_LONG_DETECT;
1174         case HPD_PORT_D:
1175                 return val & PORTD_HOTPLUG_LONG_DETECT;
1176         default:
1177                 return false;
1178         }
1179 }
1180
1181 static bool i9xx_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1182 {
1183         switch (pin) {
1184         case HPD_PORT_B:
1185                 return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1186         case HPD_PORT_C:
1187                 return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1188         case HPD_PORT_D:
1189                 return val & PORTD_HOTPLUG_INT_LONG_PULSE;
1190         default:
1191                 return false;
1192         }
1193 }
1194
1195 /*
1196  * Get a bit mask of pins that have triggered, and which ones may be long.
1197  * This can be called multiple times with the same masks to accumulate
1198  * hotplug detection results from several registers.
1199  *
1200  * Note that the caller is expected to zero out the masks initially.
1201  */
1202 static void intel_get_hpd_pins(struct drm_i915_private *dev_priv,
1203                                u32 *pin_mask, u32 *long_mask,
1204                                u32 hotplug_trigger, u32 dig_hotplug_reg,
1205                                const u32 hpd[HPD_NUM_PINS],
1206                                bool long_pulse_detect(enum hpd_pin pin, u32 val))
1207 {
1208         enum hpd_pin pin;
1209
1210         BUILD_BUG_ON(BITS_PER_TYPE(*pin_mask) < HPD_NUM_PINS);
1211
1212         for_each_hpd_pin(pin) {
1213                 if ((hpd[pin] & hotplug_trigger) == 0)
1214                         continue;
1215
1216                 *pin_mask |= BIT(pin);
1217
1218                 if (long_pulse_detect(pin, dig_hotplug_reg))
1219                         *long_mask |= BIT(pin);
1220         }
1221
1222         drm_dbg(&dev_priv->drm,
1223                 "hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x, long 0x%08x\n",
1224                 hotplug_trigger, dig_hotplug_reg, *pin_mask, *long_mask);
1225
1226 }
1227
1228 static u32 intel_hpd_enabled_irqs(struct drm_i915_private *dev_priv,
1229                                   const u32 hpd[HPD_NUM_PINS])
1230 {
1231         struct intel_encoder *encoder;
1232         u32 enabled_irqs = 0;
1233
1234         for_each_intel_encoder(&dev_priv->drm, encoder)
1235                 if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
1236                         enabled_irqs |= hpd[encoder->hpd_pin];
1237
1238         return enabled_irqs;
1239 }
1240
1241 static u32 intel_hpd_hotplug_irqs(struct drm_i915_private *dev_priv,
1242                                   const u32 hpd[HPD_NUM_PINS])
1243 {
1244         struct intel_encoder *encoder;
1245         u32 hotplug_irqs = 0;
1246
1247         for_each_intel_encoder(&dev_priv->drm, encoder)
1248                 hotplug_irqs |= hpd[encoder->hpd_pin];
1249
1250         return hotplug_irqs;
1251 }
1252
1253 static u32 intel_hpd_hotplug_enables(struct drm_i915_private *i915,
1254                                      hotplug_enables_func hotplug_enables)
1255 {
1256         struct intel_encoder *encoder;
1257         u32 hotplug = 0;
1258
1259         for_each_intel_encoder(&i915->drm, encoder)
1260                 hotplug |= hotplug_enables(i915, encoder->hpd_pin);
1261
1262         return hotplug;
1263 }
1264
1265 static void gmbus_irq_handler(struct drm_i915_private *dev_priv)
1266 {
1267         wake_up_all(&dev_priv->gmbus_wait_queue);
1268 }
1269
1270 static void dp_aux_irq_handler(struct drm_i915_private *dev_priv)
1271 {
1272         wake_up_all(&dev_priv->gmbus_wait_queue);
1273 }
1274
1275 #if defined(CONFIG_DEBUG_FS)
1276 static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1277                                          enum pipe pipe,
1278                                          u32 crc0, u32 crc1,
1279                                          u32 crc2, u32 crc3,
1280                                          u32 crc4)
1281 {
1282         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
1283         struct intel_pipe_crc *pipe_crc = &crtc->pipe_crc;
1284         u32 crcs[5] = { crc0, crc1, crc2, crc3, crc4 };
1285
1286         trace_intel_pipe_crc(crtc, crcs);
1287
1288         spin_lock(&pipe_crc->lock);
1289         /*
1290          * For some not yet identified reason, the first CRC is
1291          * bonkers. So let's just wait for the next vblank and read
1292          * out the buggy result.
1293          *
1294          * On GEN8+ sometimes the second CRC is bonkers as well, so
1295          * don't trust that one either.
1296          */
1297         if (pipe_crc->skipped <= 0 ||
1298             (INTEL_GEN(dev_priv) >= 8 && pipe_crc->skipped == 1)) {
1299                 pipe_crc->skipped++;
1300                 spin_unlock(&pipe_crc->lock);
1301                 return;
1302         }
1303         spin_unlock(&pipe_crc->lock);
1304
1305         drm_crtc_add_crc_entry(&crtc->base, true,
1306                                 drm_crtc_accurate_vblank_count(&crtc->base),
1307                                 crcs);
1308 }
1309 #else
1310 static inline void
1311 display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1312                              enum pipe pipe,
1313                              u32 crc0, u32 crc1,
1314                              u32 crc2, u32 crc3,
1315                              u32 crc4) {}
1316 #endif
1317
1318 static void flip_done_handler(struct drm_i915_private *i915,
1319                               enum pipe pipe)
1320 {
1321         struct intel_crtc *crtc = intel_get_crtc_for_pipe(i915, pipe);
1322         struct drm_crtc_state *crtc_state = crtc->base.state;
1323         struct drm_pending_vblank_event *e = crtc_state->event;
1324         struct drm_device *dev = &i915->drm;
1325         unsigned long irqflags;
1326
1327         spin_lock_irqsave(&dev->event_lock, irqflags);
1328
1329         crtc_state->event = NULL;
1330
1331         drm_crtc_send_vblank_event(&crtc->base, e);
1332
1333         spin_unlock_irqrestore(&dev->event_lock, irqflags);
1334 }
1335
1336 static void hsw_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1337                                      enum pipe pipe)
1338 {
1339         display_pipe_crc_irq_handler(dev_priv, pipe,
1340                                      I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1341                                      0, 0, 0, 0);
1342 }
1343
1344 static void ivb_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1345                                      enum pipe pipe)
1346 {
1347         display_pipe_crc_irq_handler(dev_priv, pipe,
1348                                      I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1349                                      I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
1350                                      I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
1351                                      I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
1352                                      I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1353 }
1354
1355 static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1356                                       enum pipe pipe)
1357 {
1358         u32 res1, res2;
1359
1360         if (INTEL_GEN(dev_priv) >= 3)
1361                 res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
1362         else
1363                 res1 = 0;
1364
1365         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1366                 res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
1367         else
1368                 res2 = 0;
1369
1370         display_pipe_crc_irq_handler(dev_priv, pipe,
1371                                      I915_READ(PIPE_CRC_RES_RED(pipe)),
1372                                      I915_READ(PIPE_CRC_RES_GREEN(pipe)),
1373                                      I915_READ(PIPE_CRC_RES_BLUE(pipe)),
1374                                      res1, res2);
1375 }
1376
1377 static void i9xx_pipestat_irq_reset(struct drm_i915_private *dev_priv)
1378 {
1379         enum pipe pipe;
1380
1381         for_each_pipe(dev_priv, pipe) {
1382                 I915_WRITE(PIPESTAT(pipe),
1383                            PIPESTAT_INT_STATUS_MASK |
1384                            PIPE_FIFO_UNDERRUN_STATUS);
1385
1386                 dev_priv->pipestat_irq_mask[pipe] = 0;
1387         }
1388 }
1389
1390 static void i9xx_pipestat_irq_ack(struct drm_i915_private *dev_priv,
1391                                   u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1392 {
1393         enum pipe pipe;
1394
1395         spin_lock(&dev_priv->irq_lock);
1396
1397         if (!dev_priv->display_irqs_enabled) {
1398                 spin_unlock(&dev_priv->irq_lock);
1399                 return;
1400         }
1401
1402         for_each_pipe(dev_priv, pipe) {
1403                 i915_reg_t reg;
1404                 u32 status_mask, enable_mask, iir_bit = 0;
1405
1406                 /*
1407                  * PIPESTAT bits get signalled even when the interrupt is
1408                  * disabled with the mask bits, and some of the status bits do
1409                  * not generate interrupts at all (like the underrun bit). Hence
1410                  * we need to be careful that we only handle what we want to
1411                  * handle.
1412                  */
1413
1414                 /* fifo underruns are filterered in the underrun handler. */
1415                 status_mask = PIPE_FIFO_UNDERRUN_STATUS;
1416
1417                 switch (pipe) {
1418                 default:
1419                 case PIPE_A:
1420                         iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
1421                         break;
1422                 case PIPE_B:
1423                         iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
1424                         break;
1425                 case PIPE_C:
1426                         iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
1427                         break;
1428                 }
1429                 if (iir & iir_bit)
1430                         status_mask |= dev_priv->pipestat_irq_mask[pipe];
1431
1432                 if (!status_mask)
1433                         continue;
1434
1435                 reg = PIPESTAT(pipe);
1436                 pipe_stats[pipe] = I915_READ(reg) & status_mask;
1437                 enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
1438
1439                 /*
1440                  * Clear the PIPE*STAT regs before the IIR
1441                  *
1442                  * Toggle the enable bits to make sure we get an
1443                  * edge in the ISR pipe event bit if we don't clear
1444                  * all the enabled status bits. Otherwise the edge
1445                  * triggered IIR on i965/g4x wouldn't notice that
1446                  * an interrupt is still pending.
1447                  */
1448                 if (pipe_stats[pipe]) {
1449                         I915_WRITE(reg, pipe_stats[pipe]);
1450                         I915_WRITE(reg, enable_mask);
1451                 }
1452         }
1453         spin_unlock(&dev_priv->irq_lock);
1454 }
1455
1456 static void i8xx_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1457                                       u16 iir, u32 pipe_stats[I915_MAX_PIPES])
1458 {
1459         enum pipe pipe;
1460
1461         for_each_pipe(dev_priv, pipe) {
1462                 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1463                         intel_handle_vblank(dev_priv, pipe);
1464
1465                 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1466                         i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1467
1468                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1469                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1470         }
1471 }
1472
1473 static void i915_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1474                                       u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1475 {
1476         bool blc_event = false;
1477         enum pipe pipe;
1478
1479         for_each_pipe(dev_priv, pipe) {
1480                 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1481                         intel_handle_vblank(dev_priv, pipe);
1482
1483                 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
1484                         blc_event = true;
1485
1486                 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1487                         i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1488
1489                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1490                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1491         }
1492
1493         if (blc_event || (iir & I915_ASLE_INTERRUPT))
1494                 intel_opregion_asle_intr(dev_priv);
1495 }
1496
1497 static void i965_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1498                                       u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1499 {
1500         bool blc_event = false;
1501         enum pipe pipe;
1502
1503         for_each_pipe(dev_priv, pipe) {
1504                 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
1505                         intel_handle_vblank(dev_priv, pipe);
1506
1507                 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
1508                         blc_event = true;
1509
1510                 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1511                         i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1512
1513                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1514                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1515         }
1516
1517         if (blc_event || (iir & I915_ASLE_INTERRUPT))
1518                 intel_opregion_asle_intr(dev_priv);
1519
1520         if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1521                 gmbus_irq_handler(dev_priv);
1522 }
1523
1524 static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1525                                             u32 pipe_stats[I915_MAX_PIPES])
1526 {
1527         enum pipe pipe;
1528
1529         for_each_pipe(dev_priv, pipe) {
1530                 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
1531                         intel_handle_vblank(dev_priv, pipe);
1532
1533                 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1534                         i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1535
1536                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1537                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1538         }
1539
1540         if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1541                 gmbus_irq_handler(dev_priv);
1542 }
1543
1544 static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv)
1545 {
1546         u32 hotplug_status = 0, hotplug_status_mask;
1547         int i;
1548
1549         if (IS_G4X(dev_priv) ||
1550             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1551                 hotplug_status_mask = HOTPLUG_INT_STATUS_G4X |
1552                         DP_AUX_CHANNEL_MASK_INT_STATUS_G4X;
1553         else
1554                 hotplug_status_mask = HOTPLUG_INT_STATUS_I915;
1555
1556         /*
1557          * We absolutely have to clear all the pending interrupt
1558          * bits in PORT_HOTPLUG_STAT. Otherwise the ISR port
1559          * interrupt bit won't have an edge, and the i965/g4x
1560          * edge triggered IIR will not notice that an interrupt
1561          * is still pending. We can't use PORT_HOTPLUG_EN to
1562          * guarantee the edge as the act of toggling the enable
1563          * bits can itself generate a new hotplug interrupt :(
1564          */
1565         for (i = 0; i < 10; i++) {
1566                 u32 tmp = I915_READ(PORT_HOTPLUG_STAT) & hotplug_status_mask;
1567
1568                 if (tmp == 0)
1569                         return hotplug_status;
1570
1571                 hotplug_status |= tmp;
1572                 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1573         }
1574
1575         drm_WARN_ONCE(&dev_priv->drm, 1,
1576                       "PORT_HOTPLUG_STAT did not clear (0x%08x)\n",
1577                       I915_READ(PORT_HOTPLUG_STAT));
1578
1579         return hotplug_status;
1580 }
1581
1582 static void i9xx_hpd_irq_handler(struct drm_i915_private *dev_priv,
1583                                  u32 hotplug_status)
1584 {
1585         u32 pin_mask = 0, long_mask = 0;
1586         u32 hotplug_trigger;
1587
1588         if (IS_G4X(dev_priv) ||
1589             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1590                 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1591         else
1592                 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1593
1594         if (hotplug_trigger) {
1595                 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1596                                    hotplug_trigger, hotplug_trigger,
1597                                    dev_priv->hotplug.hpd,
1598                                    i9xx_port_hotplug_long_detect);
1599
1600                 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1601         }
1602
1603         if ((IS_G4X(dev_priv) ||
1604              IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
1605             hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
1606                 dp_aux_irq_handler(dev_priv);
1607 }
1608
1609 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
1610 {
1611         struct drm_i915_private *dev_priv = arg;
1612         irqreturn_t ret = IRQ_NONE;
1613
1614         if (!intel_irqs_enabled(dev_priv))
1615                 return IRQ_NONE;
1616
1617         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
1618         disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1619
1620         do {
1621                 u32 iir, gt_iir, pm_iir;
1622                 u32 pipe_stats[I915_MAX_PIPES] = {};
1623                 u32 hotplug_status = 0;
1624                 u32 ier = 0;
1625
1626                 gt_iir = I915_READ(GTIIR);
1627                 pm_iir = I915_READ(GEN6_PMIIR);
1628                 iir = I915_READ(VLV_IIR);
1629
1630                 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1631                         break;
1632
1633                 ret = IRQ_HANDLED;
1634
1635                 /*
1636                  * Theory on interrupt generation, based on empirical evidence:
1637                  *
1638                  * x = ((VLV_IIR & VLV_IER) ||
1639                  *      (((GT_IIR & GT_IER) || (GEN6_PMIIR & GEN6_PMIER)) &&
1640                  *       (VLV_MASTER_IER & MASTER_INTERRUPT_ENABLE)));
1641                  *
1642                  * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
1643                  * Hence we clear MASTER_INTERRUPT_ENABLE and VLV_IER to
1644                  * guarantee the CPU interrupt will be raised again even if we
1645                  * don't end up clearing all the VLV_IIR, GT_IIR, GEN6_PMIIR
1646                  * bits this time around.
1647                  */
1648                 I915_WRITE(VLV_MASTER_IER, 0);
1649                 ier = I915_READ(VLV_IER);
1650                 I915_WRITE(VLV_IER, 0);
1651
1652                 if (gt_iir)
1653                         I915_WRITE(GTIIR, gt_iir);
1654                 if (pm_iir)
1655                         I915_WRITE(GEN6_PMIIR, pm_iir);
1656
1657                 if (iir & I915_DISPLAY_PORT_INTERRUPT)
1658                         hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1659
1660                 /* Call regardless, as some status bits might not be
1661                  * signalled in iir */
1662                 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1663
1664                 if (iir & (I915_LPE_PIPE_A_INTERRUPT |
1665                            I915_LPE_PIPE_B_INTERRUPT))
1666                         intel_lpe_audio_irq_handler(dev_priv);
1667
1668                 /*
1669                  * VLV_IIR is single buffered, and reflects the level
1670                  * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
1671                  */
1672                 if (iir)
1673                         I915_WRITE(VLV_IIR, iir);
1674
1675                 I915_WRITE(VLV_IER, ier);
1676                 I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
1677
1678                 if (gt_iir)
1679                         gen6_gt_irq_handler(&dev_priv->gt, gt_iir);
1680                 if (pm_iir)
1681                         gen6_rps_irq_handler(&dev_priv->gt.rps, pm_iir);
1682
1683                 if (hotplug_status)
1684                         i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1685
1686                 valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
1687         } while (0);
1688
1689         pmu_irq_stats(dev_priv, ret);
1690
1691         enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1692
1693         return ret;
1694 }
1695
1696 static irqreturn_t cherryview_irq_handler(int irq, void *arg)
1697 {
1698         struct drm_i915_private *dev_priv = arg;
1699         irqreturn_t ret = IRQ_NONE;
1700
1701         if (!intel_irqs_enabled(dev_priv))
1702                 return IRQ_NONE;
1703
1704         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
1705         disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1706
1707         do {
1708                 u32 master_ctl, iir;
1709                 u32 pipe_stats[I915_MAX_PIPES] = {};
1710                 u32 hotplug_status = 0;
1711                 u32 ier = 0;
1712
1713                 master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
1714                 iir = I915_READ(VLV_IIR);
1715
1716                 if (master_ctl == 0 && iir == 0)
1717                         break;
1718
1719                 ret = IRQ_HANDLED;
1720
1721                 /*
1722                  * Theory on interrupt generation, based on empirical evidence:
1723                  *
1724                  * x = ((VLV_IIR & VLV_IER) ||
1725                  *      ((GEN8_MASTER_IRQ & ~GEN8_MASTER_IRQ_CONTROL) &&
1726                  *       (GEN8_MASTER_IRQ & GEN8_MASTER_IRQ_CONTROL)));
1727                  *
1728                  * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
1729                  * Hence we clear GEN8_MASTER_IRQ_CONTROL and VLV_IER to
1730                  * guarantee the CPU interrupt will be raised again even if we
1731                  * don't end up clearing all the VLV_IIR and GEN8_MASTER_IRQ_CONTROL
1732                  * bits this time around.
1733                  */
1734                 I915_WRITE(GEN8_MASTER_IRQ, 0);
1735                 ier = I915_READ(VLV_IER);
1736                 I915_WRITE(VLV_IER, 0);
1737
1738                 gen8_gt_irq_handler(&dev_priv->gt, master_ctl);
1739
1740                 if (iir & I915_DISPLAY_PORT_INTERRUPT)
1741                         hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1742
1743                 /* Call regardless, as some status bits might not be
1744                  * signalled in iir */
1745                 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1746
1747                 if (iir & (I915_LPE_PIPE_A_INTERRUPT |
1748                            I915_LPE_PIPE_B_INTERRUPT |
1749                            I915_LPE_PIPE_C_INTERRUPT))
1750                         intel_lpe_audio_irq_handler(dev_priv);
1751
1752                 /*
1753                  * VLV_IIR is single buffered, and reflects the level
1754                  * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
1755                  */
1756                 if (iir)
1757                         I915_WRITE(VLV_IIR, iir);
1758
1759                 I915_WRITE(VLV_IER, ier);
1760                 I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
1761
1762                 if (hotplug_status)
1763                         i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1764
1765                 valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
1766         } while (0);
1767
1768         pmu_irq_stats(dev_priv, ret);
1769
1770         enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1771
1772         return ret;
1773 }
1774
1775 static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv,
1776                                 u32 hotplug_trigger)
1777 {
1778         u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
1779
1780         /*
1781          * Somehow the PCH doesn't seem to really ack the interrupt to the CPU
1782          * unless we touch the hotplug register, even if hotplug_trigger is
1783          * zero. Not acking leads to "The master control interrupt lied (SDE)!"
1784          * errors.
1785          */
1786         dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1787         if (!hotplug_trigger) {
1788                 u32 mask = PORTA_HOTPLUG_STATUS_MASK |
1789                         PORTD_HOTPLUG_STATUS_MASK |
1790                         PORTC_HOTPLUG_STATUS_MASK |
1791                         PORTB_HOTPLUG_STATUS_MASK;
1792                 dig_hotplug_reg &= ~mask;
1793         }
1794
1795         I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1796         if (!hotplug_trigger)
1797                 return;
1798
1799         intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1800                            hotplug_trigger, dig_hotplug_reg,
1801                            dev_priv->hotplug.pch_hpd,
1802                            pch_port_hotplug_long_detect);
1803
1804         intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1805 }
1806
1807 static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1808 {
1809         enum pipe pipe;
1810         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1811
1812         ibx_hpd_irq_handler(dev_priv, hotplug_trigger);
1813
1814         if (pch_iir & SDE_AUDIO_POWER_MASK) {
1815                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1816                                SDE_AUDIO_POWER_SHIFT);
1817                 drm_dbg(&dev_priv->drm, "PCH audio power change on port %d\n",
1818                         port_name(port));
1819         }
1820
1821         if (pch_iir & SDE_AUX_MASK)
1822                 dp_aux_irq_handler(dev_priv);
1823
1824         if (pch_iir & SDE_GMBUS)
1825                 gmbus_irq_handler(dev_priv);
1826
1827         if (pch_iir & SDE_AUDIO_HDCP_MASK)
1828                 drm_dbg(&dev_priv->drm, "PCH HDCP audio interrupt\n");
1829
1830         if (pch_iir & SDE_AUDIO_TRANS_MASK)
1831                 drm_dbg(&dev_priv->drm, "PCH transcoder audio interrupt\n");
1832
1833         if (pch_iir & SDE_POISON)
1834                 drm_err(&dev_priv->drm, "PCH poison interrupt\n");
1835
1836         if (pch_iir & SDE_FDI_MASK) {
1837                 for_each_pipe(dev_priv, pipe)
1838                         drm_dbg(&dev_priv->drm, "  pipe %c FDI IIR: 0x%08x\n",
1839                                 pipe_name(pipe),
1840                                 I915_READ(FDI_RX_IIR(pipe)));
1841         }
1842
1843         if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1844                 drm_dbg(&dev_priv->drm, "PCH transcoder CRC done interrupt\n");
1845
1846         if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1847                 drm_dbg(&dev_priv->drm,
1848                         "PCH transcoder CRC error interrupt\n");
1849
1850         if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1851                 intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_A);
1852
1853         if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1854                 intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_B);
1855 }
1856
1857 static void ivb_err_int_handler(struct drm_i915_private *dev_priv)
1858 {
1859         u32 err_int = I915_READ(GEN7_ERR_INT);
1860         enum pipe pipe;
1861
1862         if (err_int & ERR_INT_POISON)
1863                 drm_err(&dev_priv->drm, "Poison interrupt\n");
1864
1865         for_each_pipe(dev_priv, pipe) {
1866                 if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
1867                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1868
1869                 if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
1870                         if (IS_IVYBRIDGE(dev_priv))
1871                                 ivb_pipe_crc_irq_handler(dev_priv, pipe);
1872                         else
1873                                 hsw_pipe_crc_irq_handler(dev_priv, pipe);
1874                 }
1875         }
1876
1877         I915_WRITE(GEN7_ERR_INT, err_int);
1878 }
1879
1880 static void cpt_serr_int_handler(struct drm_i915_private *dev_priv)
1881 {
1882         u32 serr_int = I915_READ(SERR_INT);
1883         enum pipe pipe;
1884
1885         if (serr_int & SERR_INT_POISON)
1886                 drm_err(&dev_priv->drm, "PCH poison interrupt\n");
1887
1888         for_each_pipe(dev_priv, pipe)
1889                 if (serr_int & SERR_INT_TRANS_FIFO_UNDERRUN(pipe))
1890                         intel_pch_fifo_underrun_irq_handler(dev_priv, pipe);
1891
1892         I915_WRITE(SERR_INT, serr_int);
1893 }
1894
1895 static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1896 {
1897         enum pipe pipe;
1898         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1899
1900         ibx_hpd_irq_handler(dev_priv, hotplug_trigger);
1901
1902         if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1903                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1904                                SDE_AUDIO_POWER_SHIFT_CPT);
1905                 drm_dbg(&dev_priv->drm, "PCH audio power change on port %c\n",
1906                         port_name(port));
1907         }
1908
1909         if (pch_iir & SDE_AUX_MASK_CPT)
1910                 dp_aux_irq_handler(dev_priv);
1911
1912         if (pch_iir & SDE_GMBUS_CPT)
1913                 gmbus_irq_handler(dev_priv);
1914
1915         if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1916                 drm_dbg(&dev_priv->drm, "Audio CP request interrupt\n");
1917
1918         if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1919                 drm_dbg(&dev_priv->drm, "Audio CP change interrupt\n");
1920
1921         if (pch_iir & SDE_FDI_MASK_CPT) {
1922                 for_each_pipe(dev_priv, pipe)
1923                         drm_dbg(&dev_priv->drm, "  pipe %c FDI IIR: 0x%08x\n",
1924                                 pipe_name(pipe),
1925                                 I915_READ(FDI_RX_IIR(pipe)));
1926         }
1927
1928         if (pch_iir & SDE_ERROR_CPT)
1929                 cpt_serr_int_handler(dev_priv);
1930 }
1931
1932 static void icp_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1933 {
1934         u32 ddi_hotplug_trigger = pch_iir & SDE_DDI_HOTPLUG_MASK_ICP;
1935         u32 tc_hotplug_trigger = pch_iir & SDE_TC_HOTPLUG_MASK_ICP;
1936         u32 pin_mask = 0, long_mask = 0;
1937
1938         if (ddi_hotplug_trigger) {
1939                 u32 dig_hotplug_reg;
1940
1941                 dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_DDI);
1942                 I915_WRITE(SHOTPLUG_CTL_DDI, dig_hotplug_reg);
1943
1944                 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1945                                    ddi_hotplug_trigger, dig_hotplug_reg,
1946                                    dev_priv->hotplug.pch_hpd,
1947                                    icp_ddi_port_hotplug_long_detect);
1948         }
1949
1950         if (tc_hotplug_trigger) {
1951                 u32 dig_hotplug_reg;
1952
1953                 dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_TC);
1954                 I915_WRITE(SHOTPLUG_CTL_TC, dig_hotplug_reg);
1955
1956                 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1957                                    tc_hotplug_trigger, dig_hotplug_reg,
1958                                    dev_priv->hotplug.pch_hpd,
1959                                    icp_tc_port_hotplug_long_detect);
1960         }
1961
1962         if (pin_mask)
1963                 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1964
1965         if (pch_iir & SDE_GMBUS_ICP)
1966                 gmbus_irq_handler(dev_priv);
1967 }
1968
1969 static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1970 {
1971         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
1972                 ~SDE_PORTE_HOTPLUG_SPT;
1973         u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
1974         u32 pin_mask = 0, long_mask = 0;
1975
1976         if (hotplug_trigger) {
1977                 u32 dig_hotplug_reg;
1978
1979                 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1980                 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1981
1982                 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1983                                    hotplug_trigger, dig_hotplug_reg,
1984                                    dev_priv->hotplug.pch_hpd,
1985                                    spt_port_hotplug_long_detect);
1986         }
1987
1988         if (hotplug2_trigger) {
1989                 u32 dig_hotplug_reg;
1990
1991                 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
1992                 I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);
1993
1994                 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1995                                    hotplug2_trigger, dig_hotplug_reg,
1996                                    dev_priv->hotplug.pch_hpd,
1997                                    spt_port_hotplug2_long_detect);
1998         }
1999
2000         if (pin_mask)
2001                 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2002
2003         if (pch_iir & SDE_GMBUS_CPT)
2004                 gmbus_irq_handler(dev_priv);
2005 }
2006
2007 static void ilk_hpd_irq_handler(struct drm_i915_private *dev_priv,
2008                                 u32 hotplug_trigger)
2009 {
2010         u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2011
2012         dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
2013         I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);
2014
2015         intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2016                            hotplug_trigger, dig_hotplug_reg,
2017                            dev_priv->hotplug.hpd,
2018                            ilk_port_hotplug_long_detect);
2019
2020         intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2021 }
2022
2023 static void ilk_display_irq_handler(struct drm_i915_private *dev_priv,
2024                                     u32 de_iir)
2025 {
2026         enum pipe pipe;
2027         u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;
2028
2029         if (hotplug_trigger)
2030                 ilk_hpd_irq_handler(dev_priv, hotplug_trigger);
2031
2032         if (de_iir & DE_AUX_CHANNEL_A)
2033                 dp_aux_irq_handler(dev_priv);
2034
2035         if (de_iir & DE_GSE)
2036                 intel_opregion_asle_intr(dev_priv);
2037
2038         if (de_iir & DE_POISON)
2039                 drm_err(&dev_priv->drm, "Poison interrupt\n");
2040
2041         for_each_pipe(dev_priv, pipe) {
2042                 if (de_iir & DE_PIPE_VBLANK(pipe))
2043                         intel_handle_vblank(dev_priv, pipe);
2044
2045                 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
2046                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2047
2048                 if (de_iir & DE_PIPE_CRC_DONE(pipe))
2049                         i9xx_pipe_crc_irq_handler(dev_priv, pipe);
2050         }
2051
2052         /* check event from PCH */
2053         if (de_iir & DE_PCH_EVENT) {
2054                 u32 pch_iir = I915_READ(SDEIIR);
2055
2056                 if (HAS_PCH_CPT(dev_priv))
2057                         cpt_irq_handler(dev_priv, pch_iir);
2058                 else
2059                         ibx_irq_handler(dev_priv, pch_iir);
2060
2061                 /* should clear PCH hotplug event before clear CPU irq */
2062                 I915_WRITE(SDEIIR, pch_iir);
2063         }
2064
2065         if (IS_GEN(dev_priv, 5) && de_iir & DE_PCU_EVENT)
2066                 gen5_rps_irq_handler(&dev_priv->gt.rps);
2067 }
2068
2069 static void ivb_display_irq_handler(struct drm_i915_private *dev_priv,
2070                                     u32 de_iir)
2071 {
2072         enum pipe pipe;
2073         u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;
2074
2075         if (hotplug_trigger)
2076                 ilk_hpd_irq_handler(dev_priv, hotplug_trigger);
2077
2078         if (de_iir & DE_ERR_INT_IVB)
2079                 ivb_err_int_handler(dev_priv);
2080
2081         if (de_iir & DE_EDP_PSR_INT_HSW) {
2082                 u32 psr_iir = I915_READ(EDP_PSR_IIR);
2083
2084                 intel_psr_irq_handler(dev_priv, psr_iir);
2085                 I915_WRITE(EDP_PSR_IIR, psr_iir);
2086         }
2087
2088         if (de_iir & DE_AUX_CHANNEL_A_IVB)
2089                 dp_aux_irq_handler(dev_priv);
2090
2091         if (de_iir & DE_GSE_IVB)
2092                 intel_opregion_asle_intr(dev_priv);
2093
2094         for_each_pipe(dev_priv, pipe) {
2095                 if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)))
2096                         intel_handle_vblank(dev_priv, pipe);
2097         }
2098
2099         /* check event from PCH */
2100         if (!HAS_PCH_NOP(dev_priv) && (de_iir & DE_PCH_EVENT_IVB)) {
2101                 u32 pch_iir = I915_READ(SDEIIR);
2102
2103                 cpt_irq_handler(dev_priv, pch_iir);
2104
2105                 /* clear PCH hotplug event before clear CPU irq */
2106                 I915_WRITE(SDEIIR, pch_iir);
2107         }
2108 }
2109
2110 /*
2111  * To handle irqs with the minimum potential races with fresh interrupts, we:
2112  * 1 - Disable Master Interrupt Control.
2113  * 2 - Find the source(s) of the interrupt.
2114  * 3 - Clear the Interrupt Identity bits (IIR).
2115  * 4 - Process the interrupt(s) that had bits set in the IIRs.
2116  * 5 - Re-enable Master Interrupt Control.
2117  */
2118 static irqreturn_t ilk_irq_handler(int irq, void *arg)
2119 {
2120         struct drm_i915_private *i915 = arg;
2121         void __iomem * const regs = i915->uncore.regs;
2122         u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2123         irqreturn_t ret = IRQ_NONE;
2124
2125         if (unlikely(!intel_irqs_enabled(i915)))
2126                 return IRQ_NONE;
2127
2128         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2129         disable_rpm_wakeref_asserts(&i915->runtime_pm);
2130
2131         /* disable master interrupt before clearing iir  */
2132         de_ier = raw_reg_read(regs, DEIER);
2133         raw_reg_write(regs, DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2134
2135         /* Disable south interrupts. We'll only write to SDEIIR once, so further
2136          * interrupts will will be stored on its back queue, and then we'll be
2137          * able to process them after we restore SDEIER (as soon as we restore
2138          * it, we'll get an interrupt if SDEIIR still has something to process
2139          * due to its back queue). */
2140         if (!HAS_PCH_NOP(i915)) {
2141                 sde_ier = raw_reg_read(regs, SDEIER);
2142                 raw_reg_write(regs, SDEIER, 0);
2143         }
2144
2145         /* Find, clear, then process each source of interrupt */
2146
2147         gt_iir = raw_reg_read(regs, GTIIR);
2148         if (gt_iir) {
2149                 raw_reg_write(regs, GTIIR, gt_iir);
2150                 if (INTEL_GEN(i915) >= 6)
2151                         gen6_gt_irq_handler(&i915->gt, gt_iir);
2152                 else
2153                         gen5_gt_irq_handler(&i915->gt, gt_iir);
2154                 ret = IRQ_HANDLED;
2155         }
2156
2157         de_iir = raw_reg_read(regs, DEIIR);
2158         if (de_iir) {
2159                 raw_reg_write(regs, DEIIR, de_iir);
2160                 if (INTEL_GEN(i915) >= 7)
2161                         ivb_display_irq_handler(i915, de_iir);
2162                 else
2163                         ilk_display_irq_handler(i915, de_iir);
2164                 ret = IRQ_HANDLED;
2165         }
2166
2167         if (INTEL_GEN(i915) >= 6) {
2168                 u32 pm_iir = raw_reg_read(regs, GEN6_PMIIR);
2169                 if (pm_iir) {
2170                         raw_reg_write(regs, GEN6_PMIIR, pm_iir);
2171                         gen6_rps_irq_handler(&i915->gt.rps, pm_iir);
2172                         ret = IRQ_HANDLED;
2173                 }
2174         }
2175
2176         raw_reg_write(regs, DEIER, de_ier);
2177         if (sde_ier)
2178                 raw_reg_write(regs, SDEIER, sde_ier);
2179
2180         pmu_irq_stats(i915, ret);
2181
2182         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2183         enable_rpm_wakeref_asserts(&i915->runtime_pm);
2184
2185         return ret;
2186 }
2187
2188 static void bxt_hpd_irq_handler(struct drm_i915_private *dev_priv,
2189                                 u32 hotplug_trigger)
2190 {
2191         u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2192
2193         dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
2194         I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2195
2196         intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2197                            hotplug_trigger, dig_hotplug_reg,
2198                            dev_priv->hotplug.hpd,
2199                            bxt_port_hotplug_long_detect);
2200
2201         intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2202 }
2203
2204 static void gen11_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 iir)
2205 {
2206         u32 pin_mask = 0, long_mask = 0;
2207         u32 trigger_tc = iir & GEN11_DE_TC_HOTPLUG_MASK;
2208         u32 trigger_tbt = iir & GEN11_DE_TBT_HOTPLUG_MASK;
2209
2210         if (trigger_tc) {
2211                 u32 dig_hotplug_reg;
2212
2213                 dig_hotplug_reg = I915_READ(GEN11_TC_HOTPLUG_CTL);
2214                 I915_WRITE(GEN11_TC_HOTPLUG_CTL, dig_hotplug_reg);
2215
2216                 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2217                                    trigger_tc, dig_hotplug_reg,
2218                                    dev_priv->hotplug.hpd,
2219                                    gen11_port_hotplug_long_detect);
2220         }
2221
2222         if (trigger_tbt) {
2223                 u32 dig_hotplug_reg;
2224
2225                 dig_hotplug_reg = I915_READ(GEN11_TBT_HOTPLUG_CTL);
2226                 I915_WRITE(GEN11_TBT_HOTPLUG_CTL, dig_hotplug_reg);
2227
2228                 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2229                                    trigger_tbt, dig_hotplug_reg,
2230                                    dev_priv->hotplug.hpd,
2231                                    gen11_port_hotplug_long_detect);
2232         }
2233
2234         if (pin_mask)
2235                 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2236         else
2237                 drm_err(&dev_priv->drm,
2238                         "Unexpected DE HPD interrupt 0x%08x\n", iir);
2239 }
2240
2241 static u32 gen8_de_port_aux_mask(struct drm_i915_private *dev_priv)
2242 {
2243         u32 mask;
2244
2245         if (INTEL_GEN(dev_priv) >= 12)
2246                 return TGL_DE_PORT_AUX_DDIA |
2247                         TGL_DE_PORT_AUX_DDIB |
2248                         TGL_DE_PORT_AUX_DDIC |
2249                         TGL_DE_PORT_AUX_USBC1 |
2250                         TGL_DE_PORT_AUX_USBC2 |
2251                         TGL_DE_PORT_AUX_USBC3 |
2252                         TGL_DE_PORT_AUX_USBC4 |
2253                         TGL_DE_PORT_AUX_USBC5 |
2254                         TGL_DE_PORT_AUX_USBC6;
2255
2256
2257         mask = GEN8_AUX_CHANNEL_A;
2258         if (INTEL_GEN(dev_priv) >= 9)
2259                 mask |= GEN9_AUX_CHANNEL_B |
2260                         GEN9_AUX_CHANNEL_C |
2261                         GEN9_AUX_CHANNEL_D;
2262
2263         if (IS_CNL_WITH_PORT_F(dev_priv) || IS_GEN(dev_priv, 11))
2264                 mask |= CNL_AUX_CHANNEL_F;
2265
2266         if (IS_GEN(dev_priv, 11))
2267                 mask |= ICL_AUX_CHANNEL_E;
2268
2269         return mask;
2270 }
2271
2272 static u32 gen8_de_pipe_fault_mask(struct drm_i915_private *dev_priv)
2273 {
2274         if (IS_ROCKETLAKE(dev_priv))
2275                 return RKL_DE_PIPE_IRQ_FAULT_ERRORS;
2276         else if (INTEL_GEN(dev_priv) >= 11)
2277                 return GEN11_DE_PIPE_IRQ_FAULT_ERRORS;
2278         else if (INTEL_GEN(dev_priv) >= 9)
2279                 return GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
2280         else
2281                 return GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2282 }
2283
2284 static void
2285 gen8_de_misc_irq_handler(struct drm_i915_private *dev_priv, u32 iir)
2286 {
2287         bool found = false;
2288
2289         if (iir & GEN8_DE_MISC_GSE) {
2290                 intel_opregion_asle_intr(dev_priv);
2291                 found = true;
2292         }
2293
2294         if (iir & GEN8_DE_EDP_PSR) {
2295                 u32 psr_iir;
2296                 i915_reg_t iir_reg;
2297
2298                 if (INTEL_GEN(dev_priv) >= 12)
2299                         iir_reg = TRANS_PSR_IIR(dev_priv->psr.transcoder);
2300                 else
2301                         iir_reg = EDP_PSR_IIR;
2302
2303                 psr_iir = I915_READ(iir_reg);
2304                 I915_WRITE(iir_reg, psr_iir);
2305
2306                 if (psr_iir)
2307                         found = true;
2308
2309                 intel_psr_irq_handler(dev_priv, psr_iir);
2310         }
2311
2312         if (!found)
2313                 drm_err(&dev_priv->drm, "Unexpected DE Misc interrupt\n");
2314 }
2315
2316 static void gen11_dsi_te_interrupt_handler(struct drm_i915_private *dev_priv,
2317                                            u32 te_trigger)
2318 {
2319         enum pipe pipe = INVALID_PIPE;
2320         enum transcoder dsi_trans;
2321         enum port port;
2322         u32 val, tmp;
2323
2324         /*
2325          * Incase of dual link, TE comes from DSI_1
2326          * this is to check if dual link is enabled
2327          */
2328         val = I915_READ(TRANS_DDI_FUNC_CTL2(TRANSCODER_DSI_0));
2329         val &= PORT_SYNC_MODE_ENABLE;
2330
2331         /*
2332          * if dual link is enabled, then read DSI_0
2333          * transcoder registers
2334          */
2335         port = ((te_trigger & DSI1_TE && val) || (te_trigger & DSI0_TE)) ?
2336                                                   PORT_A : PORT_B;
2337         dsi_trans = (port == PORT_A) ? TRANSCODER_DSI_0 : TRANSCODER_DSI_1;
2338
2339         /* Check if DSI configured in command mode */
2340         val = I915_READ(DSI_TRANS_FUNC_CONF(dsi_trans));
2341         val = val & OP_MODE_MASK;
2342
2343         if (val != CMD_MODE_NO_GATE && val != CMD_MODE_TE_GATE) {
2344                 drm_err(&dev_priv->drm, "DSI trancoder not configured in command mode\n");
2345                 return;
2346         }
2347
2348         /* Get PIPE for handling VBLANK event */
2349         val = I915_READ(TRANS_DDI_FUNC_CTL(dsi_trans));
2350         switch (val & TRANS_DDI_EDP_INPUT_MASK) {
2351         case TRANS_DDI_EDP_INPUT_A_ON:
2352                 pipe = PIPE_A;
2353                 break;
2354         case TRANS_DDI_EDP_INPUT_B_ONOFF:
2355                 pipe = PIPE_B;
2356                 break;
2357         case TRANS_DDI_EDP_INPUT_C_ONOFF:
2358                 pipe = PIPE_C;
2359                 break;
2360         default:
2361                 drm_err(&dev_priv->drm, "Invalid PIPE\n");
2362                 return;
2363         }
2364
2365         intel_handle_vblank(dev_priv, pipe);
2366
2367         /* clear TE in dsi IIR */
2368         port = (te_trigger & DSI1_TE) ? PORT_B : PORT_A;
2369         tmp = I915_READ(DSI_INTR_IDENT_REG(port));
2370         I915_WRITE(DSI_INTR_IDENT_REG(port), tmp);
2371 }
2372
2373 static irqreturn_t
2374 gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
2375 {
2376         irqreturn_t ret = IRQ_NONE;
2377         u32 iir;
2378         enum pipe pipe;
2379
2380         if (master_ctl & GEN8_DE_MISC_IRQ) {
2381                 iir = I915_READ(GEN8_DE_MISC_IIR);
2382                 if (iir) {
2383                         I915_WRITE(GEN8_DE_MISC_IIR, iir);
2384                         ret = IRQ_HANDLED;
2385                         gen8_de_misc_irq_handler(dev_priv, iir);
2386                 } else {
2387                         drm_err(&dev_priv->drm,
2388                                 "The master control interrupt lied (DE MISC)!\n");
2389                 }
2390         }
2391
2392         if (INTEL_GEN(dev_priv) >= 11 && (master_ctl & GEN11_DE_HPD_IRQ)) {
2393                 iir = I915_READ(GEN11_DE_HPD_IIR);
2394                 if (iir) {
2395                         I915_WRITE(GEN11_DE_HPD_IIR, iir);
2396                         ret = IRQ_HANDLED;
2397                         gen11_hpd_irq_handler(dev_priv, iir);
2398                 } else {
2399                         drm_err(&dev_priv->drm,
2400                                 "The master control interrupt lied, (DE HPD)!\n");
2401                 }
2402         }
2403
2404         if (master_ctl & GEN8_DE_PORT_IRQ) {
2405                 iir = I915_READ(GEN8_DE_PORT_IIR);
2406                 if (iir) {
2407                         bool found = false;
2408
2409                         I915_WRITE(GEN8_DE_PORT_IIR, iir);
2410                         ret = IRQ_HANDLED;
2411
2412                         if (iir & gen8_de_port_aux_mask(dev_priv)) {
2413                                 dp_aux_irq_handler(dev_priv);
2414                                 found = true;
2415                         }
2416
2417                         if (IS_GEN9_LP(dev_priv)) {
2418                                 u32 hotplug_trigger = iir & BXT_DE_PORT_HOTPLUG_MASK;
2419
2420                                 if (hotplug_trigger) {
2421                                         bxt_hpd_irq_handler(dev_priv, hotplug_trigger);
2422                                         found = true;
2423                                 }
2424                         } else if (IS_BROADWELL(dev_priv)) {
2425                                 u32 hotplug_trigger = iir & BDW_DE_PORT_HOTPLUG_MASK;
2426
2427                                 if (hotplug_trigger) {
2428                                         ilk_hpd_irq_handler(dev_priv, hotplug_trigger);
2429                                         found = true;
2430                                 }
2431                         }
2432
2433                         if (IS_GEN9_LP(dev_priv) && (iir & BXT_DE_PORT_GMBUS)) {
2434                                 gmbus_irq_handler(dev_priv);
2435                                 found = true;
2436                         }
2437
2438                         if (INTEL_GEN(dev_priv) >= 11) {
2439                                 u32 te_trigger = iir & (DSI0_TE | DSI1_TE);
2440
2441                                 if (te_trigger) {
2442                                         gen11_dsi_te_interrupt_handler(dev_priv, te_trigger);
2443                                         found = true;
2444                                 }
2445                         }
2446
2447                         if (!found)
2448                                 drm_err(&dev_priv->drm,
2449                                         "Unexpected DE Port interrupt\n");
2450                 }
2451                 else
2452                         drm_err(&dev_priv->drm,
2453                                 "The master control interrupt lied (DE PORT)!\n");
2454         }
2455
2456         for_each_pipe(dev_priv, pipe) {
2457                 u32 fault_errors;
2458
2459                 if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
2460                         continue;
2461
2462                 iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
2463                 if (!iir) {
2464                         drm_err(&dev_priv->drm,
2465                                 "The master control interrupt lied (DE PIPE)!\n");
2466                         continue;
2467                 }
2468
2469                 ret = IRQ_HANDLED;
2470                 I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir);
2471
2472                 if (iir & GEN8_PIPE_VBLANK)
2473                         intel_handle_vblank(dev_priv, pipe);
2474
2475                 if (iir & GEN9_PIPE_PLANE1_FLIP_DONE)
2476                         flip_done_handler(dev_priv, pipe);
2477
2478                 if (iir & GEN8_PIPE_CDCLK_CRC_DONE)
2479                         hsw_pipe_crc_irq_handler(dev_priv, pipe);
2480
2481                 if (iir & GEN8_PIPE_FIFO_UNDERRUN)
2482                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2483
2484                 fault_errors = iir & gen8_de_pipe_fault_mask(dev_priv);
2485                 if (fault_errors)
2486                         drm_err(&dev_priv->drm,
2487                                 "Fault errors on pipe %c: 0x%08x\n",
2488                                 pipe_name(pipe),
2489                                 fault_errors);
2490         }
2491
2492         if (HAS_PCH_SPLIT(dev_priv) && !HAS_PCH_NOP(dev_priv) &&
2493             master_ctl & GEN8_DE_PCH_IRQ) {
2494                 /*
2495                  * FIXME(BDW): Assume for now that the new interrupt handling
2496                  * scheme also closed the SDE interrupt handling race we've seen
2497                  * on older pch-split platforms. But this needs testing.
2498                  */
2499                 iir = I915_READ(SDEIIR);
2500                 if (iir) {
2501                         I915_WRITE(SDEIIR, iir);
2502                         ret = IRQ_HANDLED;
2503
2504                         if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
2505                                 icp_irq_handler(dev_priv, iir);
2506                         else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT)
2507                                 spt_irq_handler(dev_priv, iir);
2508                         else
2509                                 cpt_irq_handler(dev_priv, iir);
2510                 } else {
2511                         /*
2512                          * Like on previous PCH there seems to be something
2513                          * fishy going on with forwarding PCH interrupts.
2514                          */
2515                         drm_dbg(&dev_priv->drm,
2516                                 "The master control interrupt lied (SDE)!\n");
2517                 }
2518         }
2519
2520         return ret;
2521 }
2522
2523 static inline u32 gen8_master_intr_disable(void __iomem * const regs)
2524 {
2525         raw_reg_write(regs, GEN8_MASTER_IRQ, 0);
2526
2527         /*
2528          * Now with master disabled, get a sample of level indications
2529          * for this interrupt. Indications will be cleared on related acks.
2530          * New indications can and will light up during processing,
2531          * and will generate new interrupt after enabling master.
2532          */
2533         return raw_reg_read(regs, GEN8_MASTER_IRQ);
2534 }
2535
2536 static inline void gen8_master_intr_enable(void __iomem * const regs)
2537 {
2538         raw_reg_write(regs, GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2539 }
2540
2541 static irqreturn_t gen8_irq_handler(int irq, void *arg)
2542 {
2543         struct drm_i915_private *dev_priv = arg;
2544         void __iomem * const regs = dev_priv->uncore.regs;
2545         u32 master_ctl;
2546
2547         if (!intel_irqs_enabled(dev_priv))
2548                 return IRQ_NONE;
2549
2550         master_ctl = gen8_master_intr_disable(regs);
2551         if (!master_ctl) {
2552                 gen8_master_intr_enable(regs);
2553                 return IRQ_NONE;
2554         }
2555
2556         /* Find, queue (onto bottom-halves), then clear each source */
2557         gen8_gt_irq_handler(&dev_priv->gt, master_ctl);
2558
2559         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2560         if (master_ctl & ~GEN8_GT_IRQS) {
2561                 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
2562                 gen8_de_irq_handler(dev_priv, master_ctl);
2563                 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
2564         }
2565
2566         gen8_master_intr_enable(regs);
2567
2568         pmu_irq_stats(dev_priv, IRQ_HANDLED);
2569
2570         return IRQ_HANDLED;
2571 }
2572
2573 static u32
2574 gen11_gu_misc_irq_ack(struct intel_gt *gt, const u32 master_ctl)
2575 {
2576         void __iomem * const regs = gt->uncore->regs;
2577         u32 iir;
2578
2579         if (!(master_ctl & GEN11_GU_MISC_IRQ))
2580                 return 0;
2581
2582         iir = raw_reg_read(regs, GEN11_GU_MISC_IIR);
2583         if (likely(iir))
2584                 raw_reg_write(regs, GEN11_GU_MISC_IIR, iir);
2585
2586         return iir;
2587 }
2588
2589 static void
2590 gen11_gu_misc_irq_handler(struct intel_gt *gt, const u32 iir)
2591 {
2592         if (iir & GEN11_GU_MISC_GSE)
2593                 intel_opregion_asle_intr(gt->i915);
2594 }
2595
2596 static inline u32 gen11_master_intr_disable(void __iomem * const regs)
2597 {
2598         raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, 0);
2599
2600         /*
2601          * Now with master disabled, get a sample of level indications
2602          * for this interrupt. Indications will be cleared on related acks.
2603          * New indications can and will light up during processing,
2604          * and will generate new interrupt after enabling master.
2605          */
2606         return raw_reg_read(regs, GEN11_GFX_MSTR_IRQ);
2607 }
2608
2609 static inline void gen11_master_intr_enable(void __iomem * const regs)
2610 {
2611         raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, GEN11_MASTER_IRQ);
2612 }
2613
2614 static void
2615 gen11_display_irq_handler(struct drm_i915_private *i915)
2616 {
2617         void __iomem * const regs = i915->uncore.regs;
2618         const u32 disp_ctl = raw_reg_read(regs, GEN11_DISPLAY_INT_CTL);
2619
2620         disable_rpm_wakeref_asserts(&i915->runtime_pm);
2621         /*
2622          * GEN11_DISPLAY_INT_CTL has same format as GEN8_MASTER_IRQ
2623          * for the display related bits.
2624          */
2625         raw_reg_write(regs, GEN11_DISPLAY_INT_CTL, 0x0);
2626         gen8_de_irq_handler(i915, disp_ctl);
2627         raw_reg_write(regs, GEN11_DISPLAY_INT_CTL,
2628                       GEN11_DISPLAY_IRQ_ENABLE);
2629
2630         enable_rpm_wakeref_asserts(&i915->runtime_pm);
2631 }
2632
2633 static __always_inline irqreturn_t
2634 __gen11_irq_handler(struct drm_i915_private * const i915,
2635                     u32 (*intr_disable)(void __iomem * const regs),
2636                     void (*intr_enable)(void __iomem * const regs))
2637 {
2638         void __iomem * const regs = i915->uncore.regs;
2639         struct intel_gt *gt = &i915->gt;
2640         u32 master_ctl;
2641         u32 gu_misc_iir;
2642
2643         if (!intel_irqs_enabled(i915))
2644                 return IRQ_NONE;
2645
2646         master_ctl = intr_disable(regs);
2647         if (!master_ctl) {
2648                 intr_enable(regs);
2649                 return IRQ_NONE;
2650         }
2651
2652         /* Find, queue (onto bottom-halves), then clear each source */
2653         gen11_gt_irq_handler(gt, master_ctl);
2654
2655         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2656         if (master_ctl & GEN11_DISPLAY_IRQ)
2657                 gen11_display_irq_handler(i915);
2658
2659         gu_misc_iir = gen11_gu_misc_irq_ack(gt, master_ctl);
2660
2661         intr_enable(regs);
2662
2663         gen11_gu_misc_irq_handler(gt, gu_misc_iir);
2664
2665         pmu_irq_stats(i915, IRQ_HANDLED);
2666
2667         return IRQ_HANDLED;
2668 }
2669
2670 static irqreturn_t gen11_irq_handler(int irq, void *arg)
2671 {
2672         return __gen11_irq_handler(arg,
2673                                    gen11_master_intr_disable,
2674                                    gen11_master_intr_enable);
2675 }
2676
2677 static u32 dg1_master_intr_disable_and_ack(void __iomem * const regs)
2678 {
2679         u32 val;
2680
2681         /* First disable interrupts */
2682         raw_reg_write(regs, DG1_MSTR_UNIT_INTR, 0);
2683
2684         /* Get the indication levels and ack the master unit */
2685         val = raw_reg_read(regs, DG1_MSTR_UNIT_INTR);
2686         if (unlikely(!val))
2687                 return 0;
2688
2689         raw_reg_write(regs, DG1_MSTR_UNIT_INTR, val);
2690
2691         /*
2692          * Now with master disabled, get a sample of level indications
2693          * for this interrupt and ack them right away - we keep GEN11_MASTER_IRQ
2694          * out as this bit doesn't exist anymore for DG1
2695          */
2696         val = raw_reg_read(regs, GEN11_GFX_MSTR_IRQ) & ~GEN11_MASTER_IRQ;
2697         if (unlikely(!val))
2698                 return 0;
2699
2700         raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, val);
2701
2702         return val;
2703 }
2704
2705 static inline void dg1_master_intr_enable(void __iomem * const regs)
2706 {
2707         raw_reg_write(regs, DG1_MSTR_UNIT_INTR, DG1_MSTR_IRQ);
2708 }
2709
2710 static irqreturn_t dg1_irq_handler(int irq, void *arg)
2711 {
2712         return __gen11_irq_handler(arg,
2713                                    dg1_master_intr_disable_and_ack,
2714                                    dg1_master_intr_enable);
2715 }
2716
2717 /* Called from drm generic code, passed 'crtc' which
2718  * we use as a pipe index
2719  */
2720 int i8xx_enable_vblank(struct drm_crtc *crtc)
2721 {
2722         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2723         enum pipe pipe = to_intel_crtc(crtc)->pipe;
2724         unsigned long irqflags;
2725
2726         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2727         i915_enable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
2728         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2729
2730         return 0;
2731 }
2732
2733 int i915gm_enable_vblank(struct drm_crtc *crtc)
2734 {
2735         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2736
2737         /*
2738          * Vblank interrupts fail to wake the device up from C2+.
2739          * Disabling render clock gating during C-states avoids
2740          * the problem. There is a small power cost so we do this
2741          * only when vblank interrupts are actually enabled.
2742          */
2743         if (dev_priv->vblank_enabled++ == 0)
2744                 I915_WRITE(SCPD0, _MASKED_BIT_ENABLE(CSTATE_RENDER_CLOCK_GATE_DISABLE));
2745
2746         return i8xx_enable_vblank(crtc);
2747 }
2748
2749 int i965_enable_vblank(struct drm_crtc *crtc)
2750 {
2751         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2752         enum pipe pipe = to_intel_crtc(crtc)->pipe;
2753         unsigned long irqflags;
2754
2755         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2756         i915_enable_pipestat(dev_priv, pipe,
2757                              PIPE_START_VBLANK_INTERRUPT_STATUS);
2758         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2759
2760         return 0;
2761 }
2762
2763 int ilk_enable_vblank(struct drm_crtc *crtc)
2764 {
2765         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2766         enum pipe pipe = to_intel_crtc(crtc)->pipe;
2767         unsigned long irqflags;
2768         u32 bit = INTEL_GEN(dev_priv) >= 7 ?
2769                 DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
2770
2771         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2772         ilk_enable_display_irq(dev_priv, bit);
2773         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2774
2775         /* Even though there is no DMC, frame counter can get stuck when
2776          * PSR is active as no frames are generated.
2777          */
2778         if (HAS_PSR(dev_priv))
2779                 drm_crtc_vblank_restore(crtc);
2780
2781         return 0;
2782 }
2783
2784 static bool gen11_dsi_configure_te(struct intel_crtc *intel_crtc,
2785                                    bool enable)
2786 {
2787         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
2788         enum port port;
2789         u32 tmp;
2790
2791         if (!(intel_crtc->mode_flags &
2792             (I915_MODE_FLAG_DSI_USE_TE1 | I915_MODE_FLAG_DSI_USE_TE0)))
2793                 return false;
2794
2795         /* for dual link cases we consider TE from slave */
2796         if (intel_crtc->mode_flags & I915_MODE_FLAG_DSI_USE_TE1)
2797                 port = PORT_B;
2798         else
2799                 port = PORT_A;
2800
2801         tmp =  I915_READ(DSI_INTR_MASK_REG(port));
2802         if (enable)
2803                 tmp &= ~DSI_TE_EVENT;
2804         else
2805                 tmp |= DSI_TE_EVENT;
2806
2807         I915_WRITE(DSI_INTR_MASK_REG(port), tmp);
2808
2809         tmp = I915_READ(DSI_INTR_IDENT_REG(port));
2810         I915_WRITE(DSI_INTR_IDENT_REG(port), tmp);
2811
2812         return true;
2813 }
2814
2815 int bdw_enable_vblank(struct drm_crtc *crtc)
2816 {
2817         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2818         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2819         enum pipe pipe = intel_crtc->pipe;
2820         unsigned long irqflags;
2821
2822         if (gen11_dsi_configure_te(intel_crtc, true))
2823                 return 0;
2824
2825         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2826         bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2827         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2828
2829         /* Even if there is no DMC, frame counter can get stuck when
2830          * PSR is active as no frames are generated, so check only for PSR.
2831          */
2832         if (HAS_PSR(dev_priv))
2833                 drm_crtc_vblank_restore(crtc);
2834
2835         return 0;
2836 }
2837
2838 void skl_enable_flip_done(struct intel_crtc *crtc)
2839 {
2840         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2841         enum pipe pipe = crtc->pipe;
2842         unsigned long irqflags;
2843
2844         spin_lock_irqsave(&i915->irq_lock, irqflags);
2845
2846         bdw_enable_pipe_irq(i915, pipe, GEN9_PIPE_PLANE1_FLIP_DONE);
2847
2848         spin_unlock_irqrestore(&i915->irq_lock, irqflags);
2849 }
2850
2851 /* Called from drm generic code, passed 'crtc' which
2852  * we use as a pipe index
2853  */
2854 void i8xx_disable_vblank(struct drm_crtc *crtc)
2855 {
2856         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2857         enum pipe pipe = to_intel_crtc(crtc)->pipe;
2858         unsigned long irqflags;
2859
2860         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2861         i915_disable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
2862         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2863 }
2864
2865 void i915gm_disable_vblank(struct drm_crtc *crtc)
2866 {
2867         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2868
2869         i8xx_disable_vblank(crtc);
2870
2871         if (--dev_priv->vblank_enabled == 0)
2872                 I915_WRITE(SCPD0, _MASKED_BIT_DISABLE(CSTATE_RENDER_CLOCK_GATE_DISABLE));
2873 }
2874
2875 void i965_disable_vblank(struct drm_crtc *crtc)
2876 {
2877         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2878         enum pipe pipe = to_intel_crtc(crtc)->pipe;
2879         unsigned long irqflags;
2880
2881         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2882         i915_disable_pipestat(dev_priv, pipe,
2883                               PIPE_START_VBLANK_INTERRUPT_STATUS);
2884         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2885 }
2886
2887 void ilk_disable_vblank(struct drm_crtc *crtc)
2888 {
2889         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2890         enum pipe pipe = to_intel_crtc(crtc)->pipe;
2891         unsigned long irqflags;
2892         u32 bit = INTEL_GEN(dev_priv) >= 7 ?
2893                 DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
2894
2895         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2896         ilk_disable_display_irq(dev_priv, bit);
2897         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2898 }
2899
2900 void bdw_disable_vblank(struct drm_crtc *crtc)
2901 {
2902         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2903         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2904         enum pipe pipe = intel_crtc->pipe;
2905         unsigned long irqflags;
2906
2907         if (gen11_dsi_configure_te(intel_crtc, false))
2908                 return;
2909
2910         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2911         bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2912         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2913 }
2914
2915 void skl_disable_flip_done(struct intel_crtc *crtc)
2916 {
2917         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2918         enum pipe pipe = crtc->pipe;
2919         unsigned long irqflags;
2920
2921         spin_lock_irqsave(&i915->irq_lock, irqflags);
2922
2923         bdw_disable_pipe_irq(i915, pipe, GEN9_PIPE_PLANE1_FLIP_DONE);
2924
2925         spin_unlock_irqrestore(&i915->irq_lock, irqflags);
2926 }
2927
2928 static void ibx_irq_reset(struct drm_i915_private *dev_priv)
2929 {
2930         struct intel_uncore *uncore = &dev_priv->uncore;
2931
2932         if (HAS_PCH_NOP(dev_priv))
2933                 return;
2934
2935         GEN3_IRQ_RESET(uncore, SDE);
2936
2937         if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
2938                 I915_WRITE(SERR_INT, 0xffffffff);
2939 }
2940
2941 static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
2942 {
2943         struct intel_uncore *uncore = &dev_priv->uncore;
2944
2945         if (IS_CHERRYVIEW(dev_priv))
2946                 intel_uncore_write(uncore, DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
2947         else
2948                 intel_uncore_write(uncore, DPINVGTT, DPINVGTT_STATUS_MASK);
2949
2950         i915_hotplug_interrupt_update_locked(dev_priv, 0xffffffff, 0);
2951         intel_uncore_write(uncore, PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2952
2953         i9xx_pipestat_irq_reset(dev_priv);
2954
2955         GEN3_IRQ_RESET(uncore, VLV_);
2956         dev_priv->irq_mask = ~0u;
2957 }
2958
2959 static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
2960 {
2961         struct intel_uncore *uncore = &dev_priv->uncore;
2962
2963         u32 pipestat_mask;
2964         u32 enable_mask;
2965         enum pipe pipe;
2966
2967         pipestat_mask = PIPE_CRC_DONE_INTERRUPT_STATUS;
2968
2969         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
2970         for_each_pipe(dev_priv, pipe)
2971                 i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
2972
2973         enable_mask = I915_DISPLAY_PORT_INTERRUPT |
2974                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2975                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2976                 I915_LPE_PIPE_A_INTERRUPT |
2977                 I915_LPE_PIPE_B_INTERRUPT;
2978
2979         if (IS_CHERRYVIEW(dev_priv))
2980                 enable_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT |
2981                         I915_LPE_PIPE_C_INTERRUPT;
2982
2983         drm_WARN_ON(&dev_priv->drm, dev_priv->irq_mask != ~0u);
2984
2985         dev_priv->irq_mask = ~enable_mask;
2986
2987         GEN3_IRQ_INIT(uncore, VLV_, dev_priv->irq_mask, enable_mask);
2988 }
2989
2990 /* drm_dma.h hooks
2991 */
2992 static void ilk_irq_reset(struct drm_i915_private *dev_priv)
2993 {
2994         struct intel_uncore *uncore = &dev_priv->uncore;
2995
2996         GEN3_IRQ_RESET(uncore, DE);
2997         dev_priv->irq_mask = ~0u;
2998
2999         if (IS_GEN(dev_priv, 7))
3000                 intel_uncore_write(uncore, GEN7_ERR_INT, 0xffffffff);
3001
3002         if (IS_HASWELL(dev_priv)) {
3003                 intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
3004                 intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
3005         }
3006
3007         gen5_gt_irq_reset(&dev_priv->gt);
3008
3009         ibx_irq_reset(dev_priv);
3010 }
3011
3012 static void valleyview_irq_reset(struct drm_i915_private *dev_priv)
3013 {
3014         I915_WRITE(VLV_MASTER_IER, 0);
3015         POSTING_READ(VLV_MASTER_IER);
3016
3017         gen5_gt_irq_reset(&dev_priv->gt);
3018
3019         spin_lock_irq(&dev_priv->irq_lock);
3020         if (dev_priv->display_irqs_enabled)
3021                 vlv_display_irq_reset(dev_priv);
3022         spin_unlock_irq(&dev_priv->irq_lock);
3023 }
3024
3025 static void gen8_irq_reset(struct drm_i915_private *dev_priv)
3026 {
3027         struct intel_uncore *uncore = &dev_priv->uncore;
3028         enum pipe pipe;
3029
3030         gen8_master_intr_disable(dev_priv->uncore.regs);
3031
3032         gen8_gt_irq_reset(&dev_priv->gt);
3033
3034         intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
3035         intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
3036
3037         for_each_pipe(dev_priv, pipe)
3038                 if (intel_display_power_is_enabled(dev_priv,
3039                                                    POWER_DOMAIN_PIPE(pipe)))
3040                         GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
3041
3042         GEN3_IRQ_RESET(uncore, GEN8_DE_PORT_);
3043         GEN3_IRQ_RESET(uncore, GEN8_DE_MISC_);
3044         GEN3_IRQ_RESET(uncore, GEN8_PCU_);
3045
3046         if (HAS_PCH_SPLIT(dev_priv))
3047                 ibx_irq_reset(dev_priv);
3048 }
3049
3050 static void gen11_display_irq_reset(struct drm_i915_private *dev_priv)
3051 {
3052         struct intel_uncore *uncore = &dev_priv->uncore;
3053         enum pipe pipe;
3054         u32 trans_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) |
3055                 BIT(TRANSCODER_C) | BIT(TRANSCODER_D);
3056
3057         intel_uncore_write(uncore, GEN11_DISPLAY_INT_CTL, 0);
3058
3059         if (INTEL_GEN(dev_priv) >= 12) {
3060                 enum transcoder trans;
3061
3062                 for_each_cpu_transcoder_masked(dev_priv, trans, trans_mask) {
3063                         enum intel_display_power_domain domain;
3064
3065                         domain = POWER_DOMAIN_TRANSCODER(trans);
3066                         if (!intel_display_power_is_enabled(dev_priv, domain))
3067                                 continue;
3068
3069                         intel_uncore_write(uncore, TRANS_PSR_IMR(trans), 0xffffffff);
3070                         intel_uncore_write(uncore, TRANS_PSR_IIR(trans), 0xffffffff);
3071                 }
3072         } else {
3073                 intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
3074                 intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
3075         }
3076
3077         for_each_pipe(dev_priv, pipe)
3078                 if (intel_display_power_is_enabled(dev_priv,
3079                                                    POWER_DOMAIN_PIPE(pipe)))
3080                         GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
3081
3082         GEN3_IRQ_RESET(uncore, GEN8_DE_PORT_);
3083         GEN3_IRQ_RESET(uncore, GEN8_DE_MISC_);
3084         GEN3_IRQ_RESET(uncore, GEN11_DE_HPD_);
3085
3086         if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
3087                 GEN3_IRQ_RESET(uncore, SDE);
3088
3089         /* Wa_14010685332:cnp/cmp,tgp,adp */
3090         if (INTEL_PCH_TYPE(dev_priv) == PCH_CNP ||
3091             (INTEL_PCH_TYPE(dev_priv) >= PCH_TGP &&
3092              INTEL_PCH_TYPE(dev_priv) < PCH_DG1)) {
3093                 intel_uncore_rmw(uncore, SOUTH_CHICKEN1,
3094                                  SBCLK_RUN_REFCLK_DIS, SBCLK_RUN_REFCLK_DIS);
3095                 intel_uncore_rmw(uncore, SOUTH_CHICKEN1,
3096                                  SBCLK_RUN_REFCLK_DIS, 0);
3097         }
3098 }
3099
3100 static void gen11_irq_reset(struct drm_i915_private *dev_priv)
3101 {
3102         struct intel_uncore *uncore = &dev_priv->uncore;
3103
3104         if (HAS_MASTER_UNIT_IRQ(dev_priv))
3105                 dg1_master_intr_disable_and_ack(dev_priv->uncore.regs);
3106         else
3107                 gen11_master_intr_disable(dev_priv->uncore.regs);
3108
3109         gen11_gt_irq_reset(&dev_priv->gt);
3110         gen11_display_irq_reset(dev_priv);
3111
3112         GEN3_IRQ_RESET(uncore, GEN11_GU_MISC_);
3113         GEN3_IRQ_RESET(uncore, GEN8_PCU_);
3114 }
3115
3116 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
3117                                      u8 pipe_mask)
3118 {
3119         struct intel_uncore *uncore = &dev_priv->uncore;
3120
3121         u32 extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
3122         enum pipe pipe;
3123
3124         if (INTEL_GEN(dev_priv) >= 9)
3125                 extra_ier |= GEN9_PIPE_PLANE1_FLIP_DONE;
3126
3127         spin_lock_irq(&dev_priv->irq_lock);
3128
3129         if (!intel_irqs_enabled(dev_priv)) {
3130                 spin_unlock_irq(&dev_priv->irq_lock);
3131                 return;
3132         }
3133
3134         for_each_pipe_masked(dev_priv, pipe, pipe_mask)
3135                 GEN8_IRQ_INIT_NDX(uncore, DE_PIPE, pipe,
3136                                   dev_priv->de_irq_mask[pipe],
3137                                   ~dev_priv->de_irq_mask[pipe] | extra_ier);
3138
3139         spin_unlock_irq(&dev_priv->irq_lock);
3140 }
3141
3142 void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
3143                                      u8 pipe_mask)
3144 {
3145         struct intel_uncore *uncore = &dev_priv->uncore;
3146         enum pipe pipe;
3147
3148         spin_lock_irq(&dev_priv->irq_lock);
3149
3150         if (!intel_irqs_enabled(dev_priv)) {
3151                 spin_unlock_irq(&dev_priv->irq_lock);
3152                 return;
3153         }
3154
3155         for_each_pipe_masked(dev_priv, pipe, pipe_mask)
3156                 GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
3157
3158         spin_unlock_irq(&dev_priv->irq_lock);
3159
3160         /* make sure we're done processing display irqs */
3161         intel_synchronize_irq(dev_priv);
3162 }
3163
3164 static void cherryview_irq_reset(struct drm_i915_private *dev_priv)
3165 {
3166         struct intel_uncore *uncore = &dev_priv->uncore;
3167
3168         I915_WRITE(GEN8_MASTER_IRQ, 0);
3169         POSTING_READ(GEN8_MASTER_IRQ);
3170
3171         gen8_gt_irq_reset(&dev_priv->gt);
3172
3173         GEN3_IRQ_RESET(uncore, GEN8_PCU_);
3174
3175         spin_lock_irq(&dev_priv->irq_lock);
3176         if (dev_priv->display_irqs_enabled)
3177                 vlv_display_irq_reset(dev_priv);
3178         spin_unlock_irq(&dev_priv->irq_lock);
3179 }
3180
3181 static u32 ibx_hotplug_enables(struct drm_i915_private *i915,
3182                                enum hpd_pin pin)
3183 {
3184         switch (pin) {
3185         case HPD_PORT_A:
3186                 /*
3187                  * When CPU and PCH are on the same package, port A
3188                  * HPD must be enabled in both north and south.
3189                  */
3190                 return HAS_PCH_LPT_LP(i915) ?
3191                         PORTA_HOTPLUG_ENABLE : 0;
3192         case HPD_PORT_B:
3193                 return PORTB_HOTPLUG_ENABLE |
3194                         PORTB_PULSE_DURATION_2ms;
3195         case HPD_PORT_C:
3196                 return PORTC_HOTPLUG_ENABLE |
3197                         PORTC_PULSE_DURATION_2ms;
3198         case HPD_PORT_D:
3199                 return PORTD_HOTPLUG_ENABLE |
3200                         PORTD_PULSE_DURATION_2ms;
3201         default:
3202                 return 0;
3203         }
3204 }
3205
3206 static void ibx_hpd_detection_setup(struct drm_i915_private *dev_priv)
3207 {
3208         u32 hotplug;
3209
3210         /*
3211          * Enable digital hotplug on the PCH, and configure the DP short pulse
3212          * duration to 2ms (which is the minimum in the Display Port spec).
3213          * The pulse duration bits are reserved on LPT+.
3214          */
3215         hotplug = I915_READ(PCH_PORT_HOTPLUG);
3216         hotplug &= ~(PORTA_HOTPLUG_ENABLE |
3217                      PORTB_HOTPLUG_ENABLE |
3218                      PORTC_HOTPLUG_ENABLE |
3219                      PORTD_HOTPLUG_ENABLE |
3220                      PORTB_PULSE_DURATION_MASK |
3221                      PORTC_PULSE_DURATION_MASK |
3222                      PORTD_PULSE_DURATION_MASK);
3223         hotplug |= intel_hpd_hotplug_enables(dev_priv, ibx_hotplug_enables);
3224         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3225 }
3226
3227 static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv)
3228 {
3229         u32 hotplug_irqs, enabled_irqs;
3230
3231         enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
3232         hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
3233
3234         ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3235
3236         ibx_hpd_detection_setup(dev_priv);
3237 }
3238
3239 static u32 icp_ddi_hotplug_enables(struct drm_i915_private *i915,
3240                                    enum hpd_pin pin)
3241 {
3242         switch (pin) {
3243         case HPD_PORT_A:
3244         case HPD_PORT_B:
3245         case HPD_PORT_C:
3246         case HPD_PORT_D:
3247                 return SHOTPLUG_CTL_DDI_HPD_ENABLE(pin);
3248         default:
3249                 return 0;
3250         }
3251 }
3252
3253 static u32 icp_tc_hotplug_enables(struct drm_i915_private *i915,
3254                                   enum hpd_pin pin)
3255 {
3256         switch (pin) {
3257         case HPD_PORT_TC1:
3258         case HPD_PORT_TC2:
3259         case HPD_PORT_TC3:
3260         case HPD_PORT_TC4:
3261         case HPD_PORT_TC5:
3262         case HPD_PORT_TC6:
3263                 return ICP_TC_HPD_ENABLE(pin);
3264         default:
3265                 return 0;
3266         }
3267 }
3268
3269 static void icp_ddi_hpd_detection_setup(struct drm_i915_private *dev_priv)
3270 {
3271         u32 hotplug;
3272
3273         hotplug = I915_READ(SHOTPLUG_CTL_DDI);
3274         hotplug &= ~(SHOTPLUG_CTL_DDI_HPD_ENABLE(HPD_PORT_A) |
3275                      SHOTPLUG_CTL_DDI_HPD_ENABLE(HPD_PORT_B) |
3276                      SHOTPLUG_CTL_DDI_HPD_ENABLE(HPD_PORT_C) |
3277                      SHOTPLUG_CTL_DDI_HPD_ENABLE(HPD_PORT_D));
3278         hotplug |= intel_hpd_hotplug_enables(dev_priv, icp_ddi_hotplug_enables);
3279         I915_WRITE(SHOTPLUG_CTL_DDI, hotplug);
3280 }
3281
3282 static void icp_tc_hpd_detection_setup(struct drm_i915_private *dev_priv)
3283 {
3284         u32 hotplug;
3285
3286         hotplug = I915_READ(SHOTPLUG_CTL_TC);
3287         hotplug &= ~(ICP_TC_HPD_ENABLE(HPD_PORT_TC1) |
3288                      ICP_TC_HPD_ENABLE(HPD_PORT_TC2) |
3289                      ICP_TC_HPD_ENABLE(HPD_PORT_TC3) |
3290                      ICP_TC_HPD_ENABLE(HPD_PORT_TC4) |
3291                      ICP_TC_HPD_ENABLE(HPD_PORT_TC5) |
3292                      ICP_TC_HPD_ENABLE(HPD_PORT_TC6));
3293         hotplug |= intel_hpd_hotplug_enables(dev_priv, icp_tc_hotplug_enables);
3294         I915_WRITE(SHOTPLUG_CTL_TC, hotplug);
3295 }
3296
3297 static void icp_hpd_irq_setup(struct drm_i915_private *dev_priv)
3298 {
3299         u32 hotplug_irqs, enabled_irqs;
3300
3301         enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
3302         hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
3303
3304         if (INTEL_PCH_TYPE(dev_priv) <= PCH_TGP)
3305                 I915_WRITE(SHPD_FILTER_CNT, SHPD_FILTER_CNT_500_ADJ);
3306
3307         ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3308
3309         icp_ddi_hpd_detection_setup(dev_priv);
3310         icp_tc_hpd_detection_setup(dev_priv);
3311 }
3312
3313 static u32 gen11_hotplug_enables(struct drm_i915_private *i915,
3314                                  enum hpd_pin pin)
3315 {
3316         switch (pin) {
3317         case HPD_PORT_TC1:
3318         case HPD_PORT_TC2:
3319         case HPD_PORT_TC3:
3320         case HPD_PORT_TC4:
3321         case HPD_PORT_TC5:
3322         case HPD_PORT_TC6:
3323                 return GEN11_HOTPLUG_CTL_ENABLE(pin);
3324         default:
3325                 return 0;
3326         }
3327 }
3328
3329 static void dg1_hpd_irq_setup(struct drm_i915_private *dev_priv)
3330 {
3331         u32 val;
3332
3333         val = I915_READ(SOUTH_CHICKEN1);
3334         val |= (INVERT_DDIA_HPD |
3335                 INVERT_DDIB_HPD |
3336                 INVERT_DDIC_HPD |
3337                 INVERT_DDID_HPD);
3338         I915_WRITE(SOUTH_CHICKEN1, val);
3339
3340         icp_hpd_irq_setup(dev_priv);
3341 }
3342
3343 static void gen11_tc_hpd_detection_setup(struct drm_i915_private *dev_priv)
3344 {
3345         u32 hotplug;
3346
3347         hotplug = I915_READ(GEN11_TC_HOTPLUG_CTL);
3348         hotplug &= ~(GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC1) |
3349                      GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC2) |
3350                      GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC3) |
3351                      GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC4) |
3352                      GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC5) |
3353                      GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC6));
3354         hotplug |= intel_hpd_hotplug_enables(dev_priv, gen11_hotplug_enables);
3355         I915_WRITE(GEN11_TC_HOTPLUG_CTL, hotplug);
3356 }
3357
3358 static void gen11_tbt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3359 {
3360         u32 hotplug;
3361
3362         hotplug = I915_READ(GEN11_TBT_HOTPLUG_CTL);
3363         hotplug &= ~(GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC1) |
3364                      GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC2) |
3365                      GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC3) |
3366                      GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC4) |
3367                      GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC5) |
3368                      GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC6));
3369         hotplug |= intel_hpd_hotplug_enables(dev_priv, gen11_hotplug_enables);
3370         I915_WRITE(GEN11_TBT_HOTPLUG_CTL, hotplug);
3371 }
3372
3373 static void gen11_hpd_irq_setup(struct drm_i915_private *dev_priv)
3374 {
3375         u32 hotplug_irqs, enabled_irqs;
3376         u32 val;
3377
3378         enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.hpd);
3379         hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.hpd);
3380
3381         val = I915_READ(GEN11_DE_HPD_IMR);
3382         val &= ~hotplug_irqs;
3383         val |= ~enabled_irqs & hotplug_irqs;
3384         I915_WRITE(GEN11_DE_HPD_IMR, val);
3385         POSTING_READ(GEN11_DE_HPD_IMR);
3386
3387         gen11_tc_hpd_detection_setup(dev_priv);
3388         gen11_tbt_hpd_detection_setup(dev_priv);
3389
3390         if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
3391                 icp_hpd_irq_setup(dev_priv);
3392 }
3393
3394 static u32 spt_hotplug_enables(struct drm_i915_private *i915,
3395                                enum hpd_pin pin)
3396 {
3397         switch (pin) {
3398         case HPD_PORT_A:
3399                 return PORTA_HOTPLUG_ENABLE;
3400         case HPD_PORT_B:
3401                 return PORTB_HOTPLUG_ENABLE;
3402         case HPD_PORT_C:
3403                 return PORTC_HOTPLUG_ENABLE;
3404         case HPD_PORT_D:
3405                 return PORTD_HOTPLUG_ENABLE;
3406         default:
3407                 return 0;
3408         }
3409 }
3410
3411 static u32 spt_hotplug2_enables(struct drm_i915_private *i915,
3412                                 enum hpd_pin pin)
3413 {
3414         switch (pin) {
3415         case HPD_PORT_E:
3416                 return PORTE_HOTPLUG_ENABLE;
3417         default:
3418                 return 0;
3419         }
3420 }
3421
3422 static void spt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3423 {
3424         u32 val, hotplug;
3425
3426         /* Display WA #1179 WaHardHangonHotPlug: cnp */
3427         if (HAS_PCH_CNP(dev_priv)) {
3428                 val = I915_READ(SOUTH_CHICKEN1);
3429                 val &= ~CHASSIS_CLK_REQ_DURATION_MASK;
3430                 val |= CHASSIS_CLK_REQ_DURATION(0xf);
3431                 I915_WRITE(SOUTH_CHICKEN1, val);
3432         }
3433
3434         /* Enable digital hotplug on the PCH */
3435         hotplug = I915_READ(PCH_PORT_HOTPLUG);
3436         hotplug &= ~(PORTA_HOTPLUG_ENABLE |
3437                      PORTB_HOTPLUG_ENABLE |
3438                      PORTC_HOTPLUG_ENABLE |
3439                      PORTD_HOTPLUG_ENABLE);
3440         hotplug |= intel_hpd_hotplug_enables(dev_priv, spt_hotplug_enables);
3441         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3442
3443         hotplug = I915_READ(PCH_PORT_HOTPLUG2);
3444         hotplug &= ~PORTE_HOTPLUG_ENABLE;
3445         hotplug |= intel_hpd_hotplug_enables(dev_priv, spt_hotplug2_enables);
3446         I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3447 }
3448
3449 static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv)
3450 {
3451         u32 hotplug_irqs, enabled_irqs;
3452
3453         if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP)
3454                 I915_WRITE(SHPD_FILTER_CNT, SHPD_FILTER_CNT_500_ADJ);
3455
3456         enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
3457         hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
3458
3459         ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3460
3461         spt_hpd_detection_setup(dev_priv);
3462 }
3463
3464 static u32 ilk_hotplug_enables(struct drm_i915_private *i915,
3465                                enum hpd_pin pin)
3466 {
3467         switch (pin) {
3468         case HPD_PORT_A:
3469                 return DIGITAL_PORTA_HOTPLUG_ENABLE |
3470                         DIGITAL_PORTA_PULSE_DURATION_2ms;
3471         default:
3472                 return 0;
3473         }
3474 }
3475
3476 static void ilk_hpd_detection_setup(struct drm_i915_private *dev_priv)
3477 {
3478         u32 hotplug;
3479
3480         /*
3481          * Enable digital hotplug on the CPU, and configure the DP short pulse
3482          * duration to 2ms (which is the minimum in the Display Port spec)
3483          * The pulse duration bits are reserved on HSW+.
3484          */
3485         hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
3486         hotplug &= ~(DIGITAL_PORTA_HOTPLUG_ENABLE |
3487                      DIGITAL_PORTA_PULSE_DURATION_MASK);
3488         hotplug |= intel_hpd_hotplug_enables(dev_priv, ilk_hotplug_enables);
3489         I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
3490 }
3491
3492 static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv)
3493 {
3494         u32 hotplug_irqs, enabled_irqs;
3495
3496         enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.hpd);
3497         hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.hpd);
3498
3499         if (INTEL_GEN(dev_priv) >= 8)
3500                 bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3501         else
3502                 ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3503
3504         ilk_hpd_detection_setup(dev_priv);
3505
3506         ibx_hpd_irq_setup(dev_priv);
3507 }
3508
3509 static u32 bxt_hotplug_enables(struct drm_i915_private *i915,
3510                                enum hpd_pin pin)
3511 {
3512         u32 hotplug;
3513
3514         switch (pin) {
3515         case HPD_PORT_A:
3516                 hotplug = PORTA_HOTPLUG_ENABLE;
3517                 if (intel_bios_is_port_hpd_inverted(i915, PORT_A))
3518                         hotplug |= BXT_DDIA_HPD_INVERT;
3519                 return hotplug;
3520         case HPD_PORT_B:
3521                 hotplug = PORTB_HOTPLUG_ENABLE;
3522                 if (intel_bios_is_port_hpd_inverted(i915, PORT_B))
3523                         hotplug |= BXT_DDIB_HPD_INVERT;
3524                 return hotplug;
3525         case HPD_PORT_C:
3526                 hotplug = PORTC_HOTPLUG_ENABLE;
3527                 if (intel_bios_is_port_hpd_inverted(i915, PORT_C))
3528                         hotplug |= BXT_DDIC_HPD_INVERT;
3529                 return hotplug;
3530         default:
3531                 return 0;
3532         }
3533 }
3534
3535 static void bxt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3536 {
3537         u32 hotplug;
3538
3539         hotplug = I915_READ(PCH_PORT_HOTPLUG);
3540         hotplug &= ~(PORTA_HOTPLUG_ENABLE |
3541                      PORTB_HOTPLUG_ENABLE |
3542                      PORTC_HOTPLUG_ENABLE |
3543                      BXT_DDIA_HPD_INVERT |
3544                      BXT_DDIB_HPD_INVERT |
3545                      BXT_DDIC_HPD_INVERT);
3546         hotplug |= intel_hpd_hotplug_enables(dev_priv, bxt_hotplug_enables);
3547         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3548 }
3549
3550 static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv)
3551 {
3552         u32 hotplug_irqs, enabled_irqs;
3553
3554         enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.hpd);
3555         hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.hpd);
3556
3557         bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3558
3559         bxt_hpd_detection_setup(dev_priv);
3560 }
3561
3562 /*
3563  * SDEIER is also touched by the interrupt handler to work around missed PCH
3564  * interrupts. Hence we can't update it after the interrupt handler is enabled -
3565  * instead we unconditionally enable all PCH interrupt sources here, but then
3566  * only unmask them as needed with SDEIMR.
3567  *
3568  * Note that we currently do this after installing the interrupt handler,
3569  * but before we enable the master interrupt. That should be sufficient
3570  * to avoid races with the irq handler, assuming we have MSI. Shared legacy
3571  * interrupts could still race.
3572  */
3573 static void ibx_irq_postinstall(struct drm_i915_private *dev_priv)
3574 {
3575         struct intel_uncore *uncore = &dev_priv->uncore;
3576         u32 mask;
3577
3578         if (HAS_PCH_NOP(dev_priv))
3579                 return;
3580
3581         if (HAS_PCH_IBX(dev_priv))
3582                 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3583         else if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
3584                 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3585         else
3586                 mask = SDE_GMBUS_CPT;
3587
3588         GEN3_IRQ_INIT(uncore, SDE, ~mask, 0xffffffff);
3589 }
3590
3591 static void ilk_irq_postinstall(struct drm_i915_private *dev_priv)
3592 {
3593         struct intel_uncore *uncore = &dev_priv->uncore;
3594         u32 display_mask, extra_mask;
3595
3596         if (INTEL_GEN(dev_priv) >= 7) {
3597                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
3598                                 DE_PCH_EVENT_IVB | DE_AUX_CHANNEL_A_IVB);
3599                 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3600                               DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
3601                               DE_DP_A_HOTPLUG_IVB);
3602         } else {
3603                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
3604                                 DE_AUX_CHANNEL_A | DE_PIPEB_CRC_DONE |
3605                                 DE_PIPEA_CRC_DONE | DE_POISON);
3606                 extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK |
3607                               DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
3608                               DE_DP_A_HOTPLUG);
3609         }
3610
3611         if (IS_HASWELL(dev_priv)) {
3612                 gen3_assert_iir_is_zero(uncore, EDP_PSR_IIR);
3613                 display_mask |= DE_EDP_PSR_INT_HSW;
3614         }
3615
3616         if (IS_IRONLAKE_M(dev_priv))
3617                 extra_mask |= DE_PCU_EVENT;
3618
3619         dev_priv->irq_mask = ~display_mask;
3620
3621         ibx_irq_postinstall(dev_priv);
3622
3623         gen5_gt_irq_postinstall(&dev_priv->gt);
3624
3625         GEN3_IRQ_INIT(uncore, DE, dev_priv->irq_mask,
3626                       display_mask | extra_mask);
3627 }
3628
3629 void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
3630 {
3631         lockdep_assert_held(&dev_priv->irq_lock);
3632
3633         if (dev_priv->display_irqs_enabled)
3634                 return;
3635
3636         dev_priv->display_irqs_enabled = true;
3637
3638         if (intel_irqs_enabled(dev_priv)) {
3639                 vlv_display_irq_reset(dev_priv);
3640                 vlv_display_irq_postinstall(dev_priv);
3641         }
3642 }
3643
3644 void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
3645 {
3646         lockdep_assert_held(&dev_priv->irq_lock);
3647
3648         if (!dev_priv->display_irqs_enabled)
3649                 return;
3650
3651         dev_priv->display_irqs_enabled = false;
3652
3653         if (intel_irqs_enabled(dev_priv))
3654                 vlv_display_irq_reset(dev_priv);
3655 }
3656
3657
3658 static void valleyview_irq_postinstall(struct drm_i915_private *dev_priv)
3659 {
3660         gen5_gt_irq_postinstall(&dev_priv->gt);
3661
3662         spin_lock_irq(&dev_priv->irq_lock);
3663         if (dev_priv->display_irqs_enabled)
3664                 vlv_display_irq_postinstall(dev_priv);
3665         spin_unlock_irq(&dev_priv->irq_lock);
3666
3667         I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3668         POSTING_READ(VLV_MASTER_IER);
3669 }
3670
3671 static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
3672 {
3673         struct intel_uncore *uncore = &dev_priv->uncore;
3674
3675         u32 de_pipe_masked = gen8_de_pipe_fault_mask(dev_priv) |
3676                 GEN8_PIPE_CDCLK_CRC_DONE;
3677         u32 de_pipe_enables;
3678         u32 de_port_masked = gen8_de_port_aux_mask(dev_priv);
3679         u32 de_port_enables;
3680         u32 de_misc_masked = GEN8_DE_EDP_PSR;
3681         u32 trans_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) |
3682                 BIT(TRANSCODER_C) | BIT(TRANSCODER_D);
3683         enum pipe pipe;
3684
3685         if (INTEL_GEN(dev_priv) <= 10)
3686                 de_misc_masked |= GEN8_DE_MISC_GSE;
3687
3688         if (IS_GEN9_LP(dev_priv))
3689                 de_port_masked |= BXT_DE_PORT_GMBUS;
3690
3691         if (INTEL_GEN(dev_priv) >= 11) {
3692                 enum port port;
3693
3694                 if (intel_bios_is_dsi_present(dev_priv, &port))
3695                         de_port_masked |= DSI0_TE | DSI1_TE;
3696         }
3697
3698         de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
3699                                            GEN8_PIPE_FIFO_UNDERRUN;
3700
3701         if (INTEL_GEN(dev_priv) >= 9)
3702                 de_pipe_enables |= GEN9_PIPE_PLANE1_FLIP_DONE;
3703
3704         de_port_enables = de_port_masked;
3705         if (IS_GEN9_LP(dev_priv))
3706                 de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
3707         else if (IS_BROADWELL(dev_priv))
3708                 de_port_enables |= BDW_DE_PORT_HOTPLUG_MASK;
3709
3710         if (INTEL_GEN(dev_priv) >= 12) {
3711                 enum transcoder trans;
3712
3713                 for_each_cpu_transcoder_masked(dev_priv, trans, trans_mask) {
3714                         enum intel_display_power_domain domain;
3715
3716                         domain = POWER_DOMAIN_TRANSCODER(trans);
3717                         if (!intel_display_power_is_enabled(dev_priv, domain))
3718                                 continue;
3719
3720                         gen3_assert_iir_is_zero(uncore, TRANS_PSR_IIR(trans));
3721                 }
3722         } else {
3723                 gen3_assert_iir_is_zero(uncore, EDP_PSR_IIR);
3724         }
3725
3726         for_each_pipe(dev_priv, pipe) {
3727                 dev_priv->de_irq_mask[pipe] = ~de_pipe_masked;
3728
3729                 if (intel_display_power_is_enabled(dev_priv,
3730                                 POWER_DOMAIN_PIPE(pipe)))
3731                         GEN8_IRQ_INIT_NDX(uncore, DE_PIPE, pipe,
3732                                           dev_priv->de_irq_mask[pipe],
3733                                           de_pipe_enables);
3734         }
3735
3736         GEN3_IRQ_INIT(uncore, GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
3737         GEN3_IRQ_INIT(uncore, GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked);
3738
3739         if (INTEL_GEN(dev_priv) >= 11) {
3740                 u32 de_hpd_masked = 0;
3741                 u32 de_hpd_enables = GEN11_DE_TC_HOTPLUG_MASK |
3742                                      GEN11_DE_TBT_HOTPLUG_MASK;
3743
3744                 GEN3_IRQ_INIT(uncore, GEN11_DE_HPD_, ~de_hpd_masked,
3745                               de_hpd_enables);
3746         }
3747 }
3748
3749 static void gen8_irq_postinstall(struct drm_i915_private *dev_priv)
3750 {
3751         if (HAS_PCH_SPLIT(dev_priv))
3752                 ibx_irq_postinstall(dev_priv);
3753
3754         gen8_gt_irq_postinstall(&dev_priv->gt);
3755         gen8_de_irq_postinstall(dev_priv);
3756
3757         gen8_master_intr_enable(dev_priv->uncore.regs);
3758 }
3759
3760 static void icp_irq_postinstall(struct drm_i915_private *dev_priv)
3761 {
3762         struct intel_uncore *uncore = &dev_priv->uncore;
3763         u32 mask = SDE_GMBUS_ICP;
3764
3765         GEN3_IRQ_INIT(uncore, SDE, ~mask, 0xffffffff);
3766 }
3767
3768 static void gen11_irq_postinstall(struct drm_i915_private *dev_priv)
3769 {
3770         struct intel_uncore *uncore = &dev_priv->uncore;
3771         u32 gu_misc_masked = GEN11_GU_MISC_GSE;
3772
3773         if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
3774                 icp_irq_postinstall(dev_priv);
3775
3776         gen11_gt_irq_postinstall(&dev_priv->gt);
3777         gen8_de_irq_postinstall(dev_priv);
3778
3779         GEN3_IRQ_INIT(uncore, GEN11_GU_MISC_, ~gu_misc_masked, gu_misc_masked);
3780
3781         I915_WRITE(GEN11_DISPLAY_INT_CTL, GEN11_DISPLAY_IRQ_ENABLE);
3782
3783         if (HAS_MASTER_UNIT_IRQ(dev_priv)) {
3784                 dg1_master_intr_enable(uncore->regs);
3785                 POSTING_READ(DG1_MSTR_UNIT_INTR);
3786         } else {
3787                 gen11_master_intr_enable(uncore->regs);
3788                 POSTING_READ(GEN11_GFX_MSTR_IRQ);
3789         }
3790 }
3791
3792 static void cherryview_irq_postinstall(struct drm_i915_private *dev_priv)
3793 {
3794         gen8_gt_irq_postinstall(&dev_priv->gt);
3795
3796         spin_lock_irq(&dev_priv->irq_lock);
3797         if (dev_priv->display_irqs_enabled)
3798                 vlv_display_irq_postinstall(dev_priv);
3799         spin_unlock_irq(&dev_priv->irq_lock);
3800
3801         I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
3802         POSTING_READ(GEN8_MASTER_IRQ);
3803 }
3804
3805 static void i8xx_irq_reset(struct drm_i915_private *dev_priv)
3806 {
3807         struct intel_uncore *uncore = &dev_priv->uncore;
3808
3809         i9xx_pipestat_irq_reset(dev_priv);
3810
3811         GEN2_IRQ_RESET(uncore);
3812         dev_priv->irq_mask = ~0u;
3813 }
3814
3815 static void i8xx_irq_postinstall(struct drm_i915_private *dev_priv)
3816 {
3817         struct intel_uncore *uncore = &dev_priv->uncore;
3818         u16 enable_mask;
3819
3820         intel_uncore_write16(uncore,
3821                              EMR,
3822                              ~(I915_ERROR_PAGE_TABLE |
3823                                I915_ERROR_MEMORY_REFRESH));
3824
3825         /* Unmask the interrupts that we always want on. */
3826         dev_priv->irq_mask =
3827                 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3828                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3829                   I915_MASTER_ERROR_INTERRUPT);
3830
3831         enable_mask =
3832                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3833                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3834                 I915_MASTER_ERROR_INTERRUPT |
3835                 I915_USER_INTERRUPT;
3836
3837         GEN2_IRQ_INIT(uncore, dev_priv->irq_mask, enable_mask);
3838
3839         /* Interrupt setup is already guaranteed to be single-threaded, this is
3840          * just to make the assert_spin_locked check happy. */
3841         spin_lock_irq(&dev_priv->irq_lock);
3842         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3843         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3844         spin_unlock_irq(&dev_priv->irq_lock);
3845 }
3846
3847 static void i8xx_error_irq_ack(struct drm_i915_private *i915,
3848                                u16 *eir, u16 *eir_stuck)
3849 {
3850         struct intel_uncore *uncore = &i915->uncore;
3851         u16 emr;
3852
3853         *eir = intel_uncore_read16(uncore, EIR);
3854
3855         if (*eir)
3856                 intel_uncore_write16(uncore, EIR, *eir);
3857
3858         *eir_stuck = intel_uncore_read16(uncore, EIR);
3859         if (*eir_stuck == 0)
3860                 return;
3861
3862         /*
3863          * Toggle all EMR bits to make sure we get an edge
3864          * in the ISR master error bit if we don't clear
3865          * all the EIR bits. Otherwise the edge triggered
3866          * IIR on i965/g4x wouldn't notice that an interrupt
3867          * is still pending. Also some EIR bits can't be
3868          * cleared except by handling the underlying error
3869          * (or by a GPU reset) so we mask any bit that
3870          * remains set.
3871          */
3872         emr = intel_uncore_read16(uncore, EMR);
3873         intel_uncore_write16(uncore, EMR, 0xffff);
3874         intel_uncore_write16(uncore, EMR, emr | *eir_stuck);
3875 }
3876
3877 static void i8xx_error_irq_handler(struct drm_i915_private *dev_priv,
3878                                    u16 eir, u16 eir_stuck)
3879 {
3880         DRM_DEBUG("Master Error: EIR 0x%04x\n", eir);
3881
3882         if (eir_stuck)
3883                 drm_dbg(&dev_priv->drm, "EIR stuck: 0x%04x, masked\n",
3884                         eir_stuck);
3885 }
3886
3887 static void i9xx_error_irq_ack(struct drm_i915_private *dev_priv,
3888                                u32 *eir, u32 *eir_stuck)
3889 {
3890         u32 emr;
3891
3892         *eir = I915_READ(EIR);
3893
3894         I915_WRITE(EIR, *eir);
3895
3896         *eir_stuck = I915_READ(EIR);
3897         if (*eir_stuck == 0)
3898                 return;
3899
3900         /*
3901          * Toggle all EMR bits to make sure we get an edge
3902          * in the ISR master error bit if we don't clear
3903          * all the EIR bits. Otherwise the edge triggered
3904          * IIR on i965/g4x wouldn't notice that an interrupt
3905          * is still pending. Also some EIR bits can't be
3906          * cleared except by handling the underlying error
3907          * (or by a GPU reset) so we mask any bit that
3908          * remains set.
3909          */
3910         emr = I915_READ(EMR);
3911         I915_WRITE(EMR, 0xffffffff);
3912         I915_WRITE(EMR, emr | *eir_stuck);
3913 }
3914
3915 static void i9xx_error_irq_handler(struct drm_i915_private *dev_priv,
3916                                    u32 eir, u32 eir_stuck)
3917 {
3918         DRM_DEBUG("Master Error, EIR 0x%08x\n", eir);
3919
3920         if (eir_stuck)
3921                 drm_dbg(&dev_priv->drm, "EIR stuck: 0x%08x, masked\n",
3922                         eir_stuck);
3923 }
3924
3925 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
3926 {
3927         struct drm_i915_private *dev_priv = arg;
3928         irqreturn_t ret = IRQ_NONE;
3929
3930         if (!intel_irqs_enabled(dev_priv))
3931                 return IRQ_NONE;
3932
3933         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
3934         disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
3935
3936         do {
3937                 u32 pipe_stats[I915_MAX_PIPES] = {};
3938                 u16 eir = 0, eir_stuck = 0;
3939                 u16 iir;
3940
3941                 iir = intel_uncore_read16(&dev_priv->uncore, GEN2_IIR);
3942                 if (iir == 0)
3943                         break;
3944
3945                 ret = IRQ_HANDLED;
3946
3947                 /* Call regardless, as some status bits might not be
3948                  * signalled in iir */
3949                 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
3950
3951                 if (iir & I915_MASTER_ERROR_INTERRUPT)
3952                         i8xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
3953
3954                 intel_uncore_write16(&dev_priv->uncore, GEN2_IIR, iir);
3955
3956                 if (iir & I915_USER_INTERRUPT)
3957                         intel_engine_signal_breadcrumbs(dev_priv->gt.engine[RCS0]);
3958
3959                 if (iir & I915_MASTER_ERROR_INTERRUPT)
3960                         i8xx_error_irq_handler(dev_priv, eir, eir_stuck);
3961
3962                 i8xx_pipestat_irq_handler(dev_priv, iir, pipe_stats);
3963         } while (0);
3964
3965         pmu_irq_stats(dev_priv, ret);
3966
3967         enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
3968
3969         return ret;
3970 }
3971
3972 static void i915_irq_reset(struct drm_i915_private *dev_priv)
3973 {
3974         struct intel_uncore *uncore = &dev_priv->uncore;
3975
3976         if (I915_HAS_HOTPLUG(dev_priv)) {
3977                 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3978                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3979         }
3980
3981         i9xx_pipestat_irq_reset(dev_priv);
3982
3983         GEN3_IRQ_RESET(uncore, GEN2_);
3984         dev_priv->irq_mask = ~0u;
3985 }
3986
3987 static void i915_irq_postinstall(struct drm_i915_private *dev_priv)
3988 {
3989         struct intel_uncore *uncore = &dev_priv->uncore;
3990         u32 enable_mask;
3991
3992         I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE |
3993                           I915_ERROR_MEMORY_REFRESH));
3994
3995         /* Unmask the interrupts that we always want on. */
3996         dev_priv->irq_mask =
3997                 ~(I915_ASLE_INTERRUPT |
3998                   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3999                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4000                   I915_MASTER_ERROR_INTERRUPT);
4001
4002         enable_mask =
4003                 I915_ASLE_INTERRUPT |
4004                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4005                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4006                 I915_MASTER_ERROR_INTERRUPT |
4007                 I915_USER_INTERRUPT;
4008
4009         if (I915_HAS_HOTPLUG(dev_priv)) {
4010                 /* Enable in IER... */
4011                 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
4012                 /* and unmask in IMR */
4013                 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
4014         }
4015
4016         GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask);
4017
4018         /* Interrupt setup is already guaranteed to be single-threaded, this is
4019          * just to make the assert_spin_locked check happy. */
4020         spin_lock_irq(&dev_priv->irq_lock);
4021         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4022         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4023         spin_unlock_irq(&dev_priv->irq_lock);
4024
4025         i915_enable_asle_pipestat(dev_priv);
4026 }
4027
4028 static irqreturn_t i915_irq_handler(int irq, void *arg)
4029 {
4030         struct drm_i915_private *dev_priv = arg;
4031         irqreturn_t ret = IRQ_NONE;
4032
4033         if (!intel_irqs_enabled(dev_priv))
4034                 return IRQ_NONE;
4035
4036         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
4037         disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
4038
4039         do {
4040                 u32 pipe_stats[I915_MAX_PIPES] = {};
4041                 u32 eir = 0, eir_stuck = 0;
4042                 u32 hotplug_status = 0;
4043                 u32 iir;
4044
4045                 iir = I915_READ(GEN2_IIR);
4046                 if (iir == 0)
4047                         break;
4048
4049                 ret = IRQ_HANDLED;
4050
4051                 if (I915_HAS_HOTPLUG(dev_priv) &&
4052                     iir & I915_DISPLAY_PORT_INTERRUPT)
4053                         hotplug_status = i9xx_hpd_irq_ack(dev_priv);
4054
4055                 /* Call regardless, as some status bits might not be
4056                  * signalled in iir */
4057                 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
4058
4059                 if (iir & I915_MASTER_ERROR_INTERRUPT)
4060                         i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
4061
4062                 I915_WRITE(GEN2_IIR, iir);
4063
4064                 if (iir & I915_USER_INTERRUPT)
4065                         intel_engine_signal_breadcrumbs(dev_priv->gt.engine[RCS0]);
4066
4067                 if (iir & I915_MASTER_ERROR_INTERRUPT)
4068                         i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
4069
4070                 if (hotplug_status)
4071                         i9xx_hpd_irq_handler(dev_priv, hotplug_status);
4072
4073                 i915_pipestat_irq_handler(dev_priv, iir, pipe_stats);
4074         } while (0);
4075
4076         pmu_irq_stats(dev_priv, ret);
4077
4078         enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
4079
4080         return ret;
4081 }
4082
4083 static void i965_irq_reset(struct drm_i915_private *dev_priv)
4084 {
4085         struct intel_uncore *uncore = &dev_priv->uncore;
4086
4087         i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4088         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4089
4090         i9xx_pipestat_irq_reset(dev_priv);
4091
4092         GEN3_IRQ_RESET(uncore, GEN2_);
4093         dev_priv->irq_mask = ~0u;
4094 }
4095
4096 static void i965_irq_postinstall(struct drm_i915_private *dev_priv)
4097 {
4098         struct intel_uncore *uncore = &dev_priv->uncore;
4099         u32 enable_mask;
4100         u32 error_mask;
4101
4102         /*
4103          * Enable some error detection, note the instruction error mask
4104          * bit is reserved, so we leave it masked.
4105          */
4106         if (IS_G4X(dev_priv)) {
4107                 error_mask = ~(GM45_ERROR_PAGE_TABLE |
4108                                GM45_ERROR_MEM_PRIV |
4109                                GM45_ERROR_CP_PRIV |
4110                                I915_ERROR_MEMORY_REFRESH);
4111         } else {
4112                 error_mask = ~(I915_ERROR_PAGE_TABLE |
4113                                I915_ERROR_MEMORY_REFRESH);
4114         }
4115         I915_WRITE(EMR, error_mask);
4116
4117         /* Unmask the interrupts that we always want on. */
4118         dev_priv->irq_mask =
4119                 ~(I915_ASLE_INTERRUPT |
4120                   I915_DISPLAY_PORT_INTERRUPT |
4121                   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4122                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4123                   I915_MASTER_ERROR_INTERRUPT);
4124
4125         enable_mask =
4126                 I915_ASLE_INTERRUPT |
4127                 I915_DISPLAY_PORT_INTERRUPT |
4128                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4129                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4130                 I915_MASTER_ERROR_INTERRUPT |
4131                 I915_USER_INTERRUPT;
4132
4133         if (IS_G4X(dev_priv))
4134                 enable_mask |= I915_BSD_USER_INTERRUPT;
4135
4136         GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask);
4137
4138         /* Interrupt setup is already guaranteed to be single-threaded, this is
4139          * just to make the assert_spin_locked check happy. */
4140         spin_lock_irq(&dev_priv->irq_lock);
4141         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
4142         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4143         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4144         spin_unlock_irq(&dev_priv->irq_lock);
4145
4146         i915_enable_asle_pipestat(dev_priv);
4147 }
4148
4149 static void i915_hpd_irq_setup(struct drm_i915_private *dev_priv)
4150 {
4151         u32 hotplug_en;
4152
4153         lockdep_assert_held(&dev_priv->irq_lock);
4154
4155         /* Note HDMI and DP share hotplug bits */
4156         /* enable bits are the same for all generations */
4157         hotplug_en = intel_hpd_enabled_irqs(dev_priv, hpd_mask_i915);
4158         /* Programming the CRT detection parameters tends
4159            to generate a spurious hotplug event about three
4160            seconds later.  So just do it once.
4161         */
4162         if (IS_G4X(dev_priv))
4163                 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
4164         hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
4165
4166         /* Ignore TV since it's buggy */
4167         i915_hotplug_interrupt_update_locked(dev_priv,
4168                                              HOTPLUG_INT_EN_MASK |
4169                                              CRT_HOTPLUG_VOLTAGE_COMPARE_MASK |
4170                                              CRT_HOTPLUG_ACTIVATION_PERIOD_64,
4171                                              hotplug_en);
4172 }
4173
4174 static irqreturn_t i965_irq_handler(int irq, void *arg)
4175 {
4176         struct drm_i915_private *dev_priv = arg;
4177         irqreturn_t ret = IRQ_NONE;
4178
4179         if (!intel_irqs_enabled(dev_priv))
4180                 return IRQ_NONE;
4181
4182         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
4183         disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
4184
4185         do {
4186                 u32 pipe_stats[I915_MAX_PIPES] = {};
4187                 u32 eir = 0, eir_stuck = 0;
4188                 u32 hotplug_status = 0;
4189                 u32 iir;
4190
4191                 iir = I915_READ(GEN2_IIR);
4192                 if (iir == 0)
4193                         break;
4194
4195                 ret = IRQ_HANDLED;
4196
4197                 if (iir & I915_DISPLAY_PORT_INTERRUPT)
4198                         hotplug_status = i9xx_hpd_irq_ack(dev_priv);
4199
4200                 /* Call regardless, as some status bits might not be
4201                  * signalled in iir */
4202                 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
4203
4204                 if (iir & I915_MASTER_ERROR_INTERRUPT)
4205                         i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
4206
4207                 I915_WRITE(GEN2_IIR, iir);
4208
4209                 if (iir & I915_USER_INTERRUPT)
4210                         intel_engine_signal_breadcrumbs(dev_priv->gt.engine[RCS0]);
4211
4212                 if (iir & I915_BSD_USER_INTERRUPT)
4213                         intel_engine_signal_breadcrumbs(dev_priv->gt.engine[VCS0]);
4214
4215                 if (iir & I915_MASTER_ERROR_INTERRUPT)
4216                         i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
4217
4218                 if (hotplug_status)
4219                         i9xx_hpd_irq_handler(dev_priv, hotplug_status);
4220
4221                 i965_pipestat_irq_handler(dev_priv, iir, pipe_stats);
4222         } while (0);
4223
4224         pmu_irq_stats(dev_priv, IRQ_HANDLED);
4225
4226         enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
4227
4228         return ret;
4229 }
4230
4231 /**
4232  * intel_irq_init - initializes irq support
4233  * @dev_priv: i915 device instance
4234  *
4235  * This function initializes all the irq support including work items, timers
4236  * and all the vtables. It does not setup the interrupt itself though.
4237  */
4238 void intel_irq_init(struct drm_i915_private *dev_priv)
4239 {
4240         struct drm_device *dev = &dev_priv->drm;
4241         int i;
4242
4243         INIT_WORK(&dev_priv->l3_parity.error_work, ivb_parity_work);
4244         for (i = 0; i < MAX_L3_SLICES; ++i)
4245                 dev_priv->l3_parity.remap_info[i] = NULL;
4246
4247         /* pre-gen11 the guc irqs bits are in the upper 16 bits of the pm reg */
4248         if (HAS_GT_UC(dev_priv) && INTEL_GEN(dev_priv) < 11)
4249                 dev_priv->gt.pm_guc_events = GUC_INTR_GUC2HOST << 16;
4250
4251         if (!HAS_DISPLAY(dev_priv))
4252                 return;
4253
4254         intel_hpd_init_pins(dev_priv);
4255
4256         intel_hpd_init_work(dev_priv);
4257
4258         dev->vblank_disable_immediate = true;
4259
4260         /* Most platforms treat the display irq block as an always-on
4261          * power domain. vlv/chv can disable it at runtime and need
4262          * special care to avoid writing any of the display block registers
4263          * outside of the power domain. We defer setting up the display irqs
4264          * in this case to the runtime pm.
4265          */
4266         dev_priv->display_irqs_enabled = true;
4267         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
4268                 dev_priv->display_irqs_enabled = false;
4269
4270         dev_priv->hotplug.hpd_storm_threshold = HPD_STORM_DEFAULT_THRESHOLD;
4271         /* If we have MST support, we want to avoid doing short HPD IRQ storm
4272          * detection, as short HPD storms will occur as a natural part of
4273          * sideband messaging with MST.
4274          * On older platforms however, IRQ storms can occur with both long and
4275          * short pulses, as seen on some G4x systems.
4276          */
4277         dev_priv->hotplug.hpd_short_storm_enabled = !HAS_DP_MST(dev_priv);
4278
4279         if (HAS_GMCH(dev_priv)) {
4280                 if (I915_HAS_HOTPLUG(dev_priv))
4281                         dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4282         } else {
4283                 if (HAS_PCH_DG1(dev_priv))
4284                         dev_priv->display.hpd_irq_setup = dg1_hpd_irq_setup;
4285                 else if (INTEL_GEN(dev_priv) >= 11)
4286                         dev_priv->display.hpd_irq_setup = gen11_hpd_irq_setup;
4287                 else if (IS_GEN9_LP(dev_priv))
4288                         dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4289                 else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT)
4290                         dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
4291                 else
4292                         dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4293         }
4294 }
4295
4296 /**
4297  * intel_irq_fini - deinitializes IRQ support
4298  * @i915: i915 device instance
4299  *
4300  * This function deinitializes all the IRQ support.
4301  */
4302 void intel_irq_fini(struct drm_i915_private *i915)
4303 {
4304         int i;
4305
4306         for (i = 0; i < MAX_L3_SLICES; ++i)
4307                 kfree(i915->l3_parity.remap_info[i]);
4308 }
4309
4310 static irq_handler_t intel_irq_handler(struct drm_i915_private *dev_priv)
4311 {
4312         if (HAS_GMCH(dev_priv)) {
4313                 if (IS_CHERRYVIEW(dev_priv))
4314                         return cherryview_irq_handler;
4315                 else if (IS_VALLEYVIEW(dev_priv))
4316                         return valleyview_irq_handler;
4317                 else if (IS_GEN(dev_priv, 4))
4318                         return i965_irq_handler;
4319                 else if (IS_GEN(dev_priv, 3))
4320                         return i915_irq_handler;
4321                 else
4322                         return i8xx_irq_handler;
4323         } else {
4324                 if (HAS_MASTER_UNIT_IRQ(dev_priv))
4325                         return dg1_irq_handler;
4326                 if (INTEL_GEN(dev_priv) >= 11)
4327                         return gen11_irq_handler;
4328                 else if (INTEL_GEN(dev_priv) >= 8)
4329                         return gen8_irq_handler;
4330                 else
4331                         return ilk_irq_handler;
4332         }
4333 }
4334
4335 static void intel_irq_reset(struct drm_i915_private *dev_priv)
4336 {
4337         if (HAS_GMCH(dev_priv)) {
4338                 if (IS_CHERRYVIEW(dev_priv))
4339                         cherryview_irq_reset(dev_priv);
4340                 else if (IS_VALLEYVIEW(dev_priv))
4341                         valleyview_irq_reset(dev_priv);
4342                 else if (IS_GEN(dev_priv, 4))
4343                         i965_irq_reset(dev_priv);
4344                 else if (IS_GEN(dev_priv, 3))
4345                         i915_irq_reset(dev_priv);
4346                 else
4347                         i8xx_irq_reset(dev_priv);
4348         } else {
4349                 if (INTEL_GEN(dev_priv) >= 11)
4350                         gen11_irq_reset(dev_priv);
4351                 else if (INTEL_GEN(dev_priv) >= 8)
4352                         gen8_irq_reset(dev_priv);
4353                 else
4354                         ilk_irq_reset(dev_priv);
4355         }
4356 }
4357
4358 static void intel_irq_postinstall(struct drm_i915_private *dev_priv)
4359 {
4360         if (HAS_GMCH(dev_priv)) {
4361                 if (IS_CHERRYVIEW(dev_priv))
4362                         cherryview_irq_postinstall(dev_priv);
4363                 else if (IS_VALLEYVIEW(dev_priv))
4364                         valleyview_irq_postinstall(dev_priv);
4365                 else if (IS_GEN(dev_priv, 4))
4366                         i965_irq_postinstall(dev_priv);
4367                 else if (IS_GEN(dev_priv, 3))
4368                         i915_irq_postinstall(dev_priv);
4369                 else
4370                         i8xx_irq_postinstall(dev_priv);
4371         } else {
4372                 if (INTEL_GEN(dev_priv) >= 11)
4373                         gen11_irq_postinstall(dev_priv);
4374                 else if (INTEL_GEN(dev_priv) >= 8)
4375                         gen8_irq_postinstall(dev_priv);
4376                 else
4377                         ilk_irq_postinstall(dev_priv);
4378         }
4379 }
4380
4381 /**
4382  * intel_irq_install - enables the hardware interrupt
4383  * @dev_priv: i915 device instance
4384  *
4385  * This function enables the hardware interrupt handling, but leaves the hotplug
4386  * handling still disabled. It is called after intel_irq_init().
4387  *
4388  * In the driver load and resume code we need working interrupts in a few places
4389  * but don't want to deal with the hassle of concurrent probe and hotplug
4390  * workers. Hence the split into this two-stage approach.
4391  */
4392 int intel_irq_install(struct drm_i915_private *dev_priv)
4393 {
4394         int irq = dev_priv->drm.pdev->irq;
4395         int ret;
4396
4397         /*
4398          * We enable some interrupt sources in our postinstall hooks, so mark
4399          * interrupts as enabled _before_ actually enabling them to avoid
4400          * special cases in our ordering checks.
4401          */
4402         dev_priv->runtime_pm.irqs_enabled = true;
4403
4404         dev_priv->drm.irq_enabled = true;
4405
4406         intel_irq_reset(dev_priv);
4407
4408         ret = request_irq(irq, intel_irq_handler(dev_priv),
4409                           IRQF_SHARED, DRIVER_NAME, dev_priv);
4410         if (ret < 0) {
4411                 dev_priv->drm.irq_enabled = false;
4412                 return ret;
4413         }
4414
4415         intel_irq_postinstall(dev_priv);
4416
4417         return ret;
4418 }
4419
4420 /**
4421  * intel_irq_uninstall - finilizes all irq handling
4422  * @dev_priv: i915 device instance
4423  *
4424  * This stops interrupt and hotplug handling and unregisters and frees all
4425  * resources acquired in the init functions.
4426  */
4427 void intel_irq_uninstall(struct drm_i915_private *dev_priv)
4428 {
4429         int irq = dev_priv->drm.pdev->irq;
4430
4431         /*
4432          * FIXME we can get called twice during driver probe
4433          * error handling as well as during driver remove due to
4434          * intel_modeset_driver_remove() calling us out of sequence.
4435          * Would be nice if it didn't do that...
4436          */
4437         if (!dev_priv->drm.irq_enabled)
4438                 return;
4439
4440         dev_priv->drm.irq_enabled = false;
4441
4442         intel_irq_reset(dev_priv);
4443
4444         free_irq(irq, dev_priv);
4445
4446         intel_hpd_cancel_work(dev_priv);
4447         dev_priv->runtime_pm.irqs_enabled = false;
4448 }
4449
4450 /**
4451  * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
4452  * @dev_priv: i915 device instance
4453  *
4454  * This function is used to disable interrupts at runtime, both in the runtime
4455  * pm and the system suspend/resume code.
4456  */
4457 void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4458 {
4459         intel_irq_reset(dev_priv);
4460         dev_priv->runtime_pm.irqs_enabled = false;
4461         intel_synchronize_irq(dev_priv);
4462 }
4463
4464 /**
4465  * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
4466  * @dev_priv: i915 device instance
4467  *
4468  * This function is used to enable interrupts at runtime, both in the runtime
4469  * pm and the system suspend/resume code.
4470  */
4471 void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4472 {
4473         dev_priv->runtime_pm.irqs_enabled = true;
4474         intel_irq_reset(dev_priv);
4475         intel_irq_postinstall(dev_priv);
4476 }
4477
4478 bool intel_irqs_enabled(struct drm_i915_private *dev_priv)
4479 {
4480         /*
4481          * We only use drm_irq_uninstall() at unload and VT switch, so
4482          * this is the only thing we need to check.
4483          */
4484         return dev_priv->runtime_pm.irqs_enabled;
4485 }
4486
4487 void intel_synchronize_irq(struct drm_i915_private *i915)
4488 {
4489         synchronize_irq(i915->drm.pdev->irq);
4490 }