drivers/gpu/drm/i915/i915_irq.c

   1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
   2  */
   3 /*
   4  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  */
  28
  29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  30
  31 #include <linux/circ_buf.h>
  32 #include <linux/slab.h>
  33 #include <linux/sysrq.h>
  34
  35 #include <drm/drm_drv.h>
  36 #include <drm/drm_irq.h>
  37
  38 #include "display/intel_display_types.h"
  39 #include "display/intel_fifo_underrun.h"
  40 #include "display/intel_hotplug.h"
  41 #include "display/intel_lpe_audio.h"
  42 #include "display/intel_psr.h"
  43
  44 #include "gt/intel_gt.h"
  45 #include "gt/intel_gt_irq.h"
  46 #include "gt/intel_gt_pm_irq.h"
  47 #include "gt/intel_rps.h"
  48
  49 #include "i915_drv.h"
  50 #include "i915_irq.h"
  51 #include "i915_trace.h"
  52 #include "intel_pm.h"
  53
  54 /**
  55  * DOC: interrupt handling
  56  *
  57  * These functions provide the basic support for enabling and disabling the
  58  * interrupt handling support. There's a lot more functionality in i915_irq.c
  59  * and related files, but that will be described in separate chapters.
  60  */
  61
  62 typedef bool (*long_pulse_detect_func)(enum hpd_pin pin, u32 val);
  63
  64 static const u32 hpd_ilk[HPD_NUM_PINS] = {
  65         [HPD_PORT_A] = DE_DP_A_HOTPLUG,
  66 };
  67
  68 static const u32 hpd_ivb[HPD_NUM_PINS] = {
  69         [HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
  70 };
  71
  72 static const u32 hpd_bdw[HPD_NUM_PINS] = {
  73         [HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
  74 };
  75
  76 static const u32 hpd_ibx[HPD_NUM_PINS] = {
  77         [HPD_CRT] = SDE_CRT_HOTPLUG,
  78         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
  79         [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
  80         [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
  81         [HPD_PORT_D] = SDE_PORTD_HOTPLUG,
  82 };
  83
  84 static const u32 hpd_cpt[HPD_NUM_PINS] = {
  85         [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
  86         [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
  87         [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
  88         [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
  89         [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
  90 };
  91
  92 static const u32 hpd_spt[HPD_NUM_PINS] = {
  93         [HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
  94         [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
  95         [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
  96         [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
  97         [HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT,
  98 };
  99
 100 static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
 101         [HPD_CRT] = CRT_HOTPLUG_INT_EN,
 102         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
 103         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
 104         [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
 105         [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
 106         [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN,
 107 };
 108
 109 static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
 110         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
 111         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
 112         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
 113         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
 114         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
 115         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS,
 116 };
 117
 118 static const u32 hpd_status_i915[HPD_NUM_PINS] = {
 119         [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
 120         [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
 121         [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
 122         [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
 123         [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
 124         [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS,
 125 };
 126
 127 /* BXT hpd list */
 128 static const u32 hpd_bxt[HPD_NUM_PINS] = {
 129         [HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
 130         [HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
 131         [HPD_PORT_C] = BXT_DE_PORT_HP_DDIC,
 132 };
 133
 134 static const u32 hpd_gen11[HPD_NUM_PINS] = {
 135         [HPD_PORT_C] = GEN11_TC1_HOTPLUG | GEN11_TBT1_HOTPLUG,
 136         [HPD_PORT_D] = GEN11_TC2_HOTPLUG | GEN11_TBT2_HOTPLUG,
 137         [HPD_PORT_E] = GEN11_TC3_HOTPLUG | GEN11_TBT3_HOTPLUG,
 138         [HPD_PORT_F] = GEN11_TC4_HOTPLUG | GEN11_TBT4_HOTPLUG,
 139 };
 140
 141 static const u32 hpd_gen12[HPD_NUM_PINS] = {
 142         [HPD_PORT_D] = GEN11_TC1_HOTPLUG | GEN11_TBT1_HOTPLUG,
 143         [HPD_PORT_E] = GEN11_TC2_HOTPLUG | GEN11_TBT2_HOTPLUG,
 144         [HPD_PORT_F] = GEN11_TC3_HOTPLUG | GEN11_TBT3_HOTPLUG,
 145         [HPD_PORT_G] = GEN11_TC4_HOTPLUG | GEN11_TBT4_HOTPLUG,
 146         [HPD_PORT_H] = GEN12_TC5_HOTPLUG | GEN12_TBT5_HOTPLUG,
 147         [HPD_PORT_I] = GEN12_TC6_HOTPLUG | GEN12_TBT6_HOTPLUG,
 148 };
 149
 150 static const u32 hpd_icp[HPD_NUM_PINS] = {
 151         [HPD_PORT_A] = SDE_DDI_HOTPLUG_ICP(PORT_A),
 152         [HPD_PORT_B] = SDE_DDI_HOTPLUG_ICP(PORT_B),
 153         [HPD_PORT_C] = SDE_TC_HOTPLUG_ICP(PORT_TC1),
 154         [HPD_PORT_D] = SDE_TC_HOTPLUG_ICP(PORT_TC2),
 155         [HPD_PORT_E] = SDE_TC_HOTPLUG_ICP(PORT_TC3),
 156         [HPD_PORT_F] = SDE_TC_HOTPLUG_ICP(PORT_TC4),
 157 };
 158
 159 static const u32 hpd_tgp[HPD_NUM_PINS] = {
 160         [HPD_PORT_A] = SDE_DDI_HOTPLUG_ICP(PORT_A),
 161         [HPD_PORT_B] = SDE_DDI_HOTPLUG_ICP(PORT_B),
 162         [HPD_PORT_C] = SDE_DDI_HOTPLUG_ICP(PORT_C),
 163         [HPD_PORT_D] = SDE_TC_HOTPLUG_ICP(PORT_TC1),
 164         [HPD_PORT_E] = SDE_TC_HOTPLUG_ICP(PORT_TC2),
 165         [HPD_PORT_F] = SDE_TC_HOTPLUG_ICP(PORT_TC3),
 166         [HPD_PORT_G] = SDE_TC_HOTPLUG_ICP(PORT_TC4),
 167         [HPD_PORT_H] = SDE_TC_HOTPLUG_ICP(PORT_TC5),
 168         [HPD_PORT_I] = SDE_TC_HOTPLUG_ICP(PORT_TC6),
 169 };
 170
 171 static void
 172 intel_handle_vblank(struct drm_i915_private *dev_priv, enum pipe pipe)
 173 {
 174         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
 175
 176         drm_crtc_handle_vblank(&crtc->base);
 177 }
 178
 179 void gen3_irq_reset(struct intel_uncore *uncore, i915_reg_t imr,
 180                     i915_reg_t iir, i915_reg_t ier)
 181 {
 182         intel_uncore_write(uncore, imr, 0xffffffff);
 183         intel_uncore_posting_read(uncore, imr);
 184
 185         intel_uncore_write(uncore, ier, 0);
 186
 187         /* IIR can theoretically queue up two events. Be paranoid. */
 188         intel_uncore_write(uncore, iir, 0xffffffff);
 189         intel_uncore_posting_read(uncore, iir);
 190         intel_uncore_write(uncore, iir, 0xffffffff);
 191         intel_uncore_posting_read(uncore, iir);
 192 }
 193
 194 void gen2_irq_reset(struct intel_uncore *uncore)
 195 {
 196         intel_uncore_write16(uncore, GEN2_IMR, 0xffff);
 197         intel_uncore_posting_read16(uncore, GEN2_IMR);
 198
 199         intel_uncore_write16(uncore, GEN2_IER, 0);
 200
 201         /* IIR can theoretically queue up two events. Be paranoid. */
 202         intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
 203         intel_uncore_posting_read16(uncore, GEN2_IIR);
 204         intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
 205         intel_uncore_posting_read16(uncore, GEN2_IIR);
 206 }
 207
 208 /*
 209  * We should clear IMR at preinstall/uninstall, and just check at postinstall.
 210  */
 211 static void gen3_assert_iir_is_zero(struct intel_uncore *uncore, i915_reg_t reg)
 212 {
 213         u32 val = intel_uncore_read(uncore, reg);
 214
 215         if (val == 0)
 216                 return;
 217
 218         drm_WARN(&uncore->i915->drm, 1,
 219                  "Interrupt register 0x%x is not zero: 0x%08x\n",
 220                  i915_mmio_reg_offset(reg), val);
 221         intel_uncore_write(uncore, reg, 0xffffffff);
 222         intel_uncore_posting_read(uncore, reg);
 223         intel_uncore_write(uncore, reg, 0xffffffff);
 224         intel_uncore_posting_read(uncore, reg);
 225 }
 226
 227 static void gen2_assert_iir_is_zero(struct intel_uncore *uncore)
 228 {
 229         u16 val = intel_uncore_read16(uncore, GEN2_IIR);
 230
 231         if (val == 0)
 232                 return;
 233
 234         drm_WARN(&uncore->i915->drm, 1,
 235                  "Interrupt register 0x%x is not zero: 0x%08x\n",
 236                  i915_mmio_reg_offset(GEN2_IIR), val);
 237         intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
 238         intel_uncore_posting_read16(uncore, GEN2_IIR);
 239         intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
 240         intel_uncore_posting_read16(uncore, GEN2_IIR);
 241 }
 242
 243 void gen3_irq_init(struct intel_uncore *uncore,
 244                    i915_reg_t imr, u32 imr_val,
 245                    i915_reg_t ier, u32 ier_val,
 246                    i915_reg_t iir)
 247 {
 248         gen3_assert_iir_is_zero(uncore, iir);
 249
 250         intel_uncore_write(uncore, ier, ier_val);
 251         intel_uncore_write(uncore, imr, imr_val);
 252         intel_uncore_posting_read(uncore, imr);
 253 }
 254
 255 void gen2_irq_init(struct intel_uncore *uncore,
 256                    u32 imr_val, u32 ier_val)
 257 {
 258         gen2_assert_iir_is_zero(uncore);
 259
 260         intel_uncore_write16(uncore, GEN2_IER, ier_val);
 261         intel_uncore_write16(uncore, GEN2_IMR, imr_val);
 262         intel_uncore_posting_read16(uncore, GEN2_IMR);
 263 }
 264
 265 /* For display hotplug interrupt */
 266 static inline void
 267 i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
 268                                      u32 mask,
 269                                      u32 bits)
 270 {
 271         u32 val;
 272
 273         lockdep_assert_held(&dev_priv->irq_lock);
 274         drm_WARN_ON(&dev_priv->drm, bits & ~mask);
 275
 276         val = I915_READ(PORT_HOTPLUG_EN);
 277         val &= ~mask;
 278         val |= bits;
 279         I915_WRITE(PORT_HOTPLUG_EN, val);
 280 }
 281
 282 /**
 283  * i915_hotplug_interrupt_update - update hotplug interrupt enable
 284  * @dev_priv: driver private
 285  * @mask: bits to update
 286  * @bits: bits to enable
 287  * NOTE: the HPD enable bits are modified both inside and outside
 288  * of an interrupt context. To avoid that read-modify-write cycles
 289  * interfer, these bits are protected by a spinlock. Since this
 290  * function is usually not called from a context where the lock is
 291  * held already, this function acquires the lock itself. A non-locking
 292  * version is also available.
 293  */
 294 void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
 295                                    u32 mask,
 296                                    u32 bits)
 297 {
 298         spin_lock_irq(&dev_priv->irq_lock);
 299         i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
 300         spin_unlock_irq(&dev_priv->irq_lock);
 301 }
 302
 303 /**
 304  * ilk_update_display_irq - update DEIMR
 305  * @dev_priv: driver private
 306  * @interrupt_mask: mask of interrupt bits to update
 307  * @enabled_irq_mask: mask of interrupt bits to enable
 308  */
 309 void ilk_update_display_irq(struct drm_i915_private *dev_priv,
 310                             u32 interrupt_mask,
 311                             u32 enabled_irq_mask)
 312 {
 313         u32 new_val;
 314
 315         lockdep_assert_held(&dev_priv->irq_lock);
 316
 317         drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
 318
 319         if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
 320                 return;
 321
 322         new_val = dev_priv->irq_mask;
 323         new_val &= ~interrupt_mask;
 324         new_val |= (~enabled_irq_mask & interrupt_mask);
 325
 326         if (new_val != dev_priv->irq_mask) {
 327                 dev_priv->irq_mask = new_val;
 328                 I915_WRITE(DEIMR, dev_priv->irq_mask);
 329                 POSTING_READ(DEIMR);
 330         }
 331 }
 332
 333 /**
 334  * bdw_update_port_irq - update DE port interrupt
 335  * @dev_priv: driver private
 336  * @interrupt_mask: mask of interrupt bits to update
 337  * @enabled_irq_mask: mask of interrupt bits to enable
 338  */
 339 static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
 340                                 u32 interrupt_mask,
 341                                 u32 enabled_irq_mask)
 342 {
 343         u32 new_val;
 344         u32 old_val;
 345
 346         lockdep_assert_held(&dev_priv->irq_lock);
 347
 348         drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
 349
 350         if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
 351                 return;
 352
 353         old_val = I915_READ(GEN8_DE_PORT_IMR);
 354
 355         new_val = old_val;
 356         new_val &= ~interrupt_mask;
 357         new_val |= (~enabled_irq_mask & interrupt_mask);
 358
 359         if (new_val != old_val) {
 360                 I915_WRITE(GEN8_DE_PORT_IMR, new_val);
 361                 POSTING_READ(GEN8_DE_PORT_IMR);
 362         }
 363 }
 364
 365 /**
 366  * bdw_update_pipe_irq - update DE pipe interrupt
 367  * @dev_priv: driver private
 368  * @pipe: pipe whose interrupt to update
 369  * @interrupt_mask: mask of interrupt bits to update
 370  * @enabled_irq_mask: mask of interrupt bits to enable
 371  */
 372 void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
 373                          enum pipe pipe,
 374                          u32 interrupt_mask,
 375                          u32 enabled_irq_mask)
 376 {
 377         u32 new_val;
 378
 379         lockdep_assert_held(&dev_priv->irq_lock);
 380
 381         drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
 382
 383         if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
 384                 return;
 385
 386         new_val = dev_priv->de_irq_mask[pipe];
 387         new_val &= ~interrupt_mask;
 388         new_val |= (~enabled_irq_mask & interrupt_mask);
 389
 390         if (new_val != dev_priv->de_irq_mask[pipe]) {
 391                 dev_priv->de_irq_mask[pipe] = new_val;
 392                 I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
 393                 POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
 394         }
 395 }
 396
 397 /**
 398  * ibx_display_interrupt_update - update SDEIMR
 399  * @dev_priv: driver private
 400  * @interrupt_mask: mask of interrupt bits to update
 401  * @enabled_irq_mask: mask of interrupt bits to enable
 402  */
 403 void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
 404                                   u32 interrupt_mask,
 405                                   u32 enabled_irq_mask)
 406 {
 407         u32 sdeimr = I915_READ(SDEIMR);
 408         sdeimr &= ~interrupt_mask;
 409         sdeimr |= (~enabled_irq_mask & interrupt_mask);
 410
 411         drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
 412
 413         lockdep_assert_held(&dev_priv->irq_lock);
 414
 415         if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
 416                 return;
 417
 418         I915_WRITE(SDEIMR, sdeimr);
 419         POSTING_READ(SDEIMR);
 420 }
 421
 422 u32 i915_pipestat_enable_mask(struct drm_i915_private *dev_priv,
 423                               enum pipe pipe)
 424 {
 425         u32 status_mask = dev_priv->pipestat_irq_mask[pipe];
 426         u32 enable_mask = status_mask << 16;
 427
 428         lockdep_assert_held(&dev_priv->irq_lock);
 429
 430         if (INTEL_GEN(dev_priv) < 5)
 431                 goto out;
 432
 433         /*
 434          * On pipe A we don't support the PSR interrupt yet,
 435          * on pipe B and C the same bit MBZ.
 436          */
 437         if (drm_WARN_ON_ONCE(&dev_priv->drm,
 438                              status_mask & PIPE_A_PSR_STATUS_VLV))
 439                 return 0;
 440         /*
 441          * On pipe B and C we don't support the PSR interrupt yet, on pipe
 442          * A the same bit is for perf counters which we don't use either.
 443          */
 444         if (drm_WARN_ON_ONCE(&dev_priv->drm,
 445                              status_mask & PIPE_B_PSR_STATUS_VLV))
 446                 return 0;
 447
 448         enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
 449                          SPRITE0_FLIP_DONE_INT_EN_VLV |
 450                          SPRITE1_FLIP_DONE_INT_EN_VLV);
 451         if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
 452                 enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
 453         if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
 454                 enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
 455
 456 out:
 457         drm_WARN_ONCE(&dev_priv->drm,
 458                       enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
 459                       status_mask & ~PIPESTAT_INT_STATUS_MASK,
 460                       "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
 461                       pipe_name(pipe), enable_mask, status_mask);
 462
 463         return enable_mask;
 464 }
 465
 466 void i915_enable_pipestat(struct drm_i915_private *dev_priv,
 467                           enum pipe pipe, u32 status_mask)
 468 {
 469         i915_reg_t reg = PIPESTAT(pipe);
 470         u32 enable_mask;
 471
 472         drm_WARN_ONCE(&dev_priv->drm, status_mask & ~PIPESTAT_INT_STATUS_MASK,
 473                       "pipe %c: status_mask=0x%x\n",
 474                       pipe_name(pipe), status_mask);
 475
 476         lockdep_assert_held(&dev_priv->irq_lock);
 477         drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv));
 478
 479         if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == status_mask)
 480                 return;
 481
 482         dev_priv->pipestat_irq_mask[pipe] |= status_mask;
 483         enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
 484
 485         I915_WRITE(reg, enable_mask | status_mask);
 486         POSTING_READ(reg);
 487 }
 488
 489 void i915_disable_pipestat(struct drm_i915_private *dev_priv,
 490                            enum pipe pipe, u32 status_mask)
 491 {
 492         i915_reg_t reg = PIPESTAT(pipe);
 493         u32 enable_mask;
 494
 495         drm_WARN_ONCE(&dev_priv->drm, status_mask & ~PIPESTAT_INT_STATUS_MASK,
 496                       "pipe %c: status_mask=0x%x\n",
 497                       pipe_name(pipe), status_mask);
 498
 499         lockdep_assert_held(&dev_priv->irq_lock);
 500         drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv));
 501
 502         if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == 0)
 503                 return;
 504
 505         dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
 506         enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
 507
 508         I915_WRITE(reg, enable_mask | status_mask);
 509         POSTING_READ(reg);
 510 }
 511
 512 static bool i915_has_asle(struct drm_i915_private *dev_priv)
 513 {
 514         if (!dev_priv->opregion.asle)
 515                 return false;
 516
 517         return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
 518 }
 519
 520 /**
 521  * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
 522  * @dev_priv: i915 device private
 523  */
 524 static void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv)
 525 {
 526         if (!i915_has_asle(dev_priv))
 527                 return;
 528
 529         spin_lock_irq(&dev_priv->irq_lock);
 530
 531         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
 532         if (INTEL_GEN(dev_priv) >= 4)
 533                 i915_enable_pipestat(dev_priv, PIPE_A,
 534                                      PIPE_LEGACY_BLC_EVENT_STATUS);
 535
 536         spin_unlock_irq(&dev_priv->irq_lock);
 537 }
 538
 539 /*
 540  * This timing diagram depicts the video signal in and
 541  * around the vertical blanking period.
 542  *
 543  * Assumptions about the fictitious mode used in this example:
 544  *  vblank_start >= 3
 545  *  vsync_start = vblank_start + 1
 546  *  vsync_end = vblank_start + 2
 547  *  vtotal = vblank_start + 3
 548  *
 549  *           start of vblank:
 550  *           latch double buffered registers
 551  *           increment frame counter (ctg+)
 552  *           generate start of vblank interrupt (gen4+)
 553  *           |
 554  *           |          frame start:
 555  *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
 556  *           |          may be shifted forward 1-3 extra lines via PIPECONF
 557  *           |          |
 558  *           |          |  start of vsync:
 559  *           |          |  generate vsync interrupt
 560  *           |          |  |
 561  * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
 562  *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
 563  * ----va---> <-----------------vb--------------------> <--------va-------------
 564  *       |          |       <----vs----->                     |
 565  * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
 566  * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
 567  * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
 568  *       |          |                                         |
 569  *       last visible pixel                                   first visible pixel
 570  *                  |                                         increment frame counter (gen3/4)
 571  *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
 572  *
 573  * x  = horizontal active
 574  * _  = horizontal blanking
 575  * hs = horizontal sync
 576  * va = vertical active
 577  * vb = vertical blanking
 578  * vs = vertical sync
 579  * vbs = vblank_start (number)
 580  *
 581  * Summary:
 582  * - most events happen at the start of horizontal sync
 583  * - frame start happens at the start of horizontal blank, 1-4 lines
 584  *   (depending on PIPECONF settings) after the start of vblank
 585  * - gen3/4 pixel and frame counter are synchronized with the start
 586  *   of horizontal active on the first line of vertical active
 587  */
 588
 589 /* Called from drm generic code, passed a 'crtc', which
 590  * we use as a pipe index
 591  */
 592 u32 i915_get_vblank_counter(struct drm_crtc *crtc)
 593 {
 594         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
 595         struct drm_vblank_crtc *vblank = &dev_priv->drm.vblank[drm_crtc_index(crtc)];
 596         const struct drm_display_mode *mode = &vblank->hwmode;
 597         enum pipe pipe = to_intel_crtc(crtc)->pipe;
 598         i915_reg_t high_frame, low_frame;
 599         u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
 600         unsigned long irqflags;
 601
 602         /*
 603          * On i965gm TV output the frame counter only works up to
 604          * the point when we enable the TV encoder. After that the
 605          * frame counter ceases to work and reads zero. We need a
 606          * vblank wait before enabling the TV encoder and so we
 607          * have to enable vblank interrupts while the frame counter
 608          * is still in a working state. However the core vblank code
 609          * does not like us returning non-zero frame counter values
 610          * when we've told it that we don't have a working frame
 611          * counter. Thus we must stop non-zero values leaking out.
 612          */
 613         if (!vblank->max_vblank_count)
 614                 return 0;
 615
 616         htotal = mode->crtc_htotal;
 617         hsync_start = mode->crtc_hsync_start;
 618         vbl_start = mode->crtc_vblank_start;
 619         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 620                 vbl_start = DIV_ROUND_UP(vbl_start, 2);
 621
 622         /* Convert to pixel count */
 623         vbl_start *= htotal;
 624
 625         /* Start of vblank event occurs at start of hsync */
 626         vbl_start -= htotal - hsync_start;
 627
 628         high_frame = PIPEFRAME(pipe);
 629         low_frame = PIPEFRAMEPIXEL(pipe);
 630
 631         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 632
 633         /*
 634          * High & low register fields aren't synchronized, so make sure
 635          * we get a low value that's stable across two reads of the high
 636          * register.
 637          */
 638         do {
 639                 high1 = intel_de_read_fw(dev_priv, high_frame) & PIPE_FRAME_HIGH_MASK;
 640                 low   = intel_de_read_fw(dev_priv, low_frame);
 641                 high2 = intel_de_read_fw(dev_priv, high_frame) & PIPE_FRAME_HIGH_MASK;
 642         } while (high1 != high2);
 643
 644         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 645
 646         high1 >>= PIPE_FRAME_HIGH_SHIFT;
 647         pixel = low & PIPE_PIXEL_MASK;
 648         low >>= PIPE_FRAME_LOW_SHIFT;
 649
 650         /*
 651          * The frame counter increments at beginning of active.
 652          * Cook up a vblank counter by also checking the pixel
 653          * counter against vblank start.
 654          */
 655         return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
 656 }
 657
 658 u32 g4x_get_vblank_counter(struct drm_crtc *crtc)
 659 {
 660         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
 661         enum pipe pipe = to_intel_crtc(crtc)->pipe;
 662
 663         return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
 664 }
 665
 666 /*
 667  * On certain encoders on certain platforms, pipe
 668  * scanline register will not work to get the scanline,
 669  * since the timings are driven from the PORT or issues
 670  * with scanline register updates.
 671  * This function will use Framestamp and current
 672  * timestamp registers to calculate the scanline.
 673  */
 674 static u32 __intel_get_crtc_scanline_from_timestamp(struct intel_crtc *crtc)
 675 {
 676         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 677         struct drm_vblank_crtc *vblank =
 678                 &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)];
 679         const struct drm_display_mode *mode = &vblank->hwmode;
 680         u32 vblank_start = mode->crtc_vblank_start;
 681         u32 vtotal = mode->crtc_vtotal;
 682         u32 htotal = mode->crtc_htotal;
 683         u32 clock = mode->crtc_clock;
 684         u32 scanline, scan_prev_time, scan_curr_time, scan_post_time;
 685
 686         /*
 687          * To avoid the race condition where we might cross into the
 688          * next vblank just between the PIPE_FRMTMSTMP and TIMESTAMP_CTR
 689          * reads. We make sure we read PIPE_FRMTMSTMP and TIMESTAMP_CTR
 690          * during the same frame.
 691          */
 692         do {
 693                 /*
 694                  * This field provides read back of the display
 695                  * pipe frame time stamp. The time stamp value
 696                  * is sampled at every start of vertical blank.
 697                  */
 698                 scan_prev_time = intel_de_read_fw(dev_priv,
 699                                                   PIPE_FRMTMSTMP(crtc->pipe));
 700
 701                 /*
 702                  * The TIMESTAMP_CTR register has the current
 703                  * time stamp value.
 704                  */
 705                 scan_curr_time = intel_de_read_fw(dev_priv, IVB_TIMESTAMP_CTR);
 706
 707                 scan_post_time = intel_de_read_fw(dev_priv,
 708                                                   PIPE_FRMTMSTMP(crtc->pipe));
 709         } while (scan_post_time != scan_prev_time);
 710
 711         scanline = div_u64(mul_u32_u32(scan_curr_time - scan_prev_time,
 712                                         clock), 1000 * htotal);
 713         scanline = min(scanline, vtotal - 1);
 714         scanline = (scanline + vblank_start) % vtotal;
 715
 716         return scanline;
 717 }
 718
 719 /*
 720  * intel_de_read_fw(), only for fast reads of display block, no need for
 721  * forcewake etc.
 722  */
 723 static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
 724 {
 725         struct drm_device *dev = crtc->base.dev;
 726         struct drm_i915_private *dev_priv = to_i915(dev);
 727         const struct drm_display_mode *mode;
 728         struct drm_vblank_crtc *vblank;
 729         enum pipe pipe = crtc->pipe;
 730         int position, vtotal;
 731
 732         if (!crtc->active)
 733                 return -1;
 734
 735         vblank = &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)];
 736         mode = &vblank->hwmode;
 737
 738         if (mode->private_flags & I915_MODE_FLAG_GET_SCANLINE_FROM_TIMESTAMP)
 739                 return __intel_get_crtc_scanline_from_timestamp(crtc);
 740
 741         vtotal = mode->crtc_vtotal;
 742         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 743                 vtotal /= 2;
 744
 745         if (IS_GEN(dev_priv, 2))
 746                 position = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
 747         else
 748                 position = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
 749
 750         /*
 751          * On HSW, the DSL reg (0x70000) appears to return 0 if we
 752          * read it just before the start of vblank.  So try it again
 753          * so we don't accidentally end up spanning a vblank frame
 754          * increment, causing the pipe_update_end() code to squak at us.
 755          *
 756          * The nature of this problem means we can't simply check the ISR
 757          * bit and return the vblank start value; nor can we use the scanline
 758          * debug register in the transcoder as it appears to have the same
 759          * problem.  We may need to extend this to include other platforms,
 760          * but so far testing only shows the problem on HSW.
 761          */
 762         if (HAS_DDI(dev_priv) && !position) {
 763                 int i, temp;
 764
 765                 for (i = 0; i < 100; i++) {
 766                         udelay(1);
 767                         temp = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
 768                         if (temp != position) {
 769                                 position = temp;
 770                                 break;
 771                         }
 772                 }
 773         }
 774
 775         /*
 776          * See update_scanline_offset() for the details on the
 777          * scanline_offset adjustment.
 778          */
 779         return (position + crtc->scanline_offset) % vtotal;
 780 }
 781
 782 static bool i915_get_crtc_scanoutpos(struct drm_crtc *_crtc,
 783                                      bool in_vblank_irq,
 784                                      int *vpos, int *hpos,
 785                                      ktime_t *stime, ktime_t *etime,
 786                                      const struct drm_display_mode *mode)
 787 {
 788         struct drm_device *dev = _crtc->dev;
 789         struct drm_i915_private *dev_priv = to_i915(dev);
 790         struct intel_crtc *crtc = to_intel_crtc(_crtc);
 791         enum pipe pipe = crtc->pipe;
 792         int position;
 793         int vbl_start, vbl_end, hsync_start, htotal, vtotal;
 794         unsigned long irqflags;
 795         bool use_scanline_counter = INTEL_GEN(dev_priv) >= 5 ||
 796                 IS_G4X(dev_priv) || IS_GEN(dev_priv, 2) ||
 797                 mode->private_flags & I915_MODE_FLAG_USE_SCANLINE_COUNTER;
 798
 799         if (drm_WARN_ON(&dev_priv->drm, !mode->crtc_clock)) {
 800                 drm_dbg(&dev_priv->drm,
 801                         "trying to get scanoutpos for disabled "
 802                         "pipe %c\n", pipe_name(pipe));
 803                 return false;
 804         }
 805
 806         htotal = mode->crtc_htotal;
 807         hsync_start = mode->crtc_hsync_start;
 808         vtotal = mode->crtc_vtotal;
 809         vbl_start = mode->crtc_vblank_start;
 810         vbl_end = mode->crtc_vblank_end;
 811
 812         if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
 813                 vbl_start = DIV_ROUND_UP(vbl_start, 2);
 814                 vbl_end /= 2;
 815                 vtotal /= 2;
 816         }
 817
 818         /*
 819          * Lock uncore.lock, as we will do multiple timing critical raw
 820          * register reads, potentially with preemption disabled, so the
 821          * following code must not block on uncore.lock.
 822          */
 823         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 824
 825         /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
 826
 827         /* Get optional system timestamp before query. */
 828         if (stime)
 829                 *stime = ktime_get();
 830
 831         if (use_scanline_counter) {
 832                 /* No obvious pixelcount register. Only query vertical
 833                  * scanout position from Display scan line register.
 834                  */
 835                 position = __intel_get_crtc_scanline(crtc);
 836         } else {
 837                 /* Have access to pixelcount since start of frame.
 838                  * We can split this into vertical and horizontal
 839                  * scanout position.
 840                  */
 841                 position = (intel_de_read_fw(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
 842
 843                 /* convert to pixel counts */
 844                 vbl_start *= htotal;
 845                 vbl_end *= htotal;
 846                 vtotal *= htotal;
 847
 848                 /*
 849                  * In interlaced modes, the pixel counter counts all pixels,
 850                  * so one field will have htotal more pixels. In order to avoid
 851                  * the reported position from jumping backwards when the pixel
 852                  * counter is beyond the length of the shorter field, just
 853                  * clamp the position the length of the shorter field. This
 854                  * matches how the scanline counter based position works since
 855                  * the scanline counter doesn't count the two half lines.
 856                  */
 857                 if (position >= vtotal)
 858                         position = vtotal - 1;
 859
 860                 /*
 861                  * Start of vblank interrupt is triggered at start of hsync,
 862                  * just prior to the first active line of vblank. However we
 863                  * consider lines to start at the leading edge of horizontal
 864                  * active. So, should we get here before we've crossed into
 865                  * the horizontal active of the first line in vblank, we would
 866                  * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
 867                  * always add htotal-hsync_start to the current pixel position.
 868                  */
 869                 position = (position + htotal - hsync_start) % vtotal;
 870         }
 871
 872         /* Get optional system timestamp after query. */
 873         if (etime)
 874                 *etime = ktime_get();
 875
 876         /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
 877
 878         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 879
 880         /*
 881          * While in vblank, position will be negative
 882          * counting up towards 0 at vbl_end. And outside
 883          * vblank, position will be positive counting
 884          * up since vbl_end.
 885          */
 886         if (position >= vbl_start)
 887                 position -= vbl_end;
 888         else
 889                 position += vtotal - vbl_end;
 890
 891         if (use_scanline_counter) {
 892                 *vpos = position;
 893                 *hpos = 0;
 894         } else {
 895                 *vpos = position / htotal;
 896                 *hpos = position - (*vpos * htotal);
 897         }
 898
 899         return true;
 900 }
 901
 902 bool intel_crtc_get_vblank_timestamp(struct drm_crtc *crtc, int *max_error,
 903                                      ktime_t *vblank_time, bool in_vblank_irq)
 904 {
 905         return drm_crtc_vblank_helper_get_vblank_timestamp_internal(
 906                 crtc, max_error, vblank_time, in_vblank_irq,
 907                 i915_get_crtc_scanoutpos);
 908 }
 909
 910 int intel_get_crtc_scanline(struct intel_crtc *crtc)
 911 {
 912         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 913         unsigned long irqflags;
 914         int position;
 915
 916         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 917         position = __intel_get_crtc_scanline(crtc);
 918         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 919
 920         return position;
 921 }
 922
 923 /**
 924  * ivb_parity_work - Workqueue called when a parity error interrupt
 925  * occurred.
 926  * @work: workqueue struct
 927  *
 928  * Doesn't actually do anything except notify userspace. As a consequence of
 929  * this event, userspace should try to remap the bad rows since statistically
 930  * it is likely the same row is more likely to go bad again.
 931  */
 932 static void ivb_parity_work(struct work_struct *work)
 933 {
 934         struct drm_i915_private *dev_priv =
 935                 container_of(work, typeof(*dev_priv), l3_parity.error_work);
 936         struct intel_gt *gt = &dev_priv->gt;
 937         u32 error_status, row, bank, subbank;
 938         char *parity_event[6];
 939         u32 misccpctl;
 940         u8 slice = 0;
 941
 942         /* We must turn off DOP level clock gating to access the L3 registers.
 943          * In order to prevent a get/put style interface, acquire struct mutex
 944          * any time we access those registers.
 945          */
 946         mutex_lock(&dev_priv->drm.struct_mutex);
 947
 948         /* If we've screwed up tracking, just let the interrupt fire again */
 949         if (drm_WARN_ON(&dev_priv->drm, !dev_priv->l3_parity.which_slice))
 950                 goto out;
 951
 952         misccpctl = I915_READ(GEN7_MISCCPCTL);
 953         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
 954         POSTING_READ(GEN7_MISCCPCTL);
 955
 956         while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
 957                 i915_reg_t reg;
 958
 959                 slice--;
 960                 if (drm_WARN_ON_ONCE(&dev_priv->drm,
 961                                      slice >= NUM_L3_SLICES(dev_priv)))
 962                         break;
 963
 964                 dev_priv->l3_parity.which_slice &= ~(1<<slice);
 965
 966                 reg = GEN7_L3CDERRST1(slice);
 967
 968                 error_status = I915_READ(reg);
 969                 row = GEN7_PARITY_ERROR_ROW(error_status);
 970                 bank = GEN7_PARITY_ERROR_BANK(error_status);
 971                 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
 972
 973                 I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
 974                 POSTING_READ(reg);
 975
 976                 parity_event[0] = I915_L3_PARITY_UEVENT "=1";
 977                 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
 978                 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
 979                 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
 980                 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
 981                 parity_event[5] = NULL;
 982
 983                 kobject_uevent_env(&dev_priv->drm.primary->kdev->kobj,
 984                                    KOBJ_CHANGE, parity_event);
 985
 986                 DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
 987                           slice, row, bank, subbank);
 988
 989                 kfree(parity_event[4]);
 990                 kfree(parity_event[3]);
 991                 kfree(parity_event[2]);
 992                 kfree(parity_event[1]);
 993         }
 994
 995         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
 996
 997 out:
 998         drm_WARN_ON(&dev_priv->drm, dev_priv->l3_parity.which_slice);
 999         spin_lock_irq(&gt->irq_lock);
1000         gen5_gt_enable_irq(gt, GT_PARITY_ERROR(dev_priv));
1001         spin_unlock_irq(&gt->irq_lock);
1002
1003         mutex_unlock(&dev_priv->drm.struct_mutex);
1004 }
1005
1006 static bool gen11_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1007 {
1008         switch (pin) {
1009         case HPD_PORT_C:
1010                 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC1);
1011         case HPD_PORT_D:
1012                 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC2);
1013         case HPD_PORT_E:
1014                 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC3);
1015         case HPD_PORT_F:
1016                 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC4);
1017         default:
1018                 return false;
1019         }
1020 }
1021
1022 static bool gen12_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1023 {
1024         switch (pin) {
1025         case HPD_PORT_D:
1026                 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC1);
1027         case HPD_PORT_E:
1028                 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC2);
1029         case HPD_PORT_F:
1030                 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC3);
1031         case HPD_PORT_G:
1032                 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC4);
1033         case HPD_PORT_H:
1034                 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC5);
1035         case HPD_PORT_I:
1036                 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(PORT_TC6);
1037         default:
1038                 return false;
1039         }
1040 }
1041
1042 static bool bxt_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1043 {
1044         switch (pin) {
1045         case HPD_PORT_A:
1046                 return val & PORTA_HOTPLUG_LONG_DETECT;
1047         case HPD_PORT_B:
1048                 return val & PORTB_HOTPLUG_LONG_DETECT;
1049         case HPD_PORT_C:
1050                 return val & PORTC_HOTPLUG_LONG_DETECT;
1051         default:
1052                 return false;
1053         }
1054 }
1055
1056 static bool icp_ddi_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1057 {
1058         switch (pin) {
1059         case HPD_PORT_A:
1060                 return val & SHOTPLUG_CTL_DDI_HPD_LONG_DETECT(PORT_A);
1061         case HPD_PORT_B:
1062                 return val & SHOTPLUG_CTL_DDI_HPD_LONG_DETECT(PORT_B);
1063         case HPD_PORT_C:
1064                 return val & SHOTPLUG_CTL_DDI_HPD_LONG_DETECT(PORT_C);
1065         default:
1066                 return false;
1067         }
1068 }
1069
1070 static bool icp_tc_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1071 {
1072         switch (pin) {
1073         case HPD_PORT_C:
1074                 return val & ICP_TC_HPD_LONG_DETECT(PORT_TC1);
1075         case HPD_PORT_D:
1076                 return val & ICP_TC_HPD_LONG_DETECT(PORT_TC2);
1077         case HPD_PORT_E:
1078                 return val & ICP_TC_HPD_LONG_DETECT(PORT_TC3);
1079         case HPD_PORT_F:
1080                 return val & ICP_TC_HPD_LONG_DETECT(PORT_TC4);
1081         default:
1082                 return false;
1083         }
1084 }
1085
1086 static bool tgp_tc_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1087 {
1088         switch (pin) {
1089         case HPD_PORT_D:
1090                 return val & ICP_TC_HPD_LONG_DETECT(PORT_TC1);
1091         case HPD_PORT_E:
1092                 return val & ICP_TC_HPD_LONG_DETECT(PORT_TC2);
1093         case HPD_PORT_F:
1094                 return val & ICP_TC_HPD_LONG_DETECT(PORT_TC3);
1095         case HPD_PORT_G:
1096                 return val & ICP_TC_HPD_LONG_DETECT(PORT_TC4);
1097         case HPD_PORT_H:
1098                 return val & ICP_TC_HPD_LONG_DETECT(PORT_TC5);
1099         case HPD_PORT_I:
1100                 return val & ICP_TC_HPD_LONG_DETECT(PORT_TC6);
1101         default:
1102                 return false;
1103         }
1104 }
1105
1106 static bool spt_port_hotplug2_long_detect(enum hpd_pin pin, u32 val)
1107 {
1108         switch (pin) {
1109         case HPD_PORT_E:
1110                 return val & PORTE_HOTPLUG_LONG_DETECT;
1111         default:
1112                 return false;
1113         }
1114 }
1115
1116 static bool spt_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1117 {
1118         switch (pin) {
1119         case HPD_PORT_A:
1120                 return val & PORTA_HOTPLUG_LONG_DETECT;
1121         case HPD_PORT_B:
1122                 return val & PORTB_HOTPLUG_LONG_DETECT;
1123         case HPD_PORT_C:
1124                 return val & PORTC_HOTPLUG_LONG_DETECT;
1125         case HPD_PORT_D:
1126                 return val & PORTD_HOTPLUG_LONG_DETECT;
1127         default:
1128                 return false;
1129         }
1130 }
1131
1132 static bool ilk_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1133 {
1134         switch (pin) {
1135         case HPD_PORT_A:
1136                 return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
1137         default:
1138                 return false;
1139         }
1140 }
1141
1142 static bool pch_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1143 {
1144         switch (pin) {
1145         case HPD_PORT_B:
1146                 return val & PORTB_HOTPLUG_LONG_DETECT;
1147         case HPD_PORT_C:
1148                 return val & PORTC_HOTPLUG_LONG_DETECT;
1149         case HPD_PORT_D:
1150                 return val & PORTD_HOTPLUG_LONG_DETECT;
1151         default:
1152                 return false;
1153         }
1154 }
1155
1156 static bool i9xx_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1157 {
1158         switch (pin) {
1159         case HPD_PORT_B:
1160                 return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1161         case HPD_PORT_C:
1162                 return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1163         case HPD_PORT_D:
1164                 return val & PORTD_HOTPLUG_INT_LONG_PULSE;
1165         default:
1166                 return false;
1167         }
1168 }
1169
1170 /*
1171  * Get a bit mask of pins that have triggered, and which ones may be long.
1172  * This can be called multiple times with the same masks to accumulate
1173  * hotplug detection results from several registers.
1174  *
1175  * Note that the caller is expected to zero out the masks initially.
1176  */
1177 static void intel_get_hpd_pins(struct drm_i915_private *dev_priv,
1178                                u32 *pin_mask, u32 *long_mask,
1179                                u32 hotplug_trigger, u32 dig_hotplug_reg,
1180                                const u32 hpd[HPD_NUM_PINS],
1181                                bool long_pulse_detect(enum hpd_pin pin, u32 val))
1182 {
1183         enum hpd_pin pin;
1184
1185         BUILD_BUG_ON(BITS_PER_TYPE(*pin_mask) < HPD_NUM_PINS);
1186
1187         for_each_hpd_pin(pin) {
1188                 if ((hpd[pin] & hotplug_trigger) == 0)
1189                         continue;
1190
1191                 *pin_mask |= BIT(pin);
1192
1193                 if (long_pulse_detect(pin, dig_hotplug_reg))
1194                         *long_mask |= BIT(pin);
1195         }
1196
1197         drm_dbg(&dev_priv->drm,
1198                 "hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x, long 0x%08x\n",
1199                 hotplug_trigger, dig_hotplug_reg, *pin_mask, *long_mask);
1200
1201 }
1202
1203 static void gmbus_irq_handler(struct drm_i915_private *dev_priv)
1204 {
1205         wake_up_all(&dev_priv->gmbus_wait_queue);
1206 }
1207
1208 static void dp_aux_irq_handler(struct drm_i915_private *dev_priv)
1209 {
1210         wake_up_all(&dev_priv->gmbus_wait_queue);
1211 }
1212
1213 #if defined(CONFIG_DEBUG_FS)
1214 static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1215                                          enum pipe pipe,
1216                                          u32 crc0, u32 crc1,
1217                                          u32 crc2, u32 crc3,
1218                                          u32 crc4)
1219 {
1220         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
1221         struct intel_pipe_crc *pipe_crc = &crtc->pipe_crc;
1222         u32 crcs[5] = { crc0, crc1, crc2, crc3, crc4 };
1223
1224         trace_intel_pipe_crc(crtc, crcs);
1225
1226         spin_lock(&pipe_crc->lock);
1227         /*
1228          * For some not yet identified reason, the first CRC is
1229          * bonkers. So let's just wait for the next vblank and read
1230          * out the buggy result.
1231          *
1232          * On GEN8+ sometimes the second CRC is bonkers as well, so
1233          * don't trust that one either.
1234          */
1235         if (pipe_crc->skipped <= 0 ||
1236             (INTEL_GEN(dev_priv) >= 8 && pipe_crc->skipped == 1)) {
1237                 pipe_crc->skipped++;
1238                 spin_unlock(&pipe_crc->lock);
1239                 return;
1240         }
1241         spin_unlock(&pipe_crc->lock);
1242
1243         drm_crtc_add_crc_entry(&crtc->base, true,
1244                                 drm_crtc_accurate_vblank_count(&crtc->base),
1245                                 crcs);
1246 }
1247 #else
1248 static inline void
1249 display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1250                              enum pipe pipe,
1251                              u32 crc0, u32 crc1,
1252                              u32 crc2, u32 crc3,
1253                              u32 crc4) {}
1254 #endif
1255
1256
1257 static void hsw_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1258                                      enum pipe pipe)
1259 {
1260         display_pipe_crc_irq_handler(dev_priv, pipe,
1261                                      I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1262                                      0, 0, 0, 0);
1263 }
1264
1265 static void ivb_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1266                                      enum pipe pipe)
1267 {
1268         display_pipe_crc_irq_handler(dev_priv, pipe,
1269                                      I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1270                                      I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
1271                                      I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
1272                                      I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
1273                                      I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1274 }
1275
1276 static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1277                                       enum pipe pipe)
1278 {
1279         u32 res1, res2;
1280
1281         if (INTEL_GEN(dev_priv) >= 3)
1282                 res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
1283         else
1284                 res1 = 0;
1285
1286         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1287                 res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
1288         else
1289                 res2 = 0;
1290
1291         display_pipe_crc_irq_handler(dev_priv, pipe,
1292                                      I915_READ(PIPE_CRC_RES_RED(pipe)),
1293                                      I915_READ(PIPE_CRC_RES_GREEN(pipe)),
1294                                      I915_READ(PIPE_CRC_RES_BLUE(pipe)),
1295                                      res1, res2);
1296 }
1297
1298 static void i9xx_pipestat_irq_reset(struct drm_i915_private *dev_priv)
1299 {
1300         enum pipe pipe;
1301
1302         for_each_pipe(dev_priv, pipe) {
1303                 I915_WRITE(PIPESTAT(pipe),
1304                            PIPESTAT_INT_STATUS_MASK |
1305                            PIPE_FIFO_UNDERRUN_STATUS);
1306
1307                 dev_priv->pipestat_irq_mask[pipe] = 0;
1308         }
1309 }
1310
1311 static void i9xx_pipestat_irq_ack(struct drm_i915_private *dev_priv,
1312                                   u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1313 {
1314         enum pipe pipe;
1315
1316         spin_lock(&dev_priv->irq_lock);
1317
1318         if (!dev_priv->display_irqs_enabled) {
1319                 spin_unlock(&dev_priv->irq_lock);
1320                 return;
1321         }
1322
1323         for_each_pipe(dev_priv, pipe) {
1324                 i915_reg_t reg;
1325                 u32 status_mask, enable_mask, iir_bit = 0;
1326
1327                 /*
1328                  * PIPESTAT bits get signalled even when the interrupt is
1329                  * disabled with the mask bits, and some of the status bits do
1330                  * not generate interrupts at all (like the underrun bit). Hence
1331                  * we need to be careful that we only handle what we want to
1332                  * handle.
1333                  */
1334
1335                 /* fifo underruns are filterered in the underrun handler. */
1336                 status_mask = PIPE_FIFO_UNDERRUN_STATUS;
1337
1338                 switch (pipe) {
1339                 default:
1340                 case PIPE_A:
1341                         iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
1342                         break;
1343                 case PIPE_B:
1344                         iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
1345                         break;
1346                 case PIPE_C:
1347                         iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
1348                         break;
1349                 }
1350                 if (iir & iir_bit)
1351                         status_mask |= dev_priv->pipestat_irq_mask[pipe];
1352
1353                 if (!status_mask)
1354                         continue;
1355
1356                 reg = PIPESTAT(pipe);
1357                 pipe_stats[pipe] = I915_READ(reg) & status_mask;
1358                 enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
1359
1360                 /*
1361                  * Clear the PIPE*STAT regs before the IIR
1362                  *
1363                  * Toggle the enable bits to make sure we get an
1364                  * edge in the ISR pipe event bit if we don't clear
1365                  * all the enabled status bits. Otherwise the edge
1366                  * triggered IIR on i965/g4x wouldn't notice that
1367                  * an interrupt is still pending.
1368                  */
1369                 if (pipe_stats[pipe]) {
1370                         I915_WRITE(reg, pipe_stats[pipe]);
1371                         I915_WRITE(reg, enable_mask);
1372                 }
1373         }
1374         spin_unlock(&dev_priv->irq_lock);
1375 }
1376
1377 static void i8xx_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1378                                       u16 iir, u32 pipe_stats[I915_MAX_PIPES])
1379 {
1380         enum pipe pipe;
1381
1382         for_each_pipe(dev_priv, pipe) {
1383                 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1384                         intel_handle_vblank(dev_priv, pipe);
1385
1386                 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1387                         i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1388
1389                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1390                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1391         }
1392 }
1393
1394 static void i915_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1395                                       u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1396 {
1397         bool blc_event = false;
1398         enum pipe pipe;
1399
1400         for_each_pipe(dev_priv, pipe) {
1401                 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1402                         intel_handle_vblank(dev_priv, pipe);
1403
1404                 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
1405                         blc_event = true;
1406
1407                 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1408                         i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1409
1410                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1411                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1412         }
1413
1414         if (blc_event || (iir & I915_ASLE_INTERRUPT))
1415                 intel_opregion_asle_intr(dev_priv);
1416 }
1417
1418 static void i965_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1419                                       u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1420 {
1421         bool blc_event = false;
1422         enum pipe pipe;
1423
1424         for_each_pipe(dev_priv, pipe) {
1425                 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
1426                         intel_handle_vblank(dev_priv, pipe);
1427
1428                 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
1429                         blc_event = true;
1430
1431                 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1432                         i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1433
1434                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1435                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1436         }
1437
1438         if (blc_event || (iir & I915_ASLE_INTERRUPT))
1439                 intel_opregion_asle_intr(dev_priv);
1440
1441         if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1442                 gmbus_irq_handler(dev_priv);
1443 }
1444
1445 static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1446                                             u32 pipe_stats[I915_MAX_PIPES])
1447 {
1448         enum pipe pipe;
1449
1450         for_each_pipe(dev_priv, pipe) {
1451                 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
1452                         intel_handle_vblank(dev_priv, pipe);
1453
1454                 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1455                         i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1456
1457                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1458                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1459         }
1460
1461         if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1462                 gmbus_irq_handler(dev_priv);
1463 }
1464
1465 static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv)
1466 {
1467         u32 hotplug_status = 0, hotplug_status_mask;
1468         int i;
1469
1470         if (IS_G4X(dev_priv) ||
1471             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1472                 hotplug_status_mask = HOTPLUG_INT_STATUS_G4X |
1473                         DP_AUX_CHANNEL_MASK_INT_STATUS_G4X;
1474         else
1475                 hotplug_status_mask = HOTPLUG_INT_STATUS_I915;
1476
1477         /*
1478          * We absolutely have to clear all the pending interrupt
1479          * bits in PORT_HOTPLUG_STAT. Otherwise the ISR port
1480          * interrupt bit won't have an edge, and the i965/g4x
1481          * edge triggered IIR will not notice that an interrupt
1482          * is still pending. We can't use PORT_HOTPLUG_EN to
1483          * guarantee the edge as the act of toggling the enable
1484          * bits can itself generate a new hotplug interrupt :(
1485          */
1486         for (i = 0; i < 10; i++) {
1487                 u32 tmp = I915_READ(PORT_HOTPLUG_STAT) & hotplug_status_mask;
1488
1489                 if (tmp == 0)
1490                         return hotplug_status;
1491
1492                 hotplug_status |= tmp;
1493                 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1494         }
1495
1496         drm_WARN_ONCE(&dev_priv->drm, 1,
1497                       "PORT_HOTPLUG_STAT did not clear (0x%08x)\n",
1498                       I915_READ(PORT_HOTPLUG_STAT));
1499
1500         return hotplug_status;
1501 }
1502
1503 static void i9xx_hpd_irq_handler(struct drm_i915_private *dev_priv,
1504                                  u32 hotplug_status)
1505 {
1506         u32 pin_mask = 0, long_mask = 0;
1507
1508         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
1509             IS_CHERRYVIEW(dev_priv)) {
1510                 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1511
1512                 if (hotplug_trigger) {
1513                         intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1514                                            hotplug_trigger, hotplug_trigger,
1515                                            hpd_status_g4x,
1516                                            i9xx_port_hotplug_long_detect);
1517
1518                         intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1519                 }
1520
1521                 if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
1522                         dp_aux_irq_handler(dev_priv);
1523         } else {
1524                 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1525
1526                 if (hotplug_trigger) {
1527                         intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1528                                            hotplug_trigger, hotplug_trigger,
1529                                            hpd_status_i915,
1530                                            i9xx_port_hotplug_long_detect);
1531                         intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1532                 }
1533         }
1534 }
1535
1536 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
1537 {
1538         struct drm_i915_private *dev_priv = arg;
1539         irqreturn_t ret = IRQ_NONE;
1540
1541         if (!intel_irqs_enabled(dev_priv))
1542                 return IRQ_NONE;
1543
1544         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
1545         disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1546
1547         do {
1548                 u32 iir, gt_iir, pm_iir;
1549                 u32 pipe_stats[I915_MAX_PIPES] = {};
1550                 u32 hotplug_status = 0;
1551                 u32 ier = 0;
1552
1553                 gt_iir = I915_READ(GTIIR);
1554                 pm_iir = I915_READ(GEN6_PMIIR);
1555                 iir = I915_READ(VLV_IIR);
1556
1557                 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1558                         break;
1559
1560                 ret = IRQ_HANDLED;
1561
1562                 /*
1563                  * Theory on interrupt generation, based on empirical evidence:
1564                  *
1565                  * x = ((VLV_IIR & VLV_IER) ||
1566                  *      (((GT_IIR & GT_IER) || (GEN6_PMIIR & GEN6_PMIER)) &&
1567                  *       (VLV_MASTER_IER & MASTER_INTERRUPT_ENABLE)));
1568                  *
1569                  * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
1570                  * Hence we clear MASTER_INTERRUPT_ENABLE and VLV_IER to
1571                  * guarantee the CPU interrupt will be raised again even if we
1572                  * don't end up clearing all the VLV_IIR, GT_IIR, GEN6_PMIIR
1573                  * bits this time around.
1574                  */
1575                 I915_WRITE(VLV_MASTER_IER, 0);
1576                 ier = I915_READ(VLV_IER);
1577                 I915_WRITE(VLV_IER, 0);
1578
1579                 if (gt_iir)
1580                         I915_WRITE(GTIIR, gt_iir);
1581                 if (pm_iir)
1582                         I915_WRITE(GEN6_PMIIR, pm_iir);
1583
1584                 if (iir & I915_DISPLAY_PORT_INTERRUPT)
1585                         hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1586
1587                 /* Call regardless, as some status bits might not be
1588                  * signalled in iir */
1589                 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1590
1591                 if (iir & (I915_LPE_PIPE_A_INTERRUPT |
1592                            I915_LPE_PIPE_B_INTERRUPT))
1593                         intel_lpe_audio_irq_handler(dev_priv);
1594
1595                 /*
1596                  * VLV_IIR is single buffered, and reflects the level
1597                  * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
1598                  */
1599                 if (iir)
1600                         I915_WRITE(VLV_IIR, iir);
1601
1602                 I915_WRITE(VLV_IER, ier);
1603                 I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
1604
1605                 if (gt_iir)
1606                         gen6_gt_irq_handler(&dev_priv->gt, gt_iir);
1607                 if (pm_iir)
1608                         gen6_rps_irq_handler(&dev_priv->gt.rps, pm_iir);
1609
1610                 if (hotplug_status)
1611                         i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1612
1613                 valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
1614         } while (0);
1615
1616         enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1617
1618         return ret;
1619 }
1620
1621 static irqreturn_t cherryview_irq_handler(int irq, void *arg)
1622 {
1623         struct drm_i915_private *dev_priv = arg;
1624         irqreturn_t ret = IRQ_NONE;
1625
1626         if (!intel_irqs_enabled(dev_priv))
1627                 return IRQ_NONE;
1628
1629         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
1630         disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1631
1632         do {
1633                 u32 master_ctl, iir;
1634                 u32 pipe_stats[I915_MAX_PIPES] = {};
1635                 u32 hotplug_status = 0;
1636                 u32 ier = 0;
1637
1638                 master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
1639                 iir = I915_READ(VLV_IIR);
1640
1641                 if (master_ctl == 0 && iir == 0)
1642                         break;
1643
1644                 ret = IRQ_HANDLED;
1645
1646                 /*
1647                  * Theory on interrupt generation, based on empirical evidence:
1648                  *
1649                  * x = ((VLV_IIR & VLV_IER) ||
1650                  *      ((GEN8_MASTER_IRQ & ~GEN8_MASTER_IRQ_CONTROL) &&
1651                  *       (GEN8_MASTER_IRQ & GEN8_MASTER_IRQ_CONTROL)));
1652                  *
1653                  * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
1654                  * Hence we clear GEN8_MASTER_IRQ_CONTROL and VLV_IER to
1655                  * guarantee the CPU interrupt will be raised again even if we
1656                  * don't end up clearing all the VLV_IIR and GEN8_MASTER_IRQ_CONTROL
1657                  * bits this time around.
1658                  */
1659                 I915_WRITE(GEN8_MASTER_IRQ, 0);
1660                 ier = I915_READ(VLV_IER);
1661                 I915_WRITE(VLV_IER, 0);
1662
1663                 gen8_gt_irq_handler(&dev_priv->gt, master_ctl);
1664
1665                 if (iir & I915_DISPLAY_PORT_INTERRUPT)
1666                         hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1667
1668                 /* Call regardless, as some status bits might not be
1669                  * signalled in iir */
1670                 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1671
1672                 if (iir & (I915_LPE_PIPE_A_INTERRUPT |
1673                            I915_LPE_PIPE_B_INTERRUPT |
1674                            I915_LPE_PIPE_C_INTERRUPT))
1675                         intel_lpe_audio_irq_handler(dev_priv);
1676
1677                 /*
1678                  * VLV_IIR is single buffered, and reflects the level
1679                  * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
1680                  */
1681                 if (iir)
1682                         I915_WRITE(VLV_IIR, iir);
1683
1684                 I915_WRITE(VLV_IER, ier);
1685                 I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
1686
1687                 if (hotplug_status)
1688                         i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1689
1690                 valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
1691         } while (0);
1692
1693         enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1694
1695         return ret;
1696 }
1697
1698 static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv,
1699                                 u32 hotplug_trigger,
1700                                 const u32 hpd[HPD_NUM_PINS])
1701 {
1702         u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
1703
1704         /*
1705          * Somehow the PCH doesn't seem to really ack the interrupt to the CPU
1706          * unless we touch the hotplug register, even if hotplug_trigger is
1707          * zero. Not acking leads to "The master control interrupt lied (SDE)!"
1708          * errors.
1709          */
1710         dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1711         if (!hotplug_trigger) {
1712                 u32 mask = PORTA_HOTPLUG_STATUS_MASK |
1713                         PORTD_HOTPLUG_STATUS_MASK |
1714                         PORTC_HOTPLUG_STATUS_MASK |
1715                         PORTB_HOTPLUG_STATUS_MASK;
1716                 dig_hotplug_reg &= ~mask;
1717         }
1718
1719         I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1720         if (!hotplug_trigger)
1721                 return;
1722
1723         intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger,
1724                            dig_hotplug_reg, hpd,
1725                            pch_port_hotplug_long_detect);
1726
1727         intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1728 }
1729
1730 static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1731 {
1732         enum pipe pipe;
1733         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1734
1735         ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ibx);
1736
1737         if (pch_iir & SDE_AUDIO_POWER_MASK) {
1738                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1739                                SDE_AUDIO_POWER_SHIFT);
1740                 drm_dbg(&dev_priv->drm, "PCH audio power change on port %d\n",
1741                         port_name(port));
1742         }
1743
1744         if (pch_iir & SDE_AUX_MASK)
1745                 dp_aux_irq_handler(dev_priv);
1746
1747         if (pch_iir & SDE_GMBUS)
1748                 gmbus_irq_handler(dev_priv);
1749
1750         if (pch_iir & SDE_AUDIO_HDCP_MASK)
1751                 drm_dbg(&dev_priv->drm, "PCH HDCP audio interrupt\n");
1752
1753         if (pch_iir & SDE_AUDIO_TRANS_MASK)
1754                 drm_dbg(&dev_priv->drm, "PCH transcoder audio interrupt\n");
1755
1756         if (pch_iir & SDE_POISON)
1757                 drm_err(&dev_priv->drm, "PCH poison interrupt\n");
1758
1759         if (pch_iir & SDE_FDI_MASK) {
1760                 for_each_pipe(dev_priv, pipe)
1761                         drm_dbg(&dev_priv->drm, "  pipe %c FDI IIR: 0x%08x\n",
1762                                 pipe_name(pipe),
1763                                 I915_READ(FDI_RX_IIR(pipe)));
1764         }
1765
1766         if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1767                 drm_dbg(&dev_priv->drm, "PCH transcoder CRC done interrupt\n");
1768
1769         if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1770                 drm_dbg(&dev_priv->drm,
1771                         "PCH transcoder CRC error interrupt\n");
1772
1773         if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1774                 intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_A);
1775
1776         if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1777                 intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_B);
1778 }
1779
1780 static void ivb_err_int_handler(struct drm_i915_private *dev_priv)
1781 {
1782         u32 err_int = I915_READ(GEN7_ERR_INT);
1783         enum pipe pipe;
1784
1785         if (err_int & ERR_INT_POISON)
1786                 drm_err(&dev_priv->drm, "Poison interrupt\n");
1787
1788         for_each_pipe(dev_priv, pipe) {
1789                 if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
1790                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1791
1792                 if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
1793                         if (IS_IVYBRIDGE(dev_priv))
1794                                 ivb_pipe_crc_irq_handler(dev_priv, pipe);
1795                         else
1796                                 hsw_pipe_crc_irq_handler(dev_priv, pipe);
1797                 }
1798         }
1799
1800         I915_WRITE(GEN7_ERR_INT, err_int);
1801 }
1802
1803 static void cpt_serr_int_handler(struct drm_i915_private *dev_priv)
1804 {
1805         u32 serr_int = I915_READ(SERR_INT);
1806         enum pipe pipe;
1807
1808         if (serr_int & SERR_INT_POISON)
1809                 drm_err(&dev_priv->drm, "PCH poison interrupt\n");
1810
1811         for_each_pipe(dev_priv, pipe)
1812                 if (serr_int & SERR_INT_TRANS_FIFO_UNDERRUN(pipe))
1813                         intel_pch_fifo_underrun_irq_handler(dev_priv, pipe);
1814
1815         I915_WRITE(SERR_INT, serr_int);
1816 }
1817
1818 static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1819 {
1820         enum pipe pipe;
1821         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1822
1823         ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_cpt);
1824
1825         if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1826                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1827                                SDE_AUDIO_POWER_SHIFT_CPT);
1828                 drm_dbg(&dev_priv->drm, "PCH audio power change on port %c\n",
1829                         port_name(port));
1830         }
1831
1832         if (pch_iir & SDE_AUX_MASK_CPT)
1833                 dp_aux_irq_handler(dev_priv);
1834
1835         if (pch_iir & SDE_GMBUS_CPT)
1836                 gmbus_irq_handler(dev_priv);
1837
1838         if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1839                 drm_dbg(&dev_priv->drm, "Audio CP request interrupt\n");
1840
1841         if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1842                 drm_dbg(&dev_priv->drm, "Audio CP change interrupt\n");
1843
1844         if (pch_iir & SDE_FDI_MASK_CPT) {
1845                 for_each_pipe(dev_priv, pipe)
1846                         drm_dbg(&dev_priv->drm, "  pipe %c FDI IIR: 0x%08x\n",
1847                                 pipe_name(pipe),
1848                                 I915_READ(FDI_RX_IIR(pipe)));
1849         }
1850
1851         if (pch_iir & SDE_ERROR_CPT)
1852                 cpt_serr_int_handler(dev_priv);
1853 }
1854
1855 static void icp_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1856 {
1857         u32 ddi_hotplug_trigger, tc_hotplug_trigger;
1858         u32 pin_mask = 0, long_mask = 0;
1859         bool (*tc_port_hotplug_long_detect)(enum hpd_pin pin, u32 val);
1860         const u32 *pins;
1861
1862         if (HAS_PCH_TGP(dev_priv)) {
1863                 ddi_hotplug_trigger = pch_iir & SDE_DDI_MASK_TGP;
1864                 tc_hotplug_trigger = pch_iir & SDE_TC_MASK_TGP;
1865                 tc_port_hotplug_long_detect = tgp_tc_port_hotplug_long_detect;
1866                 pins = hpd_tgp;
1867         } else if (HAS_PCH_JSP(dev_priv)) {
1868                 ddi_hotplug_trigger = pch_iir & SDE_DDI_MASK_TGP;
1869                 tc_hotplug_trigger = 0;
1870                 pins = hpd_tgp;
1871         } else if (HAS_PCH_MCC(dev_priv)) {
1872                 ddi_hotplug_trigger = pch_iir & SDE_DDI_MASK_ICP;
1873                 tc_hotplug_trigger = pch_iir & SDE_TC_HOTPLUG_ICP(PORT_TC1);
1874                 tc_port_hotplug_long_detect = icp_tc_port_hotplug_long_detect;
1875                 pins = hpd_icp;
1876         } else {
1877                 drm_WARN(&dev_priv->drm, !HAS_PCH_ICP(dev_priv),
1878                          "Unrecognized PCH type 0x%x\n",
1879                          INTEL_PCH_TYPE(dev_priv));
1880
1881                 ddi_hotplug_trigger = pch_iir & SDE_DDI_MASK_ICP;
1882                 tc_hotplug_trigger = pch_iir & SDE_TC_MASK_ICP;
1883                 tc_port_hotplug_long_detect = icp_tc_port_hotplug_long_detect;
1884                 pins = hpd_icp;
1885         }
1886
1887         if (ddi_hotplug_trigger) {
1888                 u32 dig_hotplug_reg;
1889
1890                 dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_DDI);
1891                 I915_WRITE(SHOTPLUG_CTL_DDI, dig_hotplug_reg);
1892
1893                 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1894                                    ddi_hotplug_trigger,
1895                                    dig_hotplug_reg, pins,
1896                                    icp_ddi_port_hotplug_long_detect);
1897         }
1898
1899         if (tc_hotplug_trigger) {
1900                 u32 dig_hotplug_reg;
1901
1902                 dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_TC);
1903                 I915_WRITE(SHOTPLUG_CTL_TC, dig_hotplug_reg);
1904
1905                 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1906                                    tc_hotplug_trigger,
1907                                    dig_hotplug_reg, pins,
1908                                    tc_port_hotplug_long_detect);
1909         }
1910
1911         if (pin_mask)
1912                 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1913
1914         if (pch_iir & SDE_GMBUS_ICP)
1915                 gmbus_irq_handler(dev_priv);
1916 }
1917
1918 static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1919 {
1920         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
1921                 ~SDE_PORTE_HOTPLUG_SPT;
1922         u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
1923         u32 pin_mask = 0, long_mask = 0;
1924
1925         if (hotplug_trigger) {
1926                 u32 dig_hotplug_reg;
1927
1928                 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1929                 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1930
1931                 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1932                                    hotplug_trigger, dig_hotplug_reg, hpd_spt,
1933                                    spt_port_hotplug_long_detect);
1934         }
1935
1936         if (hotplug2_trigger) {
1937                 u32 dig_hotplug_reg;
1938
1939                 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
1940                 I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);
1941
1942                 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1943                                    hotplug2_trigger, dig_hotplug_reg, hpd_spt,
1944                                    spt_port_hotplug2_long_detect);
1945         }
1946
1947         if (pin_mask)
1948                 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1949
1950         if (pch_iir & SDE_GMBUS_CPT)
1951                 gmbus_irq_handler(dev_priv);
1952 }
1953
1954 static void ilk_hpd_irq_handler(struct drm_i915_private *dev_priv,
1955                                 u32 hotplug_trigger,
1956                                 const u32 hpd[HPD_NUM_PINS])
1957 {
1958         u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
1959
1960         dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
1961         I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);
1962
1963         intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger,
1964                            dig_hotplug_reg, hpd,
1965                            ilk_port_hotplug_long_detect);
1966
1967         intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1968 }
1969
1970 static void ilk_display_irq_handler(struct drm_i915_private *dev_priv,
1971                                     u32 de_iir)
1972 {
1973         enum pipe pipe;
1974         u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;
1975
1976         if (hotplug_trigger)
1977                 ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ilk);
1978
1979         if (de_iir & DE_AUX_CHANNEL_A)
1980                 dp_aux_irq_handler(dev_priv);
1981
1982         if (de_iir & DE_GSE)
1983                 intel_opregion_asle_intr(dev_priv);
1984
1985         if (de_iir & DE_POISON)
1986                 drm_err(&dev_priv->drm, "Poison interrupt\n");
1987
1988         for_each_pipe(dev_priv, pipe) {
1989                 if (de_iir & DE_PIPE_VBLANK(pipe))
1990                         intel_handle_vblank(dev_priv, pipe);
1991
1992                 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
1993                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1994
1995                 if (de_iir & DE_PIPE_CRC_DONE(pipe))
1996                         i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1997         }
1998
1999         /* check event from PCH */
2000         if (de_iir & DE_PCH_EVENT) {
2001                 u32 pch_iir = I915_READ(SDEIIR);
2002
2003                 if (HAS_PCH_CPT(dev_priv))
2004                         cpt_irq_handler(dev_priv, pch_iir);
2005                 else
2006                         ibx_irq_handler(dev_priv, pch_iir);
2007
2008                 /* should clear PCH hotplug event before clear CPU irq */
2009                 I915_WRITE(SDEIIR, pch_iir);
2010         }
2011
2012         if (IS_GEN(dev_priv, 5) && de_iir & DE_PCU_EVENT)
2013                 gen5_rps_irq_handler(&dev_priv->gt.rps);
2014 }
2015
2016 static void ivb_display_irq_handler(struct drm_i915_private *dev_priv,
2017                                     u32 de_iir)
2018 {
2019         enum pipe pipe;
2020         u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;
2021
2022         if (hotplug_trigger)
2023                 ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ivb);
2024
2025         if (de_iir & DE_ERR_INT_IVB)
2026                 ivb_err_int_handler(dev_priv);
2027
2028         if (de_iir & DE_EDP_PSR_INT_HSW) {
2029                 u32 psr_iir = I915_READ(EDP_PSR_IIR);
2030
2031                 intel_psr_irq_handler(dev_priv, psr_iir);
2032                 I915_WRITE(EDP_PSR_IIR, psr_iir);
2033         }
2034
2035         if (de_iir & DE_AUX_CHANNEL_A_IVB)
2036                 dp_aux_irq_handler(dev_priv);
2037
2038         if (de_iir & DE_GSE_IVB)
2039                 intel_opregion_asle_intr(dev_priv);
2040
2041         for_each_pipe(dev_priv, pipe) {
2042                 if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)))
2043                         intel_handle_vblank(dev_priv, pipe);
2044         }
2045
2046         /* check event from PCH */
2047         if (!HAS_PCH_NOP(dev_priv) && (de_iir & DE_PCH_EVENT_IVB)) {
2048                 u32 pch_iir = I915_READ(SDEIIR);
2049
2050                 cpt_irq_handler(dev_priv, pch_iir);
2051
2052                 /* clear PCH hotplug event before clear CPU irq */
2053                 I915_WRITE(SDEIIR, pch_iir);
2054         }
2055 }
2056
2057 /*
2058  * To handle irqs with the minimum potential races with fresh interrupts, we:
2059  * 1 - Disable Master Interrupt Control.
2060  * 2 - Find the source(s) of the interrupt.
2061  * 3 - Clear the Interrupt Identity bits (IIR).
2062  * 4 - Process the interrupt(s) that had bits set in the IIRs.
2063  * 5 - Re-enable Master Interrupt Control.
2064  */
2065 static irqreturn_t ilk_irq_handler(int irq, void *arg)
2066 {
2067         struct drm_i915_private *dev_priv = arg;
2068         u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2069         irqreturn_t ret = IRQ_NONE;
2070
2071         if (!intel_irqs_enabled(dev_priv))
2072                 return IRQ_NONE;
2073
2074         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2075         disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
2076
2077         /* disable master interrupt before clearing iir  */
2078         de_ier = I915_READ(DEIER);
2079         I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2080
2081         /* Disable south interrupts. We'll only write to SDEIIR once, so further
2082          * interrupts will will be stored on its back queue, and then we'll be
2083          * able to process them after we restore SDEIER (as soon as we restore
2084          * it, we'll get an interrupt if SDEIIR still has something to process
2085          * due to its back queue). */
2086         if (!HAS_PCH_NOP(dev_priv)) {
2087                 sde_ier = I915_READ(SDEIER);
2088                 I915_WRITE(SDEIER, 0);
2089         }
2090
2091         /* Find, clear, then process each source of interrupt */
2092
2093         gt_iir = I915_READ(GTIIR);
2094         if (gt_iir) {
2095                 I915_WRITE(GTIIR, gt_iir);
2096                 ret = IRQ_HANDLED;
2097                 if (INTEL_GEN(dev_priv) >= 6)
2098                         gen6_gt_irq_handler(&dev_priv->gt, gt_iir);
2099                 else
2100                         gen5_gt_irq_handler(&dev_priv->gt, gt_iir);
2101         }
2102
2103         de_iir = I915_READ(DEIIR);
2104         if (de_iir) {
2105                 I915_WRITE(DEIIR, de_iir);
2106                 ret = IRQ_HANDLED;
2107                 if (INTEL_GEN(dev_priv) >= 7)
2108                         ivb_display_irq_handler(dev_priv, de_iir);
2109                 else
2110                         ilk_display_irq_handler(dev_priv, de_iir);
2111         }
2112
2113         if (INTEL_GEN(dev_priv) >= 6) {
2114                 u32 pm_iir = I915_READ(GEN6_PMIIR);
2115                 if (pm_iir) {
2116                         I915_WRITE(GEN6_PMIIR, pm_iir);
2117                         ret = IRQ_HANDLED;
2118                         gen6_rps_irq_handler(&dev_priv->gt.rps, pm_iir);
2119                 }
2120         }
2121
2122         I915_WRITE(DEIER, de_ier);
2123         if (!HAS_PCH_NOP(dev_priv))
2124                 I915_WRITE(SDEIER, sde_ier);
2125
2126         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2127         enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
2128
2129         return ret;
2130 }
2131
2132 static void bxt_hpd_irq_handler(struct drm_i915_private *dev_priv,
2133                                 u32 hotplug_trigger,
2134                                 const u32 hpd[HPD_NUM_PINS])
2135 {
2136         u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2137
2138         dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
2139         I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2140
2141         intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger,
2142                            dig_hotplug_reg, hpd,
2143                            bxt_port_hotplug_long_detect);
2144
2145         intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2146 }
2147
2148 static void gen11_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 iir)
2149 {
2150         u32 pin_mask = 0, long_mask = 0;
2151         u32 trigger_tc = iir & GEN11_DE_TC_HOTPLUG_MASK;
2152         u32 trigger_tbt = iir & GEN11_DE_TBT_HOTPLUG_MASK;
2153         long_pulse_detect_func long_pulse_detect;
2154         const u32 *hpd;
2155
2156         if (INTEL_GEN(dev_priv) >= 12) {
2157                 long_pulse_detect = gen12_port_hotplug_long_detect;
2158                 hpd = hpd_gen12;
2159         } else {
2160                 long_pulse_detect = gen11_port_hotplug_long_detect;
2161                 hpd = hpd_gen11;
2162         }
2163
2164         if (trigger_tc) {
2165                 u32 dig_hotplug_reg;
2166
2167                 dig_hotplug_reg = I915_READ(GEN11_TC_HOTPLUG_CTL);
2168                 I915_WRITE(GEN11_TC_HOTPLUG_CTL, dig_hotplug_reg);
2169
2170                 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, trigger_tc,
2171                                    dig_hotplug_reg, hpd, long_pulse_detect);
2172         }
2173
2174         if (trigger_tbt) {
2175                 u32 dig_hotplug_reg;
2176
2177                 dig_hotplug_reg = I915_READ(GEN11_TBT_HOTPLUG_CTL);
2178                 I915_WRITE(GEN11_TBT_HOTPLUG_CTL, dig_hotplug_reg);
2179
2180                 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, trigger_tbt,
2181                                    dig_hotplug_reg, hpd, long_pulse_detect);
2182         }
2183
2184         if (pin_mask)
2185                 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2186         else
2187                 drm_err(&dev_priv->drm,
2188                         "Unexpected DE HPD interrupt 0x%08x\n", iir);
2189 }
2190
2191 static u32 gen8_de_port_aux_mask(struct drm_i915_private *dev_priv)
2192 {
2193         u32 mask;
2194
2195         if (INTEL_GEN(dev_priv) >= 12)
2196                 return TGL_DE_PORT_AUX_DDIA |
2197                         TGL_DE_PORT_AUX_DDIB |
2198                         TGL_DE_PORT_AUX_DDIC |
2199                         TGL_DE_PORT_AUX_USBC1 |
2200                         TGL_DE_PORT_AUX_USBC2 |
2201                         TGL_DE_PORT_AUX_USBC3 |
2202                         TGL_DE_PORT_AUX_USBC4 |
2203                         TGL_DE_PORT_AUX_USBC5 |
2204                         TGL_DE_PORT_AUX_USBC6;
2205
2206
2207         mask = GEN8_AUX_CHANNEL_A;
2208         if (INTEL_GEN(dev_priv) >= 9)
2209                 mask |= GEN9_AUX_CHANNEL_B |
2210                         GEN9_AUX_CHANNEL_C |
2211                         GEN9_AUX_CHANNEL_D;
2212
2213         if (IS_CNL_WITH_PORT_F(dev_priv) || IS_GEN(dev_priv, 11))
2214                 mask |= CNL_AUX_CHANNEL_F;
2215
2216         if (IS_GEN(dev_priv, 11))
2217                 mask |= ICL_AUX_CHANNEL_E;
2218
2219         return mask;
2220 }
2221
2222 static u32 gen8_de_pipe_fault_mask(struct drm_i915_private *dev_priv)
2223 {
2224         if (INTEL_GEN(dev_priv) >= 11)
2225                 return GEN11_DE_PIPE_IRQ_FAULT_ERRORS;
2226         else if (INTEL_GEN(dev_priv) >= 9)
2227                 return GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
2228         else
2229                 return GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2230 }
2231
2232 static void
2233 gen8_de_misc_irq_handler(struct drm_i915_private *dev_priv, u32 iir)
2234 {
2235         bool found = false;
2236
2237         if (iir & GEN8_DE_MISC_GSE) {
2238                 intel_opregion_asle_intr(dev_priv);
2239                 found = true;
2240         }
2241
2242         if (iir & GEN8_DE_EDP_PSR) {
2243                 u32 psr_iir;
2244                 i915_reg_t iir_reg;
2245
2246                 if (INTEL_GEN(dev_priv) >= 12)
2247                         iir_reg = TRANS_PSR_IIR(dev_priv->psr.transcoder);
2248                 else
2249                         iir_reg = EDP_PSR_IIR;
2250
2251                 psr_iir = I915_READ(iir_reg);
2252                 I915_WRITE(iir_reg, psr_iir);
2253
2254                 if (psr_iir)
2255                         found = true;
2256
2257                 intel_psr_irq_handler(dev_priv, psr_iir);
2258         }
2259
2260         if (!found)
2261                 drm_err(&dev_priv->drm, "Unexpected DE Misc interrupt\n");
2262 }
2263
2264 static irqreturn_t
2265 gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
2266 {
2267         irqreturn_t ret = IRQ_NONE;
2268         u32 iir;
2269         enum pipe pipe;
2270
2271         if (master_ctl & GEN8_DE_MISC_IRQ) {
2272                 iir = I915_READ(GEN8_DE_MISC_IIR);
2273                 if (iir) {
2274                         I915_WRITE(GEN8_DE_MISC_IIR, iir);
2275                         ret = IRQ_HANDLED;
2276                         gen8_de_misc_irq_handler(dev_priv, iir);
2277                 } else {
2278                         drm_err(&dev_priv->drm,
2279                                 "The master control interrupt lied (DE MISC)!\n");
2280                 }
2281         }
2282
2283         if (INTEL_GEN(dev_priv) >= 11 && (master_ctl & GEN11_DE_HPD_IRQ)) {
2284                 iir = I915_READ(GEN11_DE_HPD_IIR);
2285                 if (iir) {
2286                         I915_WRITE(GEN11_DE_HPD_IIR, iir);
2287                         ret = IRQ_HANDLED;
2288                         gen11_hpd_irq_handler(dev_priv, iir);
2289                 } else {
2290                         drm_err(&dev_priv->drm,
2291                                 "The master control interrupt lied, (DE HPD)!\n");
2292                 }
2293         }
2294
2295         if (master_ctl & GEN8_DE_PORT_IRQ) {
2296                 iir = I915_READ(GEN8_DE_PORT_IIR);
2297                 if (iir) {
2298                         u32 tmp_mask;
2299                         bool found = false;
2300
2301                         I915_WRITE(GEN8_DE_PORT_IIR, iir);
2302                         ret = IRQ_HANDLED;
2303
2304                         if (iir & gen8_de_port_aux_mask(dev_priv)) {
2305                                 dp_aux_irq_handler(dev_priv);
2306                                 found = true;
2307                         }
2308
2309                         if (IS_GEN9_LP(dev_priv)) {
2310                                 tmp_mask = iir & BXT_DE_PORT_HOTPLUG_MASK;
2311                                 if (tmp_mask) {
2312                                         bxt_hpd_irq_handler(dev_priv, tmp_mask,
2313                                                             hpd_bxt);
2314                                         found = true;
2315                                 }
2316                         } else if (IS_BROADWELL(dev_priv)) {
2317                                 tmp_mask = iir & GEN8_PORT_DP_A_HOTPLUG;
2318                                 if (tmp_mask) {
2319                                         ilk_hpd_irq_handler(dev_priv,
2320                                                             tmp_mask, hpd_bdw);
2321                                         found = true;
2322                                 }
2323                         }
2324
2325                         if (IS_GEN9_LP(dev_priv) && (iir & BXT_DE_PORT_GMBUS)) {
2326                                 gmbus_irq_handler(dev_priv);
2327                                 found = true;
2328                         }
2329
2330                         if (!found)
2331                                 drm_err(&dev_priv->drm,
2332                                         "Unexpected DE Port interrupt\n");
2333                 }
2334                 else
2335                         drm_err(&dev_priv->drm,
2336                                 "The master control interrupt lied (DE PORT)!\n");
2337         }
2338
2339         for_each_pipe(dev_priv, pipe) {
2340                 u32 fault_errors;
2341
2342                 if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
2343                         continue;
2344
2345                 iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
2346                 if (!iir) {
2347                         drm_err(&dev_priv->drm,
2348                                 "The master control interrupt lied (DE PIPE)!\n");
2349                         continue;
2350                 }
2351
2352                 ret = IRQ_HANDLED;
2353                 I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir);
2354
2355                 if (iir & GEN8_PIPE_VBLANK)
2356                         intel_handle_vblank(dev_priv, pipe);
2357
2358                 if (iir & GEN8_PIPE_CDCLK_CRC_DONE)
2359                         hsw_pipe_crc_irq_handler(dev_priv, pipe);
2360
2361                 if (iir & GEN8_PIPE_FIFO_UNDERRUN)
2362                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2363
2364                 fault_errors = iir & gen8_de_pipe_fault_mask(dev_priv);
2365                 if (fault_errors)
2366                         drm_err(&dev_priv->drm,
2367                                 "Fault errors on pipe %c: 0x%08x\n",
2368                                 pipe_name(pipe),
2369                                 fault_errors);
2370         }
2371
2372         if (HAS_PCH_SPLIT(dev_priv) && !HAS_PCH_NOP(dev_priv) &&
2373             master_ctl & GEN8_DE_PCH_IRQ) {
2374                 /*
2375                  * FIXME(BDW): Assume for now that the new interrupt handling
2376                  * scheme also closed the SDE interrupt handling race we've seen
2377                  * on older pch-split platforms. But this needs testing.
2378                  */
2379                 iir = I915_READ(SDEIIR);
2380                 if (iir) {
2381                         I915_WRITE(SDEIIR, iir);
2382                         ret = IRQ_HANDLED;
2383
2384                         if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
2385                                 icp_irq_handler(dev_priv, iir);
2386                         else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT)
2387                                 spt_irq_handler(dev_priv, iir);
2388                         else
2389                                 cpt_irq_handler(dev_priv, iir);
2390                 } else {
2391                         /*
2392                          * Like on previous PCH there seems to be something
2393                          * fishy going on with forwarding PCH interrupts.
2394                          */
2395                         drm_dbg(&dev_priv->drm,
2396                                 "The master control interrupt lied (SDE)!\n");
2397                 }
2398         }
2399
2400         return ret;
2401 }
2402
2403 static inline u32 gen8_master_intr_disable(void __iomem * const regs)
2404 {
2405         raw_reg_write(regs, GEN8_MASTER_IRQ, 0);
2406
2407         /*
2408          * Now with master disabled, get a sample of level indications
2409          * for this interrupt. Indications will be cleared on related acks.
2410          * New indications can and will light up during processing,
2411          * and will generate new interrupt after enabling master.
2412          */
2413         return raw_reg_read(regs, GEN8_MASTER_IRQ);
2414 }
2415
2416 static inline void gen8_master_intr_enable(void __iomem * const regs)
2417 {
2418         raw_reg_write(regs, GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2419 }
2420
2421 static irqreturn_t gen8_irq_handler(int irq, void *arg)
2422 {
2423         struct drm_i915_private *dev_priv = arg;
2424         void __iomem * const regs = dev_priv->uncore.regs;
2425         u32 master_ctl;
2426
2427         if (!intel_irqs_enabled(dev_priv))
2428                 return IRQ_NONE;
2429
2430         master_ctl = gen8_master_intr_disable(regs);
2431         if (!master_ctl) {
2432                 gen8_master_intr_enable(regs);
2433                 return IRQ_NONE;
2434         }
2435
2436         /* Find, queue (onto bottom-halves), then clear each source */
2437         gen8_gt_irq_handler(&dev_priv->gt, master_ctl);
2438
2439         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2440         if (master_ctl & ~GEN8_GT_IRQS) {
2441                 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
2442                 gen8_de_irq_handler(dev_priv, master_ctl);
2443                 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
2444         }
2445
2446         gen8_master_intr_enable(regs);
2447
2448         return IRQ_HANDLED;
2449 }
2450
2451 static u32
2452 gen11_gu_misc_irq_ack(struct intel_gt *gt, const u32 master_ctl)
2453 {
2454         void __iomem * const regs = gt->uncore->regs;
2455         u32 iir;
2456
2457         if (!(master_ctl & GEN11_GU_MISC_IRQ))
2458                 return 0;
2459
2460         iir = raw_reg_read(regs, GEN11_GU_MISC_IIR);
2461         if (likely(iir))
2462                 raw_reg_write(regs, GEN11_GU_MISC_IIR, iir);
2463
2464         return iir;
2465 }
2466
2467 static void
2468 gen11_gu_misc_irq_handler(struct intel_gt *gt, const u32 iir)
2469 {
2470         if (iir & GEN11_GU_MISC_GSE)
2471                 intel_opregion_asle_intr(gt->i915);
2472 }
2473
2474 static inline u32 gen11_master_intr_disable(void __iomem * const regs)
2475 {
2476         raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, 0);
2477
2478         /*
2479          * Now with master disabled, get a sample of level indications
2480          * for this interrupt. Indications will be cleared on related acks.
2481          * New indications can and will light up during processing,
2482          * and will generate new interrupt after enabling master.
2483          */
2484         return raw_reg_read(regs, GEN11_GFX_MSTR_IRQ);
2485 }
2486
2487 static inline void gen11_master_intr_enable(void __iomem * const regs)
2488 {
2489         raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, GEN11_MASTER_IRQ);
2490 }
2491
2492 static void
2493 gen11_display_irq_handler(struct drm_i915_private *i915)
2494 {
2495         void __iomem * const regs = i915->uncore.regs;
2496         const u32 disp_ctl = raw_reg_read(regs, GEN11_DISPLAY_INT_CTL);
2497
2498         disable_rpm_wakeref_asserts(&i915->runtime_pm);
2499         /*
2500          * GEN11_DISPLAY_INT_CTL has same format as GEN8_MASTER_IRQ
2501          * for the display related bits.
2502          */
2503         raw_reg_write(regs, GEN11_DISPLAY_INT_CTL, 0x0);
2504         gen8_de_irq_handler(i915, disp_ctl);
2505         raw_reg_write(regs, GEN11_DISPLAY_INT_CTL,
2506                       GEN11_DISPLAY_IRQ_ENABLE);
2507
2508         enable_rpm_wakeref_asserts(&i915->runtime_pm);
2509 }
2510
2511 static __always_inline irqreturn_t
2512 __gen11_irq_handler(struct drm_i915_private * const i915,
2513                     u32 (*intr_disable)(void __iomem * const regs),
2514                     void (*intr_enable)(void __iomem * const regs))
2515 {
2516         void __iomem * const regs = i915->uncore.regs;
2517         struct intel_gt *gt = &i915->gt;
2518         u32 master_ctl;
2519         u32 gu_misc_iir;
2520
2521         if (!intel_irqs_enabled(i915))
2522                 return IRQ_NONE;
2523
2524         master_ctl = intr_disable(regs);
2525         if (!master_ctl) {
2526                 intr_enable(regs);
2527                 return IRQ_NONE;
2528         }
2529
2530         /* Find, queue (onto bottom-halves), then clear each source */
2531         gen11_gt_irq_handler(gt, master_ctl);
2532
2533         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2534         if (master_ctl & GEN11_DISPLAY_IRQ)
2535                 gen11_display_irq_handler(i915);
2536
2537         gu_misc_iir = gen11_gu_misc_irq_ack(gt, master_ctl);
2538
2539         intr_enable(regs);
2540
2541         gen11_gu_misc_irq_handler(gt, gu_misc_iir);
2542
2543         return IRQ_HANDLED;
2544 }
2545
2546 static irqreturn_t gen11_irq_handler(int irq, void *arg)
2547 {
2548         return __gen11_irq_handler(arg,
2549                                    gen11_master_intr_disable,
2550                                    gen11_master_intr_enable);
2551 }
2552
2553 /* Called from drm generic code, passed 'crtc' which
2554  * we use as a pipe index
2555  */
2556 int i8xx_enable_vblank(struct drm_crtc *crtc)
2557 {
2558         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2559         enum pipe pipe = to_intel_crtc(crtc)->pipe;
2560         unsigned long irqflags;
2561
2562         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2563         i915_enable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
2564         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2565
2566         return 0;
2567 }
2568
2569 int i915gm_enable_vblank(struct drm_crtc *crtc)
2570 {
2571         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2572
2573         /*
2574          * Vblank interrupts fail to wake the device up from C2+.
2575          * Disabling render clock gating during C-states avoids
2576          * the problem. There is a small power cost so we do this
2577          * only when vblank interrupts are actually enabled.
2578          */
2579         if (dev_priv->vblank_enabled++ == 0)
2580                 I915_WRITE(SCPD0, _MASKED_BIT_ENABLE(CSTATE_RENDER_CLOCK_GATE_DISABLE));
2581
2582         return i8xx_enable_vblank(crtc);
2583 }
2584
2585 int i965_enable_vblank(struct drm_crtc *crtc)
2586 {
2587         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2588         enum pipe pipe = to_intel_crtc(crtc)->pipe;
2589         unsigned long irqflags;
2590
2591         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2592         i915_enable_pipestat(dev_priv, pipe,
2593                              PIPE_START_VBLANK_INTERRUPT_STATUS);
2594         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2595
2596         return 0;
2597 }
2598
2599 int ilk_enable_vblank(struct drm_crtc *crtc)
2600 {
2601         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2602         enum pipe pipe = to_intel_crtc(crtc)->pipe;
2603         unsigned long irqflags;
2604         u32 bit = INTEL_GEN(dev_priv) >= 7 ?
2605                 DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
2606
2607         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2608         ilk_enable_display_irq(dev_priv, bit);
2609         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2610
2611         /* Even though there is no DMC, frame counter can get stuck when
2612          * PSR is active as no frames are generated.
2613          */
2614         if (HAS_PSR(dev_priv))
2615                 drm_crtc_vblank_restore(crtc);
2616
2617         return 0;
2618 }
2619
2620 int bdw_enable_vblank(struct drm_crtc *crtc)
2621 {
2622         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2623         enum pipe pipe = to_intel_crtc(crtc)->pipe;
2624         unsigned long irqflags;
2625
2626         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2627         bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2628         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2629
2630         /* Even if there is no DMC, frame counter can get stuck when
2631          * PSR is active as no frames are generated, so check only for PSR.
2632          */
2633         if (HAS_PSR(dev_priv))
2634                 drm_crtc_vblank_restore(crtc);
2635
2636         return 0;
2637 }
2638
2639 /* Called from drm generic code, passed 'crtc' which
2640  * we use as a pipe index
2641  */
2642 void i8xx_disable_vblank(struct drm_crtc *crtc)
2643 {
2644         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2645         enum pipe pipe = to_intel_crtc(crtc)->pipe;
2646         unsigned long irqflags;
2647
2648         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2649         i915_disable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
2650         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2651 }
2652
2653 void i915gm_disable_vblank(struct drm_crtc *crtc)
2654 {
2655         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2656
2657         i8xx_disable_vblank(crtc);
2658
2659         if (--dev_priv->vblank_enabled == 0)
2660                 I915_WRITE(SCPD0, _MASKED_BIT_DISABLE(CSTATE_RENDER_CLOCK_GATE_DISABLE));
2661 }
2662
2663 void i965_disable_vblank(struct drm_crtc *crtc)
2664 {
2665         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2666         enum pipe pipe = to_intel_crtc(crtc)->pipe;
2667         unsigned long irqflags;
2668
2669         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2670         i915_disable_pipestat(dev_priv, pipe,
2671                               PIPE_START_VBLANK_INTERRUPT_STATUS);
2672         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2673 }
2674
2675 void ilk_disable_vblank(struct drm_crtc *crtc)
2676 {
2677         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2678         enum pipe pipe = to_intel_crtc(crtc)->pipe;
2679         unsigned long irqflags;
2680         u32 bit = INTEL_GEN(dev_priv) >= 7 ?
2681                 DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
2682
2683         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2684         ilk_disable_display_irq(dev_priv, bit);
2685         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2686 }
2687
2688 void bdw_disable_vblank(struct drm_crtc *crtc)
2689 {
2690         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2691         enum pipe pipe = to_intel_crtc(crtc)->pipe;
2692         unsigned long irqflags;
2693
2694         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2695         bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2696         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2697 }
2698
2699 static void ibx_irq_reset(struct drm_i915_private *dev_priv)
2700 {
2701         struct intel_uncore *uncore = &dev_priv->uncore;
2702
2703         if (HAS_PCH_NOP(dev_priv))
2704                 return;
2705
2706         GEN3_IRQ_RESET(uncore, SDE);
2707
2708         if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
2709                 I915_WRITE(SERR_INT, 0xffffffff);
2710 }
2711
2712 /*
2713  * SDEIER is also touched by the interrupt handler to work around missed PCH
2714  * interrupts. Hence we can't update it after the interrupt handler is enabled -
2715  * instead we unconditionally enable all PCH interrupt sources here, but then
2716  * only unmask them as needed with SDEIMR.
2717  *
2718  * This function needs to be called before interrupts are enabled.
2719  */
2720 static void ibx_irq_pre_postinstall(struct drm_i915_private *dev_priv)
2721 {
2722         if (HAS_PCH_NOP(dev_priv))
2723                 return;
2724
2725         drm_WARN_ON(&dev_priv->drm, I915_READ(SDEIER) != 0);
2726         I915_WRITE(SDEIER, 0xffffffff);
2727         POSTING_READ(SDEIER);
2728 }
2729
2730 static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
2731 {
2732         struct intel_uncore *uncore = &dev_priv->uncore;
2733
2734         if (IS_CHERRYVIEW(dev_priv))
2735                 intel_uncore_write(uncore, DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
2736         else
2737                 intel_uncore_write(uncore, DPINVGTT, DPINVGTT_STATUS_MASK);
2738
2739         i915_hotplug_interrupt_update_locked(dev_priv, 0xffffffff, 0);
2740         intel_uncore_write(uncore, PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2741
2742         i9xx_pipestat_irq_reset(dev_priv);
2743
2744         GEN3_IRQ_RESET(uncore, VLV_);
2745         dev_priv->irq_mask = ~0u;
2746 }
2747
2748 static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
2749 {
2750         struct intel_uncore *uncore = &dev_priv->uncore;
2751
2752         u32 pipestat_mask;
2753         u32 enable_mask;
2754         enum pipe pipe;
2755
2756         pipestat_mask = PIPE_CRC_DONE_INTERRUPT_STATUS;
2757
2758         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
2759         for_each_pipe(dev_priv, pipe)
2760                 i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
2761
2762         enable_mask = I915_DISPLAY_PORT_INTERRUPT |
2763                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2764                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2765                 I915_LPE_PIPE_A_INTERRUPT |
2766                 I915_LPE_PIPE_B_INTERRUPT;
2767
2768         if (IS_CHERRYVIEW(dev_priv))
2769                 enable_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT |
2770                         I915_LPE_PIPE_C_INTERRUPT;
2771
2772         drm_WARN_ON(&dev_priv->drm, dev_priv->irq_mask != ~0u);
2773
2774         dev_priv->irq_mask = ~enable_mask;
2775
2776         GEN3_IRQ_INIT(uncore, VLV_, dev_priv->irq_mask, enable_mask);
2777 }
2778
2779 /* drm_dma.h hooks
2780 */
2781 static void ilk_irq_reset(struct drm_i915_private *dev_priv)
2782 {
2783         struct intel_uncore *uncore = &dev_priv->uncore;
2784
2785         GEN3_IRQ_RESET(uncore, DE);
2786         if (IS_GEN(dev_priv, 7))
2787                 intel_uncore_write(uncore, GEN7_ERR_INT, 0xffffffff);
2788
2789         if (IS_HASWELL(dev_priv)) {
2790                 intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
2791                 intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
2792         }
2793
2794         gen5_gt_irq_reset(&dev_priv->gt);
2795
2796         ibx_irq_reset(dev_priv);
2797 }
2798
2799 static void valleyview_irq_reset(struct drm_i915_private *dev_priv)
2800 {
2801         I915_WRITE(VLV_MASTER_IER, 0);
2802         POSTING_READ(VLV_MASTER_IER);
2803
2804         gen5_gt_irq_reset(&dev_priv->gt);
2805
2806         spin_lock_irq(&dev_priv->irq_lock);
2807         if (dev_priv->display_irqs_enabled)
2808                 vlv_display_irq_reset(dev_priv);
2809         spin_unlock_irq(&dev_priv->irq_lock);
2810 }
2811
2812 static void gen8_irq_reset(struct drm_i915_private *dev_priv)
2813 {
2814         struct intel_uncore *uncore = &dev_priv->uncore;
2815         enum pipe pipe;
2816
2817         gen8_master_intr_disable(dev_priv->uncore.regs);
2818
2819         gen8_gt_irq_reset(&dev_priv->gt);
2820
2821         intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
2822         intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
2823
2824         for_each_pipe(dev_priv, pipe)
2825                 if (intel_display_power_is_enabled(dev_priv,
2826                                                    POWER_DOMAIN_PIPE(pipe)))
2827                         GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
2828
2829         GEN3_IRQ_RESET(uncore, GEN8_DE_PORT_);
2830         GEN3_IRQ_RESET(uncore, GEN8_DE_MISC_);
2831         GEN3_IRQ_RESET(uncore, GEN8_PCU_);
2832
2833         if (HAS_PCH_SPLIT(dev_priv))
2834                 ibx_irq_reset(dev_priv);
2835 }
2836
2837 static void gen11_display_irq_reset(struct drm_i915_private *dev_priv)
2838 {
2839         struct intel_uncore *uncore = &dev_priv->uncore;
2840         enum pipe pipe;
2841
2842         intel_uncore_write(uncore, GEN11_DISPLAY_INT_CTL, 0);
2843
2844         if (INTEL_GEN(dev_priv) >= 12) {
2845                 enum transcoder trans;
2846
2847                 for (trans = TRANSCODER_A; trans <= TRANSCODER_D; trans++) {
2848                         enum intel_display_power_domain domain;
2849
2850                         domain = POWER_DOMAIN_TRANSCODER(trans);
2851                         if (!intel_display_power_is_enabled(dev_priv, domain))
2852                                 continue;
2853
2854                         intel_uncore_write(uncore, TRANS_PSR_IMR(trans), 0xffffffff);
2855                         intel_uncore_write(uncore, TRANS_PSR_IIR(trans), 0xffffffff);
2856                 }
2857         } else {
2858                 intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
2859                 intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
2860         }
2861
2862         for_each_pipe(dev_priv, pipe)
2863                 if (intel_display_power_is_enabled(dev_priv,
2864                                                    POWER_DOMAIN_PIPE(pipe)))
2865                         GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
2866
2867         GEN3_IRQ_RESET(uncore, GEN8_DE_PORT_);
2868         GEN3_IRQ_RESET(uncore, GEN8_DE_MISC_);
2869         GEN3_IRQ_RESET(uncore, GEN11_DE_HPD_);
2870
2871         if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
2872                 GEN3_IRQ_RESET(uncore, SDE);
2873 }
2874
2875 static void gen11_irq_reset(struct drm_i915_private *dev_priv)
2876 {
2877         struct intel_uncore *uncore = &dev_priv->uncore;
2878
2879         gen11_master_intr_disable(dev_priv->uncore.regs);
2880
2881         gen11_gt_irq_reset(&dev_priv->gt);
2882         gen11_display_irq_reset(dev_priv);
2883
2884         GEN3_IRQ_RESET(uncore, GEN11_GU_MISC_);
2885         GEN3_IRQ_RESET(uncore, GEN8_PCU_);
2886 }
2887
2888 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
2889                                      u8 pipe_mask)
2890 {
2891         struct intel_uncore *uncore = &dev_priv->uncore;
2892
2893         u32 extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
2894         enum pipe pipe;
2895
2896         spin_lock_irq(&dev_priv->irq_lock);
2897
2898         if (!intel_irqs_enabled(dev_priv)) {
2899                 spin_unlock_irq(&dev_priv->irq_lock);
2900                 return;
2901         }
2902
2903         for_each_pipe_masked(dev_priv, pipe, pipe_mask)
2904                 GEN8_IRQ_INIT_NDX(uncore, DE_PIPE, pipe,
2905                                   dev_priv->de_irq_mask[pipe],
2906                                   ~dev_priv->de_irq_mask[pipe] | extra_ier);
2907
2908         spin_unlock_irq(&dev_priv->irq_lock);
2909 }
2910
2911 void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
2912                                      u8 pipe_mask)
2913 {
2914         struct intel_uncore *uncore = &dev_priv->uncore;
2915         enum pipe pipe;
2916
2917         spin_lock_irq(&dev_priv->irq_lock);
2918
2919         if (!intel_irqs_enabled(dev_priv)) {
2920                 spin_unlock_irq(&dev_priv->irq_lock);
2921                 return;
2922         }
2923
2924         for_each_pipe_masked(dev_priv, pipe, pipe_mask)
2925                 GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
2926
2927         spin_unlock_irq(&dev_priv->irq_lock);
2928
2929         /* make sure we're done processing display irqs */
2930         intel_synchronize_irq(dev_priv);
2931 }
2932
2933 static void cherryview_irq_reset(struct drm_i915_private *dev_priv)
2934 {
2935         struct intel_uncore *uncore = &dev_priv->uncore;
2936
2937         I915_WRITE(GEN8_MASTER_IRQ, 0);
2938         POSTING_READ(GEN8_MASTER_IRQ);
2939
2940         gen8_gt_irq_reset(&dev_priv->gt);
2941
2942         GEN3_IRQ_RESET(uncore, GEN8_PCU_);
2943
2944         spin_lock_irq(&dev_priv->irq_lock);
2945         if (dev_priv->display_irqs_enabled)
2946                 vlv_display_irq_reset(dev_priv);
2947         spin_unlock_irq(&dev_priv->irq_lock);
2948 }
2949
2950 static u32 intel_hpd_enabled_irqs(struct drm_i915_private *dev_priv,
2951                                   const u32 hpd[HPD_NUM_PINS])
2952 {
2953         struct intel_encoder *encoder;
2954         u32 enabled_irqs = 0;
2955
2956         for_each_intel_encoder(&dev_priv->drm, encoder)
2957                 if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
2958                         enabled_irqs |= hpd[encoder->hpd_pin];
2959
2960         return enabled_irqs;
2961 }
2962
2963 static void ibx_hpd_detection_setup(struct drm_i915_private *dev_priv)
2964 {
2965         u32 hotplug;
2966
2967         /*
2968          * Enable digital hotplug on the PCH, and configure the DP short pulse
2969          * duration to 2ms (which is the minimum in the Display Port spec).
2970          * The pulse duration bits are reserved on LPT+.
2971          */
2972         hotplug = I915_READ(PCH_PORT_HOTPLUG);
2973         hotplug &= ~(PORTB_PULSE_DURATION_MASK |
2974                      PORTC_PULSE_DURATION_MASK |
2975                      PORTD_PULSE_DURATION_MASK);
2976         hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
2977         hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
2978         hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
2979         /*
2980          * When CPU and PCH are on the same package, port A
2981          * HPD must be enabled in both north and south.
2982          */
2983         if (HAS_PCH_LPT_LP(dev_priv))
2984                 hotplug |= PORTA_HOTPLUG_ENABLE;
2985         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
2986 }
2987
2988 static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv)
2989 {
2990         u32 hotplug_irqs, enabled_irqs;
2991
2992         if (HAS_PCH_IBX(dev_priv)) {
2993                 hotplug_irqs = SDE_HOTPLUG_MASK;
2994                 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ibx);
2995         } else {
2996                 hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
2997                 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_cpt);
2998         }
2999
3000         ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3001
3002         ibx_hpd_detection_setup(dev_priv);
3003 }
3004
3005 static void icp_hpd_detection_setup(struct drm_i915_private *dev_priv,
3006                                     u32 ddi_hotplug_enable_mask,
3007                                     u32 tc_hotplug_enable_mask)
3008 {
3009         u32 hotplug;
3010
3011         hotplug = I915_READ(SHOTPLUG_CTL_DDI);
3012         hotplug |= ddi_hotplug_enable_mask;
3013         I915_WRITE(SHOTPLUG_CTL_DDI, hotplug);
3014
3015         if (tc_hotplug_enable_mask) {
3016                 hotplug = I915_READ(SHOTPLUG_CTL_TC);
3017                 hotplug |= tc_hotplug_enable_mask;
3018                 I915_WRITE(SHOTPLUG_CTL_TC, hotplug);
3019         }
3020 }
3021
3022 static void icp_hpd_irq_setup(struct drm_i915_private *dev_priv,
3023                               u32 sde_ddi_mask, u32 sde_tc_mask,
3024                               u32 ddi_enable_mask, u32 tc_enable_mask,
3025                               const u32 *pins)
3026 {
3027         u32 hotplug_irqs, enabled_irqs;
3028
3029         hotplug_irqs = sde_ddi_mask | sde_tc_mask;
3030         enabled_irqs = intel_hpd_enabled_irqs(dev_priv, pins);
3031
3032         I915_WRITE(SHPD_FILTER_CNT, SHPD_FILTER_CNT_500_ADJ);
3033
3034         ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3035
3036         icp_hpd_detection_setup(dev_priv, ddi_enable_mask, tc_enable_mask);
3037 }
3038
3039 /*
3040  * EHL doesn't need most of gen11_hpd_irq_setup, it's handling only the
3041  * equivalent of SDE.
3042  */
3043 static void mcc_hpd_irq_setup(struct drm_i915_private *dev_priv)
3044 {
3045         icp_hpd_irq_setup(dev_priv,
3046                           SDE_DDI_MASK_ICP, SDE_TC_HOTPLUG_ICP(PORT_TC1),
3047                           ICP_DDI_HPD_ENABLE_MASK, ICP_TC_HPD_ENABLE(PORT_TC1),
3048                           hpd_icp);
3049 }
3050
3051 /*
3052  * JSP behaves exactly the same as MCC above except that port C is mapped to
3053  * the DDI-C pins instead of the TC1 pins.  This means we should follow TGP's
3054  * masks & tables rather than ICP's masks & tables.
3055  */
3056 static void jsp_hpd_irq_setup(struct drm_i915_private *dev_priv)
3057 {
3058         icp_hpd_irq_setup(dev_priv,
3059                           SDE_DDI_MASK_TGP, 0,
3060                           TGP_DDI_HPD_ENABLE_MASK, 0,
3061                           hpd_tgp);
3062 }
3063
3064 static void gen11_hpd_detection_setup(struct drm_i915_private *dev_priv)
3065 {
3066         u32 hotplug;
3067
3068         hotplug = I915_READ(GEN11_TC_HOTPLUG_CTL);
3069         hotplug |= GEN11_HOTPLUG_CTL_ENABLE(PORT_TC1) |
3070                    GEN11_HOTPLUG_CTL_ENABLE(PORT_TC2) |
3071                    GEN11_HOTPLUG_CTL_ENABLE(PORT_TC3) |
3072                    GEN11_HOTPLUG_CTL_ENABLE(PORT_TC4);
3073         I915_WRITE(GEN11_TC_HOTPLUG_CTL, hotplug);
3074
3075         hotplug = I915_READ(GEN11_TBT_HOTPLUG_CTL);
3076         hotplug |= GEN11_HOTPLUG_CTL_ENABLE(PORT_TC1) |
3077                    GEN11_HOTPLUG_CTL_ENABLE(PORT_TC2) |
3078                    GEN11_HOTPLUG_CTL_ENABLE(PORT_TC3) |
3079                    GEN11_HOTPLUG_CTL_ENABLE(PORT_TC4);
3080         I915_WRITE(GEN11_TBT_HOTPLUG_CTL, hotplug);
3081 }
3082
3083 static void gen11_hpd_irq_setup(struct drm_i915_private *dev_priv)
3084 {
3085         u32 hotplug_irqs, enabled_irqs;
3086         const u32 *hpd;
3087         u32 val;
3088
3089         hpd = INTEL_GEN(dev_priv) >= 12 ? hpd_gen12 : hpd_gen11;
3090         enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd);
3091         hotplug_irqs = GEN11_DE_TC_HOTPLUG_MASK | GEN11_DE_TBT_HOTPLUG_MASK;
3092
3093         val = I915_READ(GEN11_DE_HPD_IMR);
3094         val &= ~hotplug_irqs;
3095         I915_WRITE(GEN11_DE_HPD_IMR, val);
3096         POSTING_READ(GEN11_DE_HPD_IMR);
3097
3098         gen11_hpd_detection_setup(dev_priv);
3099
3100         if (INTEL_PCH_TYPE(dev_priv) >= PCH_TGP)
3101                 icp_hpd_irq_setup(dev_priv, SDE_DDI_MASK_TGP, SDE_TC_MASK_TGP,
3102                                   TGP_DDI_HPD_ENABLE_MASK,
3103                                   TGP_TC_HPD_ENABLE_MASK, hpd_tgp);
3104         else if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
3105                 icp_hpd_irq_setup(dev_priv, SDE_DDI_MASK_ICP, SDE_TC_MASK_ICP,
3106                                   ICP_DDI_HPD_ENABLE_MASK,
3107                                   ICP_TC_HPD_ENABLE_MASK, hpd_icp);
3108 }
3109
3110 static void spt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3111 {
3112         u32 val, hotplug;
3113
3114         /* Display WA #1179 WaHardHangonHotPlug: cnp */
3115         if (HAS_PCH_CNP(dev_priv)) {
3116                 val = I915_READ(SOUTH_CHICKEN1);
3117                 val &= ~CHASSIS_CLK_REQ_DURATION_MASK;
3118                 val |= CHASSIS_CLK_REQ_DURATION(0xf);
3119                 I915_WRITE(SOUTH_CHICKEN1, val);
3120         }
3121
3122         /* Enable digital hotplug on the PCH */
3123         hotplug = I915_READ(PCH_PORT_HOTPLUG);
3124         hotplug |= PORTA_HOTPLUG_ENABLE |
3125                    PORTB_HOTPLUG_ENABLE |
3126                    PORTC_HOTPLUG_ENABLE |
3127                    PORTD_HOTPLUG_ENABLE;
3128         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3129
3130         hotplug = I915_READ(PCH_PORT_HOTPLUG2);
3131         hotplug |= PORTE_HOTPLUG_ENABLE;
3132         I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3133 }
3134
3135 static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv)
3136 {
3137         u32 hotplug_irqs, enabled_irqs;
3138
3139         if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP)
3140                 I915_WRITE(SHPD_FILTER_CNT, SHPD_FILTER_CNT_500_ADJ);
3141
3142         hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
3143         enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_spt);
3144
3145         ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3146
3147         spt_hpd_detection_setup(dev_priv);
3148 }
3149
3150 static void ilk_hpd_detection_setup(struct drm_i915_private *dev_priv)
3151 {
3152         u32 hotplug;
3153
3154         /*
3155          * Enable digital hotplug on the CPU, and configure the DP short pulse
3156          * duration to 2ms (which is the minimum in the Display Port spec)
3157          * The pulse duration bits are reserved on HSW+.
3158          */
3159         hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
3160         hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
3161         hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE |
3162                    DIGITAL_PORTA_PULSE_DURATION_2ms;
3163         I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
3164 }
3165
3166 static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv)
3167 {
3168         u32 hotplug_irqs, enabled_irqs;
3169
3170         if (INTEL_GEN(dev_priv) >= 8) {
3171                 hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
3172                 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bdw);
3173
3174                 bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3175         } else if (INTEL_GEN(dev_priv) >= 7) {
3176                 hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
3177                 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ivb);
3178
3179                 ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3180         } else {
3181                 hotplug_irqs = DE_DP_A_HOTPLUG;
3182                 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ilk);
3183
3184                 ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3185         }
3186
3187         ilk_hpd_detection_setup(dev_priv);
3188
3189         ibx_hpd_irq_setup(dev_priv);
3190 }
3191
3192 static void __bxt_hpd_detection_setup(struct drm_i915_private *dev_priv,
3193                                       u32 enabled_irqs)
3194 {
3195         u32 hotplug;
3196
3197         hotplug = I915_READ(PCH_PORT_HOTPLUG);
3198         hotplug |= PORTA_HOTPLUG_ENABLE |
3199                    PORTB_HOTPLUG_ENABLE |
3200                    PORTC_HOTPLUG_ENABLE;
3201
3202         drm_dbg_kms(&dev_priv->drm,
3203                     "Invert bit setting: hp_ctl:%x hp_port:%x\n",
3204                     hotplug, enabled_irqs);
3205         hotplug &= ~BXT_DDI_HPD_INVERT_MASK;
3206
3207         /*
3208          * For BXT invert bit has to be set based on AOB design
3209          * for HPD detection logic, update it based on VBT fields.
3210          */
3211         if ((enabled_irqs & BXT_DE_PORT_HP_DDIA) &&
3212             intel_bios_is_port_hpd_inverted(dev_priv, PORT_A))
3213                 hotplug |= BXT_DDIA_HPD_INVERT;
3214         if ((enabled_irqs & BXT_DE_PORT_HP_DDIB) &&
3215             intel_bios_is_port_hpd_inverted(dev_priv, PORT_B))
3216                 hotplug |= BXT_DDIB_HPD_INVERT;
3217         if ((enabled_irqs & BXT_DE_PORT_HP_DDIC) &&
3218             intel_bios_is_port_hpd_inverted(dev_priv, PORT_C))
3219                 hotplug |= BXT_DDIC_HPD_INVERT;
3220
3221         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3222 }
3223
3224 static void bxt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3225 {
3226         __bxt_hpd_detection_setup(dev_priv, BXT_DE_PORT_HOTPLUG_MASK);
3227 }
3228
3229 static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv)
3230 {
3231         u32 hotplug_irqs, enabled_irqs;
3232
3233         enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bxt);
3234         hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;
3235
3236         bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3237
3238         __bxt_hpd_detection_setup(dev_priv, enabled_irqs);
3239 }
3240
3241 static void ibx_irq_postinstall(struct drm_i915_private *dev_priv)
3242 {
3243         u32 mask;
3244
3245         if (HAS_PCH_NOP(dev_priv))
3246                 return;
3247
3248         if (HAS_PCH_IBX(dev_priv))
3249                 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3250         else if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
3251                 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3252         else
3253                 mask = SDE_GMBUS_CPT;
3254
3255         gen3_assert_iir_is_zero(&dev_priv->uncore, SDEIIR);
3256         I915_WRITE(SDEIMR, ~mask);
3257
3258         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
3259             HAS_PCH_LPT(dev_priv))
3260                 ibx_hpd_detection_setup(dev_priv);
3261         else
3262                 spt_hpd_detection_setup(dev_priv);
3263 }
3264
3265 static void ilk_irq_postinstall(struct drm_i915_private *dev_priv)
3266 {
3267         struct intel_uncore *uncore = &dev_priv->uncore;
3268         u32 display_mask, extra_mask;
3269
3270         if (INTEL_GEN(dev_priv) >= 7) {
3271                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
3272                                 DE_PCH_EVENT_IVB | DE_AUX_CHANNEL_A_IVB);
3273                 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3274                               DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
3275                               DE_DP_A_HOTPLUG_IVB);
3276         } else {
3277                 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
3278                                 DE_AUX_CHANNEL_A | DE_PIPEB_CRC_DONE |
3279                                 DE_PIPEA_CRC_DONE | DE_POISON);
3280                 extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
3281                               DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
3282                               DE_DP_A_HOTPLUG);
3283         }
3284
3285         if (IS_HASWELL(dev_priv)) {
3286                 gen3_assert_iir_is_zero(uncore, EDP_PSR_IIR);
3287                 display_mask |= DE_EDP_PSR_INT_HSW;
3288         }
3289
3290         dev_priv->irq_mask = ~display_mask;
3291
3292         ibx_irq_pre_postinstall(dev_priv);
3293
3294         GEN3_IRQ_INIT(uncore, DE, dev_priv->irq_mask,
3295                       display_mask | extra_mask);
3296
3297         gen5_gt_irq_postinstall(&dev_priv->gt);
3298
3299         ilk_hpd_detection_setup(dev_priv);
3300
3301         ibx_irq_postinstall(dev_priv);
3302
3303         if (IS_IRONLAKE_M(dev_priv)) {
3304                 /* Enable PCU event interrupts
3305                  *
3306                  * spinlocking not required here for correctness since interrupt
3307                  * setup is guaranteed to run in single-threaded context. But we
3308                  * need it to make the assert_spin_locked happy. */
3309                 spin_lock_irq(&dev_priv->irq_lock);
3310                 ilk_enable_display_irq(dev_priv, DE_PCU_EVENT);
3311                 spin_unlock_irq(&dev_priv->irq_lock);
3312         }
3313 }
3314
3315 void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
3316 {
3317         lockdep_assert_held(&dev_priv->irq_lock);
3318
3319         if (dev_priv->display_irqs_enabled)
3320                 return;
3321
3322         dev_priv->display_irqs_enabled = true;
3323
3324         if (intel_irqs_enabled(dev_priv)) {
3325                 vlv_display_irq_reset(dev_priv);
3326                 vlv_display_irq_postinstall(dev_priv);
3327         }
3328 }
3329
3330 void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
3331 {
3332         lockdep_assert_held(&dev_priv->irq_lock);
3333
3334         if (!dev_priv->display_irqs_enabled)
3335                 return;
3336
3337         dev_priv->display_irqs_enabled = false;
3338
3339         if (intel_irqs_enabled(dev_priv))
3340                 vlv_display_irq_reset(dev_priv);
3341 }
3342
3343
3344 static void valleyview_irq_postinstall(struct drm_i915_private *dev_priv)
3345 {
3346         gen5_gt_irq_postinstall(&dev_priv->gt);
3347
3348         spin_lock_irq(&dev_priv->irq_lock);
3349         if (dev_priv->display_irqs_enabled)
3350                 vlv_display_irq_postinstall(dev_priv);
3351         spin_unlock_irq(&dev_priv->irq_lock);
3352
3353         I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3354         POSTING_READ(VLV_MASTER_IER);
3355 }
3356
3357 static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
3358 {
3359         struct intel_uncore *uncore = &dev_priv->uncore;
3360
3361         u32 de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
3362         u32 de_pipe_enables;
3363         u32 de_port_masked = GEN8_AUX_CHANNEL_A;
3364         u32 de_port_enables;
3365         u32 de_misc_masked = GEN8_DE_EDP_PSR;
3366         enum pipe pipe;
3367
3368         if (INTEL_GEN(dev_priv) <= 10)
3369                 de_misc_masked |= GEN8_DE_MISC_GSE;
3370
3371         if (INTEL_GEN(dev_priv) >= 9) {
3372                 de_pipe_masked |= GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
3373                 de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
3374                                   GEN9_AUX_CHANNEL_D;
3375                 if (IS_GEN9_LP(dev_priv))
3376                         de_port_masked |= BXT_DE_PORT_GMBUS;
3377         } else {
3378                 de_pipe_masked |= GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3379         }
3380
3381         if (INTEL_GEN(dev_priv) >= 11)
3382                 de_port_masked |= ICL_AUX_CHANNEL_E;
3383
3384         if (IS_CNL_WITH_PORT_F(dev_priv) || INTEL_GEN(dev_priv) >= 11)
3385                 de_port_masked |= CNL_AUX_CHANNEL_F;
3386
3387         de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
3388                                            GEN8_PIPE_FIFO_UNDERRUN;
3389
3390         de_port_enables = de_port_masked;
3391         if (IS_GEN9_LP(dev_priv))
3392                 de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
3393         else if (IS_BROADWELL(dev_priv))
3394                 de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;
3395
3396         if (INTEL_GEN(dev_priv) >= 12) {
3397                 enum transcoder trans;
3398
3399                 for (trans = TRANSCODER_A; trans <= TRANSCODER_D; trans++) {
3400                         enum intel_display_power_domain domain;
3401
3402                         domain = POWER_DOMAIN_TRANSCODER(trans);
3403                         if (!intel_display_power_is_enabled(dev_priv, domain))
3404                                 continue;
3405
3406                         gen3_assert_iir_is_zero(uncore, TRANS_PSR_IIR(trans));
3407                 }
3408         } else {
3409                 gen3_assert_iir_is_zero(uncore, EDP_PSR_IIR);
3410         }
3411
3412         for_each_pipe(dev_priv, pipe) {
3413                 dev_priv->de_irq_mask[pipe] = ~de_pipe_masked;
3414
3415                 if (intel_display_power_is_enabled(dev_priv,
3416                                 POWER_DOMAIN_PIPE(pipe)))
3417                         GEN8_IRQ_INIT_NDX(uncore, DE_PIPE, pipe,
3418                                           dev_priv->de_irq_mask[pipe],
3419                                           de_pipe_enables);
3420         }
3421
3422         GEN3_IRQ_INIT(uncore, GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
3423         GEN3_IRQ_INIT(uncore, GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked);
3424
3425         if (INTEL_GEN(dev_priv) >= 11) {
3426                 u32 de_hpd_masked = 0;
3427                 u32 de_hpd_enables = GEN11_DE_TC_HOTPLUG_MASK |
3428                                      GEN11_DE_TBT_HOTPLUG_MASK;
3429
3430                 GEN3_IRQ_INIT(uncore, GEN11_DE_HPD_, ~de_hpd_masked,
3431                               de_hpd_enables);
3432                 gen11_hpd_detection_setup(dev_priv);
3433         } else if (IS_GEN9_LP(dev_priv)) {
3434                 bxt_hpd_detection_setup(dev_priv);
3435         } else if (IS_BROADWELL(dev_priv)) {
3436                 ilk_hpd_detection_setup(dev_priv);
3437         }
3438 }
3439
3440 static void gen8_irq_postinstall(struct drm_i915_private *dev_priv)
3441 {
3442         if (HAS_PCH_SPLIT(dev_priv))
3443                 ibx_irq_pre_postinstall(dev_priv);
3444
3445         gen8_gt_irq_postinstall(&dev_priv->gt);
3446         gen8_de_irq_postinstall(dev_priv);
3447
3448         if (HAS_PCH_SPLIT(dev_priv))
3449                 ibx_irq_postinstall(dev_priv);
3450
3451         gen8_master_intr_enable(dev_priv->uncore.regs);
3452 }
3453
3454 static void icp_irq_postinstall(struct drm_i915_private *dev_priv)
3455 {
3456         u32 mask = SDE_GMBUS_ICP;
3457
3458         drm_WARN_ON(&dev_priv->drm, I915_READ(SDEIER) != 0);
3459         I915_WRITE(SDEIER, 0xffffffff);
3460         POSTING_READ(SDEIER);
3461
3462         gen3_assert_iir_is_zero(&dev_priv->uncore, SDEIIR);
3463         I915_WRITE(SDEIMR, ~mask);
3464
3465         if (HAS_PCH_TGP(dev_priv))
3466                 icp_hpd_detection_setup(dev_priv, TGP_DDI_HPD_ENABLE_MASK,
3467                                         TGP_TC_HPD_ENABLE_MASK);
3468         else if (HAS_PCH_JSP(dev_priv))
3469                 icp_hpd_detection_setup(dev_priv, TGP_DDI_HPD_ENABLE_MASK, 0);
3470         else if (HAS_PCH_MCC(dev_priv))
3471                 icp_hpd_detection_setup(dev_priv, ICP_DDI_HPD_ENABLE_MASK,
3472                                         ICP_TC_HPD_ENABLE(PORT_TC1));
3473         else
3474                 icp_hpd_detection_setup(dev_priv, ICP_DDI_HPD_ENABLE_MASK,
3475                                         ICP_TC_HPD_ENABLE_MASK);
3476 }
3477
3478 static void gen11_irq_postinstall(struct drm_i915_private *dev_priv)
3479 {
3480         struct intel_uncore *uncore = &dev_priv->uncore;
3481         u32 gu_misc_masked = GEN11_GU_MISC_GSE;
3482
3483         if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
3484                 icp_irq_postinstall(dev_priv);
3485
3486         gen11_gt_irq_postinstall(&dev_priv->gt);
3487         gen8_de_irq_postinstall(dev_priv);
3488
3489         GEN3_IRQ_INIT(uncore, GEN11_GU_MISC_, ~gu_misc_masked, gu_misc_masked);
3490
3491         I915_WRITE(GEN11_DISPLAY_INT_CTL, GEN11_DISPLAY_IRQ_ENABLE);
3492
3493         gen11_master_intr_enable(uncore->regs);
3494         POSTING_READ(GEN11_GFX_MSTR_IRQ);
3495 }
3496
3497 static void cherryview_irq_postinstall(struct drm_i915_private *dev_priv)
3498 {
3499         gen8_gt_irq_postinstall(&dev_priv->gt);
3500
3501         spin_lock_irq(&dev_priv->irq_lock);
3502         if (dev_priv->display_irqs_enabled)
3503                 vlv_display_irq_postinstall(dev_priv);
3504         spin_unlock_irq(&dev_priv->irq_lock);
3505
3506         I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
3507         POSTING_READ(GEN8_MASTER_IRQ);
3508 }
3509
3510 static void i8xx_irq_reset(struct drm_i915_private *dev_priv)
3511 {
3512         struct intel_uncore *uncore = &dev_priv->uncore;
3513
3514         i9xx_pipestat_irq_reset(dev_priv);
3515
3516         GEN2_IRQ_RESET(uncore);
3517 }
3518
3519 static void i8xx_irq_postinstall(struct drm_i915_private *dev_priv)
3520 {
3521         struct intel_uncore *uncore = &dev_priv->uncore;
3522         u16 enable_mask;
3523
3524         intel_uncore_write16(uncore,
3525                              EMR,
3526                              ~(I915_ERROR_PAGE_TABLE |
3527                                I915_ERROR_MEMORY_REFRESH));
3528
3529         /* Unmask the interrupts that we always want on. */
3530         dev_priv->irq_mask =
3531                 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3532                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3533                   I915_MASTER_ERROR_INTERRUPT);
3534
3535         enable_mask =
3536                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3537                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3538                 I915_MASTER_ERROR_INTERRUPT |
3539                 I915_USER_INTERRUPT;
3540
3541         GEN2_IRQ_INIT(uncore, dev_priv->irq_mask, enable_mask);
3542
3543         /* Interrupt setup is already guaranteed to be single-threaded, this is
3544          * just to make the assert_spin_locked check happy. */
3545         spin_lock_irq(&dev_priv->irq_lock);
3546         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3547         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3548         spin_unlock_irq(&dev_priv->irq_lock);
3549 }
3550
3551 static void i8xx_error_irq_ack(struct drm_i915_private *i915,
3552                                u16 *eir, u16 *eir_stuck)
3553 {
3554         struct intel_uncore *uncore = &i915->uncore;
3555         u16 emr;
3556
3557         *eir = intel_uncore_read16(uncore, EIR);
3558
3559         if (*eir)
3560                 intel_uncore_write16(uncore, EIR, *eir);
3561
3562         *eir_stuck = intel_uncore_read16(uncore, EIR);
3563         if (*eir_stuck == 0)
3564                 return;
3565
3566         /*
3567          * Toggle all EMR bits to make sure we get an edge
3568          * in the ISR master error bit if we don't clear
3569          * all the EIR bits. Otherwise the edge triggered
3570          * IIR on i965/g4x wouldn't notice that an interrupt
3571          * is still pending. Also some EIR bits can't be
3572          * cleared except by handling the underlying error
3573          * (or by a GPU reset) so we mask any bit that
3574          * remains set.
3575          */
3576         emr = intel_uncore_read16(uncore, EMR);
3577         intel_uncore_write16(uncore, EMR, 0xffff);
3578         intel_uncore_write16(uncore, EMR, emr | *eir_stuck);
3579 }
3580
3581 static void i8xx_error_irq_handler(struct drm_i915_private *dev_priv,
3582                                    u16 eir, u16 eir_stuck)
3583 {
3584         DRM_DEBUG("Master Error: EIR 0x%04x\n", eir);
3585
3586         if (eir_stuck)
3587                 drm_dbg(&dev_priv->drm, "EIR stuck: 0x%04x, masked\n",
3588                         eir_stuck);
3589 }
3590
3591 static void i9xx_error_irq_ack(struct drm_i915_private *dev_priv,
3592                                u32 *eir, u32 *eir_stuck)
3593 {
3594         u32 emr;
3595
3596         *eir = I915_READ(EIR);
3597
3598         I915_WRITE(EIR, *eir);
3599
3600         *eir_stuck = I915_READ(EIR);
3601         if (*eir_stuck == 0)
3602                 return;
3603
3604         /*
3605          * Toggle all EMR bits to make sure we get an edge
3606          * in the ISR master error bit if we don't clear
3607          * all the EIR bits. Otherwise the edge triggered
3608          * IIR on i965/g4x wouldn't notice that an interrupt
3609          * is still pending. Also some EIR bits can't be
3610          * cleared except by handling the underlying error
3611          * (or by a GPU reset) so we mask any bit that
3612          * remains set.
3613          */
3614         emr = I915_READ(EMR);
3615         I915_WRITE(EMR, 0xffffffff);
3616         I915_WRITE(EMR, emr | *eir_stuck);
3617 }
3618
3619 static void i9xx_error_irq_handler(struct drm_i915_private *dev_priv,
3620                                    u32 eir, u32 eir_stuck)
3621 {
3622         DRM_DEBUG("Master Error, EIR 0x%08x\n", eir);
3623
3624         if (eir_stuck)
3625                 drm_dbg(&dev_priv->drm, "EIR stuck: 0x%08x, masked\n",
3626                         eir_stuck);
3627 }
3628
3629 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
3630 {
3631         struct drm_i915_private *dev_priv = arg;
3632         irqreturn_t ret = IRQ_NONE;
3633
3634         if (!intel_irqs_enabled(dev_priv))
3635                 return IRQ_NONE;
3636
3637         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
3638         disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
3639
3640         do {
3641                 u32 pipe_stats[I915_MAX_PIPES] = {};
3642                 u16 eir = 0, eir_stuck = 0;
3643                 u16 iir;
3644
3645                 iir = intel_uncore_read16(&dev_priv->uncore, GEN2_IIR);
3646                 if (iir == 0)
3647                         break;
3648
3649                 ret = IRQ_HANDLED;
3650
3651                 /* Call regardless, as some status bits might not be
3652                  * signalled in iir */
3653                 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
3654
3655                 if (iir & I915_MASTER_ERROR_INTERRUPT)
3656                         i8xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
3657
3658                 intel_uncore_write16(&dev_priv->uncore, GEN2_IIR, iir);
3659
3660                 if (iir & I915_USER_INTERRUPT)
3661                         intel_engine_signal_breadcrumbs(dev_priv->engine[RCS0]);
3662
3663                 if (iir & I915_MASTER_ERROR_INTERRUPT)
3664                         i8xx_error_irq_handler(dev_priv, eir, eir_stuck);
3665
3666                 i8xx_pipestat_irq_handler(dev_priv, iir, pipe_stats);
3667         } while (0);
3668
3669         enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
3670
3671         return ret;
3672 }
3673
3674 static void i915_irq_reset(struct drm_i915_private *dev_priv)
3675 {
3676         struct intel_uncore *uncore = &dev_priv->uncore;
3677
3678         if (I915_HAS_HOTPLUG(dev_priv)) {
3679                 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3680                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3681         }
3682
3683         i9xx_pipestat_irq_reset(dev_priv);
3684
3685         GEN3_IRQ_RESET(uncore, GEN2_);
3686 }
3687
3688 static void i915_irq_postinstall(struct drm_i915_private *dev_priv)
3689 {
3690         struct intel_uncore *uncore = &dev_priv->uncore;
3691         u32 enable_mask;
3692
3693         I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE |
3694                           I915_ERROR_MEMORY_REFRESH));
3695
3696         /* Unmask the interrupts that we always want on. */
3697         dev_priv->irq_mask =
3698                 ~(I915_ASLE_INTERRUPT |
3699                   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3700                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3701                   I915_MASTER_ERROR_INTERRUPT);
3702
3703         enable_mask =
3704                 I915_ASLE_INTERRUPT |
3705                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3706                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3707                 I915_MASTER_ERROR_INTERRUPT |
3708                 I915_USER_INTERRUPT;
3709
3710         if (I915_HAS_HOTPLUG(dev_priv)) {
3711                 /* Enable in IER... */
3712                 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
3713                 /* and unmask in IMR */
3714                 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
3715         }
3716
3717         GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask);
3718
3719         /* Interrupt setup is already guaranteed to be single-threaded, this is
3720          * just to make the assert_spin_locked check happy. */
3721         spin_lock_irq(&dev_priv->irq_lock);
3722         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3723         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3724         spin_unlock_irq(&dev_priv->irq_lock);
3725
3726         i915_enable_asle_pipestat(dev_priv);
3727 }
3728
3729 static irqreturn_t i915_irq_handler(int irq, void *arg)
3730 {
3731         struct drm_i915_private *dev_priv = arg;
3732         irqreturn_t ret = IRQ_NONE;
3733
3734         if (!intel_irqs_enabled(dev_priv))
3735                 return IRQ_NONE;
3736
3737         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
3738         disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
3739
3740         do {
3741                 u32 pipe_stats[I915_MAX_PIPES] = {};
3742                 u32 eir = 0, eir_stuck = 0;
3743                 u32 hotplug_status = 0;
3744                 u32 iir;
3745
3746                 iir = I915_READ(GEN2_IIR);
3747                 if (iir == 0)
3748                         break;
3749
3750                 ret = IRQ_HANDLED;
3751
3752                 if (I915_HAS_HOTPLUG(dev_priv) &&
3753                     iir & I915_DISPLAY_PORT_INTERRUPT)
3754                         hotplug_status = i9xx_hpd_irq_ack(dev_priv);
3755
3756                 /* Call regardless, as some status bits might not be
3757                  * signalled in iir */
3758                 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
3759
3760                 if (iir & I915_MASTER_ERROR_INTERRUPT)
3761                         i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
3762
3763                 I915_WRITE(GEN2_IIR, iir);
3764
3765                 if (iir & I915_USER_INTERRUPT)
3766                         intel_engine_signal_breadcrumbs(dev_priv->engine[RCS0]);
3767
3768                 if (iir & I915_MASTER_ERROR_INTERRUPT)
3769                         i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
3770
3771                 if (hotplug_status)
3772                         i9xx_hpd_irq_handler(dev_priv, hotplug_status);
3773
3774                 i915_pipestat_irq_handler(dev_priv, iir, pipe_stats);
3775         } while (0);
3776
3777         enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
3778
3779         return ret;
3780 }
3781
3782 static void i965_irq_reset(struct drm_i915_private *dev_priv)
3783 {
3784         struct intel_uncore *uncore = &dev_priv->uncore;
3785
3786         i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3787         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3788
3789         i9xx_pipestat_irq_reset(dev_priv);
3790
3791         GEN3_IRQ_RESET(uncore, GEN2_);
3792 }
3793
3794 static void i965_irq_postinstall(struct drm_i915_private *dev_priv)
3795 {
3796         struct intel_uncore *uncore = &dev_priv->uncore;
3797         u32 enable_mask;
3798         u32 error_mask;
3799
3800         /*
3801          * Enable some error detection, note the instruction error mask
3802          * bit is reserved, so we leave it masked.
3803          */
3804         if (IS_G4X(dev_priv)) {
3805                 error_mask = ~(GM45_ERROR_PAGE_TABLE |
3806                                GM45_ERROR_MEM_PRIV |
3807                                GM45_ERROR_CP_PRIV |
3808                                I915_ERROR_MEMORY_REFRESH);
3809         } else {
3810                 error_mask = ~(I915_ERROR_PAGE_TABLE |
3811                                I915_ERROR_MEMORY_REFRESH);
3812         }
3813         I915_WRITE(EMR, error_mask);
3814
3815         /* Unmask the interrupts that we always want on. */
3816         dev_priv->irq_mask =
3817                 ~(I915_ASLE_INTERRUPT |
3818                   I915_DISPLAY_PORT_INTERRUPT |
3819                   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3820                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3821                   I915_MASTER_ERROR_INTERRUPT);
3822
3823         enable_mask =
3824                 I915_ASLE_INTERRUPT |
3825                 I915_DISPLAY_PORT_INTERRUPT |
3826                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3827                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3828                 I915_MASTER_ERROR_INTERRUPT |
3829                 I915_USER_INTERRUPT;
3830
3831         if (IS_G4X(dev_priv))
3832                 enable_mask |= I915_BSD_USER_INTERRUPT;
3833
3834         GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask);
3835
3836         /* Interrupt setup is already guaranteed to be single-threaded, this is
3837          * just to make the assert_spin_locked check happy. */
3838         spin_lock_irq(&dev_priv->irq_lock);
3839         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3840         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3841         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3842         spin_unlock_irq(&dev_priv->irq_lock);
3843
3844         i915_enable_asle_pipestat(dev_priv);
3845 }
3846
3847 static void i915_hpd_irq_setup(struct drm_i915_private *dev_priv)
3848 {
3849         u32 hotplug_en;
3850
3851         lockdep_assert_held(&dev_priv->irq_lock);
3852
3853         /* Note HDMI and DP share hotplug bits */
3854         /* enable bits are the same for all generations */
3855         hotplug_en = intel_hpd_enabled_irqs(dev_priv, hpd_mask_i915);
3856         /* Programming the CRT detection parameters tends
3857            to generate a spurious hotplug event about three
3858            seconds later.  So just do it once.
3859         */
3860         if (IS_G4X(dev_priv))
3861                 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
3862         hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
3863
3864         /* Ignore TV since it's buggy */
3865         i915_hotplug_interrupt_update_locked(dev_priv,
3866                                              HOTPLUG_INT_EN_MASK |
3867                                              CRT_HOTPLUG_VOLTAGE_COMPARE_MASK |
3868                                              CRT_HOTPLUG_ACTIVATION_PERIOD_64,
3869                                              hotplug_en);
3870 }
3871
3872 static irqreturn_t i965_irq_handler(int irq, void *arg)
3873 {
3874         struct drm_i915_private *dev_priv = arg;
3875         irqreturn_t ret = IRQ_NONE;
3876
3877         if (!intel_irqs_enabled(dev_priv))
3878                 return IRQ_NONE;
3879
3880         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
3881         disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
3882
3883         do {
3884                 u32 pipe_stats[I915_MAX_PIPES] = {};
3885                 u32 eir = 0, eir_stuck = 0;
3886                 u32 hotplug_status = 0;
3887                 u32 iir;
3888
3889                 iir = I915_READ(GEN2_IIR);
3890                 if (iir == 0)
3891                         break;
3892
3893                 ret = IRQ_HANDLED;
3894
3895                 if (iir & I915_DISPLAY_PORT_INTERRUPT)
3896                         hotplug_status = i9xx_hpd_irq_ack(dev_priv);
3897
3898                 /* Call regardless, as some status bits might not be
3899                  * signalled in iir */
3900                 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
3901
3902                 if (iir & I915_MASTER_ERROR_INTERRUPT)
3903                         i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
3904
3905                 I915_WRITE(GEN2_IIR, iir);
3906
3907                 if (iir & I915_USER_INTERRUPT)
3908                         intel_engine_signal_breadcrumbs(dev_priv->engine[RCS0]);
3909
3910                 if (iir & I915_BSD_USER_INTERRUPT)
3911                         intel_engine_signal_breadcrumbs(dev_priv->engine[VCS0]);
3912
3913                 if (iir & I915_MASTER_ERROR_INTERRUPT)
3914                         i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
3915
3916                 if (hotplug_status)
3917                         i9xx_hpd_irq_handler(dev_priv, hotplug_status);
3918
3919                 i965_pipestat_irq_handler(dev_priv, iir, pipe_stats);
3920         } while (0);
3921
3922         enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
3923
3924         return ret;
3925 }
3926
3927 /**
3928  * intel_irq_init - initializes irq support
3929  * @dev_priv: i915 device instance
3930  *
3931  * This function initializes all the irq support including work items, timers
3932  * and all the vtables. It does not setup the interrupt itself though.
3933  */
3934 void intel_irq_init(struct drm_i915_private *dev_priv)
3935 {
3936         struct drm_device *dev = &dev_priv->drm;
3937         int i;
3938
3939         intel_hpd_init_work(dev_priv);
3940
3941         INIT_WORK(&dev_priv->l3_parity.error_work, ivb_parity_work);
3942         for (i = 0; i < MAX_L3_SLICES; ++i)
3943                 dev_priv->l3_parity.remap_info[i] = NULL;
3944
3945         /* pre-gen11 the guc irqs bits are in the upper 16 bits of the pm reg */
3946         if (HAS_GT_UC(dev_priv) && INTEL_GEN(dev_priv) < 11)
3947                 dev_priv->gt.pm_guc_events = GUC_INTR_GUC2HOST << 16;
3948
3949         dev->vblank_disable_immediate = true;
3950
3951         /* Most platforms treat the display irq block as an always-on
3952          * power domain. vlv/chv can disable it at runtime and need
3953          * special care to avoid writing any of the display block registers
3954          * outside of the power domain. We defer setting up the display irqs
3955          * in this case to the runtime pm.
3956          */
3957         dev_priv->display_irqs_enabled = true;
3958         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
3959                 dev_priv->display_irqs_enabled = false;
3960
3961         dev_priv->hotplug.hpd_storm_threshold = HPD_STORM_DEFAULT_THRESHOLD;
3962         /* If we have MST support, we want to avoid doing short HPD IRQ storm
3963          * detection, as short HPD storms will occur as a natural part of
3964          * sideband messaging with MST.
3965          * On older platforms however, IRQ storms can occur with both long and
3966          * short pulses, as seen on some G4x systems.
3967          */
3968         dev_priv->hotplug.hpd_short_storm_enabled = !HAS_DP_MST(dev_priv);
3969
3970         if (HAS_GMCH(dev_priv)) {
3971                 if (I915_HAS_HOTPLUG(dev_priv))
3972                         dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
3973         } else {
3974                 if (HAS_PCH_JSP(dev_priv))
3975                         dev_priv->display.hpd_irq_setup = jsp_hpd_irq_setup;
3976                 else if (HAS_PCH_MCC(dev_priv))
3977                         dev_priv->display.hpd_irq_setup = mcc_hpd_irq_setup;
3978                 else if (INTEL_GEN(dev_priv) >= 11)
3979                         dev_priv->display.hpd_irq_setup = gen11_hpd_irq_setup;
3980                 else if (IS_GEN9_LP(dev_priv))
3981                         dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
3982                 else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT)
3983                         dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
3984                 else
3985                         dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
3986         }
3987 }
3988
3989 /**
3990  * intel_irq_fini - deinitializes IRQ support
3991  * @i915: i915 device instance
3992  *
3993  * This function deinitializes all the IRQ support.
3994  */
3995 void intel_irq_fini(struct drm_i915_private *i915)
3996 {
3997         int i;
3998
3999         for (i = 0; i < MAX_L3_SLICES; ++i)
4000                 kfree(i915->l3_parity.remap_info[i]);
4001 }
4002
4003 static irq_handler_t intel_irq_handler(struct drm_i915_private *dev_priv)
4004 {
4005         if (HAS_GMCH(dev_priv)) {
4006                 if (IS_CHERRYVIEW(dev_priv))
4007                         return cherryview_irq_handler;
4008                 else if (IS_VALLEYVIEW(dev_priv))
4009                         return valleyview_irq_handler;
4010                 else if (IS_GEN(dev_priv, 4))
4011                         return i965_irq_handler;
4012                 else if (IS_GEN(dev_priv, 3))
4013                         return i915_irq_handler;
4014                 else
4015                         return i8xx_irq_handler;
4016         } else {
4017                 if (INTEL_GEN(dev_priv) >= 11)
4018                         return gen11_irq_handler;
4019                 else if (INTEL_GEN(dev_priv) >= 8)
4020                         return gen8_irq_handler;
4021                 else
4022                         return ilk_irq_handler;
4023         }
4024 }
4025
4026 static void intel_irq_reset(struct drm_i915_private *dev_priv)
4027 {
4028         if (HAS_GMCH(dev_priv)) {
4029                 if (IS_CHERRYVIEW(dev_priv))
4030                         cherryview_irq_reset(dev_priv);
4031                 else if (IS_VALLEYVIEW(dev_priv))
4032                         valleyview_irq_reset(dev_priv);
4033                 else if (IS_GEN(dev_priv, 4))
4034                         i965_irq_reset(dev_priv);
4035                 else if (IS_GEN(dev_priv, 3))
4036                         i915_irq_reset(dev_priv);
4037                 else
4038                         i8xx_irq_reset(dev_priv);
4039         } else {
4040                 if (INTEL_GEN(dev_priv) >= 11)
4041                         gen11_irq_reset(dev_priv);
4042                 else if (INTEL_GEN(dev_priv) >= 8)
4043                         gen8_irq_reset(dev_priv);
4044                 else
4045                         ilk_irq_reset(dev_priv);
4046         }
4047 }
4048
4049 static void intel_irq_postinstall(struct drm_i915_private *dev_priv)
4050 {
4051         if (HAS_GMCH(dev_priv)) {
4052                 if (IS_CHERRYVIEW(dev_priv))
4053                         cherryview_irq_postinstall(dev_priv);
4054                 else if (IS_VALLEYVIEW(dev_priv))
4055                         valleyview_irq_postinstall(dev_priv);
4056                 else if (IS_GEN(dev_priv, 4))
4057                         i965_irq_postinstall(dev_priv);
4058                 else if (IS_GEN(dev_priv, 3))
4059                         i915_irq_postinstall(dev_priv);
4060                 else
4061                         i8xx_irq_postinstall(dev_priv);
4062         } else {
4063                 if (INTEL_GEN(dev_priv) >= 11)
4064                         gen11_irq_postinstall(dev_priv);
4065                 else if (INTEL_GEN(dev_priv) >= 8)
4066                         gen8_irq_postinstall(dev_priv);
4067                 else
4068                         ilk_irq_postinstall(dev_priv);
4069         }
4070 }
4071
4072 /**
4073  * intel_irq_install - enables the hardware interrupt
4074  * @dev_priv: i915 device instance
4075  *
4076  * This function enables the hardware interrupt handling, but leaves the hotplug
4077  * handling still disabled. It is called after intel_irq_init().
4078  *
4079  * In the driver load and resume code we need working interrupts in a few places
4080  * but don't want to deal with the hassle of concurrent probe and hotplug
4081  * workers. Hence the split into this two-stage approach.
4082  */
4083 int intel_irq_install(struct drm_i915_private *dev_priv)
4084 {
4085         int irq = dev_priv->drm.pdev->irq;
4086         int ret;
4087
4088         /*
4089          * We enable some interrupt sources in our postinstall hooks, so mark
4090          * interrupts as enabled _before_ actually enabling them to avoid
4091          * special cases in our ordering checks.
4092          */
4093         dev_priv->runtime_pm.irqs_enabled = true;
4094
4095         dev_priv->drm.irq_enabled = true;
4096
4097         intel_irq_reset(dev_priv);
4098
4099         ret = request_irq(irq, intel_irq_handler(dev_priv),
4100                           IRQF_SHARED, DRIVER_NAME, dev_priv);
4101         if (ret < 0) {
4102                 dev_priv->drm.irq_enabled = false;
4103                 return ret;
4104         }
4105
4106         intel_irq_postinstall(dev_priv);
4107
4108         return ret;
4109 }
4110
4111 /**
4112  * intel_irq_uninstall - finilizes all irq handling
4113  * @dev_priv: i915 device instance
4114  *
4115  * This stops interrupt and hotplug handling and unregisters and frees all
4116  * resources acquired in the init functions.
4117  */
4118 void intel_irq_uninstall(struct drm_i915_private *dev_priv)
4119 {
4120         int irq = dev_priv->drm.pdev->irq;
4121
4122         /*
4123          * FIXME we can get called twice during driver probe
4124          * error handling as well as during driver remove due to
4125          * intel_modeset_driver_remove() calling us out of sequence.
4126          * Would be nice if it didn't do that...
4127          */
4128         if (!dev_priv->drm.irq_enabled)
4129                 return;
4130
4131         dev_priv->drm.irq_enabled = false;
4132
4133         intel_irq_reset(dev_priv);
4134
4135         free_irq(irq, dev_priv);
4136
4137         intel_hpd_cancel_work(dev_priv);
4138         dev_priv->runtime_pm.irqs_enabled = false;
4139 }
4140
4141 /**
4142  * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
4143  * @dev_priv: i915 device instance
4144  *
4145  * This function is used to disable interrupts at runtime, both in the runtime
4146  * pm and the system suspend/resume code.
4147  */
4148 void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4149 {
4150         intel_irq_reset(dev_priv);
4151         dev_priv->runtime_pm.irqs_enabled = false;
4152         intel_synchronize_irq(dev_priv);
4153 }
4154
4155 /**
4156  * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
4157  * @dev_priv: i915 device instance
4158  *
4159  * This function is used to enable interrupts at runtime, both in the runtime
4160  * pm and the system suspend/resume code.
4161  */
4162 void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4163 {
4164         dev_priv->runtime_pm.irqs_enabled = true;
4165         intel_irq_reset(dev_priv);
4166         intel_irq_postinstall(dev_priv);
4167 }
4168
4169 bool intel_irqs_enabled(struct drm_i915_private *dev_priv)
4170 {
4171         /*
4172          * We only use drm_irq_uninstall() at unload and VT switch, so
4173          * this is the only thing we need to check.
4174          */
4175         return dev_priv->runtime_pm.irqs_enabled;
4176 }
4177
4178 void intel_synchronize_irq(struct drm_i915_private *i915)
4179 {
4180         synchronize_irq(i915->drm.pdev->irq);
4181 }