1 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
4 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
31 #include <linux/circ_buf.h>
32 #include <linux/slab.h>
33 #include <linux/sysrq.h>
35 #include <drm/drm_drv.h>
36 #include <drm/drm_irq.h>
38 #include "display/intel_display_types.h"
39 #include "display/intel_fifo_underrun.h"
40 #include "display/intel_hotplug.h"
41 #include "display/intel_lpe_audio.h"
42 #include "display/intel_psr.h"
44 #include "gt/intel_breadcrumbs.h"
45 #include "gt/intel_gt.h"
46 #include "gt/intel_gt_irq.h"
47 #include "gt/intel_gt_pm_irq.h"
48 #include "gt/intel_rps.h"
52 #include "i915_trace.h"
56 * DOC: interrupt handling
58 * These functions provide the basic support for enabling and disabling the
59 * interrupt handling support. There's a lot more functionality in i915_irq.c
60 * and related files, but that will be described in separate chapters.
63 typedef bool (*long_pulse_detect_func)(enum hpd_pin pin, u32 val);
64 typedef u32 (*hotplug_enables_func)(struct drm_i915_private *i915,
67 static const u32 hpd_ilk[HPD_NUM_PINS] = {
68 [HPD_PORT_A] = DE_DP_A_HOTPLUG,
71 static const u32 hpd_ivb[HPD_NUM_PINS] = {
72 [HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
75 static const u32 hpd_bdw[HPD_NUM_PINS] = {
76 [HPD_PORT_A] = GEN8_DE_PORT_HOTPLUG(HPD_PORT_A),
79 static const u32 hpd_ibx[HPD_NUM_PINS] = {
80 [HPD_CRT] = SDE_CRT_HOTPLUG,
81 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
82 [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
83 [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
84 [HPD_PORT_D] = SDE_PORTD_HOTPLUG,
87 static const u32 hpd_cpt[HPD_NUM_PINS] = {
88 [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
89 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
90 [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
91 [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
92 [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
95 static const u32 hpd_spt[HPD_NUM_PINS] = {
96 [HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
97 [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
98 [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
99 [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
100 [HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT,
103 static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
104 [HPD_CRT] = CRT_HOTPLUG_INT_EN,
105 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
106 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
107 [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
108 [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
109 [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN,
112 static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
113 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
114 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
115 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
116 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
117 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
118 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS,
121 static const u32 hpd_status_i915[HPD_NUM_PINS] = {
122 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
123 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
124 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
125 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
126 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
127 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS,
130 static const u32 hpd_bxt[HPD_NUM_PINS] = {
131 [HPD_PORT_A] = GEN8_DE_PORT_HOTPLUG(HPD_PORT_A),
132 [HPD_PORT_B] = GEN8_DE_PORT_HOTPLUG(HPD_PORT_B),
133 [HPD_PORT_C] = GEN8_DE_PORT_HOTPLUG(HPD_PORT_C),
136 static const u32 hpd_gen11[HPD_NUM_PINS] = {
137 [HPD_PORT_TC1] = GEN11_TC_HOTPLUG(HPD_PORT_TC1) | GEN11_TBT_HOTPLUG(HPD_PORT_TC1),
138 [HPD_PORT_TC2] = GEN11_TC_HOTPLUG(HPD_PORT_TC2) | GEN11_TBT_HOTPLUG(HPD_PORT_TC2),
139 [HPD_PORT_TC3] = GEN11_TC_HOTPLUG(HPD_PORT_TC3) | GEN11_TBT_HOTPLUG(HPD_PORT_TC3),
140 [HPD_PORT_TC4] = GEN11_TC_HOTPLUG(HPD_PORT_TC4) | GEN11_TBT_HOTPLUG(HPD_PORT_TC4),
141 [HPD_PORT_TC5] = GEN11_TC_HOTPLUG(HPD_PORT_TC5) | GEN11_TBT_HOTPLUG(HPD_PORT_TC5),
142 [HPD_PORT_TC6] = GEN11_TC_HOTPLUG(HPD_PORT_TC6) | GEN11_TBT_HOTPLUG(HPD_PORT_TC6),
145 static const u32 hpd_icp[HPD_NUM_PINS] = {
146 [HPD_PORT_A] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_A),
147 [HPD_PORT_B] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_B),
148 [HPD_PORT_C] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_C),
149 [HPD_PORT_TC1] = SDE_TC_HOTPLUG_ICP(HPD_PORT_TC1),
150 [HPD_PORT_TC2] = SDE_TC_HOTPLUG_ICP(HPD_PORT_TC2),
151 [HPD_PORT_TC3] = SDE_TC_HOTPLUG_ICP(HPD_PORT_TC3),
152 [HPD_PORT_TC4] = SDE_TC_HOTPLUG_ICP(HPD_PORT_TC4),
153 [HPD_PORT_TC5] = SDE_TC_HOTPLUG_ICP(HPD_PORT_TC5),
154 [HPD_PORT_TC6] = SDE_TC_HOTPLUG_ICP(HPD_PORT_TC6),
157 static const u32 hpd_sde_dg1[HPD_NUM_PINS] = {
158 [HPD_PORT_A] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_A),
159 [HPD_PORT_B] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_B),
160 [HPD_PORT_C] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_C),
161 [HPD_PORT_D] = SDE_DDI_HOTPLUG_ICP(HPD_PORT_D),
164 static void intel_hpd_init_pins(struct drm_i915_private *dev_priv)
166 struct i915_hotplug *hpd = &dev_priv->hotplug;
168 if (HAS_GMCH(dev_priv)) {
169 if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
170 IS_CHERRYVIEW(dev_priv))
171 hpd->hpd = hpd_status_g4x;
173 hpd->hpd = hpd_status_i915;
177 if (INTEL_GEN(dev_priv) >= 11)
178 hpd->hpd = hpd_gen11;
179 else if (IS_GEN9_LP(dev_priv))
181 else if (INTEL_GEN(dev_priv) >= 8)
183 else if (INTEL_GEN(dev_priv) >= 7)
188 if ((INTEL_PCH_TYPE(dev_priv) < PCH_DG1) &&
189 (!HAS_PCH_SPLIT(dev_priv) || HAS_PCH_NOP(dev_priv)))
192 if (HAS_PCH_DG1(dev_priv))
193 hpd->pch_hpd = hpd_sde_dg1;
194 else if (HAS_PCH_TGP(dev_priv) || HAS_PCH_JSP(dev_priv) ||
195 HAS_PCH_ICP(dev_priv) || HAS_PCH_MCC(dev_priv))
196 hpd->pch_hpd = hpd_icp;
197 else if (HAS_PCH_CNP(dev_priv) || HAS_PCH_SPT(dev_priv))
198 hpd->pch_hpd = hpd_spt;
199 else if (HAS_PCH_LPT(dev_priv) || HAS_PCH_CPT(dev_priv))
200 hpd->pch_hpd = hpd_cpt;
201 else if (HAS_PCH_IBX(dev_priv))
202 hpd->pch_hpd = hpd_ibx;
204 MISSING_CASE(INTEL_PCH_TYPE(dev_priv));
208 intel_handle_vblank(struct drm_i915_private *dev_priv, enum pipe pipe)
210 struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
212 drm_crtc_handle_vblank(&crtc->base);
215 void gen3_irq_reset(struct intel_uncore *uncore, i915_reg_t imr,
216 i915_reg_t iir, i915_reg_t ier)
218 intel_uncore_write(uncore, imr, 0xffffffff);
219 intel_uncore_posting_read(uncore, imr);
221 intel_uncore_write(uncore, ier, 0);
223 /* IIR can theoretically queue up two events. Be paranoid. */
224 intel_uncore_write(uncore, iir, 0xffffffff);
225 intel_uncore_posting_read(uncore, iir);
226 intel_uncore_write(uncore, iir, 0xffffffff);
227 intel_uncore_posting_read(uncore, iir);
230 void gen2_irq_reset(struct intel_uncore *uncore)
232 intel_uncore_write16(uncore, GEN2_IMR, 0xffff);
233 intel_uncore_posting_read16(uncore, GEN2_IMR);
235 intel_uncore_write16(uncore, GEN2_IER, 0);
237 /* IIR can theoretically queue up two events. Be paranoid. */
238 intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
239 intel_uncore_posting_read16(uncore, GEN2_IIR);
240 intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
241 intel_uncore_posting_read16(uncore, GEN2_IIR);
245 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
247 static void gen3_assert_iir_is_zero(struct intel_uncore *uncore, i915_reg_t reg)
249 u32 val = intel_uncore_read(uncore, reg);
254 drm_WARN(&uncore->i915->drm, 1,
255 "Interrupt register 0x%x is not zero: 0x%08x\n",
256 i915_mmio_reg_offset(reg), val);
257 intel_uncore_write(uncore, reg, 0xffffffff);
258 intel_uncore_posting_read(uncore, reg);
259 intel_uncore_write(uncore, reg, 0xffffffff);
260 intel_uncore_posting_read(uncore, reg);
263 static void gen2_assert_iir_is_zero(struct intel_uncore *uncore)
265 u16 val = intel_uncore_read16(uncore, GEN2_IIR);
270 drm_WARN(&uncore->i915->drm, 1,
271 "Interrupt register 0x%x is not zero: 0x%08x\n",
272 i915_mmio_reg_offset(GEN2_IIR), val);
273 intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
274 intel_uncore_posting_read16(uncore, GEN2_IIR);
275 intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
276 intel_uncore_posting_read16(uncore, GEN2_IIR);
279 void gen3_irq_init(struct intel_uncore *uncore,
280 i915_reg_t imr, u32 imr_val,
281 i915_reg_t ier, u32 ier_val,
284 gen3_assert_iir_is_zero(uncore, iir);
286 intel_uncore_write(uncore, ier, ier_val);
287 intel_uncore_write(uncore, imr, imr_val);
288 intel_uncore_posting_read(uncore, imr);
291 void gen2_irq_init(struct intel_uncore *uncore,
292 u32 imr_val, u32 ier_val)
294 gen2_assert_iir_is_zero(uncore);
296 intel_uncore_write16(uncore, GEN2_IER, ier_val);
297 intel_uncore_write16(uncore, GEN2_IMR, imr_val);
298 intel_uncore_posting_read16(uncore, GEN2_IMR);
301 /* For display hotplug interrupt */
303 i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
309 lockdep_assert_held(&dev_priv->irq_lock);
310 drm_WARN_ON(&dev_priv->drm, bits & ~mask);
312 val = I915_READ(PORT_HOTPLUG_EN);
315 I915_WRITE(PORT_HOTPLUG_EN, val);
319 * i915_hotplug_interrupt_update - update hotplug interrupt enable
320 * @dev_priv: driver private
321 * @mask: bits to update
322 * @bits: bits to enable
323 * NOTE: the HPD enable bits are modified both inside and outside
324 * of an interrupt context. To avoid that read-modify-write cycles
325 * interfer, these bits are protected by a spinlock. Since this
326 * function is usually not called from a context where the lock is
327 * held already, this function acquires the lock itself. A non-locking
328 * version is also available.
330 void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
334 spin_lock_irq(&dev_priv->irq_lock);
335 i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
336 spin_unlock_irq(&dev_priv->irq_lock);
340 * ilk_update_display_irq - update DEIMR
341 * @dev_priv: driver private
342 * @interrupt_mask: mask of interrupt bits to update
343 * @enabled_irq_mask: mask of interrupt bits to enable
345 void ilk_update_display_irq(struct drm_i915_private *dev_priv,
347 u32 enabled_irq_mask)
351 lockdep_assert_held(&dev_priv->irq_lock);
352 drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
354 new_val = dev_priv->irq_mask;
355 new_val &= ~interrupt_mask;
356 new_val |= (~enabled_irq_mask & interrupt_mask);
358 if (new_val != dev_priv->irq_mask &&
359 !drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv))) {
360 dev_priv->irq_mask = new_val;
361 I915_WRITE(DEIMR, dev_priv->irq_mask);
367 * bdw_update_port_irq - update DE port interrupt
368 * @dev_priv: driver private
369 * @interrupt_mask: mask of interrupt bits to update
370 * @enabled_irq_mask: mask of interrupt bits to enable
372 static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
374 u32 enabled_irq_mask)
379 lockdep_assert_held(&dev_priv->irq_lock);
381 drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
383 if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
386 old_val = I915_READ(GEN8_DE_PORT_IMR);
389 new_val &= ~interrupt_mask;
390 new_val |= (~enabled_irq_mask & interrupt_mask);
392 if (new_val != old_val) {
393 I915_WRITE(GEN8_DE_PORT_IMR, new_val);
394 POSTING_READ(GEN8_DE_PORT_IMR);
399 * bdw_update_pipe_irq - update DE pipe interrupt
400 * @dev_priv: driver private
401 * @pipe: pipe whose interrupt to update
402 * @interrupt_mask: mask of interrupt bits to update
403 * @enabled_irq_mask: mask of interrupt bits to enable
405 void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
408 u32 enabled_irq_mask)
412 lockdep_assert_held(&dev_priv->irq_lock);
414 drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
416 if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
419 new_val = dev_priv->de_irq_mask[pipe];
420 new_val &= ~interrupt_mask;
421 new_val |= (~enabled_irq_mask & interrupt_mask);
423 if (new_val != dev_priv->de_irq_mask[pipe]) {
424 dev_priv->de_irq_mask[pipe] = new_val;
425 I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
426 POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
431 * ibx_display_interrupt_update - update SDEIMR
432 * @dev_priv: driver private
433 * @interrupt_mask: mask of interrupt bits to update
434 * @enabled_irq_mask: mask of interrupt bits to enable
436 void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
438 u32 enabled_irq_mask)
440 u32 sdeimr = I915_READ(SDEIMR);
441 sdeimr &= ~interrupt_mask;
442 sdeimr |= (~enabled_irq_mask & interrupt_mask);
444 drm_WARN_ON(&dev_priv->drm, enabled_irq_mask & ~interrupt_mask);
446 lockdep_assert_held(&dev_priv->irq_lock);
448 if (drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)))
451 I915_WRITE(SDEIMR, sdeimr);
452 POSTING_READ(SDEIMR);
455 u32 i915_pipestat_enable_mask(struct drm_i915_private *dev_priv,
458 u32 status_mask = dev_priv->pipestat_irq_mask[pipe];
459 u32 enable_mask = status_mask << 16;
461 lockdep_assert_held(&dev_priv->irq_lock);
463 if (INTEL_GEN(dev_priv) < 5)
467 * On pipe A we don't support the PSR interrupt yet,
468 * on pipe B and C the same bit MBZ.
470 if (drm_WARN_ON_ONCE(&dev_priv->drm,
471 status_mask & PIPE_A_PSR_STATUS_VLV))
474 * On pipe B and C we don't support the PSR interrupt yet, on pipe
475 * A the same bit is for perf counters which we don't use either.
477 if (drm_WARN_ON_ONCE(&dev_priv->drm,
478 status_mask & PIPE_B_PSR_STATUS_VLV))
481 enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
482 SPRITE0_FLIP_DONE_INT_EN_VLV |
483 SPRITE1_FLIP_DONE_INT_EN_VLV);
484 if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
485 enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
486 if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
487 enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
490 drm_WARN_ONCE(&dev_priv->drm,
491 enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
492 status_mask & ~PIPESTAT_INT_STATUS_MASK,
493 "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
494 pipe_name(pipe), enable_mask, status_mask);
499 void i915_enable_pipestat(struct drm_i915_private *dev_priv,
500 enum pipe pipe, u32 status_mask)
502 i915_reg_t reg = PIPESTAT(pipe);
505 drm_WARN_ONCE(&dev_priv->drm, status_mask & ~PIPESTAT_INT_STATUS_MASK,
506 "pipe %c: status_mask=0x%x\n",
507 pipe_name(pipe), status_mask);
509 lockdep_assert_held(&dev_priv->irq_lock);
510 drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv));
512 if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == status_mask)
515 dev_priv->pipestat_irq_mask[pipe] |= status_mask;
516 enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
518 I915_WRITE(reg, enable_mask | status_mask);
522 void i915_disable_pipestat(struct drm_i915_private *dev_priv,
523 enum pipe pipe, u32 status_mask)
525 i915_reg_t reg = PIPESTAT(pipe);
528 drm_WARN_ONCE(&dev_priv->drm, status_mask & ~PIPESTAT_INT_STATUS_MASK,
529 "pipe %c: status_mask=0x%x\n",
530 pipe_name(pipe), status_mask);
532 lockdep_assert_held(&dev_priv->irq_lock);
533 drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv));
535 if ((dev_priv->pipestat_irq_mask[pipe] & status_mask) == 0)
538 dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
539 enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
541 I915_WRITE(reg, enable_mask | status_mask);
545 static bool i915_has_asle(struct drm_i915_private *dev_priv)
547 if (!dev_priv->opregion.asle)
550 return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
554 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
555 * @dev_priv: i915 device private
557 static void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv)
559 if (!i915_has_asle(dev_priv))
562 spin_lock_irq(&dev_priv->irq_lock);
564 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
565 if (INTEL_GEN(dev_priv) >= 4)
566 i915_enable_pipestat(dev_priv, PIPE_A,
567 PIPE_LEGACY_BLC_EVENT_STATUS);
569 spin_unlock_irq(&dev_priv->irq_lock);
573 * This timing diagram depicts the video signal in and
574 * around the vertical blanking period.
576 * Assumptions about the fictitious mode used in this example:
578 * vsync_start = vblank_start + 1
579 * vsync_end = vblank_start + 2
580 * vtotal = vblank_start + 3
583 * latch double buffered registers
584 * increment frame counter (ctg+)
585 * generate start of vblank interrupt (gen4+)
588 * | generate frame start interrupt (aka. vblank interrupt) (gmch)
589 * | may be shifted forward 1-3 extra lines via PIPECONF
591 * | | start of vsync:
592 * | | generate vsync interrupt
594 * ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx
595 * . \hs/ . \hs/ \hs/ \hs/ . \hs/
596 * ----va---> <-----------------vb--------------------> <--------va-------------
597 * | | <----vs-----> |
598 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
599 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
600 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
602 * last visible pixel first visible pixel
603 * | increment frame counter (gen3/4)
604 * pixel counter = vblank_start * htotal pixel counter = 0 (gen3/4)
606 * x = horizontal active
607 * _ = horizontal blanking
608 * hs = horizontal sync
609 * va = vertical active
610 * vb = vertical blanking
612 * vbs = vblank_start (number)
615 * - most events happen at the start of horizontal sync
616 * - frame start happens at the start of horizontal blank, 1-4 lines
617 * (depending on PIPECONF settings) after the start of vblank
618 * - gen3/4 pixel and frame counter are synchronized with the start
619 * of horizontal active on the first line of vertical active
622 /* Called from drm generic code, passed a 'crtc', which
623 * we use as a pipe index
625 u32 i915_get_vblank_counter(struct drm_crtc *crtc)
627 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
628 struct drm_vblank_crtc *vblank = &dev_priv->drm.vblank[drm_crtc_index(crtc)];
629 const struct drm_display_mode *mode = &vblank->hwmode;
630 enum pipe pipe = to_intel_crtc(crtc)->pipe;
631 i915_reg_t high_frame, low_frame;
632 u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
633 unsigned long irqflags;
636 * On i965gm TV output the frame counter only works up to
637 * the point when we enable the TV encoder. After that the
638 * frame counter ceases to work and reads zero. We need a
639 * vblank wait before enabling the TV encoder and so we
640 * have to enable vblank interrupts while the frame counter
641 * is still in a working state. However the core vblank code
642 * does not like us returning non-zero frame counter values
643 * when we've told it that we don't have a working frame
644 * counter. Thus we must stop non-zero values leaking out.
646 if (!vblank->max_vblank_count)
649 htotal = mode->crtc_htotal;
650 hsync_start = mode->crtc_hsync_start;
651 vbl_start = mode->crtc_vblank_start;
652 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
653 vbl_start = DIV_ROUND_UP(vbl_start, 2);
655 /* Convert to pixel count */
658 /* Start of vblank event occurs at start of hsync */
659 vbl_start -= htotal - hsync_start;
661 high_frame = PIPEFRAME(pipe);
662 low_frame = PIPEFRAMEPIXEL(pipe);
664 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
667 * High & low register fields aren't synchronized, so make sure
668 * we get a low value that's stable across two reads of the high
672 high1 = intel_de_read_fw(dev_priv, high_frame) & PIPE_FRAME_HIGH_MASK;
673 low = intel_de_read_fw(dev_priv, low_frame);
674 high2 = intel_de_read_fw(dev_priv, high_frame) & PIPE_FRAME_HIGH_MASK;
675 } while (high1 != high2);
677 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
679 high1 >>= PIPE_FRAME_HIGH_SHIFT;
680 pixel = low & PIPE_PIXEL_MASK;
681 low >>= PIPE_FRAME_LOW_SHIFT;
684 * The frame counter increments at beginning of active.
685 * Cook up a vblank counter by also checking the pixel
686 * counter against vblank start.
688 return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
691 u32 g4x_get_vblank_counter(struct drm_crtc *crtc)
693 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
694 struct drm_vblank_crtc *vblank = &dev_priv->drm.vblank[drm_crtc_index(crtc)];
695 enum pipe pipe = to_intel_crtc(crtc)->pipe;
697 if (!vblank->max_vblank_count)
700 return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
704 * On certain encoders on certain platforms, pipe
705 * scanline register will not work to get the scanline,
706 * since the timings are driven from the PORT or issues
707 * with scanline register updates.
708 * This function will use Framestamp and current
709 * timestamp registers to calculate the scanline.
711 static u32 __intel_get_crtc_scanline_from_timestamp(struct intel_crtc *crtc)
713 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
714 struct drm_vblank_crtc *vblank =
715 &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)];
716 const struct drm_display_mode *mode = &vblank->hwmode;
717 u32 vblank_start = mode->crtc_vblank_start;
718 u32 vtotal = mode->crtc_vtotal;
719 u32 htotal = mode->crtc_htotal;
720 u32 clock = mode->crtc_clock;
721 u32 scanline, scan_prev_time, scan_curr_time, scan_post_time;
724 * To avoid the race condition where we might cross into the
725 * next vblank just between the PIPE_FRMTMSTMP and TIMESTAMP_CTR
726 * reads. We make sure we read PIPE_FRMTMSTMP and TIMESTAMP_CTR
727 * during the same frame.
731 * This field provides read back of the display
732 * pipe frame time stamp. The time stamp value
733 * is sampled at every start of vertical blank.
735 scan_prev_time = intel_de_read_fw(dev_priv,
736 PIPE_FRMTMSTMP(crtc->pipe));
739 * The TIMESTAMP_CTR register has the current
742 scan_curr_time = intel_de_read_fw(dev_priv, IVB_TIMESTAMP_CTR);
744 scan_post_time = intel_de_read_fw(dev_priv,
745 PIPE_FRMTMSTMP(crtc->pipe));
746 } while (scan_post_time != scan_prev_time);
748 scanline = div_u64(mul_u32_u32(scan_curr_time - scan_prev_time,
749 clock), 1000 * htotal);
750 scanline = min(scanline, vtotal - 1);
751 scanline = (scanline + vblank_start) % vtotal;
757 * intel_de_read_fw(), only for fast reads of display block, no need for
760 static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
762 struct drm_device *dev = crtc->base.dev;
763 struct drm_i915_private *dev_priv = to_i915(dev);
764 const struct drm_display_mode *mode;
765 struct drm_vblank_crtc *vblank;
766 enum pipe pipe = crtc->pipe;
767 int position, vtotal;
772 vblank = &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)];
773 mode = &vblank->hwmode;
775 if (crtc->mode_flags & I915_MODE_FLAG_GET_SCANLINE_FROM_TIMESTAMP)
776 return __intel_get_crtc_scanline_from_timestamp(crtc);
778 vtotal = mode->crtc_vtotal;
779 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
782 if (IS_GEN(dev_priv, 2))
783 position = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
785 position = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
788 * On HSW, the DSL reg (0x70000) appears to return 0 if we
789 * read it just before the start of vblank. So try it again
790 * so we don't accidentally end up spanning a vblank frame
791 * increment, causing the pipe_update_end() code to squak at us.
793 * The nature of this problem means we can't simply check the ISR
794 * bit and return the vblank start value; nor can we use the scanline
795 * debug register in the transcoder as it appears to have the same
796 * problem. We may need to extend this to include other platforms,
797 * but so far testing only shows the problem on HSW.
799 if (HAS_DDI(dev_priv) && !position) {
802 for (i = 0; i < 100; i++) {
804 temp = intel_de_read_fw(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
805 if (temp != position) {
813 * See update_scanline_offset() for the details on the
814 * scanline_offset adjustment.
816 return (position + crtc->scanline_offset) % vtotal;
819 static bool i915_get_crtc_scanoutpos(struct drm_crtc *_crtc,
821 int *vpos, int *hpos,
822 ktime_t *stime, ktime_t *etime,
823 const struct drm_display_mode *mode)
825 struct drm_device *dev = _crtc->dev;
826 struct drm_i915_private *dev_priv = to_i915(dev);
827 struct intel_crtc *crtc = to_intel_crtc(_crtc);
828 enum pipe pipe = crtc->pipe;
830 int vbl_start, vbl_end, hsync_start, htotal, vtotal;
831 unsigned long irqflags;
832 bool use_scanline_counter = INTEL_GEN(dev_priv) >= 5 ||
833 IS_G4X(dev_priv) || IS_GEN(dev_priv, 2) ||
834 crtc->mode_flags & I915_MODE_FLAG_USE_SCANLINE_COUNTER;
836 if (drm_WARN_ON(&dev_priv->drm, !mode->crtc_clock)) {
837 drm_dbg(&dev_priv->drm,
838 "trying to get scanoutpos for disabled "
839 "pipe %c\n", pipe_name(pipe));
843 htotal = mode->crtc_htotal;
844 hsync_start = mode->crtc_hsync_start;
845 vtotal = mode->crtc_vtotal;
846 vbl_start = mode->crtc_vblank_start;
847 vbl_end = mode->crtc_vblank_end;
849 if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
850 vbl_start = DIV_ROUND_UP(vbl_start, 2);
856 * Lock uncore.lock, as we will do multiple timing critical raw
857 * register reads, potentially with preemption disabled, so the
858 * following code must not block on uncore.lock.
860 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
862 /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
864 /* Get optional system timestamp before query. */
866 *stime = ktime_get();
868 if (use_scanline_counter) {
869 /* No obvious pixelcount register. Only query vertical
870 * scanout position from Display scan line register.
872 position = __intel_get_crtc_scanline(crtc);
874 /* Have access to pixelcount since start of frame.
875 * We can split this into vertical and horizontal
878 position = (intel_de_read_fw(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
880 /* convert to pixel counts */
886 * In interlaced modes, the pixel counter counts all pixels,
887 * so one field will have htotal more pixels. In order to avoid
888 * the reported position from jumping backwards when the pixel
889 * counter is beyond the length of the shorter field, just
890 * clamp the position the length of the shorter field. This
891 * matches how the scanline counter based position works since
892 * the scanline counter doesn't count the two half lines.
894 if (position >= vtotal)
895 position = vtotal - 1;
898 * Start of vblank interrupt is triggered at start of hsync,
899 * just prior to the first active line of vblank. However we
900 * consider lines to start at the leading edge of horizontal
901 * active. So, should we get here before we've crossed into
902 * the horizontal active of the first line in vblank, we would
903 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
904 * always add htotal-hsync_start to the current pixel position.
906 position = (position + htotal - hsync_start) % vtotal;
909 /* Get optional system timestamp after query. */
911 *etime = ktime_get();
913 /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
915 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
918 * While in vblank, position will be negative
919 * counting up towards 0 at vbl_end. And outside
920 * vblank, position will be positive counting
923 if (position >= vbl_start)
926 position += vtotal - vbl_end;
928 if (use_scanline_counter) {
932 *vpos = position / htotal;
933 *hpos = position - (*vpos * htotal);
939 bool intel_crtc_get_vblank_timestamp(struct drm_crtc *crtc, int *max_error,
940 ktime_t *vblank_time, bool in_vblank_irq)
942 return drm_crtc_vblank_helper_get_vblank_timestamp_internal(
943 crtc, max_error, vblank_time, in_vblank_irq,
944 i915_get_crtc_scanoutpos);
947 int intel_get_crtc_scanline(struct intel_crtc *crtc)
949 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
950 unsigned long irqflags;
953 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
954 position = __intel_get_crtc_scanline(crtc);
955 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
961 * ivb_parity_work - Workqueue called when a parity error interrupt
963 * @work: workqueue struct
965 * Doesn't actually do anything except notify userspace. As a consequence of
966 * this event, userspace should try to remap the bad rows since statistically
967 * it is likely the same row is more likely to go bad again.
969 static void ivb_parity_work(struct work_struct *work)
971 struct drm_i915_private *dev_priv =
972 container_of(work, typeof(*dev_priv), l3_parity.error_work);
973 struct intel_gt *gt = &dev_priv->gt;
974 u32 error_status, row, bank, subbank;
975 char *parity_event[6];
979 /* We must turn off DOP level clock gating to access the L3 registers.
980 * In order to prevent a get/put style interface, acquire struct mutex
981 * any time we access those registers.
983 mutex_lock(&dev_priv->drm.struct_mutex);
985 /* If we've screwed up tracking, just let the interrupt fire again */
986 if (drm_WARN_ON(&dev_priv->drm, !dev_priv->l3_parity.which_slice))
989 misccpctl = I915_READ(GEN7_MISCCPCTL);
990 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
991 POSTING_READ(GEN7_MISCCPCTL);
993 while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
997 if (drm_WARN_ON_ONCE(&dev_priv->drm,
998 slice >= NUM_L3_SLICES(dev_priv)))
1001 dev_priv->l3_parity.which_slice &= ~(1<<slice);
1003 reg = GEN7_L3CDERRST1(slice);
1005 error_status = I915_READ(reg);
1006 row = GEN7_PARITY_ERROR_ROW(error_status);
1007 bank = GEN7_PARITY_ERROR_BANK(error_status);
1008 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
1010 I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
1013 parity_event[0] = I915_L3_PARITY_UEVENT "=1";
1014 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
1015 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
1016 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
1017 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
1018 parity_event[5] = NULL;
1020 kobject_uevent_env(&dev_priv->drm.primary->kdev->kobj,
1021 KOBJ_CHANGE, parity_event);
1023 DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
1024 slice, row, bank, subbank);
1026 kfree(parity_event[4]);
1027 kfree(parity_event[3]);
1028 kfree(parity_event[2]);
1029 kfree(parity_event[1]);
1032 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1035 drm_WARN_ON(&dev_priv->drm, dev_priv->l3_parity.which_slice);
1036 spin_lock_irq(>->irq_lock);
1037 gen5_gt_enable_irq(gt, GT_PARITY_ERROR(dev_priv));
1038 spin_unlock_irq(>->irq_lock);
1040 mutex_unlock(&dev_priv->drm.struct_mutex);
1043 static bool gen11_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1047 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(HPD_PORT_TC1);
1049 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(HPD_PORT_TC2);
1051 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(HPD_PORT_TC3);
1053 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(HPD_PORT_TC4);
1055 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(HPD_PORT_TC5);
1057 return val & GEN11_HOTPLUG_CTL_LONG_DETECT(HPD_PORT_TC6);
1063 static bool bxt_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1067 return val & PORTA_HOTPLUG_LONG_DETECT;
1069 return val & PORTB_HOTPLUG_LONG_DETECT;
1071 return val & PORTC_HOTPLUG_LONG_DETECT;
1077 static bool icp_ddi_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1081 return val & SHOTPLUG_CTL_DDI_HPD_LONG_DETECT(HPD_PORT_A);
1083 return val & SHOTPLUG_CTL_DDI_HPD_LONG_DETECT(HPD_PORT_B);
1085 return val & SHOTPLUG_CTL_DDI_HPD_LONG_DETECT(HPD_PORT_C);
1087 return val & SHOTPLUG_CTL_DDI_HPD_LONG_DETECT(HPD_PORT_D);
1093 static bool icp_tc_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1097 return val & ICP_TC_HPD_LONG_DETECT(HPD_PORT_TC1);
1099 return val & ICP_TC_HPD_LONG_DETECT(HPD_PORT_TC2);
1101 return val & ICP_TC_HPD_LONG_DETECT(HPD_PORT_TC3);
1103 return val & ICP_TC_HPD_LONG_DETECT(HPD_PORT_TC4);
1105 return val & ICP_TC_HPD_LONG_DETECT(HPD_PORT_TC5);
1107 return val & ICP_TC_HPD_LONG_DETECT(HPD_PORT_TC6);
1113 static bool spt_port_hotplug2_long_detect(enum hpd_pin pin, u32 val)
1117 return val & PORTE_HOTPLUG_LONG_DETECT;
1123 static bool spt_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1127 return val & PORTA_HOTPLUG_LONG_DETECT;
1129 return val & PORTB_HOTPLUG_LONG_DETECT;
1131 return val & PORTC_HOTPLUG_LONG_DETECT;
1133 return val & PORTD_HOTPLUG_LONG_DETECT;
1139 static bool ilk_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1143 return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
1149 static bool pch_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1153 return val & PORTB_HOTPLUG_LONG_DETECT;
1155 return val & PORTC_HOTPLUG_LONG_DETECT;
1157 return val & PORTD_HOTPLUG_LONG_DETECT;
1163 static bool i9xx_port_hotplug_long_detect(enum hpd_pin pin, u32 val)
1167 return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1169 return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1171 return val & PORTD_HOTPLUG_INT_LONG_PULSE;
1178 * Get a bit mask of pins that have triggered, and which ones may be long.
1179 * This can be called multiple times with the same masks to accumulate
1180 * hotplug detection results from several registers.
1182 * Note that the caller is expected to zero out the masks initially.
1184 static void intel_get_hpd_pins(struct drm_i915_private *dev_priv,
1185 u32 *pin_mask, u32 *long_mask,
1186 u32 hotplug_trigger, u32 dig_hotplug_reg,
1187 const u32 hpd[HPD_NUM_PINS],
1188 bool long_pulse_detect(enum hpd_pin pin, u32 val))
1192 BUILD_BUG_ON(BITS_PER_TYPE(*pin_mask) < HPD_NUM_PINS);
1194 for_each_hpd_pin(pin) {
1195 if ((hpd[pin] & hotplug_trigger) == 0)
1198 *pin_mask |= BIT(pin);
1200 if (long_pulse_detect(pin, dig_hotplug_reg))
1201 *long_mask |= BIT(pin);
1204 drm_dbg(&dev_priv->drm,
1205 "hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x, long 0x%08x\n",
1206 hotplug_trigger, dig_hotplug_reg, *pin_mask, *long_mask);
1210 static u32 intel_hpd_enabled_irqs(struct drm_i915_private *dev_priv,
1211 const u32 hpd[HPD_NUM_PINS])
1213 struct intel_encoder *encoder;
1214 u32 enabled_irqs = 0;
1216 for_each_intel_encoder(&dev_priv->drm, encoder)
1217 if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
1218 enabled_irqs |= hpd[encoder->hpd_pin];
1220 return enabled_irqs;
1223 static u32 intel_hpd_hotplug_irqs(struct drm_i915_private *dev_priv,
1224 const u32 hpd[HPD_NUM_PINS])
1226 struct intel_encoder *encoder;
1227 u32 hotplug_irqs = 0;
1229 for_each_intel_encoder(&dev_priv->drm, encoder)
1230 hotplug_irqs |= hpd[encoder->hpd_pin];
1232 return hotplug_irqs;
1235 static u32 intel_hpd_hotplug_enables(struct drm_i915_private *i915,
1236 hotplug_enables_func hotplug_enables)
1238 struct intel_encoder *encoder;
1241 for_each_intel_encoder(&i915->drm, encoder)
1242 hotplug |= hotplug_enables(i915, encoder->hpd_pin);
1247 static void gmbus_irq_handler(struct drm_i915_private *dev_priv)
1249 wake_up_all(&dev_priv->gmbus_wait_queue);
1252 static void dp_aux_irq_handler(struct drm_i915_private *dev_priv)
1254 wake_up_all(&dev_priv->gmbus_wait_queue);
1257 #if defined(CONFIG_DEBUG_FS)
1258 static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1264 struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
1265 struct intel_pipe_crc *pipe_crc = &crtc->pipe_crc;
1266 u32 crcs[5] = { crc0, crc1, crc2, crc3, crc4 };
1268 trace_intel_pipe_crc(crtc, crcs);
1270 spin_lock(&pipe_crc->lock);
1272 * For some not yet identified reason, the first CRC is
1273 * bonkers. So let's just wait for the next vblank and read
1274 * out the buggy result.
1276 * On GEN8+ sometimes the second CRC is bonkers as well, so
1277 * don't trust that one either.
1279 if (pipe_crc->skipped <= 0 ||
1280 (INTEL_GEN(dev_priv) >= 8 && pipe_crc->skipped == 1)) {
1281 pipe_crc->skipped++;
1282 spin_unlock(&pipe_crc->lock);
1285 spin_unlock(&pipe_crc->lock);
1287 drm_crtc_add_crc_entry(&crtc->base, true,
1288 drm_crtc_accurate_vblank_count(&crtc->base),
1293 display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1300 static void flip_done_handler(struct drm_i915_private *i915,
1303 struct intel_crtc *crtc = intel_get_crtc_for_pipe(i915, pipe);
1304 struct drm_crtc_state *crtc_state = crtc->base.state;
1305 struct drm_pending_vblank_event *e = crtc_state->event;
1306 struct drm_device *dev = &i915->drm;
1307 unsigned long irqflags;
1309 spin_lock_irqsave(&dev->event_lock, irqflags);
1311 crtc_state->event = NULL;
1313 drm_crtc_send_vblank_event(&crtc->base, e);
1315 spin_unlock_irqrestore(&dev->event_lock, irqflags);
1318 static void hsw_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1321 display_pipe_crc_irq_handler(dev_priv, pipe,
1322 I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1326 static void ivb_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1329 display_pipe_crc_irq_handler(dev_priv, pipe,
1330 I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1331 I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
1332 I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
1333 I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
1334 I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1337 static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
1342 if (INTEL_GEN(dev_priv) >= 3)
1343 res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
1347 if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1348 res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
1352 display_pipe_crc_irq_handler(dev_priv, pipe,
1353 I915_READ(PIPE_CRC_RES_RED(pipe)),
1354 I915_READ(PIPE_CRC_RES_GREEN(pipe)),
1355 I915_READ(PIPE_CRC_RES_BLUE(pipe)),
1359 static void i9xx_pipestat_irq_reset(struct drm_i915_private *dev_priv)
1363 for_each_pipe(dev_priv, pipe) {
1364 I915_WRITE(PIPESTAT(pipe),
1365 PIPESTAT_INT_STATUS_MASK |
1366 PIPE_FIFO_UNDERRUN_STATUS);
1368 dev_priv->pipestat_irq_mask[pipe] = 0;
1372 static void i9xx_pipestat_irq_ack(struct drm_i915_private *dev_priv,
1373 u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1377 spin_lock(&dev_priv->irq_lock);
1379 if (!dev_priv->display_irqs_enabled) {
1380 spin_unlock(&dev_priv->irq_lock);
1384 for_each_pipe(dev_priv, pipe) {
1386 u32 status_mask, enable_mask, iir_bit = 0;
1389 * PIPESTAT bits get signalled even when the interrupt is
1390 * disabled with the mask bits, and some of the status bits do
1391 * not generate interrupts at all (like the underrun bit). Hence
1392 * we need to be careful that we only handle what we want to
1396 /* fifo underruns are filterered in the underrun handler. */
1397 status_mask = PIPE_FIFO_UNDERRUN_STATUS;
1402 iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
1405 iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
1408 iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
1412 status_mask |= dev_priv->pipestat_irq_mask[pipe];
1417 reg = PIPESTAT(pipe);
1418 pipe_stats[pipe] = I915_READ(reg) & status_mask;
1419 enable_mask = i915_pipestat_enable_mask(dev_priv, pipe);
1422 * Clear the PIPE*STAT regs before the IIR
1424 * Toggle the enable bits to make sure we get an
1425 * edge in the ISR pipe event bit if we don't clear
1426 * all the enabled status bits. Otherwise the edge
1427 * triggered IIR on i965/g4x wouldn't notice that
1428 * an interrupt is still pending.
1430 if (pipe_stats[pipe]) {
1431 I915_WRITE(reg, pipe_stats[pipe]);
1432 I915_WRITE(reg, enable_mask);
1435 spin_unlock(&dev_priv->irq_lock);
1438 static void i8xx_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1439 u16 iir, u32 pipe_stats[I915_MAX_PIPES])
1443 for_each_pipe(dev_priv, pipe) {
1444 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1445 intel_handle_vblank(dev_priv, pipe);
1447 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1448 i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1450 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1451 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1455 static void i915_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1456 u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1458 bool blc_event = false;
1461 for_each_pipe(dev_priv, pipe) {
1462 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
1463 intel_handle_vblank(dev_priv, pipe);
1465 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
1468 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1469 i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1471 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1472 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1475 if (blc_event || (iir & I915_ASLE_INTERRUPT))
1476 intel_opregion_asle_intr(dev_priv);
1479 static void i965_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1480 u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1482 bool blc_event = false;
1485 for_each_pipe(dev_priv, pipe) {
1486 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
1487 intel_handle_vblank(dev_priv, pipe);
1489 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
1492 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1493 i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1495 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1496 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1499 if (blc_event || (iir & I915_ASLE_INTERRUPT))
1500 intel_opregion_asle_intr(dev_priv);
1502 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1503 gmbus_irq_handler(dev_priv);
1506 static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1507 u32 pipe_stats[I915_MAX_PIPES])
1511 for_each_pipe(dev_priv, pipe) {
1512 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
1513 intel_handle_vblank(dev_priv, pipe);
1515 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1516 i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1518 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1519 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1522 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1523 gmbus_irq_handler(dev_priv);
1526 static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv)
1528 u32 hotplug_status = 0, hotplug_status_mask;
1531 if (IS_G4X(dev_priv) ||
1532 IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1533 hotplug_status_mask = HOTPLUG_INT_STATUS_G4X |
1534 DP_AUX_CHANNEL_MASK_INT_STATUS_G4X;
1536 hotplug_status_mask = HOTPLUG_INT_STATUS_I915;
1539 * We absolutely have to clear all the pending interrupt
1540 * bits in PORT_HOTPLUG_STAT. Otherwise the ISR port
1541 * interrupt bit won't have an edge, and the i965/g4x
1542 * edge triggered IIR will not notice that an interrupt
1543 * is still pending. We can't use PORT_HOTPLUG_EN to
1544 * guarantee the edge as the act of toggling the enable
1545 * bits can itself generate a new hotplug interrupt :(
1547 for (i = 0; i < 10; i++) {
1548 u32 tmp = I915_READ(PORT_HOTPLUG_STAT) & hotplug_status_mask;
1551 return hotplug_status;
1553 hotplug_status |= tmp;
1554 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1557 drm_WARN_ONCE(&dev_priv->drm, 1,
1558 "PORT_HOTPLUG_STAT did not clear (0x%08x)\n",
1559 I915_READ(PORT_HOTPLUG_STAT));
1561 return hotplug_status;
1564 static void i9xx_hpd_irq_handler(struct drm_i915_private *dev_priv,
1567 u32 pin_mask = 0, long_mask = 0;
1568 u32 hotplug_trigger;
1570 if (IS_G4X(dev_priv) ||
1571 IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1572 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1574 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1576 if (hotplug_trigger) {
1577 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1578 hotplug_trigger, hotplug_trigger,
1579 dev_priv->hotplug.hpd,
1580 i9xx_port_hotplug_long_detect);
1582 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1585 if ((IS_G4X(dev_priv) ||
1586 IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
1587 hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
1588 dp_aux_irq_handler(dev_priv);
1591 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
1593 struct drm_i915_private *dev_priv = arg;
1594 irqreturn_t ret = IRQ_NONE;
1596 if (!intel_irqs_enabled(dev_priv))
1599 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
1600 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1603 u32 iir, gt_iir, pm_iir;
1604 u32 pipe_stats[I915_MAX_PIPES] = {};
1605 u32 hotplug_status = 0;
1608 gt_iir = I915_READ(GTIIR);
1609 pm_iir = I915_READ(GEN6_PMIIR);
1610 iir = I915_READ(VLV_IIR);
1612 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1618 * Theory on interrupt generation, based on empirical evidence:
1620 * x = ((VLV_IIR & VLV_IER) ||
1621 * (((GT_IIR & GT_IER) || (GEN6_PMIIR & GEN6_PMIER)) &&
1622 * (VLV_MASTER_IER & MASTER_INTERRUPT_ENABLE)));
1624 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
1625 * Hence we clear MASTER_INTERRUPT_ENABLE and VLV_IER to
1626 * guarantee the CPU interrupt will be raised again even if we
1627 * don't end up clearing all the VLV_IIR, GT_IIR, GEN6_PMIIR
1628 * bits this time around.
1630 I915_WRITE(VLV_MASTER_IER, 0);
1631 ier = I915_READ(VLV_IER);
1632 I915_WRITE(VLV_IER, 0);
1635 I915_WRITE(GTIIR, gt_iir);
1637 I915_WRITE(GEN6_PMIIR, pm_iir);
1639 if (iir & I915_DISPLAY_PORT_INTERRUPT)
1640 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1642 /* Call regardless, as some status bits might not be
1643 * signalled in iir */
1644 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1646 if (iir & (I915_LPE_PIPE_A_INTERRUPT |
1647 I915_LPE_PIPE_B_INTERRUPT))
1648 intel_lpe_audio_irq_handler(dev_priv);
1651 * VLV_IIR is single buffered, and reflects the level
1652 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
1655 I915_WRITE(VLV_IIR, iir);
1657 I915_WRITE(VLV_IER, ier);
1658 I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
1661 gen6_gt_irq_handler(&dev_priv->gt, gt_iir);
1663 gen6_rps_irq_handler(&dev_priv->gt.rps, pm_iir);
1666 i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1668 valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
1671 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1676 static irqreturn_t cherryview_irq_handler(int irq, void *arg)
1678 struct drm_i915_private *dev_priv = arg;
1679 irqreturn_t ret = IRQ_NONE;
1681 if (!intel_irqs_enabled(dev_priv))
1684 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
1685 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1688 u32 master_ctl, iir;
1689 u32 pipe_stats[I915_MAX_PIPES] = {};
1690 u32 hotplug_status = 0;
1693 master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
1694 iir = I915_READ(VLV_IIR);
1696 if (master_ctl == 0 && iir == 0)
1702 * Theory on interrupt generation, based on empirical evidence:
1704 * x = ((VLV_IIR & VLV_IER) ||
1705 * ((GEN8_MASTER_IRQ & ~GEN8_MASTER_IRQ_CONTROL) &&
1706 * (GEN8_MASTER_IRQ & GEN8_MASTER_IRQ_CONTROL)));
1708 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
1709 * Hence we clear GEN8_MASTER_IRQ_CONTROL and VLV_IER to
1710 * guarantee the CPU interrupt will be raised again even if we
1711 * don't end up clearing all the VLV_IIR and GEN8_MASTER_IRQ_CONTROL
1712 * bits this time around.
1714 I915_WRITE(GEN8_MASTER_IRQ, 0);
1715 ier = I915_READ(VLV_IER);
1716 I915_WRITE(VLV_IER, 0);
1718 gen8_gt_irq_handler(&dev_priv->gt, master_ctl);
1720 if (iir & I915_DISPLAY_PORT_INTERRUPT)
1721 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1723 /* Call regardless, as some status bits might not be
1724 * signalled in iir */
1725 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1727 if (iir & (I915_LPE_PIPE_A_INTERRUPT |
1728 I915_LPE_PIPE_B_INTERRUPT |
1729 I915_LPE_PIPE_C_INTERRUPT))
1730 intel_lpe_audio_irq_handler(dev_priv);
1733 * VLV_IIR is single buffered, and reflects the level
1734 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
1737 I915_WRITE(VLV_IIR, iir);
1739 I915_WRITE(VLV_IER, ier);
1740 I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
1743 i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1745 valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
1748 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1753 static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv,
1754 u32 hotplug_trigger)
1756 u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
1759 * Somehow the PCH doesn't seem to really ack the interrupt to the CPU
1760 * unless we touch the hotplug register, even if hotplug_trigger is
1761 * zero. Not acking leads to "The master control interrupt lied (SDE)!"
1764 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1765 if (!hotplug_trigger) {
1766 u32 mask = PORTA_HOTPLUG_STATUS_MASK |
1767 PORTD_HOTPLUG_STATUS_MASK |
1768 PORTC_HOTPLUG_STATUS_MASK |
1769 PORTB_HOTPLUG_STATUS_MASK;
1770 dig_hotplug_reg &= ~mask;
1773 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1774 if (!hotplug_trigger)
1777 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1778 hotplug_trigger, dig_hotplug_reg,
1779 dev_priv->hotplug.pch_hpd,
1780 pch_port_hotplug_long_detect);
1782 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1785 static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1788 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1790 ibx_hpd_irq_handler(dev_priv, hotplug_trigger);
1792 if (pch_iir & SDE_AUDIO_POWER_MASK) {
1793 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1794 SDE_AUDIO_POWER_SHIFT);
1795 drm_dbg(&dev_priv->drm, "PCH audio power change on port %d\n",
1799 if (pch_iir & SDE_AUX_MASK)
1800 dp_aux_irq_handler(dev_priv);
1802 if (pch_iir & SDE_GMBUS)
1803 gmbus_irq_handler(dev_priv);
1805 if (pch_iir & SDE_AUDIO_HDCP_MASK)
1806 drm_dbg(&dev_priv->drm, "PCH HDCP audio interrupt\n");
1808 if (pch_iir & SDE_AUDIO_TRANS_MASK)
1809 drm_dbg(&dev_priv->drm, "PCH transcoder audio interrupt\n");
1811 if (pch_iir & SDE_POISON)
1812 drm_err(&dev_priv->drm, "PCH poison interrupt\n");
1814 if (pch_iir & SDE_FDI_MASK) {
1815 for_each_pipe(dev_priv, pipe)
1816 drm_dbg(&dev_priv->drm, " pipe %c FDI IIR: 0x%08x\n",
1818 I915_READ(FDI_RX_IIR(pipe)));
1821 if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1822 drm_dbg(&dev_priv->drm, "PCH transcoder CRC done interrupt\n");
1824 if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1825 drm_dbg(&dev_priv->drm,
1826 "PCH transcoder CRC error interrupt\n");
1828 if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1829 intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_A);
1831 if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1832 intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_B);
1835 static void ivb_err_int_handler(struct drm_i915_private *dev_priv)
1837 u32 err_int = I915_READ(GEN7_ERR_INT);
1840 if (err_int & ERR_INT_POISON)
1841 drm_err(&dev_priv->drm, "Poison interrupt\n");
1843 for_each_pipe(dev_priv, pipe) {
1844 if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
1845 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1847 if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
1848 if (IS_IVYBRIDGE(dev_priv))
1849 ivb_pipe_crc_irq_handler(dev_priv, pipe);
1851 hsw_pipe_crc_irq_handler(dev_priv, pipe);
1855 I915_WRITE(GEN7_ERR_INT, err_int);
1858 static void cpt_serr_int_handler(struct drm_i915_private *dev_priv)
1860 u32 serr_int = I915_READ(SERR_INT);
1863 if (serr_int & SERR_INT_POISON)
1864 drm_err(&dev_priv->drm, "PCH poison interrupt\n");
1866 for_each_pipe(dev_priv, pipe)
1867 if (serr_int & SERR_INT_TRANS_FIFO_UNDERRUN(pipe))
1868 intel_pch_fifo_underrun_irq_handler(dev_priv, pipe);
1870 I915_WRITE(SERR_INT, serr_int);
1873 static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1876 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1878 ibx_hpd_irq_handler(dev_priv, hotplug_trigger);
1880 if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1881 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1882 SDE_AUDIO_POWER_SHIFT_CPT);
1883 drm_dbg(&dev_priv->drm, "PCH audio power change on port %c\n",
1887 if (pch_iir & SDE_AUX_MASK_CPT)
1888 dp_aux_irq_handler(dev_priv);
1890 if (pch_iir & SDE_GMBUS_CPT)
1891 gmbus_irq_handler(dev_priv);
1893 if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1894 drm_dbg(&dev_priv->drm, "Audio CP request interrupt\n");
1896 if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1897 drm_dbg(&dev_priv->drm, "Audio CP change interrupt\n");
1899 if (pch_iir & SDE_FDI_MASK_CPT) {
1900 for_each_pipe(dev_priv, pipe)
1901 drm_dbg(&dev_priv->drm, " pipe %c FDI IIR: 0x%08x\n",
1903 I915_READ(FDI_RX_IIR(pipe)));
1906 if (pch_iir & SDE_ERROR_CPT)
1907 cpt_serr_int_handler(dev_priv);
1910 static void icp_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1912 u32 ddi_hotplug_trigger = pch_iir & SDE_DDI_HOTPLUG_MASK_ICP;
1913 u32 tc_hotplug_trigger = pch_iir & SDE_TC_HOTPLUG_MASK_ICP;
1914 u32 pin_mask = 0, long_mask = 0;
1916 if (ddi_hotplug_trigger) {
1917 u32 dig_hotplug_reg;
1919 dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_DDI);
1920 I915_WRITE(SHOTPLUG_CTL_DDI, dig_hotplug_reg);
1922 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1923 ddi_hotplug_trigger, dig_hotplug_reg,
1924 dev_priv->hotplug.pch_hpd,
1925 icp_ddi_port_hotplug_long_detect);
1928 if (tc_hotplug_trigger) {
1929 u32 dig_hotplug_reg;
1931 dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_TC);
1932 I915_WRITE(SHOTPLUG_CTL_TC, dig_hotplug_reg);
1934 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1935 tc_hotplug_trigger, dig_hotplug_reg,
1936 dev_priv->hotplug.pch_hpd,
1937 icp_tc_port_hotplug_long_detect);
1941 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1943 if (pch_iir & SDE_GMBUS_ICP)
1944 gmbus_irq_handler(dev_priv);
1947 static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1949 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
1950 ~SDE_PORTE_HOTPLUG_SPT;
1951 u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
1952 u32 pin_mask = 0, long_mask = 0;
1954 if (hotplug_trigger) {
1955 u32 dig_hotplug_reg;
1957 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1958 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1960 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1961 hotplug_trigger, dig_hotplug_reg,
1962 dev_priv->hotplug.pch_hpd,
1963 spt_port_hotplug_long_detect);
1966 if (hotplug2_trigger) {
1967 u32 dig_hotplug_reg;
1969 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
1970 I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);
1972 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1973 hotplug2_trigger, dig_hotplug_reg,
1974 dev_priv->hotplug.pch_hpd,
1975 spt_port_hotplug2_long_detect);
1979 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1981 if (pch_iir & SDE_GMBUS_CPT)
1982 gmbus_irq_handler(dev_priv);
1985 static void ilk_hpd_irq_handler(struct drm_i915_private *dev_priv,
1986 u32 hotplug_trigger)
1988 u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
1990 dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
1991 I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);
1993 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
1994 hotplug_trigger, dig_hotplug_reg,
1995 dev_priv->hotplug.hpd,
1996 ilk_port_hotplug_long_detect);
1998 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2001 static void ilk_display_irq_handler(struct drm_i915_private *dev_priv,
2005 u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;
2007 if (hotplug_trigger)
2008 ilk_hpd_irq_handler(dev_priv, hotplug_trigger);
2010 if (de_iir & DE_AUX_CHANNEL_A)
2011 dp_aux_irq_handler(dev_priv);
2013 if (de_iir & DE_GSE)
2014 intel_opregion_asle_intr(dev_priv);
2016 if (de_iir & DE_POISON)
2017 drm_err(&dev_priv->drm, "Poison interrupt\n");
2019 for_each_pipe(dev_priv, pipe) {
2020 if (de_iir & DE_PIPE_VBLANK(pipe))
2021 intel_handle_vblank(dev_priv, pipe);
2023 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
2024 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2026 if (de_iir & DE_PIPE_CRC_DONE(pipe))
2027 i9xx_pipe_crc_irq_handler(dev_priv, pipe);
2030 /* check event from PCH */
2031 if (de_iir & DE_PCH_EVENT) {
2032 u32 pch_iir = I915_READ(SDEIIR);
2034 if (HAS_PCH_CPT(dev_priv))
2035 cpt_irq_handler(dev_priv, pch_iir);
2037 ibx_irq_handler(dev_priv, pch_iir);
2039 /* should clear PCH hotplug event before clear CPU irq */
2040 I915_WRITE(SDEIIR, pch_iir);
2043 if (IS_GEN(dev_priv, 5) && de_iir & DE_PCU_EVENT)
2044 gen5_rps_irq_handler(&dev_priv->gt.rps);
2047 static void ivb_display_irq_handler(struct drm_i915_private *dev_priv,
2051 u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;
2053 if (hotplug_trigger)
2054 ilk_hpd_irq_handler(dev_priv, hotplug_trigger);
2056 if (de_iir & DE_ERR_INT_IVB)
2057 ivb_err_int_handler(dev_priv);
2059 if (de_iir & DE_EDP_PSR_INT_HSW) {
2060 u32 psr_iir = I915_READ(EDP_PSR_IIR);
2062 intel_psr_irq_handler(dev_priv, psr_iir);
2063 I915_WRITE(EDP_PSR_IIR, psr_iir);
2066 if (de_iir & DE_AUX_CHANNEL_A_IVB)
2067 dp_aux_irq_handler(dev_priv);
2069 if (de_iir & DE_GSE_IVB)
2070 intel_opregion_asle_intr(dev_priv);
2072 for_each_pipe(dev_priv, pipe) {
2073 if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)))
2074 intel_handle_vblank(dev_priv, pipe);
2077 /* check event from PCH */
2078 if (!HAS_PCH_NOP(dev_priv) && (de_iir & DE_PCH_EVENT_IVB)) {
2079 u32 pch_iir = I915_READ(SDEIIR);
2081 cpt_irq_handler(dev_priv, pch_iir);
2083 /* clear PCH hotplug event before clear CPU irq */
2084 I915_WRITE(SDEIIR, pch_iir);
2089 * To handle irqs with the minimum potential races with fresh interrupts, we:
2090 * 1 - Disable Master Interrupt Control.
2091 * 2 - Find the source(s) of the interrupt.
2092 * 3 - Clear the Interrupt Identity bits (IIR).
2093 * 4 - Process the interrupt(s) that had bits set in the IIRs.
2094 * 5 - Re-enable Master Interrupt Control.
2096 static irqreturn_t ilk_irq_handler(int irq, void *arg)
2098 struct drm_i915_private *i915 = arg;
2099 void __iomem * const regs = i915->uncore.regs;
2100 u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2101 irqreturn_t ret = IRQ_NONE;
2103 if (unlikely(!intel_irqs_enabled(i915)))
2106 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2107 disable_rpm_wakeref_asserts(&i915->runtime_pm);
2109 /* disable master interrupt before clearing iir */
2110 de_ier = raw_reg_read(regs, DEIER);
2111 raw_reg_write(regs, DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2113 /* Disable south interrupts. We'll only write to SDEIIR once, so further
2114 * interrupts will will be stored on its back queue, and then we'll be
2115 * able to process them after we restore SDEIER (as soon as we restore
2116 * it, we'll get an interrupt if SDEIIR still has something to process
2117 * due to its back queue). */
2118 if (!HAS_PCH_NOP(i915)) {
2119 sde_ier = raw_reg_read(regs, SDEIER);
2120 raw_reg_write(regs, SDEIER, 0);
2123 /* Find, clear, then process each source of interrupt */
2125 gt_iir = raw_reg_read(regs, GTIIR);
2127 raw_reg_write(regs, GTIIR, gt_iir);
2128 if (INTEL_GEN(i915) >= 6)
2129 gen6_gt_irq_handler(&i915->gt, gt_iir);
2131 gen5_gt_irq_handler(&i915->gt, gt_iir);
2135 de_iir = raw_reg_read(regs, DEIIR);
2137 raw_reg_write(regs, DEIIR, de_iir);
2138 if (INTEL_GEN(i915) >= 7)
2139 ivb_display_irq_handler(i915, de_iir);
2141 ilk_display_irq_handler(i915, de_iir);
2145 if (INTEL_GEN(i915) >= 6) {
2146 u32 pm_iir = raw_reg_read(regs, GEN6_PMIIR);
2148 raw_reg_write(regs, GEN6_PMIIR, pm_iir);
2149 gen6_rps_irq_handler(&i915->gt.rps, pm_iir);
2154 raw_reg_write(regs, DEIER, de_ier);
2156 raw_reg_write(regs, SDEIER, sde_ier);
2158 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2159 enable_rpm_wakeref_asserts(&i915->runtime_pm);
2164 static void bxt_hpd_irq_handler(struct drm_i915_private *dev_priv,
2165 u32 hotplug_trigger)
2167 u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2169 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
2170 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2172 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2173 hotplug_trigger, dig_hotplug_reg,
2174 dev_priv->hotplug.hpd,
2175 bxt_port_hotplug_long_detect);
2177 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2180 static void gen11_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 iir)
2182 u32 pin_mask = 0, long_mask = 0;
2183 u32 trigger_tc = iir & GEN11_DE_TC_HOTPLUG_MASK;
2184 u32 trigger_tbt = iir & GEN11_DE_TBT_HOTPLUG_MASK;
2187 u32 dig_hotplug_reg;
2189 dig_hotplug_reg = I915_READ(GEN11_TC_HOTPLUG_CTL);
2190 I915_WRITE(GEN11_TC_HOTPLUG_CTL, dig_hotplug_reg);
2192 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2193 trigger_tc, dig_hotplug_reg,
2194 dev_priv->hotplug.hpd,
2195 gen11_port_hotplug_long_detect);
2199 u32 dig_hotplug_reg;
2201 dig_hotplug_reg = I915_READ(GEN11_TBT_HOTPLUG_CTL);
2202 I915_WRITE(GEN11_TBT_HOTPLUG_CTL, dig_hotplug_reg);
2204 intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
2205 trigger_tbt, dig_hotplug_reg,
2206 dev_priv->hotplug.hpd,
2207 gen11_port_hotplug_long_detect);
2211 intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2213 drm_err(&dev_priv->drm,
2214 "Unexpected DE HPD interrupt 0x%08x\n", iir);
2217 static u32 gen8_de_port_aux_mask(struct drm_i915_private *dev_priv)
2221 if (INTEL_GEN(dev_priv) >= 12)
2222 return TGL_DE_PORT_AUX_DDIA |
2223 TGL_DE_PORT_AUX_DDIB |
2224 TGL_DE_PORT_AUX_DDIC |
2225 TGL_DE_PORT_AUX_USBC1 |
2226 TGL_DE_PORT_AUX_USBC2 |
2227 TGL_DE_PORT_AUX_USBC3 |
2228 TGL_DE_PORT_AUX_USBC4 |
2229 TGL_DE_PORT_AUX_USBC5 |
2230 TGL_DE_PORT_AUX_USBC6;
2233 mask = GEN8_AUX_CHANNEL_A;
2234 if (INTEL_GEN(dev_priv) >= 9)
2235 mask |= GEN9_AUX_CHANNEL_B |
2236 GEN9_AUX_CHANNEL_C |
2239 if (IS_CNL_WITH_PORT_F(dev_priv) || IS_GEN(dev_priv, 11))
2240 mask |= CNL_AUX_CHANNEL_F;
2242 if (IS_GEN(dev_priv, 11))
2243 mask |= ICL_AUX_CHANNEL_E;
2248 static u32 gen8_de_pipe_fault_mask(struct drm_i915_private *dev_priv)
2250 if (IS_ROCKETLAKE(dev_priv))
2251 return RKL_DE_PIPE_IRQ_FAULT_ERRORS;
2252 else if (INTEL_GEN(dev_priv) >= 11)
2253 return GEN11_DE_PIPE_IRQ_FAULT_ERRORS;
2254 else if (INTEL_GEN(dev_priv) >= 9)
2255 return GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
2257 return GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2261 gen8_de_misc_irq_handler(struct drm_i915_private *dev_priv, u32 iir)
2265 if (iir & GEN8_DE_MISC_GSE) {
2266 intel_opregion_asle_intr(dev_priv);
2270 if (iir & GEN8_DE_EDP_PSR) {
2274 if (INTEL_GEN(dev_priv) >= 12)
2275 iir_reg = TRANS_PSR_IIR(dev_priv->psr.transcoder);
2277 iir_reg = EDP_PSR_IIR;
2279 psr_iir = I915_READ(iir_reg);
2280 I915_WRITE(iir_reg, psr_iir);
2285 intel_psr_irq_handler(dev_priv, psr_iir);
2289 drm_err(&dev_priv->drm, "Unexpected DE Misc interrupt\n");
2292 static void gen11_dsi_te_interrupt_handler(struct drm_i915_private *dev_priv,
2295 enum pipe pipe = INVALID_PIPE;
2296 enum transcoder dsi_trans;
2301 * Incase of dual link, TE comes from DSI_1
2302 * this is to check if dual link is enabled
2304 val = I915_READ(TRANS_DDI_FUNC_CTL2(TRANSCODER_DSI_0));
2305 val &= PORT_SYNC_MODE_ENABLE;
2308 * if dual link is enabled, then read DSI_0
2309 * transcoder registers
2311 port = ((te_trigger & DSI1_TE && val) || (te_trigger & DSI0_TE)) ?
2313 dsi_trans = (port == PORT_A) ? TRANSCODER_DSI_0 : TRANSCODER_DSI_1;
2315 /* Check if DSI configured in command mode */
2316 val = I915_READ(DSI_TRANS_FUNC_CONF(dsi_trans));
2317 val = val & OP_MODE_MASK;
2319 if (val != CMD_MODE_NO_GATE && val != CMD_MODE_TE_GATE) {
2320 drm_err(&dev_priv->drm, "DSI trancoder not configured in command mode\n");
2324 /* Get PIPE for handling VBLANK event */
2325 val = I915_READ(TRANS_DDI_FUNC_CTL(dsi_trans));
2326 switch (val & TRANS_DDI_EDP_INPUT_MASK) {
2327 case TRANS_DDI_EDP_INPUT_A_ON:
2330 case TRANS_DDI_EDP_INPUT_B_ONOFF:
2333 case TRANS_DDI_EDP_INPUT_C_ONOFF:
2337 drm_err(&dev_priv->drm, "Invalid PIPE\n");
2341 intel_handle_vblank(dev_priv, pipe);
2343 /* clear TE in dsi IIR */
2344 port = (te_trigger & DSI1_TE) ? PORT_B : PORT_A;
2345 tmp = I915_READ(DSI_INTR_IDENT_REG(port));
2346 I915_WRITE(DSI_INTR_IDENT_REG(port), tmp);
2350 gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
2352 irqreturn_t ret = IRQ_NONE;
2356 if (master_ctl & GEN8_DE_MISC_IRQ) {
2357 iir = I915_READ(GEN8_DE_MISC_IIR);
2359 I915_WRITE(GEN8_DE_MISC_IIR, iir);
2361 gen8_de_misc_irq_handler(dev_priv, iir);
2363 drm_err(&dev_priv->drm,
2364 "The master control interrupt lied (DE MISC)!\n");
2368 if (INTEL_GEN(dev_priv) >= 11 && (master_ctl & GEN11_DE_HPD_IRQ)) {
2369 iir = I915_READ(GEN11_DE_HPD_IIR);
2371 I915_WRITE(GEN11_DE_HPD_IIR, iir);
2373 gen11_hpd_irq_handler(dev_priv, iir);
2375 drm_err(&dev_priv->drm,
2376 "The master control interrupt lied, (DE HPD)!\n");
2380 if (master_ctl & GEN8_DE_PORT_IRQ) {
2381 iir = I915_READ(GEN8_DE_PORT_IIR);
2385 I915_WRITE(GEN8_DE_PORT_IIR, iir);
2388 if (iir & gen8_de_port_aux_mask(dev_priv)) {
2389 dp_aux_irq_handler(dev_priv);
2393 if (IS_GEN9_LP(dev_priv)) {
2394 u32 hotplug_trigger = iir & BXT_DE_PORT_HOTPLUG_MASK;
2396 if (hotplug_trigger) {
2397 bxt_hpd_irq_handler(dev_priv, hotplug_trigger);
2400 } else if (IS_BROADWELL(dev_priv)) {
2401 u32 hotplug_trigger = iir & BDW_DE_PORT_HOTPLUG_MASK;
2403 if (hotplug_trigger) {
2404 ilk_hpd_irq_handler(dev_priv, hotplug_trigger);
2409 if (IS_GEN9_LP(dev_priv) && (iir & BXT_DE_PORT_GMBUS)) {
2410 gmbus_irq_handler(dev_priv);
2414 if (INTEL_GEN(dev_priv) >= 11) {
2415 u32 te_trigger = iir & (DSI0_TE | DSI1_TE);
2418 gen11_dsi_te_interrupt_handler(dev_priv, te_trigger);
2424 drm_err(&dev_priv->drm,
2425 "Unexpected DE Port interrupt\n");
2428 drm_err(&dev_priv->drm,
2429 "The master control interrupt lied (DE PORT)!\n");
2432 for_each_pipe(dev_priv, pipe) {
2435 if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
2438 iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
2440 drm_err(&dev_priv->drm,
2441 "The master control interrupt lied (DE PIPE)!\n");
2446 I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir);
2448 if (iir & GEN8_PIPE_VBLANK)
2449 intel_handle_vblank(dev_priv, pipe);
2451 if (iir & GEN9_PIPE_PLANE1_FLIP_DONE)
2452 flip_done_handler(dev_priv, pipe);
2454 if (iir & GEN8_PIPE_CDCLK_CRC_DONE)
2455 hsw_pipe_crc_irq_handler(dev_priv, pipe);
2457 if (iir & GEN8_PIPE_FIFO_UNDERRUN)
2458 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2460 fault_errors = iir & gen8_de_pipe_fault_mask(dev_priv);
2462 drm_err(&dev_priv->drm,
2463 "Fault errors on pipe %c: 0x%08x\n",
2468 if (HAS_PCH_SPLIT(dev_priv) && !HAS_PCH_NOP(dev_priv) &&
2469 master_ctl & GEN8_DE_PCH_IRQ) {
2471 * FIXME(BDW): Assume for now that the new interrupt handling
2472 * scheme also closed the SDE interrupt handling race we've seen
2473 * on older pch-split platforms. But this needs testing.
2475 iir = I915_READ(SDEIIR);
2477 I915_WRITE(SDEIIR, iir);
2480 if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
2481 icp_irq_handler(dev_priv, iir);
2482 else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT)
2483 spt_irq_handler(dev_priv, iir);
2485 cpt_irq_handler(dev_priv, iir);
2488 * Like on previous PCH there seems to be something
2489 * fishy going on with forwarding PCH interrupts.
2491 drm_dbg(&dev_priv->drm,
2492 "The master control interrupt lied (SDE)!\n");
2499 static inline u32 gen8_master_intr_disable(void __iomem * const regs)
2501 raw_reg_write(regs, GEN8_MASTER_IRQ, 0);
2504 * Now with master disabled, get a sample of level indications
2505 * for this interrupt. Indications will be cleared on related acks.
2506 * New indications can and will light up during processing,
2507 * and will generate new interrupt after enabling master.
2509 return raw_reg_read(regs, GEN8_MASTER_IRQ);
2512 static inline void gen8_master_intr_enable(void __iomem * const regs)
2514 raw_reg_write(regs, GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2517 static irqreturn_t gen8_irq_handler(int irq, void *arg)
2519 struct drm_i915_private *dev_priv = arg;
2520 void __iomem * const regs = dev_priv->uncore.regs;
2523 if (!intel_irqs_enabled(dev_priv))
2526 master_ctl = gen8_master_intr_disable(regs);
2528 gen8_master_intr_enable(regs);
2532 /* Find, queue (onto bottom-halves), then clear each source */
2533 gen8_gt_irq_handler(&dev_priv->gt, master_ctl);
2535 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2536 if (master_ctl & ~GEN8_GT_IRQS) {
2537 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
2538 gen8_de_irq_handler(dev_priv, master_ctl);
2539 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
2542 gen8_master_intr_enable(regs);
2548 gen11_gu_misc_irq_ack(struct intel_gt *gt, const u32 master_ctl)
2550 void __iomem * const regs = gt->uncore->regs;
2553 if (!(master_ctl & GEN11_GU_MISC_IRQ))
2556 iir = raw_reg_read(regs, GEN11_GU_MISC_IIR);
2558 raw_reg_write(regs, GEN11_GU_MISC_IIR, iir);
2564 gen11_gu_misc_irq_handler(struct intel_gt *gt, const u32 iir)
2566 if (iir & GEN11_GU_MISC_GSE)
2567 intel_opregion_asle_intr(gt->i915);
2570 static inline u32 gen11_master_intr_disable(void __iomem * const regs)
2572 raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, 0);
2575 * Now with master disabled, get a sample of level indications
2576 * for this interrupt. Indications will be cleared on related acks.
2577 * New indications can and will light up during processing,
2578 * and will generate new interrupt after enabling master.
2580 return raw_reg_read(regs, GEN11_GFX_MSTR_IRQ);
2583 static inline void gen11_master_intr_enable(void __iomem * const regs)
2585 raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, GEN11_MASTER_IRQ);
2589 gen11_display_irq_handler(struct drm_i915_private *i915)
2591 void __iomem * const regs = i915->uncore.regs;
2592 const u32 disp_ctl = raw_reg_read(regs, GEN11_DISPLAY_INT_CTL);
2594 disable_rpm_wakeref_asserts(&i915->runtime_pm);
2596 * GEN11_DISPLAY_INT_CTL has same format as GEN8_MASTER_IRQ
2597 * for the display related bits.
2599 raw_reg_write(regs, GEN11_DISPLAY_INT_CTL, 0x0);
2600 gen8_de_irq_handler(i915, disp_ctl);
2601 raw_reg_write(regs, GEN11_DISPLAY_INT_CTL,
2602 GEN11_DISPLAY_IRQ_ENABLE);
2604 enable_rpm_wakeref_asserts(&i915->runtime_pm);
2607 static __always_inline irqreturn_t
2608 __gen11_irq_handler(struct drm_i915_private * const i915,
2609 u32 (*intr_disable)(void __iomem * const regs),
2610 void (*intr_enable)(void __iomem * const regs))
2612 void __iomem * const regs = i915->uncore.regs;
2613 struct intel_gt *gt = &i915->gt;
2617 if (!intel_irqs_enabled(i915))
2620 master_ctl = intr_disable(regs);
2626 /* Find, queue (onto bottom-halves), then clear each source */
2627 gen11_gt_irq_handler(gt, master_ctl);
2629 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2630 if (master_ctl & GEN11_DISPLAY_IRQ)
2631 gen11_display_irq_handler(i915);
2633 gu_misc_iir = gen11_gu_misc_irq_ack(gt, master_ctl);
2637 gen11_gu_misc_irq_handler(gt, gu_misc_iir);
2642 static irqreturn_t gen11_irq_handler(int irq, void *arg)
2644 return __gen11_irq_handler(arg,
2645 gen11_master_intr_disable,
2646 gen11_master_intr_enable);
2649 static u32 dg1_master_intr_disable_and_ack(void __iomem * const regs)
2653 /* First disable interrupts */
2654 raw_reg_write(regs, DG1_MSTR_UNIT_INTR, 0);
2656 /* Get the indication levels and ack the master unit */
2657 val = raw_reg_read(regs, DG1_MSTR_UNIT_INTR);
2661 raw_reg_write(regs, DG1_MSTR_UNIT_INTR, val);
2664 * Now with master disabled, get a sample of level indications
2665 * for this interrupt and ack them right away - we keep GEN11_MASTER_IRQ
2666 * out as this bit doesn't exist anymore for DG1
2668 val = raw_reg_read(regs, GEN11_GFX_MSTR_IRQ) & ~GEN11_MASTER_IRQ;
2672 raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, val);
2677 static inline void dg1_master_intr_enable(void __iomem * const regs)
2679 raw_reg_write(regs, DG1_MSTR_UNIT_INTR, DG1_MSTR_IRQ);
2682 static irqreturn_t dg1_irq_handler(int irq, void *arg)
2684 return __gen11_irq_handler(arg,
2685 dg1_master_intr_disable_and_ack,
2686 dg1_master_intr_enable);
2689 /* Called from drm generic code, passed 'crtc' which
2690 * we use as a pipe index
2692 int i8xx_enable_vblank(struct drm_crtc *crtc)
2694 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2695 enum pipe pipe = to_intel_crtc(crtc)->pipe;
2696 unsigned long irqflags;
2698 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2699 i915_enable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
2700 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2705 int i915gm_enable_vblank(struct drm_crtc *crtc)
2707 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2710 * Vblank interrupts fail to wake the device up from C2+.
2711 * Disabling render clock gating during C-states avoids
2712 * the problem. There is a small power cost so we do this
2713 * only when vblank interrupts are actually enabled.
2715 if (dev_priv->vblank_enabled++ == 0)
2716 I915_WRITE(SCPD0, _MASKED_BIT_ENABLE(CSTATE_RENDER_CLOCK_GATE_DISABLE));
2718 return i8xx_enable_vblank(crtc);
2721 int i965_enable_vblank(struct drm_crtc *crtc)
2723 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2724 enum pipe pipe = to_intel_crtc(crtc)->pipe;
2725 unsigned long irqflags;
2727 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2728 i915_enable_pipestat(dev_priv, pipe,
2729 PIPE_START_VBLANK_INTERRUPT_STATUS);
2730 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2735 int ilk_enable_vblank(struct drm_crtc *crtc)
2737 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2738 enum pipe pipe = to_intel_crtc(crtc)->pipe;
2739 unsigned long irqflags;
2740 u32 bit = INTEL_GEN(dev_priv) >= 7 ?
2741 DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
2743 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2744 ilk_enable_display_irq(dev_priv, bit);
2745 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2747 /* Even though there is no DMC, frame counter can get stuck when
2748 * PSR is active as no frames are generated.
2750 if (HAS_PSR(dev_priv))
2751 drm_crtc_vblank_restore(crtc);
2756 static bool gen11_dsi_configure_te(struct intel_crtc *intel_crtc,
2759 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
2763 if (!(intel_crtc->mode_flags &
2764 (I915_MODE_FLAG_DSI_USE_TE1 | I915_MODE_FLAG_DSI_USE_TE0)))
2767 /* for dual link cases we consider TE from slave */
2768 if (intel_crtc->mode_flags & I915_MODE_FLAG_DSI_USE_TE1)
2773 tmp = I915_READ(DSI_INTR_MASK_REG(port));
2775 tmp &= ~DSI_TE_EVENT;
2777 tmp |= DSI_TE_EVENT;
2779 I915_WRITE(DSI_INTR_MASK_REG(port), tmp);
2781 tmp = I915_READ(DSI_INTR_IDENT_REG(port));
2782 I915_WRITE(DSI_INTR_IDENT_REG(port), tmp);
2787 int bdw_enable_vblank(struct drm_crtc *crtc)
2789 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2790 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2791 enum pipe pipe = intel_crtc->pipe;
2792 unsigned long irqflags;
2794 if (gen11_dsi_configure_te(intel_crtc, true))
2797 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2798 bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2799 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2801 /* Even if there is no DMC, frame counter can get stuck when
2802 * PSR is active as no frames are generated, so check only for PSR.
2804 if (HAS_PSR(dev_priv))
2805 drm_crtc_vblank_restore(crtc);
2810 void skl_enable_flip_done(struct intel_crtc *crtc)
2812 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2813 enum pipe pipe = crtc->pipe;
2814 unsigned long irqflags;
2816 spin_lock_irqsave(&i915->irq_lock, irqflags);
2818 bdw_enable_pipe_irq(i915, pipe, GEN9_PIPE_PLANE1_FLIP_DONE);
2820 spin_unlock_irqrestore(&i915->irq_lock, irqflags);
2823 /* Called from drm generic code, passed 'crtc' which
2824 * we use as a pipe index
2826 void i8xx_disable_vblank(struct drm_crtc *crtc)
2828 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2829 enum pipe pipe = to_intel_crtc(crtc)->pipe;
2830 unsigned long irqflags;
2832 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2833 i915_disable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
2834 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2837 void i915gm_disable_vblank(struct drm_crtc *crtc)
2839 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2841 i8xx_disable_vblank(crtc);
2843 if (--dev_priv->vblank_enabled == 0)
2844 I915_WRITE(SCPD0, _MASKED_BIT_DISABLE(CSTATE_RENDER_CLOCK_GATE_DISABLE));
2847 void i965_disable_vblank(struct drm_crtc *crtc)
2849 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2850 enum pipe pipe = to_intel_crtc(crtc)->pipe;
2851 unsigned long irqflags;
2853 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2854 i915_disable_pipestat(dev_priv, pipe,
2855 PIPE_START_VBLANK_INTERRUPT_STATUS);
2856 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2859 void ilk_disable_vblank(struct drm_crtc *crtc)
2861 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2862 enum pipe pipe = to_intel_crtc(crtc)->pipe;
2863 unsigned long irqflags;
2864 u32 bit = INTEL_GEN(dev_priv) >= 7 ?
2865 DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
2867 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2868 ilk_disable_display_irq(dev_priv, bit);
2869 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2872 void bdw_disable_vblank(struct drm_crtc *crtc)
2874 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
2875 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2876 enum pipe pipe = intel_crtc->pipe;
2877 unsigned long irqflags;
2879 if (gen11_dsi_configure_te(intel_crtc, false))
2882 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2883 bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2884 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2887 void skl_disable_flip_done(struct intel_crtc *crtc)
2889 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2890 enum pipe pipe = crtc->pipe;
2891 unsigned long irqflags;
2893 spin_lock_irqsave(&i915->irq_lock, irqflags);
2895 bdw_disable_pipe_irq(i915, pipe, GEN9_PIPE_PLANE1_FLIP_DONE);
2897 spin_unlock_irqrestore(&i915->irq_lock, irqflags);
2900 static void ibx_irq_reset(struct drm_i915_private *dev_priv)
2902 struct intel_uncore *uncore = &dev_priv->uncore;
2904 if (HAS_PCH_NOP(dev_priv))
2907 GEN3_IRQ_RESET(uncore, SDE);
2909 if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
2910 I915_WRITE(SERR_INT, 0xffffffff);
2913 static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
2915 struct intel_uncore *uncore = &dev_priv->uncore;
2917 if (IS_CHERRYVIEW(dev_priv))
2918 intel_uncore_write(uncore, DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
2920 intel_uncore_write(uncore, DPINVGTT, DPINVGTT_STATUS_MASK);
2922 i915_hotplug_interrupt_update_locked(dev_priv, 0xffffffff, 0);
2923 intel_uncore_write(uncore, PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
2925 i9xx_pipestat_irq_reset(dev_priv);
2927 GEN3_IRQ_RESET(uncore, VLV_);
2928 dev_priv->irq_mask = ~0u;
2931 static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
2933 struct intel_uncore *uncore = &dev_priv->uncore;
2939 pipestat_mask = PIPE_CRC_DONE_INTERRUPT_STATUS;
2941 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
2942 for_each_pipe(dev_priv, pipe)
2943 i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
2945 enable_mask = I915_DISPLAY_PORT_INTERRUPT |
2946 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2947 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
2948 I915_LPE_PIPE_A_INTERRUPT |
2949 I915_LPE_PIPE_B_INTERRUPT;
2951 if (IS_CHERRYVIEW(dev_priv))
2952 enable_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT |
2953 I915_LPE_PIPE_C_INTERRUPT;
2955 drm_WARN_ON(&dev_priv->drm, dev_priv->irq_mask != ~0u);
2957 dev_priv->irq_mask = ~enable_mask;
2959 GEN3_IRQ_INIT(uncore, VLV_, dev_priv->irq_mask, enable_mask);
2964 static void ilk_irq_reset(struct drm_i915_private *dev_priv)
2966 struct intel_uncore *uncore = &dev_priv->uncore;
2968 GEN3_IRQ_RESET(uncore, DE);
2969 dev_priv->irq_mask = ~0u;
2971 if (IS_GEN(dev_priv, 7))
2972 intel_uncore_write(uncore, GEN7_ERR_INT, 0xffffffff);
2974 if (IS_HASWELL(dev_priv)) {
2975 intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
2976 intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
2979 gen5_gt_irq_reset(&dev_priv->gt);
2981 ibx_irq_reset(dev_priv);
2984 static void valleyview_irq_reset(struct drm_i915_private *dev_priv)
2986 I915_WRITE(VLV_MASTER_IER, 0);
2987 POSTING_READ(VLV_MASTER_IER);
2989 gen5_gt_irq_reset(&dev_priv->gt);
2991 spin_lock_irq(&dev_priv->irq_lock);
2992 if (dev_priv->display_irqs_enabled)
2993 vlv_display_irq_reset(dev_priv);
2994 spin_unlock_irq(&dev_priv->irq_lock);
2997 static void gen8_irq_reset(struct drm_i915_private *dev_priv)
2999 struct intel_uncore *uncore = &dev_priv->uncore;
3002 gen8_master_intr_disable(dev_priv->uncore.regs);
3004 gen8_gt_irq_reset(&dev_priv->gt);
3006 intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
3007 intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
3009 for_each_pipe(dev_priv, pipe)
3010 if (intel_display_power_is_enabled(dev_priv,
3011 POWER_DOMAIN_PIPE(pipe)))
3012 GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
3014 GEN3_IRQ_RESET(uncore, GEN8_DE_PORT_);
3015 GEN3_IRQ_RESET(uncore, GEN8_DE_MISC_);
3016 GEN3_IRQ_RESET(uncore, GEN8_PCU_);
3018 if (HAS_PCH_SPLIT(dev_priv))
3019 ibx_irq_reset(dev_priv);
3022 static void gen11_display_irq_reset(struct drm_i915_private *dev_priv)
3024 struct intel_uncore *uncore = &dev_priv->uncore;
3026 u32 trans_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) |
3027 BIT(TRANSCODER_C) | BIT(TRANSCODER_D);
3029 intel_uncore_write(uncore, GEN11_DISPLAY_INT_CTL, 0);
3031 if (INTEL_GEN(dev_priv) >= 12) {
3032 enum transcoder trans;
3034 for_each_cpu_transcoder_masked(dev_priv, trans, trans_mask) {
3035 enum intel_display_power_domain domain;
3037 domain = POWER_DOMAIN_TRANSCODER(trans);
3038 if (!intel_display_power_is_enabled(dev_priv, domain))
3041 intel_uncore_write(uncore, TRANS_PSR_IMR(trans), 0xffffffff);
3042 intel_uncore_write(uncore, TRANS_PSR_IIR(trans), 0xffffffff);
3045 intel_uncore_write(uncore, EDP_PSR_IMR, 0xffffffff);
3046 intel_uncore_write(uncore, EDP_PSR_IIR, 0xffffffff);
3049 for_each_pipe(dev_priv, pipe)
3050 if (intel_display_power_is_enabled(dev_priv,
3051 POWER_DOMAIN_PIPE(pipe)))
3052 GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
3054 GEN3_IRQ_RESET(uncore, GEN8_DE_PORT_);
3055 GEN3_IRQ_RESET(uncore, GEN8_DE_MISC_);
3056 GEN3_IRQ_RESET(uncore, GEN11_DE_HPD_);
3058 if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
3059 GEN3_IRQ_RESET(uncore, SDE);
3061 /* Wa_14010685332:cnp/cmp,tgp,adp */
3062 if (INTEL_PCH_TYPE(dev_priv) == PCH_CNP ||
3063 (INTEL_PCH_TYPE(dev_priv) >= PCH_TGP &&
3064 INTEL_PCH_TYPE(dev_priv) < PCH_DG1)) {
3065 intel_uncore_rmw(uncore, SOUTH_CHICKEN1,
3066 SBCLK_RUN_REFCLK_DIS, SBCLK_RUN_REFCLK_DIS);
3067 intel_uncore_rmw(uncore, SOUTH_CHICKEN1,
3068 SBCLK_RUN_REFCLK_DIS, 0);
3072 static void gen11_irq_reset(struct drm_i915_private *dev_priv)
3074 struct intel_uncore *uncore = &dev_priv->uncore;
3076 if (HAS_MASTER_UNIT_IRQ(dev_priv))
3077 dg1_master_intr_disable_and_ack(dev_priv->uncore.regs);
3079 gen11_master_intr_disable(dev_priv->uncore.regs);
3081 gen11_gt_irq_reset(&dev_priv->gt);
3082 gen11_display_irq_reset(dev_priv);
3084 GEN3_IRQ_RESET(uncore, GEN11_GU_MISC_);
3085 GEN3_IRQ_RESET(uncore, GEN8_PCU_);
3088 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
3091 struct intel_uncore *uncore = &dev_priv->uncore;
3093 u32 extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
3096 if (INTEL_GEN(dev_priv) >= 9)
3097 extra_ier |= GEN9_PIPE_PLANE1_FLIP_DONE;
3099 spin_lock_irq(&dev_priv->irq_lock);
3101 if (!intel_irqs_enabled(dev_priv)) {
3102 spin_unlock_irq(&dev_priv->irq_lock);
3106 for_each_pipe_masked(dev_priv, pipe, pipe_mask)
3107 GEN8_IRQ_INIT_NDX(uncore, DE_PIPE, pipe,
3108 dev_priv->de_irq_mask[pipe],
3109 ~dev_priv->de_irq_mask[pipe] | extra_ier);
3111 spin_unlock_irq(&dev_priv->irq_lock);
3114 void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
3117 struct intel_uncore *uncore = &dev_priv->uncore;
3120 spin_lock_irq(&dev_priv->irq_lock);
3122 if (!intel_irqs_enabled(dev_priv)) {
3123 spin_unlock_irq(&dev_priv->irq_lock);
3127 for_each_pipe_masked(dev_priv, pipe, pipe_mask)
3128 GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
3130 spin_unlock_irq(&dev_priv->irq_lock);
3132 /* make sure we're done processing display irqs */
3133 intel_synchronize_irq(dev_priv);
3136 static void cherryview_irq_reset(struct drm_i915_private *dev_priv)
3138 struct intel_uncore *uncore = &dev_priv->uncore;
3140 I915_WRITE(GEN8_MASTER_IRQ, 0);
3141 POSTING_READ(GEN8_MASTER_IRQ);
3143 gen8_gt_irq_reset(&dev_priv->gt);
3145 GEN3_IRQ_RESET(uncore, GEN8_PCU_);
3147 spin_lock_irq(&dev_priv->irq_lock);
3148 if (dev_priv->display_irqs_enabled)
3149 vlv_display_irq_reset(dev_priv);
3150 spin_unlock_irq(&dev_priv->irq_lock);
3153 static u32 ibx_hotplug_enables(struct drm_i915_private *i915,
3159 * When CPU and PCH are on the same package, port A
3160 * HPD must be enabled in both north and south.
3162 return HAS_PCH_LPT_LP(i915) ?
3163 PORTA_HOTPLUG_ENABLE : 0;
3165 return PORTB_HOTPLUG_ENABLE |
3166 PORTB_PULSE_DURATION_2ms;
3168 return PORTC_HOTPLUG_ENABLE |
3169 PORTC_PULSE_DURATION_2ms;
3171 return PORTD_HOTPLUG_ENABLE |
3172 PORTD_PULSE_DURATION_2ms;
3178 static void ibx_hpd_detection_setup(struct drm_i915_private *dev_priv)
3183 * Enable digital hotplug on the PCH, and configure the DP short pulse
3184 * duration to 2ms (which is the minimum in the Display Port spec).
3185 * The pulse duration bits are reserved on LPT+.
3187 hotplug = I915_READ(PCH_PORT_HOTPLUG);
3188 hotplug &= ~(PORTA_HOTPLUG_ENABLE |
3189 PORTB_HOTPLUG_ENABLE |
3190 PORTC_HOTPLUG_ENABLE |
3191 PORTD_HOTPLUG_ENABLE |
3192 PORTB_PULSE_DURATION_MASK |
3193 PORTC_PULSE_DURATION_MASK |
3194 PORTD_PULSE_DURATION_MASK);
3195 hotplug |= intel_hpd_hotplug_enables(dev_priv, ibx_hotplug_enables);
3196 I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3199 static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv)
3201 u32 hotplug_irqs, enabled_irqs;
3203 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
3204 hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
3206 ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3208 ibx_hpd_detection_setup(dev_priv);
3211 static u32 icp_ddi_hotplug_enables(struct drm_i915_private *i915,
3219 return SHOTPLUG_CTL_DDI_HPD_ENABLE(pin);
3225 static u32 icp_tc_hotplug_enables(struct drm_i915_private *i915,
3235 return ICP_TC_HPD_ENABLE(pin);
3241 static void icp_ddi_hpd_detection_setup(struct drm_i915_private *dev_priv)
3245 hotplug = I915_READ(SHOTPLUG_CTL_DDI);
3246 hotplug &= ~(SHOTPLUG_CTL_DDI_HPD_ENABLE(HPD_PORT_A) |
3247 SHOTPLUG_CTL_DDI_HPD_ENABLE(HPD_PORT_B) |
3248 SHOTPLUG_CTL_DDI_HPD_ENABLE(HPD_PORT_C) |
3249 SHOTPLUG_CTL_DDI_HPD_ENABLE(HPD_PORT_D));
3250 hotplug |= intel_hpd_hotplug_enables(dev_priv, icp_ddi_hotplug_enables);
3251 I915_WRITE(SHOTPLUG_CTL_DDI, hotplug);
3254 static void icp_tc_hpd_detection_setup(struct drm_i915_private *dev_priv)
3258 hotplug = I915_READ(SHOTPLUG_CTL_TC);
3259 hotplug &= ~(ICP_TC_HPD_ENABLE(HPD_PORT_TC1) |
3260 ICP_TC_HPD_ENABLE(HPD_PORT_TC2) |
3261 ICP_TC_HPD_ENABLE(HPD_PORT_TC3) |
3262 ICP_TC_HPD_ENABLE(HPD_PORT_TC4) |
3263 ICP_TC_HPD_ENABLE(HPD_PORT_TC5) |
3264 ICP_TC_HPD_ENABLE(HPD_PORT_TC6));
3265 hotplug |= intel_hpd_hotplug_enables(dev_priv, icp_tc_hotplug_enables);
3266 I915_WRITE(SHOTPLUG_CTL_TC, hotplug);
3269 static void icp_hpd_irq_setup(struct drm_i915_private *dev_priv)
3271 u32 hotplug_irqs, enabled_irqs;
3273 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
3274 hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
3276 if (INTEL_PCH_TYPE(dev_priv) <= PCH_TGP)
3277 I915_WRITE(SHPD_FILTER_CNT, SHPD_FILTER_CNT_500_ADJ);
3279 ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3281 icp_ddi_hpd_detection_setup(dev_priv);
3282 icp_tc_hpd_detection_setup(dev_priv);
3285 static u32 gen11_hotplug_enables(struct drm_i915_private *i915,
3295 return GEN11_HOTPLUG_CTL_ENABLE(pin);
3301 static void dg1_hpd_irq_setup(struct drm_i915_private *dev_priv)
3305 val = I915_READ(SOUTH_CHICKEN1);
3306 val |= (INVERT_DDIA_HPD |
3310 I915_WRITE(SOUTH_CHICKEN1, val);
3312 icp_hpd_irq_setup(dev_priv);
3315 static void gen11_tc_hpd_detection_setup(struct drm_i915_private *dev_priv)
3319 hotplug = I915_READ(GEN11_TC_HOTPLUG_CTL);
3320 hotplug &= ~(GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC1) |
3321 GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC2) |
3322 GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC3) |
3323 GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC4) |
3324 GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC5) |
3325 GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC6));
3326 hotplug |= intel_hpd_hotplug_enables(dev_priv, gen11_hotplug_enables);
3327 I915_WRITE(GEN11_TC_HOTPLUG_CTL, hotplug);
3330 static void gen11_tbt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3334 hotplug = I915_READ(GEN11_TBT_HOTPLUG_CTL);
3335 hotplug &= ~(GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC1) |
3336 GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC2) |
3337 GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC3) |
3338 GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC4) |
3339 GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC5) |
3340 GEN11_HOTPLUG_CTL_ENABLE(HPD_PORT_TC6));
3341 hotplug |= intel_hpd_hotplug_enables(dev_priv, gen11_hotplug_enables);
3342 I915_WRITE(GEN11_TBT_HOTPLUG_CTL, hotplug);
3345 static void gen11_hpd_irq_setup(struct drm_i915_private *dev_priv)
3347 u32 hotplug_irqs, enabled_irqs;
3350 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.hpd);
3351 hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.hpd);
3353 val = I915_READ(GEN11_DE_HPD_IMR);
3354 val &= ~hotplug_irqs;
3355 val |= ~enabled_irqs & hotplug_irqs;
3356 I915_WRITE(GEN11_DE_HPD_IMR, val);
3357 POSTING_READ(GEN11_DE_HPD_IMR);
3359 gen11_tc_hpd_detection_setup(dev_priv);
3360 gen11_tbt_hpd_detection_setup(dev_priv);
3362 if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
3363 icp_hpd_irq_setup(dev_priv);
3366 static u32 spt_hotplug_enables(struct drm_i915_private *i915,
3371 return PORTA_HOTPLUG_ENABLE;
3373 return PORTB_HOTPLUG_ENABLE;
3375 return PORTC_HOTPLUG_ENABLE;
3377 return PORTD_HOTPLUG_ENABLE;
3383 static u32 spt_hotplug2_enables(struct drm_i915_private *i915,
3388 return PORTE_HOTPLUG_ENABLE;
3394 static void spt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3398 /* Display WA #1179 WaHardHangonHotPlug: cnp */
3399 if (HAS_PCH_CNP(dev_priv)) {
3400 val = I915_READ(SOUTH_CHICKEN1);
3401 val &= ~CHASSIS_CLK_REQ_DURATION_MASK;
3402 val |= CHASSIS_CLK_REQ_DURATION(0xf);
3403 I915_WRITE(SOUTH_CHICKEN1, val);
3406 /* Enable digital hotplug on the PCH */
3407 hotplug = I915_READ(PCH_PORT_HOTPLUG);
3408 hotplug &= ~(PORTA_HOTPLUG_ENABLE |
3409 PORTB_HOTPLUG_ENABLE |
3410 PORTC_HOTPLUG_ENABLE |
3411 PORTD_HOTPLUG_ENABLE);
3412 hotplug |= intel_hpd_hotplug_enables(dev_priv, spt_hotplug_enables);
3413 I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3415 hotplug = I915_READ(PCH_PORT_HOTPLUG2);
3416 hotplug &= ~PORTE_HOTPLUG_ENABLE;
3417 hotplug |= intel_hpd_hotplug_enables(dev_priv, spt_hotplug2_enables);
3418 I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3421 static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv)
3423 u32 hotplug_irqs, enabled_irqs;
3425 if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP)
3426 I915_WRITE(SHPD_FILTER_CNT, SHPD_FILTER_CNT_500_ADJ);
3428 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
3429 hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
3431 ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3433 spt_hpd_detection_setup(dev_priv);
3436 static u32 ilk_hotplug_enables(struct drm_i915_private *i915,
3441 return DIGITAL_PORTA_HOTPLUG_ENABLE |
3442 DIGITAL_PORTA_PULSE_DURATION_2ms;
3448 static void ilk_hpd_detection_setup(struct drm_i915_private *dev_priv)
3453 * Enable digital hotplug on the CPU, and configure the DP short pulse
3454 * duration to 2ms (which is the minimum in the Display Port spec)
3455 * The pulse duration bits are reserved on HSW+.
3457 hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
3458 hotplug &= ~(DIGITAL_PORTA_HOTPLUG_ENABLE |
3459 DIGITAL_PORTA_PULSE_DURATION_MASK);
3460 hotplug |= intel_hpd_hotplug_enables(dev_priv, ilk_hotplug_enables);
3461 I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
3464 static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv)
3466 u32 hotplug_irqs, enabled_irqs;
3468 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.hpd);
3469 hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.hpd);
3471 if (INTEL_GEN(dev_priv) >= 8)
3472 bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3474 ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3476 ilk_hpd_detection_setup(dev_priv);
3478 ibx_hpd_irq_setup(dev_priv);
3481 static u32 bxt_hotplug_enables(struct drm_i915_private *i915,
3488 hotplug = PORTA_HOTPLUG_ENABLE;
3489 if (intel_bios_is_port_hpd_inverted(i915, PORT_A))
3490 hotplug |= BXT_DDIA_HPD_INVERT;
3493 hotplug = PORTB_HOTPLUG_ENABLE;
3494 if (intel_bios_is_port_hpd_inverted(i915, PORT_B))
3495 hotplug |= BXT_DDIB_HPD_INVERT;
3498 hotplug = PORTC_HOTPLUG_ENABLE;
3499 if (intel_bios_is_port_hpd_inverted(i915, PORT_C))
3500 hotplug |= BXT_DDIC_HPD_INVERT;
3507 static void bxt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3511 hotplug = I915_READ(PCH_PORT_HOTPLUG);
3512 hotplug &= ~(PORTA_HOTPLUG_ENABLE |
3513 PORTB_HOTPLUG_ENABLE |
3514 PORTC_HOTPLUG_ENABLE |
3515 BXT_DDIA_HPD_INVERT |
3516 BXT_DDIB_HPD_INVERT |
3517 BXT_DDIC_HPD_INVERT);
3518 hotplug |= intel_hpd_hotplug_enables(dev_priv, bxt_hotplug_enables);
3519 I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3522 static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv)
3524 u32 hotplug_irqs, enabled_irqs;
3526 enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.hpd);
3527 hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.hpd);
3529 bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3531 bxt_hpd_detection_setup(dev_priv);
3535 * SDEIER is also touched by the interrupt handler to work around missed PCH
3536 * interrupts. Hence we can't update it after the interrupt handler is enabled -
3537 * instead we unconditionally enable all PCH interrupt sources here, but then
3538 * only unmask them as needed with SDEIMR.
3540 * Note that we currently do this after installing the interrupt handler,
3541 * but before we enable the master interrupt. That should be sufficient
3542 * to avoid races with the irq handler, assuming we have MSI. Shared legacy
3543 * interrupts could still race.
3545 static void ibx_irq_postinstall(struct drm_i915_private *dev_priv)
3547 struct intel_uncore *uncore = &dev_priv->uncore;
3550 if (HAS_PCH_NOP(dev_priv))
3553 if (HAS_PCH_IBX(dev_priv))
3554 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3555 else if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
3556 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3558 mask = SDE_GMBUS_CPT;
3560 GEN3_IRQ_INIT(uncore, SDE, ~mask, 0xffffffff);
3563 static void ilk_irq_postinstall(struct drm_i915_private *dev_priv)
3565 struct intel_uncore *uncore = &dev_priv->uncore;
3566 u32 display_mask, extra_mask;
3568 if (INTEL_GEN(dev_priv) >= 7) {
3569 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
3570 DE_PCH_EVENT_IVB | DE_AUX_CHANNEL_A_IVB);
3571 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3572 DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
3573 DE_DP_A_HOTPLUG_IVB);
3575 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
3576 DE_AUX_CHANNEL_A | DE_PIPEB_CRC_DONE |
3577 DE_PIPEA_CRC_DONE | DE_POISON);
3578 extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK |
3579 DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
3583 if (IS_HASWELL(dev_priv)) {
3584 gen3_assert_iir_is_zero(uncore, EDP_PSR_IIR);
3585 display_mask |= DE_EDP_PSR_INT_HSW;
3588 if (IS_IRONLAKE_M(dev_priv))
3589 extra_mask |= DE_PCU_EVENT;
3591 dev_priv->irq_mask = ~display_mask;
3593 ibx_irq_postinstall(dev_priv);
3595 gen5_gt_irq_postinstall(&dev_priv->gt);
3597 GEN3_IRQ_INIT(uncore, DE, dev_priv->irq_mask,
3598 display_mask | extra_mask);
3601 void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
3603 lockdep_assert_held(&dev_priv->irq_lock);
3605 if (dev_priv->display_irqs_enabled)
3608 dev_priv->display_irqs_enabled = true;
3610 if (intel_irqs_enabled(dev_priv)) {
3611 vlv_display_irq_reset(dev_priv);
3612 vlv_display_irq_postinstall(dev_priv);
3616 void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
3618 lockdep_assert_held(&dev_priv->irq_lock);
3620 if (!dev_priv->display_irqs_enabled)
3623 dev_priv->display_irqs_enabled = false;
3625 if (intel_irqs_enabled(dev_priv))
3626 vlv_display_irq_reset(dev_priv);
3630 static void valleyview_irq_postinstall(struct drm_i915_private *dev_priv)
3632 gen5_gt_irq_postinstall(&dev_priv->gt);
3634 spin_lock_irq(&dev_priv->irq_lock);
3635 if (dev_priv->display_irqs_enabled)
3636 vlv_display_irq_postinstall(dev_priv);
3637 spin_unlock_irq(&dev_priv->irq_lock);
3639 I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3640 POSTING_READ(VLV_MASTER_IER);
3643 static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
3645 struct intel_uncore *uncore = &dev_priv->uncore;
3647 u32 de_pipe_masked = gen8_de_pipe_fault_mask(dev_priv) |
3648 GEN8_PIPE_CDCLK_CRC_DONE;
3649 u32 de_pipe_enables;
3650 u32 de_port_masked = gen8_de_port_aux_mask(dev_priv);
3651 u32 de_port_enables;
3652 u32 de_misc_masked = GEN8_DE_EDP_PSR;
3653 u32 trans_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) |
3654 BIT(TRANSCODER_C) | BIT(TRANSCODER_D);
3657 if (INTEL_GEN(dev_priv) <= 10)
3658 de_misc_masked |= GEN8_DE_MISC_GSE;
3660 if (IS_GEN9_LP(dev_priv))
3661 de_port_masked |= BXT_DE_PORT_GMBUS;
3663 if (INTEL_GEN(dev_priv) >= 11) {
3666 if (intel_bios_is_dsi_present(dev_priv, &port))
3667 de_port_masked |= DSI0_TE | DSI1_TE;
3670 de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
3671 GEN8_PIPE_FIFO_UNDERRUN;
3673 if (INTEL_GEN(dev_priv) >= 9)
3674 de_pipe_enables |= GEN9_PIPE_PLANE1_FLIP_DONE;
3676 de_port_enables = de_port_masked;
3677 if (IS_GEN9_LP(dev_priv))
3678 de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
3679 else if (IS_BROADWELL(dev_priv))
3680 de_port_enables |= BDW_DE_PORT_HOTPLUG_MASK;
3682 if (INTEL_GEN(dev_priv) >= 12) {
3683 enum transcoder trans;
3685 for_each_cpu_transcoder_masked(dev_priv, trans, trans_mask) {
3686 enum intel_display_power_domain domain;
3688 domain = POWER_DOMAIN_TRANSCODER(trans);
3689 if (!intel_display_power_is_enabled(dev_priv, domain))
3692 gen3_assert_iir_is_zero(uncore, TRANS_PSR_IIR(trans));
3695 gen3_assert_iir_is_zero(uncore, EDP_PSR_IIR);
3698 for_each_pipe(dev_priv, pipe) {
3699 dev_priv->de_irq_mask[pipe] = ~de_pipe_masked;
3701 if (intel_display_power_is_enabled(dev_priv,
3702 POWER_DOMAIN_PIPE(pipe)))
3703 GEN8_IRQ_INIT_NDX(uncore, DE_PIPE, pipe,
3704 dev_priv->de_irq_mask[pipe],
3708 GEN3_IRQ_INIT(uncore, GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
3709 GEN3_IRQ_INIT(uncore, GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked);
3711 if (INTEL_GEN(dev_priv) >= 11) {
3712 u32 de_hpd_masked = 0;
3713 u32 de_hpd_enables = GEN11_DE_TC_HOTPLUG_MASK |
3714 GEN11_DE_TBT_HOTPLUG_MASK;
3716 GEN3_IRQ_INIT(uncore, GEN11_DE_HPD_, ~de_hpd_masked,
3721 static void gen8_irq_postinstall(struct drm_i915_private *dev_priv)
3723 if (HAS_PCH_SPLIT(dev_priv))
3724 ibx_irq_postinstall(dev_priv);
3726 gen8_gt_irq_postinstall(&dev_priv->gt);
3727 gen8_de_irq_postinstall(dev_priv);
3729 gen8_master_intr_enable(dev_priv->uncore.regs);
3732 static void icp_irq_postinstall(struct drm_i915_private *dev_priv)
3734 struct intel_uncore *uncore = &dev_priv->uncore;
3735 u32 mask = SDE_GMBUS_ICP;
3737 GEN3_IRQ_INIT(uncore, SDE, ~mask, 0xffffffff);
3740 static void gen11_irq_postinstall(struct drm_i915_private *dev_priv)
3742 struct intel_uncore *uncore = &dev_priv->uncore;
3743 u32 gu_misc_masked = GEN11_GU_MISC_GSE;
3745 if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
3746 icp_irq_postinstall(dev_priv);
3748 gen11_gt_irq_postinstall(&dev_priv->gt);
3749 gen8_de_irq_postinstall(dev_priv);
3751 GEN3_IRQ_INIT(uncore, GEN11_GU_MISC_, ~gu_misc_masked, gu_misc_masked);
3753 I915_WRITE(GEN11_DISPLAY_INT_CTL, GEN11_DISPLAY_IRQ_ENABLE);
3755 if (HAS_MASTER_UNIT_IRQ(dev_priv)) {
3756 dg1_master_intr_enable(uncore->regs);
3757 POSTING_READ(DG1_MSTR_UNIT_INTR);
3759 gen11_master_intr_enable(uncore->regs);
3760 POSTING_READ(GEN11_GFX_MSTR_IRQ);
3764 static void cherryview_irq_postinstall(struct drm_i915_private *dev_priv)
3766 gen8_gt_irq_postinstall(&dev_priv->gt);
3768 spin_lock_irq(&dev_priv->irq_lock);
3769 if (dev_priv->display_irqs_enabled)
3770 vlv_display_irq_postinstall(dev_priv);
3771 spin_unlock_irq(&dev_priv->irq_lock);
3773 I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
3774 POSTING_READ(GEN8_MASTER_IRQ);
3777 static void i8xx_irq_reset(struct drm_i915_private *dev_priv)
3779 struct intel_uncore *uncore = &dev_priv->uncore;
3781 i9xx_pipestat_irq_reset(dev_priv);
3783 GEN2_IRQ_RESET(uncore);
3784 dev_priv->irq_mask = ~0u;
3787 static void i8xx_irq_postinstall(struct drm_i915_private *dev_priv)
3789 struct intel_uncore *uncore = &dev_priv->uncore;
3792 intel_uncore_write16(uncore,
3794 ~(I915_ERROR_PAGE_TABLE |
3795 I915_ERROR_MEMORY_REFRESH));
3797 /* Unmask the interrupts that we always want on. */
3798 dev_priv->irq_mask =
3799 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3800 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3801 I915_MASTER_ERROR_INTERRUPT);
3804 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3805 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3806 I915_MASTER_ERROR_INTERRUPT |
3807 I915_USER_INTERRUPT;
3809 GEN2_IRQ_INIT(uncore, dev_priv->irq_mask, enable_mask);
3811 /* Interrupt setup is already guaranteed to be single-threaded, this is
3812 * just to make the assert_spin_locked check happy. */
3813 spin_lock_irq(&dev_priv->irq_lock);
3814 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3815 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3816 spin_unlock_irq(&dev_priv->irq_lock);
3819 static void i8xx_error_irq_ack(struct drm_i915_private *i915,
3820 u16 *eir, u16 *eir_stuck)
3822 struct intel_uncore *uncore = &i915->uncore;
3825 *eir = intel_uncore_read16(uncore, EIR);
3828 intel_uncore_write16(uncore, EIR, *eir);
3830 *eir_stuck = intel_uncore_read16(uncore, EIR);
3831 if (*eir_stuck == 0)
3835 * Toggle all EMR bits to make sure we get an edge
3836 * in the ISR master error bit if we don't clear
3837 * all the EIR bits. Otherwise the edge triggered
3838 * IIR on i965/g4x wouldn't notice that an interrupt
3839 * is still pending. Also some EIR bits can't be
3840 * cleared except by handling the underlying error
3841 * (or by a GPU reset) so we mask any bit that
3844 emr = intel_uncore_read16(uncore, EMR);
3845 intel_uncore_write16(uncore, EMR, 0xffff);
3846 intel_uncore_write16(uncore, EMR, emr | *eir_stuck);
3849 static void i8xx_error_irq_handler(struct drm_i915_private *dev_priv,
3850 u16 eir, u16 eir_stuck)
3852 DRM_DEBUG("Master Error: EIR 0x%04x\n", eir);
3855 drm_dbg(&dev_priv->drm, "EIR stuck: 0x%04x, masked\n",
3859 static void i9xx_error_irq_ack(struct drm_i915_private *dev_priv,
3860 u32 *eir, u32 *eir_stuck)
3864 *eir = I915_READ(EIR);
3866 I915_WRITE(EIR, *eir);
3868 *eir_stuck = I915_READ(EIR);
3869 if (*eir_stuck == 0)
3873 * Toggle all EMR bits to make sure we get an edge
3874 * in the ISR master error bit if we don't clear
3875 * all the EIR bits. Otherwise the edge triggered
3876 * IIR on i965/g4x wouldn't notice that an interrupt
3877 * is still pending. Also some EIR bits can't be
3878 * cleared except by handling the underlying error
3879 * (or by a GPU reset) so we mask any bit that
3882 emr = I915_READ(EMR);
3883 I915_WRITE(EMR, 0xffffffff);
3884 I915_WRITE(EMR, emr | *eir_stuck);
3887 static void i9xx_error_irq_handler(struct drm_i915_private *dev_priv,
3888 u32 eir, u32 eir_stuck)
3890 DRM_DEBUG("Master Error, EIR 0x%08x\n", eir);
3893 drm_dbg(&dev_priv->drm, "EIR stuck: 0x%08x, masked\n",
3897 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
3899 struct drm_i915_private *dev_priv = arg;
3900 irqreturn_t ret = IRQ_NONE;
3902 if (!intel_irqs_enabled(dev_priv))
3905 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
3906 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
3909 u32 pipe_stats[I915_MAX_PIPES] = {};
3910 u16 eir = 0, eir_stuck = 0;
3913 iir = intel_uncore_read16(&dev_priv->uncore, GEN2_IIR);
3919 /* Call regardless, as some status bits might not be
3920 * signalled in iir */
3921 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
3923 if (iir & I915_MASTER_ERROR_INTERRUPT)
3924 i8xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
3926 intel_uncore_write16(&dev_priv->uncore, GEN2_IIR, iir);
3928 if (iir & I915_USER_INTERRUPT)
3929 intel_engine_signal_breadcrumbs(dev_priv->gt.engine[RCS0]);
3931 if (iir & I915_MASTER_ERROR_INTERRUPT)
3932 i8xx_error_irq_handler(dev_priv, eir, eir_stuck);
3934 i8xx_pipestat_irq_handler(dev_priv, iir, pipe_stats);
3937 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
3942 static void i915_irq_reset(struct drm_i915_private *dev_priv)
3944 struct intel_uncore *uncore = &dev_priv->uncore;
3946 if (I915_HAS_HOTPLUG(dev_priv)) {
3947 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3948 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3951 i9xx_pipestat_irq_reset(dev_priv);
3953 GEN3_IRQ_RESET(uncore, GEN2_);
3954 dev_priv->irq_mask = ~0u;
3957 static void i915_irq_postinstall(struct drm_i915_private *dev_priv)
3959 struct intel_uncore *uncore = &dev_priv->uncore;
3962 I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE |
3963 I915_ERROR_MEMORY_REFRESH));
3965 /* Unmask the interrupts that we always want on. */
3966 dev_priv->irq_mask =
3967 ~(I915_ASLE_INTERRUPT |
3968 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3969 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3970 I915_MASTER_ERROR_INTERRUPT);
3973 I915_ASLE_INTERRUPT |
3974 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3975 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3976 I915_MASTER_ERROR_INTERRUPT |
3977 I915_USER_INTERRUPT;
3979 if (I915_HAS_HOTPLUG(dev_priv)) {
3980 /* Enable in IER... */
3981 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
3982 /* and unmask in IMR */
3983 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
3986 GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask);
3988 /* Interrupt setup is already guaranteed to be single-threaded, this is
3989 * just to make the assert_spin_locked check happy. */
3990 spin_lock_irq(&dev_priv->irq_lock);
3991 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3992 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3993 spin_unlock_irq(&dev_priv->irq_lock);
3995 i915_enable_asle_pipestat(dev_priv);
3998 static irqreturn_t i915_irq_handler(int irq, void *arg)
4000 struct drm_i915_private *dev_priv = arg;
4001 irqreturn_t ret = IRQ_NONE;
4003 if (!intel_irqs_enabled(dev_priv))
4006 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
4007 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
4010 u32 pipe_stats[I915_MAX_PIPES] = {};
4011 u32 eir = 0, eir_stuck = 0;
4012 u32 hotplug_status = 0;
4015 iir = I915_READ(GEN2_IIR);
4021 if (I915_HAS_HOTPLUG(dev_priv) &&
4022 iir & I915_DISPLAY_PORT_INTERRUPT)
4023 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
4025 /* Call regardless, as some status bits might not be
4026 * signalled in iir */
4027 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
4029 if (iir & I915_MASTER_ERROR_INTERRUPT)
4030 i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
4032 I915_WRITE(GEN2_IIR, iir);
4034 if (iir & I915_USER_INTERRUPT)
4035 intel_engine_signal_breadcrumbs(dev_priv->gt.engine[RCS0]);
4037 if (iir & I915_MASTER_ERROR_INTERRUPT)
4038 i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
4041 i9xx_hpd_irq_handler(dev_priv, hotplug_status);
4043 i915_pipestat_irq_handler(dev_priv, iir, pipe_stats);
4046 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
4051 static void i965_irq_reset(struct drm_i915_private *dev_priv)
4053 struct intel_uncore *uncore = &dev_priv->uncore;
4055 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4056 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4058 i9xx_pipestat_irq_reset(dev_priv);
4060 GEN3_IRQ_RESET(uncore, GEN2_);
4061 dev_priv->irq_mask = ~0u;
4064 static void i965_irq_postinstall(struct drm_i915_private *dev_priv)
4066 struct intel_uncore *uncore = &dev_priv->uncore;
4071 * Enable some error detection, note the instruction error mask
4072 * bit is reserved, so we leave it masked.
4074 if (IS_G4X(dev_priv)) {
4075 error_mask = ~(GM45_ERROR_PAGE_TABLE |
4076 GM45_ERROR_MEM_PRIV |
4077 GM45_ERROR_CP_PRIV |
4078 I915_ERROR_MEMORY_REFRESH);
4080 error_mask = ~(I915_ERROR_PAGE_TABLE |
4081 I915_ERROR_MEMORY_REFRESH);
4083 I915_WRITE(EMR, error_mask);
4085 /* Unmask the interrupts that we always want on. */
4086 dev_priv->irq_mask =
4087 ~(I915_ASLE_INTERRUPT |
4088 I915_DISPLAY_PORT_INTERRUPT |
4089 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4090 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4091 I915_MASTER_ERROR_INTERRUPT);
4094 I915_ASLE_INTERRUPT |
4095 I915_DISPLAY_PORT_INTERRUPT |
4096 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4097 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4098 I915_MASTER_ERROR_INTERRUPT |
4099 I915_USER_INTERRUPT;
4101 if (IS_G4X(dev_priv))
4102 enable_mask |= I915_BSD_USER_INTERRUPT;
4104 GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask);
4106 /* Interrupt setup is already guaranteed to be single-threaded, this is
4107 * just to make the assert_spin_locked check happy. */
4108 spin_lock_irq(&dev_priv->irq_lock);
4109 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
4110 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4111 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4112 spin_unlock_irq(&dev_priv->irq_lock);
4114 i915_enable_asle_pipestat(dev_priv);
4117 static void i915_hpd_irq_setup(struct drm_i915_private *dev_priv)
4121 lockdep_assert_held(&dev_priv->irq_lock);
4123 /* Note HDMI and DP share hotplug bits */
4124 /* enable bits are the same for all generations */
4125 hotplug_en = intel_hpd_enabled_irqs(dev_priv, hpd_mask_i915);
4126 /* Programming the CRT detection parameters tends
4127 to generate a spurious hotplug event about three
4128 seconds later. So just do it once.
4130 if (IS_G4X(dev_priv))
4131 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
4132 hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
4134 /* Ignore TV since it's buggy */
4135 i915_hotplug_interrupt_update_locked(dev_priv,
4136 HOTPLUG_INT_EN_MASK |
4137 CRT_HOTPLUG_VOLTAGE_COMPARE_MASK |
4138 CRT_HOTPLUG_ACTIVATION_PERIOD_64,
4142 static irqreturn_t i965_irq_handler(int irq, void *arg)
4144 struct drm_i915_private *dev_priv = arg;
4145 irqreturn_t ret = IRQ_NONE;
4147 if (!intel_irqs_enabled(dev_priv))
4150 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
4151 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
4154 u32 pipe_stats[I915_MAX_PIPES] = {};
4155 u32 eir = 0, eir_stuck = 0;
4156 u32 hotplug_status = 0;
4159 iir = I915_READ(GEN2_IIR);
4165 if (iir & I915_DISPLAY_PORT_INTERRUPT)
4166 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
4168 /* Call regardless, as some status bits might not be
4169 * signalled in iir */
4170 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
4172 if (iir & I915_MASTER_ERROR_INTERRUPT)
4173 i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
4175 I915_WRITE(GEN2_IIR, iir);
4177 if (iir & I915_USER_INTERRUPT)
4178 intel_engine_signal_breadcrumbs(dev_priv->gt.engine[RCS0]);
4180 if (iir & I915_BSD_USER_INTERRUPT)
4181 intel_engine_signal_breadcrumbs(dev_priv->gt.engine[VCS0]);
4183 if (iir & I915_MASTER_ERROR_INTERRUPT)
4184 i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
4187 i9xx_hpd_irq_handler(dev_priv, hotplug_status);
4189 i965_pipestat_irq_handler(dev_priv, iir, pipe_stats);
4192 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
4198 * intel_irq_init - initializes irq support
4199 * @dev_priv: i915 device instance
4201 * This function initializes all the irq support including work items, timers
4202 * and all the vtables. It does not setup the interrupt itself though.
4204 void intel_irq_init(struct drm_i915_private *dev_priv)
4206 struct drm_device *dev = &dev_priv->drm;
4209 INIT_WORK(&dev_priv->l3_parity.error_work, ivb_parity_work);
4210 for (i = 0; i < MAX_L3_SLICES; ++i)
4211 dev_priv->l3_parity.remap_info[i] = NULL;
4213 /* pre-gen11 the guc irqs bits are in the upper 16 bits of the pm reg */
4214 if (HAS_GT_UC(dev_priv) && INTEL_GEN(dev_priv) < 11)
4215 dev_priv->gt.pm_guc_events = GUC_INTR_GUC2HOST << 16;
4217 if (!HAS_DISPLAY(dev_priv))
4220 intel_hpd_init_pins(dev_priv);
4222 intel_hpd_init_work(dev_priv);
4224 dev->vblank_disable_immediate = true;
4226 /* Most platforms treat the display irq block as an always-on
4227 * power domain. vlv/chv can disable it at runtime and need
4228 * special care to avoid writing any of the display block registers
4229 * outside of the power domain. We defer setting up the display irqs
4230 * in this case to the runtime pm.
4232 dev_priv->display_irqs_enabled = true;
4233 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
4234 dev_priv->display_irqs_enabled = false;
4236 dev_priv->hotplug.hpd_storm_threshold = HPD_STORM_DEFAULT_THRESHOLD;
4237 /* If we have MST support, we want to avoid doing short HPD IRQ storm
4238 * detection, as short HPD storms will occur as a natural part of
4239 * sideband messaging with MST.
4240 * On older platforms however, IRQ storms can occur with both long and
4241 * short pulses, as seen on some G4x systems.
4243 dev_priv->hotplug.hpd_short_storm_enabled = !HAS_DP_MST(dev_priv);
4245 if (HAS_GMCH(dev_priv)) {
4246 if (I915_HAS_HOTPLUG(dev_priv))
4247 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4249 if (HAS_PCH_DG1(dev_priv))
4250 dev_priv->display.hpd_irq_setup = dg1_hpd_irq_setup;
4251 else if (INTEL_GEN(dev_priv) >= 11)
4252 dev_priv->display.hpd_irq_setup = gen11_hpd_irq_setup;
4253 else if (IS_GEN9_LP(dev_priv))
4254 dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4255 else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT)
4256 dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
4258 dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4263 * intel_irq_fini - deinitializes IRQ support
4264 * @i915: i915 device instance
4266 * This function deinitializes all the IRQ support.
4268 void intel_irq_fini(struct drm_i915_private *i915)
4272 for (i = 0; i < MAX_L3_SLICES; ++i)
4273 kfree(i915->l3_parity.remap_info[i]);
4276 static irq_handler_t intel_irq_handler(struct drm_i915_private *dev_priv)
4278 if (HAS_GMCH(dev_priv)) {
4279 if (IS_CHERRYVIEW(dev_priv))
4280 return cherryview_irq_handler;
4281 else if (IS_VALLEYVIEW(dev_priv))
4282 return valleyview_irq_handler;
4283 else if (IS_GEN(dev_priv, 4))
4284 return i965_irq_handler;
4285 else if (IS_GEN(dev_priv, 3))
4286 return i915_irq_handler;
4288 return i8xx_irq_handler;
4290 if (HAS_MASTER_UNIT_IRQ(dev_priv))
4291 return dg1_irq_handler;
4292 if (INTEL_GEN(dev_priv) >= 11)
4293 return gen11_irq_handler;
4294 else if (INTEL_GEN(dev_priv) >= 8)
4295 return gen8_irq_handler;
4297 return ilk_irq_handler;
4301 static void intel_irq_reset(struct drm_i915_private *dev_priv)
4303 if (HAS_GMCH(dev_priv)) {
4304 if (IS_CHERRYVIEW(dev_priv))
4305 cherryview_irq_reset(dev_priv);
4306 else if (IS_VALLEYVIEW(dev_priv))
4307 valleyview_irq_reset(dev_priv);
4308 else if (IS_GEN(dev_priv, 4))
4309 i965_irq_reset(dev_priv);
4310 else if (IS_GEN(dev_priv, 3))
4311 i915_irq_reset(dev_priv);
4313 i8xx_irq_reset(dev_priv);
4315 if (INTEL_GEN(dev_priv) >= 11)
4316 gen11_irq_reset(dev_priv);
4317 else if (INTEL_GEN(dev_priv) >= 8)
4318 gen8_irq_reset(dev_priv);
4320 ilk_irq_reset(dev_priv);
4324 static void intel_irq_postinstall(struct drm_i915_private *dev_priv)
4326 if (HAS_GMCH(dev_priv)) {
4327 if (IS_CHERRYVIEW(dev_priv))
4328 cherryview_irq_postinstall(dev_priv);
4329 else if (IS_VALLEYVIEW(dev_priv))
4330 valleyview_irq_postinstall(dev_priv);
4331 else if (IS_GEN(dev_priv, 4))
4332 i965_irq_postinstall(dev_priv);
4333 else if (IS_GEN(dev_priv, 3))
4334 i915_irq_postinstall(dev_priv);
4336 i8xx_irq_postinstall(dev_priv);
4338 if (INTEL_GEN(dev_priv) >= 11)
4339 gen11_irq_postinstall(dev_priv);
4340 else if (INTEL_GEN(dev_priv) >= 8)
4341 gen8_irq_postinstall(dev_priv);
4343 ilk_irq_postinstall(dev_priv);
4348 * intel_irq_install - enables the hardware interrupt
4349 * @dev_priv: i915 device instance
4351 * This function enables the hardware interrupt handling, but leaves the hotplug
4352 * handling still disabled. It is called after intel_irq_init().
4354 * In the driver load and resume code we need working interrupts in a few places
4355 * but don't want to deal with the hassle of concurrent probe and hotplug
4356 * workers. Hence the split into this two-stage approach.
4358 int intel_irq_install(struct drm_i915_private *dev_priv)
4360 int irq = dev_priv->drm.pdev->irq;
4364 * We enable some interrupt sources in our postinstall hooks, so mark
4365 * interrupts as enabled _before_ actually enabling them to avoid
4366 * special cases in our ordering checks.
4368 dev_priv->runtime_pm.irqs_enabled = true;
4370 dev_priv->drm.irq_enabled = true;
4372 intel_irq_reset(dev_priv);
4374 ret = request_irq(irq, intel_irq_handler(dev_priv),
4375 IRQF_SHARED, DRIVER_NAME, dev_priv);
4377 dev_priv->drm.irq_enabled = false;
4381 intel_irq_postinstall(dev_priv);
4387 * intel_irq_uninstall - finilizes all irq handling
4388 * @dev_priv: i915 device instance
4390 * This stops interrupt and hotplug handling and unregisters and frees all
4391 * resources acquired in the init functions.
4393 void intel_irq_uninstall(struct drm_i915_private *dev_priv)
4395 int irq = dev_priv->drm.pdev->irq;
4398 * FIXME we can get called twice during driver probe
4399 * error handling as well as during driver remove due to
4400 * intel_modeset_driver_remove() calling us out of sequence.
4401 * Would be nice if it didn't do that...
4403 if (!dev_priv->drm.irq_enabled)
4406 dev_priv->drm.irq_enabled = false;
4408 intel_irq_reset(dev_priv);
4410 free_irq(irq, dev_priv);
4412 intel_hpd_cancel_work(dev_priv);
4413 dev_priv->runtime_pm.irqs_enabled = false;
4417 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
4418 * @dev_priv: i915 device instance
4420 * This function is used to disable interrupts at runtime, both in the runtime
4421 * pm and the system suspend/resume code.
4423 void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4425 intel_irq_reset(dev_priv);
4426 dev_priv->runtime_pm.irqs_enabled = false;
4427 intel_synchronize_irq(dev_priv);
4431 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
4432 * @dev_priv: i915 device instance
4434 * This function is used to enable interrupts at runtime, both in the runtime
4435 * pm and the system suspend/resume code.
4437 void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4439 dev_priv->runtime_pm.irqs_enabled = true;
4440 intel_irq_reset(dev_priv);
4441 intel_irq_postinstall(dev_priv);
4444 bool intel_irqs_enabled(struct drm_i915_private *dev_priv)
4447 * We only use drm_irq_uninstall() at unload and VT switch, so
4448 * this is the only thing we need to check.
4450 return dev_priv->runtime_pm.irqs_enabled;
4453 void intel_synchronize_irq(struct drm_i915_private *i915)
4455 synchronize_irq(i915->drm.pdev->irq);