drivers/gpu/drm/i915/display/intel_psr.c

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include <drm/drm_atomic_helper.h>
  25
  26 #include "display/intel_dp.h"
  27
  28 #include "i915_drv.h"
  29 #include "intel_atomic.h"
  30 #include "intel_de.h"
  31 #include "intel_display_types.h"
  32 #include "intel_dp_aux.h"
  33 #include "intel_hdmi.h"
  34 #include "intel_psr.h"
  35 #include "intel_sprite.h"
  36 #include "skl_universal_plane.h"
  37
  38 /**
  39  * DOC: Panel Self Refresh (PSR/SRD)
  40  *
  41  * Since Haswell Display controller supports Panel Self-Refresh on display
  42  * panels witch have a remote frame buffer (RFB) implemented according to PSR
  43  * spec in eDP1.3. PSR feature allows the display to go to lower standby states
  44  * when system is idle but display is on as it eliminates display refresh
  45  * request to DDR memory completely as long as the frame buffer for that
  46  * display is unchanged.
  47  *
  48  * Panel Self Refresh must be supported by both Hardware (source) and
  49  * Panel (sink).
  50  *
  51  * PSR saves power by caching the framebuffer in the panel RFB, which allows us
  52  * to power down the link and memory controller. For DSI panels the same idea
  53  * is called "manual mode".
  54  *
  55  * The implementation uses the hardware-based PSR support which automatically
  56  * enters/exits self-refresh mode. The hardware takes care of sending the
  57  * required DP aux message and could even retrain the link (that part isn't
  58  * enabled yet though). The hardware also keeps track of any frontbuffer
  59  * changes to know when to exit self-refresh mode again. Unfortunately that
  60  * part doesn't work too well, hence why the i915 PSR support uses the
  61  * software frontbuffer tracking to make sure it doesn't miss a screen
  62  * update. For this integration intel_psr_invalidate() and intel_psr_flush()
  63  * get called by the frontbuffer tracking code. Note that because of locking
  64  * issues the self-refresh re-enable code is done from a work queue, which
  65  * must be correctly synchronized/cancelled when shutting down the pipe."
  66  *
  67  * DC3CO (DC3 clock off)
  68  *
  69  * On top of PSR2, GEN12 adds a intermediate power savings state that turns
  70  * clock off automatically during PSR2 idle state.
  71  * The smaller overhead of DC3co entry/exit vs. the overhead of PSR2 deep sleep
  72  * entry/exit allows the HW to enter a low-power state even when page flipping
  73  * periodically (for instance a 30fps video playback scenario).
  74  *
  75  * Every time a flips occurs PSR2 will get out of deep sleep state(if it was),
  76  * so DC3CO is enabled and tgl_dc3co_disable_work is schedule to run after 6
  77  * frames, if no other flip occurs and the function above is executed, DC3CO is
  78  * disabled and PSR2 is configured to enter deep sleep, resetting again in case
  79  * of another flip.
  80  * Front buffer modifications do not trigger DC3CO activation on purpose as it
  81  * would bring a lot of complexity and most of the moderns systems will only
  82  * use page flips.
  83  */
  84
  85 static bool psr_global_enabled(struct intel_dp *intel_dp)
  86 {
  87         struct drm_i915_private *i915 = dp_to_i915(intel_dp);
  88
  89         switch (intel_dp->psr.debug & I915_PSR_DEBUG_MODE_MASK) {
  90         case I915_PSR_DEBUG_DEFAULT:
  91                 return i915->params.enable_psr;
  92         case I915_PSR_DEBUG_DISABLE:
  93                 return false;
  94         default:
  95                 return true;
  96         }
  97 }
  98
  99 static bool psr2_global_enabled(struct intel_dp *intel_dp)
 100 {
 101         switch (intel_dp->psr.debug & I915_PSR_DEBUG_MODE_MASK) {
 102         case I915_PSR_DEBUG_DISABLE:
 103         case I915_PSR_DEBUG_FORCE_PSR1:
 104                 return false;
 105         default:
 106                 return true;
 107         }
 108 }
 109
 110 static void psr_irq_control(struct intel_dp *intel_dp)
 111 {
 112         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 113         enum transcoder trans_shift;
 114         i915_reg_t imr_reg;
 115         u32 mask, val;
 116
 117         /*
 118          * gen12+ has registers relative to transcoder and one per transcoder
 119          * using the same bit definition: handle it as TRANSCODER_EDP to force
 120          * 0 shift in bit definition
 121          */
 122         if (DISPLAY_VER(dev_priv) >= 12) {
 123                 trans_shift = 0;
 124                 imr_reg = TRANS_PSR_IMR(intel_dp->psr.transcoder);
 125         } else {
 126                 trans_shift = intel_dp->psr.transcoder;
 127                 imr_reg = EDP_PSR_IMR;
 128         }
 129
 130         mask = EDP_PSR_ERROR(trans_shift);
 131         if (intel_dp->psr.debug & I915_PSR_DEBUG_IRQ)
 132                 mask |= EDP_PSR_POST_EXIT(trans_shift) |
 133                         EDP_PSR_PRE_ENTRY(trans_shift);
 134
 135         /* Warning: it is masking/setting reserved bits too */
 136         val = intel_de_read(dev_priv, imr_reg);
 137         val &= ~EDP_PSR_TRANS_MASK(trans_shift);
 138         val |= ~mask;
 139         intel_de_write(dev_priv, imr_reg, val);
 140 }
 141
 142 static void psr_event_print(struct drm_i915_private *i915,
 143                             u32 val, bool psr2_enabled)
 144 {
 145         drm_dbg_kms(&i915->drm, "PSR exit events: 0x%x\n", val);
 146         if (val & PSR_EVENT_PSR2_WD_TIMER_EXPIRE)
 147                 drm_dbg_kms(&i915->drm, "\tPSR2 watchdog timer expired\n");
 148         if ((val & PSR_EVENT_PSR2_DISABLED) && psr2_enabled)
 149                 drm_dbg_kms(&i915->drm, "\tPSR2 disabled\n");
 150         if (val & PSR_EVENT_SU_DIRTY_FIFO_UNDERRUN)
 151                 drm_dbg_kms(&i915->drm, "\tSU dirty FIFO underrun\n");
 152         if (val & PSR_EVENT_SU_CRC_FIFO_UNDERRUN)
 153                 drm_dbg_kms(&i915->drm, "\tSU CRC FIFO underrun\n");
 154         if (val & PSR_EVENT_GRAPHICS_RESET)
 155                 drm_dbg_kms(&i915->drm, "\tGraphics reset\n");
 156         if (val & PSR_EVENT_PCH_INTERRUPT)
 157                 drm_dbg_kms(&i915->drm, "\tPCH interrupt\n");
 158         if (val & PSR_EVENT_MEMORY_UP)
 159                 drm_dbg_kms(&i915->drm, "\tMemory up\n");
 160         if (val & PSR_EVENT_FRONT_BUFFER_MODIFY)
 161                 drm_dbg_kms(&i915->drm, "\tFront buffer modification\n");
 162         if (val & PSR_EVENT_WD_TIMER_EXPIRE)
 163                 drm_dbg_kms(&i915->drm, "\tPSR watchdog timer expired\n");
 164         if (val & PSR_EVENT_PIPE_REGISTERS_UPDATE)
 165                 drm_dbg_kms(&i915->drm, "\tPIPE registers updated\n");
 166         if (val & PSR_EVENT_REGISTER_UPDATE)
 167                 drm_dbg_kms(&i915->drm, "\tRegister updated\n");
 168         if (val & PSR_EVENT_HDCP_ENABLE)
 169                 drm_dbg_kms(&i915->drm, "\tHDCP enabled\n");
 170         if (val & PSR_EVENT_KVMR_SESSION_ENABLE)
 171                 drm_dbg_kms(&i915->drm, "\tKVMR session enabled\n");
 172         if (val & PSR_EVENT_VBI_ENABLE)
 173                 drm_dbg_kms(&i915->drm, "\tVBI enabled\n");
 174         if (val & PSR_EVENT_LPSP_MODE_EXIT)
 175                 drm_dbg_kms(&i915->drm, "\tLPSP mode exited\n");
 176         if ((val & PSR_EVENT_PSR_DISABLE) && !psr2_enabled)
 177                 drm_dbg_kms(&i915->drm, "\tPSR disabled\n");
 178 }
 179
 180 void intel_psr_irq_handler(struct intel_dp *intel_dp, u32 psr_iir)
 181 {
 182         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
 183         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 184         ktime_t time_ns =  ktime_get();
 185         enum transcoder trans_shift;
 186         i915_reg_t imr_reg;
 187
 188         if (DISPLAY_VER(dev_priv) >= 12) {
 189                 trans_shift = 0;
 190                 imr_reg = TRANS_PSR_IMR(intel_dp->psr.transcoder);
 191         } else {
 192                 trans_shift = intel_dp->psr.transcoder;
 193                 imr_reg = EDP_PSR_IMR;
 194         }
 195
 196         if (psr_iir & EDP_PSR_PRE_ENTRY(trans_shift)) {
 197                 intel_dp->psr.last_entry_attempt = time_ns;
 198                 drm_dbg_kms(&dev_priv->drm,
 199                             "[transcoder %s] PSR entry attempt in 2 vblanks\n",
 200                             transcoder_name(cpu_transcoder));
 201         }
 202
 203         if (psr_iir & EDP_PSR_POST_EXIT(trans_shift)) {
 204                 intel_dp->psr.last_exit = time_ns;
 205                 drm_dbg_kms(&dev_priv->drm,
 206                             "[transcoder %s] PSR exit completed\n",
 207                             transcoder_name(cpu_transcoder));
 208
 209                 if (DISPLAY_VER(dev_priv) >= 9) {
 210                         u32 val = intel_de_read(dev_priv,
 211                                                 PSR_EVENT(cpu_transcoder));
 212                         bool psr2_enabled = intel_dp->psr.psr2_enabled;
 213
 214                         intel_de_write(dev_priv, PSR_EVENT(cpu_transcoder),
 215                                        val);
 216                         psr_event_print(dev_priv, val, psr2_enabled);
 217                 }
 218         }
 219
 220         if (psr_iir & EDP_PSR_ERROR(trans_shift)) {
 221                 u32 val;
 222
 223                 drm_warn(&dev_priv->drm, "[transcoder %s] PSR aux error\n",
 224                          transcoder_name(cpu_transcoder));
 225
 226                 intel_dp->psr.irq_aux_error = true;
 227
 228                 /*
 229                  * If this interruption is not masked it will keep
 230                  * interrupting so fast that it prevents the scheduled
 231                  * work to run.
 232                  * Also after a PSR error, we don't want to arm PSR
 233                  * again so we don't care about unmask the interruption
 234                  * or unset irq_aux_error.
 235                  */
 236                 val = intel_de_read(dev_priv, imr_reg);
 237                 val |= EDP_PSR_ERROR(trans_shift);
 238                 intel_de_write(dev_priv, imr_reg, val);
 239
 240                 schedule_work(&intel_dp->psr.work);
 241         }
 242 }
 243
 244 static bool intel_dp_get_alpm_status(struct intel_dp *intel_dp)
 245 {
 246         u8 alpm_caps = 0;
 247
 248         if (drm_dp_dpcd_readb(&intel_dp->aux, DP_RECEIVER_ALPM_CAP,
 249                               &alpm_caps) != 1)
 250                 return false;
 251         return alpm_caps & DP_ALPM_CAP;
 252 }
 253
 254 static u8 intel_dp_get_sink_sync_latency(struct intel_dp *intel_dp)
 255 {
 256         struct drm_i915_private *i915 = dp_to_i915(intel_dp);
 257         u8 val = 8; /* assume the worst if we can't read the value */
 258
 259         if (drm_dp_dpcd_readb(&intel_dp->aux,
 260                               DP_SYNCHRONIZATION_LATENCY_IN_SINK, &val) == 1)
 261                 val &= DP_MAX_RESYNC_FRAME_COUNT_MASK;
 262         else
 263                 drm_dbg_kms(&i915->drm,
 264                             "Unable to get sink synchronization latency, assuming 8 frames\n");
 265         return val;
 266 }
 267
 268 static void intel_dp_get_su_granularity(struct intel_dp *intel_dp)
 269 {
 270         struct drm_i915_private *i915 = dp_to_i915(intel_dp);
 271         ssize_t r;
 272         u16 w;
 273         u8 y;
 274
 275         /* If sink don't have specific granularity requirements set legacy ones */
 276         if (!(intel_dp->psr_dpcd[1] & DP_PSR2_SU_GRANULARITY_REQUIRED)) {
 277                 /* As PSR2 HW sends full lines, we do not care about x granularity */
 278                 w = 4;
 279                 y = 4;
 280                 goto exit;
 281         }
 282
 283         r = drm_dp_dpcd_read(&intel_dp->aux, DP_PSR2_SU_X_GRANULARITY, &w, 2);
 284         if (r != 2)
 285                 drm_dbg_kms(&i915->drm,
 286                             "Unable to read DP_PSR2_SU_X_GRANULARITY\n");
 287         /*
 288          * Spec says that if the value read is 0 the default granularity should
 289          * be used instead.
 290          */
 291         if (r != 2 || w == 0)
 292                 w = 4;
 293
 294         r = drm_dp_dpcd_read(&intel_dp->aux, DP_PSR2_SU_Y_GRANULARITY, &y, 1);
 295         if (r != 1) {
 296                 drm_dbg_kms(&i915->drm,
 297                             "Unable to read DP_PSR2_SU_Y_GRANULARITY\n");
 298                 y = 4;
 299         }
 300         if (y == 0)
 301                 y = 1;
 302
 303 exit:
 304         intel_dp->psr.su_w_granularity = w;
 305         intel_dp->psr.su_y_granularity = y;
 306 }
 307
 308 void intel_psr_init_dpcd(struct intel_dp *intel_dp)
 309 {
 310         struct drm_i915_private *dev_priv =
 311                 to_i915(dp_to_dig_port(intel_dp)->base.base.dev);
 312
 313         drm_dp_dpcd_read(&intel_dp->aux, DP_PSR_SUPPORT, intel_dp->psr_dpcd,
 314                          sizeof(intel_dp->psr_dpcd));
 315
 316         if (!intel_dp->psr_dpcd[0])
 317                 return;
 318         drm_dbg_kms(&dev_priv->drm, "eDP panel supports PSR version %x\n",
 319                     intel_dp->psr_dpcd[0]);
 320
 321         if (drm_dp_has_quirk(&intel_dp->desc, DP_DPCD_QUIRK_NO_PSR)) {
 322                 drm_dbg_kms(&dev_priv->drm,
 323                             "PSR support not currently available for this panel\n");
 324                 return;
 325         }
 326
 327         if (!(intel_dp->edp_dpcd[1] & DP_EDP_SET_POWER_CAP)) {
 328                 drm_dbg_kms(&dev_priv->drm,
 329                             "Panel lacks power state control, PSR cannot be enabled\n");
 330                 return;
 331         }
 332
 333         intel_dp->psr.sink_support = true;
 334         intel_dp->psr.sink_sync_latency =
 335                 intel_dp_get_sink_sync_latency(intel_dp);
 336
 337         if (DISPLAY_VER(dev_priv) >= 9 &&
 338             (intel_dp->psr_dpcd[0] == DP_PSR2_WITH_Y_COORD_IS_SUPPORTED)) {
 339                 bool y_req = intel_dp->psr_dpcd[1] &
 340                              DP_PSR2_SU_Y_COORDINATE_REQUIRED;
 341                 bool alpm = intel_dp_get_alpm_status(intel_dp);
 342
 343                 /*
 344                  * All panels that supports PSR version 03h (PSR2 +
 345                  * Y-coordinate) can handle Y-coordinates in VSC but we are
 346                  * only sure that it is going to be used when required by the
 347                  * panel. This way panel is capable to do selective update
 348                  * without a aux frame sync.
 349                  *
 350                  * To support PSR version 02h and PSR version 03h without
 351                  * Y-coordinate requirement panels we would need to enable
 352                  * GTC first.
 353                  */
 354                 intel_dp->psr.sink_psr2_support = y_req && alpm;
 355                 drm_dbg_kms(&dev_priv->drm, "PSR2 %ssupported\n",
 356                             intel_dp->psr.sink_psr2_support ? "" : "not ");
 357
 358                 if (intel_dp->psr.sink_psr2_support) {
 359                         intel_dp->psr.colorimetry_support =
 360                                 intel_dp_get_colorimetry_status(intel_dp);
 361                         intel_dp_get_su_granularity(intel_dp);
 362                 }
 363         }
 364 }
 365
 366 static void hsw_psr_setup_aux(struct intel_dp *intel_dp)
 367 {
 368         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 369         u32 aux_clock_divider, aux_ctl;
 370         int i;
 371         static const u8 aux_msg[] = {
 372                 [0] = DP_AUX_NATIVE_WRITE << 4,
 373                 [1] = DP_SET_POWER >> 8,
 374                 [2] = DP_SET_POWER & 0xff,
 375                 [3] = 1 - 1,
 376                 [4] = DP_SET_POWER_D0,
 377         };
 378         u32 psr_aux_mask = EDP_PSR_AUX_CTL_TIME_OUT_MASK |
 379                            EDP_PSR_AUX_CTL_MESSAGE_SIZE_MASK |
 380                            EDP_PSR_AUX_CTL_PRECHARGE_2US_MASK |
 381                            EDP_PSR_AUX_CTL_BIT_CLOCK_2X_MASK;
 382
 383         BUILD_BUG_ON(sizeof(aux_msg) > 20);
 384         for (i = 0; i < sizeof(aux_msg); i += 4)
 385                 intel_de_write(dev_priv,
 386                                EDP_PSR_AUX_DATA(intel_dp->psr.transcoder, i >> 2),
 387                                intel_dp_pack_aux(&aux_msg[i], sizeof(aux_msg) - i));
 388
 389         aux_clock_divider = intel_dp->get_aux_clock_divider(intel_dp, 0);
 390
 391         /* Start with bits set for DDI_AUX_CTL register */
 392         aux_ctl = intel_dp->get_aux_send_ctl(intel_dp, sizeof(aux_msg),
 393                                              aux_clock_divider);
 394
 395         /* Select only valid bits for SRD_AUX_CTL */
 396         aux_ctl &= psr_aux_mask;
 397         intel_de_write(dev_priv, EDP_PSR_AUX_CTL(intel_dp->psr.transcoder),
 398                        aux_ctl);
 399 }
 400
 401 static void intel_psr_enable_sink(struct intel_dp *intel_dp)
 402 {
 403         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 404         u8 dpcd_val = DP_PSR_ENABLE;
 405
 406         /* Enable ALPM at sink for psr2 */
 407         if (intel_dp->psr.psr2_enabled) {
 408                 drm_dp_dpcd_writeb(&intel_dp->aux, DP_RECEIVER_ALPM_CONFIG,
 409                                    DP_ALPM_ENABLE |
 410                                    DP_ALPM_LOCK_ERROR_IRQ_HPD_ENABLE);
 411
 412                 dpcd_val |= DP_PSR_ENABLE_PSR2 | DP_PSR_IRQ_HPD_WITH_CRC_ERRORS;
 413         } else {
 414                 if (intel_dp->psr.link_standby)
 415                         dpcd_val |= DP_PSR_MAIN_LINK_ACTIVE;
 416
 417                 if (DISPLAY_VER(dev_priv) >= 8)
 418                         dpcd_val |= DP_PSR_CRC_VERIFICATION;
 419         }
 420
 421         if (intel_dp->psr.req_psr2_sdp_prior_scanline)
 422                 dpcd_val |= DP_PSR_SU_REGION_SCANLINE_CAPTURE;
 423
 424         drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, dpcd_val);
 425
 426         drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
 427 }
 428
 429 static u32 intel_psr1_get_tp_time(struct intel_dp *intel_dp)
 430 {
 431         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 432         u32 val = 0;
 433
 434         if (DISPLAY_VER(dev_priv) >= 11)
 435                 val |= EDP_PSR_TP4_TIME_0US;
 436
 437         if (dev_priv->params.psr_safest_params) {
 438                 val |= EDP_PSR_TP1_TIME_2500us;
 439                 val |= EDP_PSR_TP2_TP3_TIME_2500us;
 440                 goto check_tp3_sel;
 441         }
 442
 443         if (dev_priv->vbt.psr.tp1_wakeup_time_us == 0)
 444                 val |= EDP_PSR_TP1_TIME_0us;
 445         else if (dev_priv->vbt.psr.tp1_wakeup_time_us <= 100)
 446                 val |= EDP_PSR_TP1_TIME_100us;
 447         else if (dev_priv->vbt.psr.tp1_wakeup_time_us <= 500)
 448                 val |= EDP_PSR_TP1_TIME_500us;
 449         else
 450                 val |= EDP_PSR_TP1_TIME_2500us;
 451
 452         if (dev_priv->vbt.psr.tp2_tp3_wakeup_time_us == 0)
 453                 val |= EDP_PSR_TP2_TP3_TIME_0us;
 454         else if (dev_priv->vbt.psr.tp2_tp3_wakeup_time_us <= 100)
 455                 val |= EDP_PSR_TP2_TP3_TIME_100us;
 456         else if (dev_priv->vbt.psr.tp2_tp3_wakeup_time_us <= 500)
 457                 val |= EDP_PSR_TP2_TP3_TIME_500us;
 458         else
 459                 val |= EDP_PSR_TP2_TP3_TIME_2500us;
 460
 461 check_tp3_sel:
 462         if (intel_dp_source_supports_hbr2(intel_dp) &&
 463             drm_dp_tps3_supported(intel_dp->dpcd))
 464                 val |= EDP_PSR_TP1_TP3_SEL;
 465         else
 466                 val |= EDP_PSR_TP1_TP2_SEL;
 467
 468         return val;
 469 }
 470
 471 static u8 psr_compute_idle_frames(struct intel_dp *intel_dp)
 472 {
 473         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 474         int idle_frames;
 475
 476         /* Let's use 6 as the minimum to cover all known cases including the
 477          * off-by-one issue that HW has in some cases.
 478          */
 479         idle_frames = max(6, dev_priv->vbt.psr.idle_frames);
 480         idle_frames = max(idle_frames, intel_dp->psr.sink_sync_latency + 1);
 481
 482         if (drm_WARN_ON(&dev_priv->drm, idle_frames > 0xf))
 483                 idle_frames = 0xf;
 484
 485         return idle_frames;
 486 }
 487
 488 static void hsw_activate_psr1(struct intel_dp *intel_dp)
 489 {
 490         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 491         u32 max_sleep_time = 0x1f;
 492         u32 val = EDP_PSR_ENABLE;
 493
 494         val |= psr_compute_idle_frames(intel_dp) << EDP_PSR_IDLE_FRAME_SHIFT;
 495
 496         val |= max_sleep_time << EDP_PSR_MAX_SLEEP_TIME_SHIFT;
 497         if (IS_HASWELL(dev_priv))
 498                 val |= EDP_PSR_MIN_LINK_ENTRY_TIME_8_LINES;
 499
 500         if (intel_dp->psr.link_standby)
 501                 val |= EDP_PSR_LINK_STANDBY;
 502
 503         val |= intel_psr1_get_tp_time(intel_dp);
 504
 505         if (DISPLAY_VER(dev_priv) >= 8)
 506                 val |= EDP_PSR_CRC_ENABLE;
 507
 508         val |= (intel_de_read(dev_priv, EDP_PSR_CTL(intel_dp->psr.transcoder)) &
 509                 EDP_PSR_RESTORE_PSR_ACTIVE_CTX_MASK);
 510         intel_de_write(dev_priv, EDP_PSR_CTL(intel_dp->psr.transcoder), val);
 511 }
 512
 513 static u32 intel_psr2_get_tp_time(struct intel_dp *intel_dp)
 514 {
 515         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 516         u32 val = 0;
 517
 518         if (dev_priv->params.psr_safest_params)
 519                 return EDP_PSR2_TP2_TIME_2500us;
 520
 521         if (dev_priv->vbt.psr.psr2_tp2_tp3_wakeup_time_us >= 0 &&
 522             dev_priv->vbt.psr.psr2_tp2_tp3_wakeup_time_us <= 50)
 523                 val |= EDP_PSR2_TP2_TIME_50us;
 524         else if (dev_priv->vbt.psr.psr2_tp2_tp3_wakeup_time_us <= 100)
 525                 val |= EDP_PSR2_TP2_TIME_100us;
 526         else if (dev_priv->vbt.psr.psr2_tp2_tp3_wakeup_time_us <= 500)
 527                 val |= EDP_PSR2_TP2_TIME_500us;
 528         else
 529                 val |= EDP_PSR2_TP2_TIME_2500us;
 530
 531         return val;
 532 }
 533
 534 static void hsw_activate_psr2(struct intel_dp *intel_dp)
 535 {
 536         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 537         u32 val = EDP_PSR2_ENABLE;
 538
 539         val |= psr_compute_idle_frames(intel_dp) << EDP_PSR2_IDLE_FRAME_SHIFT;
 540
 541         if (!IS_ALDERLAKE_P(dev_priv))
 542                 val |= EDP_SU_TRACK_ENABLE;
 543
 544         if (DISPLAY_VER(dev_priv) >= 10 && DISPLAY_VER(dev_priv) <= 12)
 545                 val |= EDP_Y_COORDINATE_ENABLE;
 546
 547         val |= EDP_PSR2_FRAME_BEFORE_SU(intel_dp->psr.sink_sync_latency + 1);
 548         val |= intel_psr2_get_tp_time(intel_dp);
 549
 550         /* Wa_22012278275:adl-p */
 551         if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_E0)) {
 552                 static const u8 map[] = {
 553                         2, /* 5 lines */
 554                         1, /* 6 lines */
 555                         0, /* 7 lines */
 556                         3, /* 8 lines */
 557                         6, /* 9 lines */
 558                         5, /* 10 lines */
 559                         4, /* 11 lines */
 560                         7, /* 12 lines */
 561                 };
 562                 /*
 563                  * Still using the default IO_BUFFER_WAKE and FAST_WAKE, see
 564                  * comments bellow for more information
 565                  */
 566                 u32 tmp, lines = 7;
 567
 568                 val |= TGL_EDP_PSR2_BLOCK_COUNT_NUM_2;
 569
 570                 tmp = map[lines - TGL_EDP_PSR2_IO_BUFFER_WAKE_MIN_LINES];
 571                 tmp = tmp << TGL_EDP_PSR2_IO_BUFFER_WAKE_SHIFT;
 572                 val |= tmp;
 573
 574                 tmp = map[lines - TGL_EDP_PSR2_FAST_WAKE_MIN_LINES];
 575                 tmp = tmp << TGL_EDP_PSR2_FAST_WAKE_MIN_SHIFT;
 576                 val |= tmp;
 577         } else if (DISPLAY_VER(dev_priv) >= 12) {
 578                 /*
 579                  * TODO: 7 lines of IO_BUFFER_WAKE and FAST_WAKE are default
 580                  * values from BSpec. In order to setting an optimal power
 581                  * consumption, lower than 4k resoluition mode needs to decrese
 582                  * IO_BUFFER_WAKE and FAST_WAKE. And higher than 4K resolution
 583                  * mode needs to increase IO_BUFFER_WAKE and FAST_WAKE.
 584                  */
 585                 val |= TGL_EDP_PSR2_BLOCK_COUNT_NUM_2;
 586                 val |= TGL_EDP_PSR2_IO_BUFFER_WAKE(7);
 587                 val |= TGL_EDP_PSR2_FAST_WAKE(7);
 588         } else if (DISPLAY_VER(dev_priv) >= 9) {
 589                 val |= EDP_PSR2_IO_BUFFER_WAKE(7);
 590                 val |= EDP_PSR2_FAST_WAKE(7);
 591         }
 592
 593         if (intel_dp->psr.req_psr2_sdp_prior_scanline)
 594                 val |= EDP_PSR2_SU_SDP_SCANLINE;
 595
 596         if (intel_dp->psr.psr2_sel_fetch_enabled) {
 597                 /* Wa_1408330847 */
 598                 if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
 599                         intel_de_rmw(dev_priv, CHICKEN_PAR1_1,
 600                                      DIS_RAM_BYPASS_PSR2_MAN_TRACK,
 601                                      DIS_RAM_BYPASS_PSR2_MAN_TRACK);
 602
 603                 intel_de_write(dev_priv,
 604                                PSR2_MAN_TRK_CTL(intel_dp->psr.transcoder),
 605                                PSR2_MAN_TRK_CTL_ENABLE);
 606         } else if (HAS_PSR2_SEL_FETCH(dev_priv)) {
 607                 intel_de_write(dev_priv,
 608                                PSR2_MAN_TRK_CTL(intel_dp->psr.transcoder), 0);
 609         }
 610
 611         /*
 612          * PSR2 HW is incorrectly using EDP_PSR_TP1_TP3_SEL and BSpec is
 613          * recommending keep this bit unset while PSR2 is enabled.
 614          */
 615         intel_de_write(dev_priv, EDP_PSR_CTL(intel_dp->psr.transcoder), 0);
 616
 617         intel_de_write(dev_priv, EDP_PSR2_CTL(intel_dp->psr.transcoder), val);
 618 }
 619
 620 static bool
 621 transcoder_has_psr2(struct drm_i915_private *dev_priv, enum transcoder trans)
 622 {
 623         if (DISPLAY_VER(dev_priv) < 9)
 624                 return false;
 625         else if (DISPLAY_VER(dev_priv) >= 12)
 626                 return trans == TRANSCODER_A;
 627         else
 628                 return trans == TRANSCODER_EDP;
 629 }
 630
 631 static u32 intel_get_frame_time_us(const struct intel_crtc_state *cstate)
 632 {
 633         if (!cstate || !cstate->hw.active)
 634                 return 0;
 635
 636         return DIV_ROUND_UP(1000 * 1000,
 637                             drm_mode_vrefresh(&cstate->hw.adjusted_mode));
 638 }
 639
 640 static void psr2_program_idle_frames(struct intel_dp *intel_dp,
 641                                      u32 idle_frames)
 642 {
 643         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 644         u32 val;
 645
 646         idle_frames <<=  EDP_PSR2_IDLE_FRAME_SHIFT;
 647         val = intel_de_read(dev_priv, EDP_PSR2_CTL(intel_dp->psr.transcoder));
 648         val &= ~EDP_PSR2_IDLE_FRAME_MASK;
 649         val |= idle_frames;
 650         intel_de_write(dev_priv, EDP_PSR2_CTL(intel_dp->psr.transcoder), val);
 651 }
 652
 653 static void tgl_psr2_enable_dc3co(struct intel_dp *intel_dp)
 654 {
 655         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 656
 657         psr2_program_idle_frames(intel_dp, 0);
 658         intel_display_power_set_target_dc_state(dev_priv, DC_STATE_EN_DC3CO);
 659 }
 660
 661 static void tgl_psr2_disable_dc3co(struct intel_dp *intel_dp)
 662 {
 663         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 664
 665         intel_display_power_set_target_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
 666         psr2_program_idle_frames(intel_dp, psr_compute_idle_frames(intel_dp));
 667 }
 668
 669 static void tgl_dc3co_disable_work(struct work_struct *work)
 670 {
 671         struct intel_dp *intel_dp =
 672                 container_of(work, typeof(*intel_dp), psr.dc3co_work.work);
 673
 674         mutex_lock(&intel_dp->psr.lock);
 675         /* If delayed work is pending, it is not idle */
 676         if (delayed_work_pending(&intel_dp->psr.dc3co_work))
 677                 goto unlock;
 678
 679         tgl_psr2_disable_dc3co(intel_dp);
 680 unlock:
 681         mutex_unlock(&intel_dp->psr.lock);
 682 }
 683
 684 static void tgl_disallow_dc3co_on_psr2_exit(struct intel_dp *intel_dp)
 685 {
 686         if (!intel_dp->psr.dc3co_exitline)
 687                 return;
 688
 689         cancel_delayed_work(&intel_dp->psr.dc3co_work);
 690         /* Before PSR2 exit disallow dc3co*/
 691         tgl_psr2_disable_dc3co(intel_dp);
 692 }
 693
 694 static bool
 695 dc3co_is_pipe_port_compatible(struct intel_dp *intel_dp,
 696                               struct intel_crtc_state *crtc_state)
 697 {
 698         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 699         enum pipe pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe;
 700         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 701         enum port port = dig_port->base.port;
 702
 703         if (IS_ALDERLAKE_P(dev_priv))
 704                 return pipe <= PIPE_B && port <= PORT_B;
 705         else
 706                 return pipe == PIPE_A && port == PORT_A;
 707 }
 708
 709 static void
 710 tgl_dc3co_exitline_compute_config(struct intel_dp *intel_dp,
 711                                   struct intel_crtc_state *crtc_state)
 712 {
 713         const u32 crtc_vdisplay = crtc_state->uapi.adjusted_mode.crtc_vdisplay;
 714         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 715         u32 exit_scanlines;
 716
 717         /*
 718          * FIXME: Due to the changed sequence of activating/deactivating DC3CO,
 719          * disable DC3CO until the changed dc3co activating/deactivating sequence
 720          * is applied. B.Specs:49196
 721          */
 722         return;
 723
 724         /*
 725          * DMC's DC3CO exit mechanism has an issue with Selective Fecth
 726          * TODO: when the issue is addressed, this restriction should be removed.
 727          */
 728         if (crtc_state->enable_psr2_sel_fetch)
 729                 return;
 730
 731         if (!(dev_priv->dmc.allowed_dc_mask & DC_STATE_EN_DC3CO))
 732                 return;
 733
 734         if (!dc3co_is_pipe_port_compatible(intel_dp, crtc_state))
 735                 return;
 736
 737         /* Wa_16011303918:adl-p */
 738         if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
 739                 return;
 740
 741         /*
 742          * DC3CO Exit time 200us B.Spec 49196
 743          * PSR2 transcoder Early Exit scanlines = ROUNDUP(200 / line time) + 1
 744          */
 745         exit_scanlines =
 746                 intel_usecs_to_scanlines(&crtc_state->uapi.adjusted_mode, 200) + 1;
 747
 748         if (drm_WARN_ON(&dev_priv->drm, exit_scanlines > crtc_vdisplay))
 749                 return;
 750
 751         crtc_state->dc3co_exitline = crtc_vdisplay - exit_scanlines;
 752 }
 753
 754 static bool intel_psr2_sel_fetch_config_valid(struct intel_dp *intel_dp,
 755                                               struct intel_crtc_state *crtc_state)
 756 {
 757         struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->uapi.state);
 758         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 759         struct intel_plane_state *plane_state;
 760         struct intel_plane *plane;
 761         int i;
 762
 763         if (!dev_priv->params.enable_psr2_sel_fetch &&
 764             intel_dp->psr.debug != I915_PSR_DEBUG_ENABLE_SEL_FETCH) {
 765                 drm_dbg_kms(&dev_priv->drm,
 766                             "PSR2 sel fetch not enabled, disabled by parameter\n");
 767                 return false;
 768         }
 769
 770         if (crtc_state->uapi.async_flip) {
 771                 drm_dbg_kms(&dev_priv->drm,
 772                             "PSR2 sel fetch not enabled, async flip enabled\n");
 773                 return false;
 774         }
 775
 776         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
 777                 if (plane_state->uapi.rotation != DRM_MODE_ROTATE_0) {
 778                         drm_dbg_kms(&dev_priv->drm,
 779                                     "PSR2 sel fetch not enabled, plane rotated\n");
 780                         return false;
 781                 }
 782         }
 783
 784         /* Wa_14010254185 Wa_14010103792 */
 785         if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0)) {
 786                 drm_dbg_kms(&dev_priv->drm,
 787                             "PSR2 sel fetch not enabled, missing the implementation of WAs\n");
 788                 return false;
 789         }
 790
 791         return crtc_state->enable_psr2_sel_fetch = true;
 792 }
 793
 794 static bool psr2_granularity_check(struct intel_dp *intel_dp,
 795                                    struct intel_crtc_state *crtc_state)
 796 {
 797         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 798         const int crtc_hdisplay = crtc_state->hw.adjusted_mode.crtc_hdisplay;
 799         const int crtc_vdisplay = crtc_state->hw.adjusted_mode.crtc_vdisplay;
 800         u16 y_granularity = 0;
 801
 802         /* PSR2 HW only send full lines so we only need to validate the width */
 803         if (crtc_hdisplay % intel_dp->psr.su_w_granularity)
 804                 return false;
 805
 806         if (crtc_vdisplay % intel_dp->psr.su_y_granularity)
 807                 return false;
 808
 809         /* HW tracking is only aligned to 4 lines */
 810         if (!crtc_state->enable_psr2_sel_fetch)
 811                 return intel_dp->psr.su_y_granularity == 4;
 812
 813         /*
 814          * adl_p has 1 line granularity. For other platforms with SW tracking we
 815          * can adjust the y coordinates to match sink requirement if multiple of
 816          * 4.
 817          */
 818         if (IS_ALDERLAKE_P(dev_priv))
 819                 y_granularity = intel_dp->psr.su_y_granularity;
 820         else if (intel_dp->psr.su_y_granularity <= 2)
 821                 y_granularity = 4;
 822         else if ((intel_dp->psr.su_y_granularity % 4) == 0)
 823                 y_granularity = intel_dp->psr.su_y_granularity;
 824
 825         if (y_granularity == 0 || crtc_vdisplay % y_granularity)
 826                 return false;
 827
 828         crtc_state->su_y_granularity = y_granularity;
 829         return true;
 830 }
 831
 832 static bool _compute_psr2_sdp_prior_scanline_indication(struct intel_dp *intel_dp,
 833                                                         struct intel_crtc_state *crtc_state)
 834 {
 835         const struct drm_display_mode *adjusted_mode = &crtc_state->uapi.adjusted_mode;
 836         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 837         u32 hblank_total, hblank_ns, req_ns;
 838
 839         hblank_total = adjusted_mode->crtc_hblank_end - adjusted_mode->crtc_hblank_start;
 840         hblank_ns = div_u64(1000000ULL * hblank_total, adjusted_mode->crtc_clock);
 841
 842         /* From spec: (72 / number of lanes) * 1000 / symbol clock frequency MHz */
 843         req_ns = (72 / crtc_state->lane_count) * 1000 / (crtc_state->port_clock / 1000);
 844
 845         if ((hblank_ns - req_ns) > 100)
 846                 return true;
 847
 848         if (DISPLAY_VER(dev_priv) < 13 || intel_dp->edp_dpcd[0] < DP_EDP_14b)
 849                 return false;
 850
 851         crtc_state->req_psr2_sdp_prior_scanline = true;
 852         return true;
 853 }
 854
 855 static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
 856                                     struct intel_crtc_state *crtc_state)
 857 {
 858         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 859         int crtc_hdisplay = crtc_state->hw.adjusted_mode.crtc_hdisplay;
 860         int crtc_vdisplay = crtc_state->hw.adjusted_mode.crtc_vdisplay;
 861         int psr_max_h = 0, psr_max_v = 0, max_bpp = 0;
 862
 863         if (!intel_dp->psr.sink_psr2_support)
 864                 return false;
 865
 866         /* JSL and EHL only supports eDP 1.3 */
 867         if (IS_JSL_EHL(dev_priv)) {
 868                 drm_dbg_kms(&dev_priv->drm, "PSR2 not supported by phy\n");
 869                 return false;
 870         }
 871
 872         /* Wa_16011181250 */
 873         if (IS_ROCKETLAKE(dev_priv) || IS_ALDERLAKE_S(dev_priv) ||
 874             IS_DG2(dev_priv)) {
 875                 drm_dbg_kms(&dev_priv->drm, "PSR2 is defeatured for this platform\n");
 876                 return false;
 877         }
 878
 879         /*
 880          * We are missing the implementation of some workarounds to enabled PSR2
 881          * in Alderlake_P, until ready PSR2 should be kept disabled.
 882          */
 883         if (IS_ALDERLAKE_P(dev_priv)) {
 884                 drm_dbg_kms(&dev_priv->drm, "PSR2 is missing the implementation of workarounds\n");
 885                 return false;
 886         }
 887
 888         if (!transcoder_has_psr2(dev_priv, crtc_state->cpu_transcoder)) {
 889                 drm_dbg_kms(&dev_priv->drm,
 890                             "PSR2 not supported in transcoder %s\n",
 891                             transcoder_name(crtc_state->cpu_transcoder));
 892                 return false;
 893         }
 894
 895         if (!psr2_global_enabled(intel_dp)) {
 896                 drm_dbg_kms(&dev_priv->drm, "PSR2 disabled by flag\n");
 897                 return false;
 898         }
 899
 900         /*
 901          * DSC and PSR2 cannot be enabled simultaneously. If a requested
 902          * resolution requires DSC to be enabled, priority is given to DSC
 903          * over PSR2.
 904          */
 905         if (crtc_state->dsc.compression_enable) {
 906                 drm_dbg_kms(&dev_priv->drm,
 907                             "PSR2 cannot be enabled since DSC is enabled\n");
 908                 return false;
 909         }
 910
 911         if (crtc_state->crc_enabled) {
 912                 drm_dbg_kms(&dev_priv->drm,
 913                             "PSR2 not enabled because it would inhibit pipe CRC calculation\n");
 914                 return false;
 915         }
 916
 917         if (DISPLAY_VER(dev_priv) >= 12) {
 918                 psr_max_h = 5120;
 919                 psr_max_v = 3200;
 920                 max_bpp = 30;
 921         } else if (DISPLAY_VER(dev_priv) >= 10) {
 922                 psr_max_h = 4096;
 923                 psr_max_v = 2304;
 924                 max_bpp = 24;
 925         } else if (DISPLAY_VER(dev_priv) == 9) {
 926                 psr_max_h = 3640;
 927                 psr_max_v = 2304;
 928                 max_bpp = 24;
 929         }
 930
 931         if (crtc_state->pipe_bpp > max_bpp) {
 932                 drm_dbg_kms(&dev_priv->drm,
 933                             "PSR2 not enabled, pipe bpp %d > max supported %d\n",
 934                             crtc_state->pipe_bpp, max_bpp);
 935                 return false;
 936         }
 937
 938         if (HAS_PSR2_SEL_FETCH(dev_priv)) {
 939                 if (!intel_psr2_sel_fetch_config_valid(intel_dp, crtc_state) &&
 940                     !HAS_PSR_HW_TRACKING(dev_priv)) {
 941                         drm_dbg_kms(&dev_priv->drm,
 942                                     "PSR2 not enabled, selective fetch not valid and no HW tracking available\n");
 943                         return false;
 944                 }
 945         }
 946
 947         /* Wa_2209313811 */
 948         if (!crtc_state->enable_psr2_sel_fetch &&
 949             IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0)) {
 950                 drm_dbg_kms(&dev_priv->drm, "PSR2 HW tracking is not supported this Display stepping\n");
 951                 return false;
 952         }
 953
 954         if (!psr2_granularity_check(intel_dp, crtc_state)) {
 955                 drm_dbg_kms(&dev_priv->drm, "PSR2 not enabled, SU granularity not compatible\n");
 956                 return false;
 957         }
 958
 959         if (!crtc_state->enable_psr2_sel_fetch &&
 960             (crtc_hdisplay > psr_max_h || crtc_vdisplay > psr_max_v)) {
 961                 drm_dbg_kms(&dev_priv->drm,
 962                             "PSR2 not enabled, resolution %dx%d > max supported %dx%d\n",
 963                             crtc_hdisplay, crtc_vdisplay,
 964                             psr_max_h, psr_max_v);
 965                 return false;
 966         }
 967
 968         if (!_compute_psr2_sdp_prior_scanline_indication(intel_dp, crtc_state)) {
 969                 drm_dbg_kms(&dev_priv->drm,
 970                             "PSR2 not enabled, PSR2 SDP indication do not fit in hblank\n");
 971                 return false;
 972         }
 973
 974         /* Wa_16011303918:adl-p */
 975         if (crtc_state->vrr.enable &&
 976             IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) {
 977                 drm_dbg_kms(&dev_priv->drm,
 978                             "PSR2 not enabled, not compatible with HW stepping + VRR\n");
 979                 return false;
 980         }
 981
 982         tgl_dc3co_exitline_compute_config(intel_dp, crtc_state);
 983         return true;
 984 }
 985
 986 void intel_psr_compute_config(struct intel_dp *intel_dp,
 987                               struct intel_crtc_state *crtc_state)
 988 {
 989         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 990         const struct drm_display_mode *adjusted_mode =
 991                 &crtc_state->hw.adjusted_mode;
 992         int psr_setup_time;
 993
 994         /*
 995          * Current PSR panels dont work reliably with VRR enabled
 996          * So if VRR is enabled, do not enable PSR.
 997          */
 998         if (crtc_state->vrr.enable)
 999                 return;
1000
1001         if (!CAN_PSR(intel_dp))
1002                 return;
1003
1004         if (!psr_global_enabled(intel_dp)) {
1005                 drm_dbg_kms(&dev_priv->drm, "PSR disabled by flag\n");
1006                 return;
1007         }
1008
1009         if (intel_dp->psr.sink_not_reliable) {
1010                 drm_dbg_kms(&dev_priv->drm,
1011                             "PSR sink implementation is not reliable\n");
1012                 return;
1013         }
1014
1015         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
1016                 drm_dbg_kms(&dev_priv->drm,
1017                             "PSR condition failed: Interlaced mode enabled\n");
1018                 return;
1019         }
1020
1021         psr_setup_time = drm_dp_psr_setup_time(intel_dp->psr_dpcd);
1022         if (psr_setup_time < 0) {
1023                 drm_dbg_kms(&dev_priv->drm,
1024                             "PSR condition failed: Invalid PSR setup time (0x%02x)\n",
1025                             intel_dp->psr_dpcd[1]);
1026                 return;
1027         }
1028
1029         if (intel_usecs_to_scanlines(adjusted_mode, psr_setup_time) >
1030             adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vdisplay - 1) {
1031                 drm_dbg_kms(&dev_priv->drm,
1032                             "PSR condition failed: PSR setup time (%d us) too long\n",
1033                             psr_setup_time);
1034                 return;
1035         }
1036
1037         crtc_state->has_psr = true;
1038         crtc_state->has_psr2 = intel_psr2_config_valid(intel_dp, crtc_state);
1039         crtc_state->infoframes.enable |= intel_hdmi_infoframe_enable(DP_SDP_VSC);
1040 }
1041
1042 void intel_psr_get_config(struct intel_encoder *encoder,
1043                           struct intel_crtc_state *pipe_config)
1044 {
1045         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
1046         struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
1047         struct intel_dp *intel_dp;
1048         u32 val;
1049
1050         if (!dig_port)
1051                 return;
1052
1053         intel_dp = &dig_port->dp;
1054         if (!CAN_PSR(intel_dp))
1055                 return;
1056
1057         mutex_lock(&intel_dp->psr.lock);
1058         if (!intel_dp->psr.enabled)
1059                 goto unlock;
1060
1061         /*
1062          * Not possible to read EDP_PSR/PSR2_CTL registers as it is
1063          * enabled/disabled because of frontbuffer tracking and others.
1064          */
1065         pipe_config->has_psr = true;
1066         pipe_config->has_psr2 = intel_dp->psr.psr2_enabled;
1067         pipe_config->infoframes.enable |= intel_hdmi_infoframe_enable(DP_SDP_VSC);
1068
1069         if (!intel_dp->psr.psr2_enabled)
1070                 goto unlock;
1071
1072         if (HAS_PSR2_SEL_FETCH(dev_priv)) {
1073                 val = intel_de_read(dev_priv, PSR2_MAN_TRK_CTL(intel_dp->psr.transcoder));
1074                 if (val & PSR2_MAN_TRK_CTL_ENABLE)
1075                         pipe_config->enable_psr2_sel_fetch = true;
1076         }
1077
1078         if (DISPLAY_VER(dev_priv) >= 12) {
1079                 val = intel_de_read(dev_priv, EXITLINE(intel_dp->psr.transcoder));
1080                 val &= EXITLINE_MASK;
1081                 pipe_config->dc3co_exitline = val;
1082         }
1083 unlock:
1084         mutex_unlock(&intel_dp->psr.lock);
1085 }
1086
1087 static void intel_psr_activate(struct intel_dp *intel_dp)
1088 {
1089         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1090         enum transcoder transcoder = intel_dp->psr.transcoder;
1091
1092         if (transcoder_has_psr2(dev_priv, transcoder))
1093                 drm_WARN_ON(&dev_priv->drm,
1094                             intel_de_read(dev_priv, EDP_PSR2_CTL(transcoder)) & EDP_PSR2_ENABLE);
1095
1096         drm_WARN_ON(&dev_priv->drm,
1097                     intel_de_read(dev_priv, EDP_PSR_CTL(transcoder)) & EDP_PSR_ENABLE);
1098         drm_WARN_ON(&dev_priv->drm, intel_dp->psr.active);
1099         lockdep_assert_held(&intel_dp->psr.lock);
1100
1101         /* psr1 and psr2 are mutually exclusive.*/
1102         if (intel_dp->psr.psr2_enabled)
1103                 hsw_activate_psr2(intel_dp);
1104         else
1105                 hsw_activate_psr1(intel_dp);
1106
1107         intel_dp->psr.active = true;
1108 }
1109
1110 static void intel_psr_enable_source(struct intel_dp *intel_dp)
1111 {
1112         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1113         enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
1114         u32 mask;
1115
1116         /* Only HSW and BDW have PSR AUX registers that need to be setup. SKL+
1117          * use hardcoded values PSR AUX transactions
1118          */
1119         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
1120                 hsw_psr_setup_aux(intel_dp);
1121
1122         if (intel_dp->psr.psr2_enabled && DISPLAY_VER(dev_priv) == 9) {
1123                 i915_reg_t reg = CHICKEN_TRANS(cpu_transcoder);
1124                 u32 chicken = intel_de_read(dev_priv, reg);
1125
1126                 chicken |= PSR2_VSC_ENABLE_PROG_HEADER |
1127                            PSR2_ADD_VERTICAL_LINE_COUNT;
1128                 intel_de_write(dev_priv, reg, chicken);
1129         }
1130
1131         /*
1132          * Per Spec: Avoid continuous PSR exit by masking MEMUP and HPD also
1133          * mask LPSP to avoid dependency on other drivers that might block
1134          * runtime_pm besides preventing  other hw tracking issues now we
1135          * can rely on frontbuffer tracking.
1136          */
1137         mask = EDP_PSR_DEBUG_MASK_MEMUP |
1138                EDP_PSR_DEBUG_MASK_HPD |
1139                EDP_PSR_DEBUG_MASK_LPSP |
1140                EDP_PSR_DEBUG_MASK_MAX_SLEEP;
1141
1142         if (DISPLAY_VER(dev_priv) < 11)
1143                 mask |= EDP_PSR_DEBUG_MASK_DISP_REG_WRITE;
1144
1145         intel_de_write(dev_priv, EDP_PSR_DEBUG(intel_dp->psr.transcoder),
1146                        mask);
1147
1148         psr_irq_control(intel_dp);
1149
1150         if (intel_dp->psr.dc3co_exitline) {
1151                 u32 val;
1152
1153                 /*
1154                  * TODO: if future platforms supports DC3CO in more than one
1155                  * transcoder, EXITLINE will need to be unset when disabling PSR
1156                  */
1157                 val = intel_de_read(dev_priv, EXITLINE(cpu_transcoder));
1158                 val &= ~EXITLINE_MASK;
1159                 val |= intel_dp->psr.dc3co_exitline << EXITLINE_SHIFT;
1160                 val |= EXITLINE_ENABLE;
1161                 intel_de_write(dev_priv, EXITLINE(cpu_transcoder), val);
1162         }
1163
1164         if (HAS_PSR_HW_TRACKING(dev_priv) && HAS_PSR2_SEL_FETCH(dev_priv))
1165                 intel_de_rmw(dev_priv, CHICKEN_PAR1_1, IGNORE_PSR2_HW_TRACKING,
1166                              intel_dp->psr.psr2_sel_fetch_enabled ?
1167                              IGNORE_PSR2_HW_TRACKING : 0);
1168
1169         /* Wa_16011168373:adl-p */
1170         if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0) &&
1171             intel_dp->psr.psr2_enabled)
1172                 intel_de_rmw(dev_priv,
1173                              TRANS_SET_CONTEXT_LATENCY(intel_dp->psr.transcoder),
1174                              TRANS_SET_CONTEXT_LATENCY_MASK,
1175                              TRANS_SET_CONTEXT_LATENCY_VALUE(1));
1176 }
1177
1178 static bool psr_interrupt_error_check(struct intel_dp *intel_dp)
1179 {
1180         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1181         u32 val;
1182
1183         /*
1184          * If a PSR error happened and the driver is reloaded, the EDP_PSR_IIR
1185          * will still keep the error set even after the reset done in the
1186          * irq_preinstall and irq_uninstall hooks.
1187          * And enabling in this situation cause the screen to freeze in the
1188          * first time that PSR HW tries to activate so lets keep PSR disabled
1189          * to avoid any rendering problems.
1190          */
1191         if (DISPLAY_VER(dev_priv) >= 12) {
1192                 val = intel_de_read(dev_priv,
1193                                     TRANS_PSR_IIR(intel_dp->psr.transcoder));
1194                 val &= EDP_PSR_ERROR(0);
1195         } else {
1196                 val = intel_de_read(dev_priv, EDP_PSR_IIR);
1197                 val &= EDP_PSR_ERROR(intel_dp->psr.transcoder);
1198         }
1199         if (val) {
1200                 intel_dp->psr.sink_not_reliable = true;
1201                 drm_dbg_kms(&dev_priv->drm,
1202                             "PSR interruption error set, not enabling PSR\n");
1203                 return false;
1204         }
1205
1206         return true;
1207 }
1208
1209 static void intel_psr_enable_locked(struct intel_dp *intel_dp,
1210                                     const struct intel_crtc_state *crtc_state,
1211                                     const struct drm_connector_state *conn_state)
1212 {
1213         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
1214         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1215         struct intel_encoder *encoder = &dig_port->base;
1216         u32 val;
1217
1218         drm_WARN_ON(&dev_priv->drm, intel_dp->psr.enabled);
1219
1220         intel_dp->psr.psr2_enabled = crtc_state->has_psr2;
1221         intel_dp->psr.busy_frontbuffer_bits = 0;
1222         intel_dp->psr.pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe;
1223         intel_dp->psr.transcoder = crtc_state->cpu_transcoder;
1224         /* DC5/DC6 requires at least 6 idle frames */
1225         val = usecs_to_jiffies(intel_get_frame_time_us(crtc_state) * 6);
1226         intel_dp->psr.dc3co_exit_delay = val;
1227         intel_dp->psr.dc3co_exitline = crtc_state->dc3co_exitline;
1228         intel_dp->psr.psr2_sel_fetch_enabled = crtc_state->enable_psr2_sel_fetch;
1229         intel_dp->psr.req_psr2_sdp_prior_scanline =
1230                 crtc_state->req_psr2_sdp_prior_scanline;
1231
1232         if (!psr_interrupt_error_check(intel_dp))
1233                 return;
1234
1235         drm_dbg_kms(&dev_priv->drm, "Enabling PSR%s\n",
1236                     intel_dp->psr.psr2_enabled ? "2" : "1");
1237         intel_dp_compute_psr_vsc_sdp(intel_dp, crtc_state, conn_state,
1238                                      &intel_dp->psr.vsc);
1239         intel_write_dp_vsc_sdp(encoder, crtc_state, &intel_dp->psr.vsc);
1240         intel_psr_enable_sink(intel_dp);
1241         intel_psr_enable_source(intel_dp);
1242         intel_dp->psr.enabled = true;
1243         intel_dp->psr.paused = false;
1244
1245         intel_psr_activate(intel_dp);
1246 }
1247
1248 /**
1249  * intel_psr_enable - Enable PSR
1250  * @intel_dp: Intel DP
1251  * @crtc_state: new CRTC state
1252  * @conn_state: new CONNECTOR state
1253  *
1254  * This function can only be called after the pipe is fully trained and enabled.
1255  */
1256 void intel_psr_enable(struct intel_dp *intel_dp,
1257                       const struct intel_crtc_state *crtc_state,
1258                       const struct drm_connector_state *conn_state)
1259 {
1260         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1261
1262         if (!CAN_PSR(intel_dp))
1263                 return;
1264
1265         if (!crtc_state->has_psr)
1266                 return;
1267
1268         drm_WARN_ON(&dev_priv->drm, dev_priv->drrs.dp);
1269
1270         mutex_lock(&intel_dp->psr.lock);
1271         intel_psr_enable_locked(intel_dp, crtc_state, conn_state);
1272         mutex_unlock(&intel_dp->psr.lock);
1273 }
1274
1275 static void intel_psr_exit(struct intel_dp *intel_dp)
1276 {
1277         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1278         u32 val;
1279
1280         if (!intel_dp->psr.active) {
1281                 if (transcoder_has_psr2(dev_priv, intel_dp->psr.transcoder)) {
1282                         val = intel_de_read(dev_priv,
1283                                             EDP_PSR2_CTL(intel_dp->psr.transcoder));
1284                         drm_WARN_ON(&dev_priv->drm, val & EDP_PSR2_ENABLE);
1285                 }
1286
1287                 val = intel_de_read(dev_priv,
1288                                     EDP_PSR_CTL(intel_dp->psr.transcoder));
1289                 drm_WARN_ON(&dev_priv->drm, val & EDP_PSR_ENABLE);
1290
1291                 return;
1292         }
1293
1294         if (intel_dp->psr.psr2_enabled) {
1295                 tgl_disallow_dc3co_on_psr2_exit(intel_dp);
1296                 val = intel_de_read(dev_priv,
1297                                     EDP_PSR2_CTL(intel_dp->psr.transcoder));
1298                 drm_WARN_ON(&dev_priv->drm, !(val & EDP_PSR2_ENABLE));
1299                 val &= ~EDP_PSR2_ENABLE;
1300                 intel_de_write(dev_priv,
1301                                EDP_PSR2_CTL(intel_dp->psr.transcoder), val);
1302         } else {
1303                 val = intel_de_read(dev_priv,
1304                                     EDP_PSR_CTL(intel_dp->psr.transcoder));
1305                 drm_WARN_ON(&dev_priv->drm, !(val & EDP_PSR_ENABLE));
1306                 val &= ~EDP_PSR_ENABLE;
1307                 intel_de_write(dev_priv,
1308                                EDP_PSR_CTL(intel_dp->psr.transcoder), val);
1309         }
1310         intel_dp->psr.active = false;
1311 }
1312
1313 static void intel_psr_wait_exit_locked(struct intel_dp *intel_dp)
1314 {
1315         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1316         i915_reg_t psr_status;
1317         u32 psr_status_mask;
1318
1319         if (intel_dp->psr.psr2_enabled) {
1320                 psr_status = EDP_PSR2_STATUS(intel_dp->psr.transcoder);
1321                 psr_status_mask = EDP_PSR2_STATUS_STATE_MASK;
1322         } else {
1323                 psr_status = EDP_PSR_STATUS(intel_dp->psr.transcoder);
1324                 psr_status_mask = EDP_PSR_STATUS_STATE_MASK;
1325         }
1326
1327         /* Wait till PSR is idle */
1328         if (intel_de_wait_for_clear(dev_priv, psr_status,
1329                                     psr_status_mask, 2000))
1330                 drm_err(&dev_priv->drm, "Timed out waiting PSR idle state\n");
1331 }
1332
1333 static void intel_psr_disable_locked(struct intel_dp *intel_dp)
1334 {
1335         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1336
1337         lockdep_assert_held(&intel_dp->psr.lock);
1338
1339         if (!intel_dp->psr.enabled)
1340                 return;
1341
1342         drm_dbg_kms(&dev_priv->drm, "Disabling PSR%s\n",
1343                     intel_dp->psr.psr2_enabled ? "2" : "1");
1344
1345         intel_psr_exit(intel_dp);
1346         intel_psr_wait_exit_locked(intel_dp);
1347
1348         /* Wa_1408330847 */
1349         if (intel_dp->psr.psr2_sel_fetch_enabled &&
1350             IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
1351                 intel_de_rmw(dev_priv, CHICKEN_PAR1_1,
1352                              DIS_RAM_BYPASS_PSR2_MAN_TRACK, 0);
1353
1354         /* Wa_16011168373:adl-p */
1355         if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0) &&
1356             intel_dp->psr.psr2_enabled)
1357                 intel_de_rmw(dev_priv,
1358                              TRANS_SET_CONTEXT_LATENCY(intel_dp->psr.transcoder),
1359                              TRANS_SET_CONTEXT_LATENCY_MASK, 0);
1360
1361         /* Disable PSR on Sink */
1362         drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, 0);
1363
1364         if (intel_dp->psr.psr2_enabled)
1365                 drm_dp_dpcd_writeb(&intel_dp->aux, DP_RECEIVER_ALPM_CONFIG, 0);
1366
1367         intel_dp->psr.enabled = false;
1368 }
1369
1370 /**
1371  * intel_psr_disable - Disable PSR
1372  * @intel_dp: Intel DP
1373  * @old_crtc_state: old CRTC state
1374  *
1375  * This function needs to be called before disabling pipe.
1376  */
1377 void intel_psr_disable(struct intel_dp *intel_dp,
1378                        const struct intel_crtc_state *old_crtc_state)
1379 {
1380         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1381
1382         if (!old_crtc_state->has_psr)
1383                 return;
1384
1385         if (drm_WARN_ON(&dev_priv->drm, !CAN_PSR(intel_dp)))
1386                 return;
1387
1388         mutex_lock(&intel_dp->psr.lock);
1389
1390         intel_psr_disable_locked(intel_dp);
1391
1392         mutex_unlock(&intel_dp->psr.lock);
1393         cancel_work_sync(&intel_dp->psr.work);
1394         cancel_delayed_work_sync(&intel_dp->psr.dc3co_work);
1395 }
1396
1397 /**
1398  * intel_psr_pause - Pause PSR
1399  * @intel_dp: Intel DP
1400  *
1401  * This function need to be called after enabling psr.
1402  */
1403 void intel_psr_pause(struct intel_dp *intel_dp)
1404 {
1405         struct intel_psr *psr = &intel_dp->psr;
1406
1407         if (!CAN_PSR(intel_dp))
1408                 return;
1409
1410         mutex_lock(&psr->lock);
1411
1412         if (!psr->enabled) {
1413                 mutex_unlock(&psr->lock);
1414                 return;
1415         }
1416
1417         intel_psr_exit(intel_dp);
1418         intel_psr_wait_exit_locked(intel_dp);
1419         psr->paused = true;
1420
1421         mutex_unlock(&psr->lock);
1422
1423         cancel_work_sync(&psr->work);
1424         cancel_delayed_work_sync(&psr->dc3co_work);
1425 }
1426
1427 /**
1428  * intel_psr_resume - Resume PSR
1429  * @intel_dp: Intel DP
1430  *
1431  * This function need to be called after pausing psr.
1432  */
1433 void intel_psr_resume(struct intel_dp *intel_dp)
1434 {
1435         struct intel_psr *psr = &intel_dp->psr;
1436
1437         if (!CAN_PSR(intel_dp))
1438                 return;
1439
1440         mutex_lock(&psr->lock);
1441
1442         if (!psr->paused)
1443                 goto unlock;
1444
1445         psr->paused = false;
1446         intel_psr_activate(intel_dp);
1447
1448 unlock:
1449         mutex_unlock(&psr->lock);
1450 }
1451
1452 static void psr_force_hw_tracking_exit(struct intel_dp *intel_dp)
1453 {
1454         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1455
1456         if (DISPLAY_VER(dev_priv) >= 9)
1457                 /*
1458                  * Display WA #0884: skl+
1459                  * This documented WA for bxt can be safely applied
1460                  * broadly so we can force HW tracking to exit PSR
1461                  * instead of disabling and re-enabling.
1462                  * Workaround tells us to write 0 to CUR_SURFLIVE_A,
1463                  * but it makes more sense write to the current active
1464                  * pipe.
1465                  */
1466                 intel_de_write(dev_priv, CURSURFLIVE(intel_dp->psr.pipe), 0);
1467         else
1468                 /*
1469                  * A write to CURSURFLIVE do not cause HW tracking to exit PSR
1470                  * on older gens so doing the manual exit instead.
1471                  */
1472                 intel_psr_exit(intel_dp);
1473 }
1474
1475 void intel_psr2_program_plane_sel_fetch(struct intel_plane *plane,
1476                                         const struct intel_crtc_state *crtc_state,
1477                                         const struct intel_plane_state *plane_state,
1478                                         int color_plane)
1479 {
1480         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1481         enum pipe pipe = plane->pipe;
1482         const struct drm_rect *clip;
1483         u32 val, offset;
1484         int ret, x, y;
1485
1486         if (!crtc_state->enable_psr2_sel_fetch)
1487                 return;
1488
1489         val = plane_state ? plane_state->ctl : 0;
1490         val &= plane->id == PLANE_CURSOR ? val : PLANE_SEL_FETCH_CTL_ENABLE;
1491         intel_de_write_fw(dev_priv, PLANE_SEL_FETCH_CTL(pipe, plane->id), val);
1492         if (!val || plane->id == PLANE_CURSOR)
1493                 return;
1494
1495         clip = &plane_state->psr2_sel_fetch_area;
1496
1497         val = (clip->y1 + plane_state->uapi.dst.y1) << 16;
1498         val |= plane_state->uapi.dst.x1;
1499         intel_de_write_fw(dev_priv, PLANE_SEL_FETCH_POS(pipe, plane->id), val);
1500
1501         /* TODO: consider auxiliary surfaces */
1502         x = plane_state->uapi.src.x1 >> 16;
1503         y = (plane_state->uapi.src.y1 >> 16) + clip->y1;
1504         ret = skl_calc_main_surface_offset(plane_state, &x, &y, &offset);
1505         if (ret)
1506                 drm_warn_once(&dev_priv->drm, "skl_calc_main_surface_offset() returned %i\n",
1507                               ret);
1508         val = y << 16 | x;
1509         intel_de_write_fw(dev_priv, PLANE_SEL_FETCH_OFFSET(pipe, plane->id),
1510                           val);
1511
1512         /* Sizes are 0 based */
1513         val = (drm_rect_height(clip) - 1) << 16;
1514         val |= (drm_rect_width(&plane_state->uapi.src) >> 16) - 1;
1515         intel_de_write_fw(dev_priv, PLANE_SEL_FETCH_SIZE(pipe, plane->id), val);
1516 }
1517
1518 void intel_psr2_program_trans_man_trk_ctl(const struct intel_crtc_state *crtc_state)
1519 {
1520         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1521
1522         if (!HAS_PSR2_SEL_FETCH(dev_priv) ||
1523             !crtc_state->enable_psr2_sel_fetch)
1524                 return;
1525
1526         intel_de_write(dev_priv, PSR2_MAN_TRK_CTL(crtc_state->cpu_transcoder),
1527                        crtc_state->psr2_man_track_ctl);
1528 }
1529
1530 static void psr2_man_trk_ctl_calc(struct intel_crtc_state *crtc_state,
1531                                   struct drm_rect *clip, bool full_update)
1532 {
1533         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1534         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1535         u32 val = PSR2_MAN_TRK_CTL_ENABLE;
1536
1537         if (full_update) {
1538                 if (IS_ALDERLAKE_P(dev_priv))
1539                         val |= ADLP_PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME;
1540                 else
1541                         val |= PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME;
1542
1543                 goto exit;
1544         }
1545
1546         if (clip->y1 == -1)
1547                 goto exit;
1548
1549         if (IS_ALDERLAKE_P(dev_priv)) {
1550                 val |= ADLP_PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR(clip->y1);
1551                 val |= ADLP_PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR(clip->y2);
1552         } else {
1553                 drm_WARN_ON(crtc_state->uapi.crtc->dev, clip->y1 % 4 || clip->y2 % 4);
1554
1555                 val |= PSR2_MAN_TRK_CTL_SF_PARTIAL_FRAME_UPDATE;
1556                 val |= PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR(clip->y1 / 4 + 1);
1557                 val |= PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR(clip->y2 / 4 + 1);
1558         }
1559 exit:
1560         crtc_state->psr2_man_track_ctl = val;
1561 }
1562
1563 static void clip_area_update(struct drm_rect *overlap_damage_area,
1564                              struct drm_rect *damage_area)
1565 {
1566         if (overlap_damage_area->y1 == -1) {
1567                 overlap_damage_area->y1 = damage_area->y1;
1568                 overlap_damage_area->y2 = damage_area->y2;
1569                 return;
1570         }
1571
1572         if (damage_area->y1 < overlap_damage_area->y1)
1573                 overlap_damage_area->y1 = damage_area->y1;
1574
1575         if (damage_area->y2 > overlap_damage_area->y2)
1576                 overlap_damage_area->y2 = damage_area->y2;
1577 }
1578
1579 static void intel_psr2_sel_fetch_pipe_alignment(const struct intel_crtc_state *crtc_state,
1580                                                 struct drm_rect *pipe_clip)
1581 {
1582         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1583         const u16 y_alignment = crtc_state->su_y_granularity;
1584
1585         pipe_clip->y1 -= pipe_clip->y1 % y_alignment;
1586         if (pipe_clip->y2 % y_alignment)
1587                 pipe_clip->y2 = ((pipe_clip->y2 / y_alignment) + 1) * y_alignment;
1588
1589         if (IS_ALDERLAKE_P(dev_priv) && crtc_state->dsc.compression_enable)
1590                 drm_warn(&dev_priv->drm, "Missing PSR2 sel fetch alignment with DSC\n");
1591 }
1592
1593 int intel_psr2_sel_fetch_update(struct intel_atomic_state *state,
1594                                 struct intel_crtc *crtc)
1595 {
1596         struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
1597         struct drm_rect pipe_clip = { .x1 = 0, .y1 = -1, .x2 = INT_MAX, .y2 = -1 };
1598         struct intel_plane_state *new_plane_state, *old_plane_state;
1599         struct intel_plane *plane;
1600         bool full_update = false;
1601         int i, ret;
1602
1603         if (!crtc_state->enable_psr2_sel_fetch)
1604                 return 0;
1605
1606         ret = drm_atomic_add_affected_planes(&state->base, &crtc->base);
1607         if (ret)
1608                 return ret;
1609
1610         /*
1611          * Calculate minimal selective fetch area of each plane and calculate
1612          * the pipe damaged area.
1613          * In the next loop the plane selective fetch area will actually be set
1614          * using whole pipe damaged area.
1615          */
1616         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
1617                                              new_plane_state, i) {
1618                 struct drm_rect src, damaged_area = { .y1 = -1 };
1619                 struct drm_mode_rect *damaged_clips;
1620                 u32 num_clips, j;
1621
1622                 if (new_plane_state->uapi.crtc != crtc_state->uapi.crtc)
1623                         continue;
1624
1625                 if (!new_plane_state->uapi.visible &&
1626                     !old_plane_state->uapi.visible)
1627                         continue;
1628
1629                 /*
1630                  * TODO: Not clear how to handle planes with negative position,
1631                  * also planes are not updated if they have a negative X
1632                  * position so for now doing a full update in this cases
1633                  */
1634                 if (new_plane_state->uapi.dst.y1 < 0 ||
1635                     new_plane_state->uapi.dst.x1 < 0) {
1636                         full_update = true;
1637                         break;
1638                 }
1639
1640                 num_clips = drm_plane_get_damage_clips_count(&new_plane_state->uapi);
1641
1642                 /*
1643                  * If visibility or plane moved, mark the whole plane area as
1644                  * damaged as it needs to be complete redraw in the new and old
1645                  * position.
1646                  */
1647                 if (new_plane_state->uapi.visible != old_plane_state->uapi.visible ||
1648                     !drm_rect_equals(&new_plane_state->uapi.dst,
1649                                      &old_plane_state->uapi.dst)) {
1650                         if (old_plane_state->uapi.visible) {
1651                                 damaged_area.y1 = old_plane_state->uapi.dst.y1;
1652                                 damaged_area.y2 = old_plane_state->uapi.dst.y2;
1653                                 clip_area_update(&pipe_clip, &damaged_area);
1654                         }
1655
1656                         if (new_plane_state->uapi.visible) {
1657                                 damaged_area.y1 = new_plane_state->uapi.dst.y1;
1658                                 damaged_area.y2 = new_plane_state->uapi.dst.y2;
1659                                 clip_area_update(&pipe_clip, &damaged_area);
1660                         }
1661                         continue;
1662                 } else if (new_plane_state->uapi.alpha != old_plane_state->uapi.alpha ||
1663                            (!num_clips &&
1664                             new_plane_state->uapi.fb != old_plane_state->uapi.fb)) {
1665                         /*
1666                          * If the plane don't have damaged areas but the
1667                          * framebuffer changed or alpha changed, mark the whole
1668                          * plane area as damaged.
1669                          */
1670                         damaged_area.y1 = new_plane_state->uapi.dst.y1;
1671                         damaged_area.y2 = new_plane_state->uapi.dst.y2;
1672                         clip_area_update(&pipe_clip, &damaged_area);
1673                         continue;
1674                 }
1675
1676                 drm_rect_fp_to_int(&src, &new_plane_state->uapi.src);
1677                 damaged_clips = drm_plane_get_damage_clips(&new_plane_state->uapi);
1678
1679                 for (j = 0; j < num_clips; j++) {
1680                         struct drm_rect clip;
1681
1682                         clip.x1 = damaged_clips[j].x1;
1683                         clip.y1 = damaged_clips[j].y1;
1684                         clip.x2 = damaged_clips[j].x2;
1685                         clip.y2 = damaged_clips[j].y2;
1686                         if (drm_rect_intersect(&clip, &src))
1687                                 clip_area_update(&damaged_area, &clip);
1688                 }
1689
1690                 if (damaged_area.y1 == -1)
1691                         continue;
1692
1693                 damaged_area.y1 += new_plane_state->uapi.dst.y1 - src.y1;
1694                 damaged_area.y2 += new_plane_state->uapi.dst.y1 - src.y1;
1695                 clip_area_update(&pipe_clip, &damaged_area);
1696         }
1697
1698         if (full_update)
1699                 goto skip_sel_fetch_set_loop;
1700
1701         intel_psr2_sel_fetch_pipe_alignment(crtc_state, &pipe_clip);
1702
1703         /*
1704          * Now that we have the pipe damaged area check if it intersect with
1705          * every plane, if it does set the plane selective fetch area.
1706          */
1707         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
1708                                              new_plane_state, i) {
1709                 struct drm_rect *sel_fetch_area, inter;
1710
1711                 if (new_plane_state->uapi.crtc != crtc_state->uapi.crtc ||
1712                     !new_plane_state->uapi.visible)
1713                         continue;
1714
1715                 inter = pipe_clip;
1716                 if (!drm_rect_intersect(&inter, &new_plane_state->uapi.dst))
1717                         continue;
1718
1719                 sel_fetch_area = &new_plane_state->psr2_sel_fetch_area;
1720                 sel_fetch_area->y1 = inter.y1 - new_plane_state->uapi.dst.y1;
1721                 sel_fetch_area->y2 = inter.y2 - new_plane_state->uapi.dst.y1;
1722         }
1723
1724 skip_sel_fetch_set_loop:
1725         psr2_man_trk_ctl_calc(crtc_state, &pipe_clip, full_update);
1726         return 0;
1727 }
1728
1729 /**
1730  * intel_psr_update - Update PSR state
1731  * @intel_dp: Intel DP
1732  * @crtc_state: new CRTC state
1733  * @conn_state: new CONNECTOR state
1734  *
1735  * This functions will update PSR states, disabling, enabling or switching PSR
1736  * version when executing fastsets. For full modeset, intel_psr_disable() and
1737  * intel_psr_enable() should be called instead.
1738  */
1739 void intel_psr_update(struct intel_dp *intel_dp,
1740                       const struct intel_crtc_state *crtc_state,
1741                       const struct drm_connector_state *conn_state)
1742 {
1743         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1744         struct intel_psr *psr = &intel_dp->psr;
1745         bool enable, psr2_enable;
1746
1747         if (!CAN_PSR(intel_dp))
1748                 return;
1749
1750         mutex_lock(&intel_dp->psr.lock);
1751
1752         enable = crtc_state->has_psr;
1753         psr2_enable = crtc_state->has_psr2;
1754
1755         if (enable == psr->enabled && psr2_enable == psr->psr2_enabled &&
1756             crtc_state->enable_psr2_sel_fetch == psr->psr2_sel_fetch_enabled) {
1757                 /* Force a PSR exit when enabling CRC to avoid CRC timeouts */
1758                 if (crtc_state->crc_enabled && psr->enabled)
1759                         psr_force_hw_tracking_exit(intel_dp);
1760                 else if (DISPLAY_VER(dev_priv) < 9 && psr->enabled) {
1761                         /*
1762                          * Activate PSR again after a force exit when enabling
1763                          * CRC in older gens
1764                          */
1765                         if (!intel_dp->psr.active &&
1766                             !intel_dp->psr.busy_frontbuffer_bits)
1767                                 schedule_work(&intel_dp->psr.work);
1768                 }
1769
1770                 goto unlock;
1771         }
1772
1773         if (psr->enabled)
1774                 intel_psr_disable_locked(intel_dp);
1775
1776         if (enable)
1777                 intel_psr_enable_locked(intel_dp, crtc_state, conn_state);
1778
1779 unlock:
1780         mutex_unlock(&intel_dp->psr.lock);
1781 }
1782
1783 /**
1784  * psr_wait_for_idle - wait for PSR1 to idle
1785  * @intel_dp: Intel DP
1786  * @out_value: PSR status in case of failure
1787  *
1788  * Returns: 0 on success or -ETIMEOUT if PSR status does not idle.
1789  *
1790  */
1791 static int psr_wait_for_idle(struct intel_dp *intel_dp, u32 *out_value)
1792 {
1793         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1794
1795         /*
1796          * From bspec: Panel Self Refresh (BDW+)
1797          * Max. time for PSR to idle = Inverse of the refresh rate + 6 ms of
1798          * exit training time + 1.5 ms of aux channel handshake. 50 ms is
1799          * defensive enough to cover everything.
1800          */
1801         return __intel_wait_for_register(&dev_priv->uncore,
1802                                          EDP_PSR_STATUS(intel_dp->psr.transcoder),
1803                                          EDP_PSR_STATUS_STATE_MASK,
1804                                          EDP_PSR_STATUS_STATE_IDLE, 2, 50,
1805                                          out_value);
1806 }
1807
1808 /**
1809  * intel_psr_wait_for_idle - wait for PSR1 to idle
1810  * @new_crtc_state: new CRTC state
1811  *
1812  * This function is expected to be called from pipe_update_start() where it is
1813  * not expected to race with PSR enable or disable.
1814  */
1815 void intel_psr_wait_for_idle(const struct intel_crtc_state *new_crtc_state)
1816 {
1817         struct drm_i915_private *dev_priv = to_i915(new_crtc_state->uapi.crtc->dev);
1818         struct intel_encoder *encoder;
1819
1820         if (!new_crtc_state->has_psr)
1821                 return;
1822
1823         for_each_intel_encoder_mask_with_psr(&dev_priv->drm, encoder,
1824                                              new_crtc_state->uapi.encoder_mask) {
1825                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
1826                 u32 psr_status;
1827
1828                 mutex_lock(&intel_dp->psr.lock);
1829                 if (!intel_dp->psr.enabled || intel_dp->psr.psr2_enabled) {
1830                         mutex_unlock(&intel_dp->psr.lock);
1831                         continue;
1832                 }
1833
1834                 /* when the PSR1 is enabled */
1835                 if (psr_wait_for_idle(intel_dp, &psr_status))
1836                         drm_err(&dev_priv->drm,
1837                                 "PSR idle timed out 0x%x, atomic update may fail\n",
1838                                 psr_status);
1839                 mutex_unlock(&intel_dp->psr.lock);
1840         }
1841 }
1842
1843 static bool __psr_wait_for_idle_locked(struct intel_dp *intel_dp)
1844 {
1845         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1846         i915_reg_t reg;
1847         u32 mask;
1848         int err;
1849
1850         if (!intel_dp->psr.enabled)
1851                 return false;
1852
1853         if (intel_dp->psr.psr2_enabled) {
1854                 reg = EDP_PSR2_STATUS(intel_dp->psr.transcoder);
1855                 mask = EDP_PSR2_STATUS_STATE_MASK;
1856         } else {
1857                 reg = EDP_PSR_STATUS(intel_dp->psr.transcoder);
1858                 mask = EDP_PSR_STATUS_STATE_MASK;
1859         }
1860
1861         mutex_unlock(&intel_dp->psr.lock);
1862
1863         err = intel_de_wait_for_clear(dev_priv, reg, mask, 50);
1864         if (err)
1865                 drm_err(&dev_priv->drm,
1866                         "Timed out waiting for PSR Idle for re-enable\n");
1867
1868         /* After the unlocked wait, verify that PSR is still wanted! */
1869         mutex_lock(&intel_dp->psr.lock);
1870         return err == 0 && intel_dp->psr.enabled;
1871 }
1872
1873 static int intel_psr_fastset_force(struct drm_i915_private *dev_priv)
1874 {
1875         struct drm_connector_list_iter conn_iter;
1876         struct drm_device *dev = &dev_priv->drm;
1877         struct drm_modeset_acquire_ctx ctx;
1878         struct drm_atomic_state *state;
1879         struct drm_connector *conn;
1880         int err = 0;
1881
1882         state = drm_atomic_state_alloc(dev);
1883         if (!state)
1884                 return -ENOMEM;
1885
1886         drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE);
1887         state->acquire_ctx = &ctx;
1888
1889 retry:
1890
1891         drm_connector_list_iter_begin(dev, &conn_iter);
1892         drm_for_each_connector_iter(conn, &conn_iter) {
1893                 struct drm_connector_state *conn_state;
1894                 struct drm_crtc_state *crtc_state;
1895
1896                 if (conn->connector_type != DRM_MODE_CONNECTOR_eDP)
1897                         continue;
1898
1899                 conn_state = drm_atomic_get_connector_state(state, conn);
1900                 if (IS_ERR(conn_state)) {
1901                         err = PTR_ERR(conn_state);
1902                         break;
1903                 }
1904
1905                 if (!conn_state->crtc)
1906                         continue;
1907
1908                 crtc_state = drm_atomic_get_crtc_state(state, conn_state->crtc);
1909                 if (IS_ERR(crtc_state)) {
1910                         err = PTR_ERR(crtc_state);
1911                         break;
1912                 }
1913
1914                 /* Mark mode as changed to trigger a pipe->update() */
1915                 crtc_state->mode_changed = true;
1916         }
1917         drm_connector_list_iter_end(&conn_iter);
1918
1919         if (err == 0)
1920                 err = drm_atomic_commit(state);
1921
1922         if (err == -EDEADLK) {
1923                 drm_atomic_state_clear(state);
1924                 err = drm_modeset_backoff(&ctx);
1925                 if (!err)
1926                         goto retry;
1927         }
1928
1929         drm_modeset_drop_locks(&ctx);
1930         drm_modeset_acquire_fini(&ctx);
1931         drm_atomic_state_put(state);
1932
1933         return err;
1934 }
1935
1936 int intel_psr_debug_set(struct intel_dp *intel_dp, u64 val)
1937 {
1938         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1939         const u32 mode = val & I915_PSR_DEBUG_MODE_MASK;
1940         u32 old_mode;
1941         int ret;
1942
1943         if (val & ~(I915_PSR_DEBUG_IRQ | I915_PSR_DEBUG_MODE_MASK) ||
1944             mode > I915_PSR_DEBUG_ENABLE_SEL_FETCH) {
1945                 drm_dbg_kms(&dev_priv->drm, "Invalid debug mask %llx\n", val);
1946                 return -EINVAL;
1947         }
1948
1949         ret = mutex_lock_interruptible(&intel_dp->psr.lock);
1950         if (ret)
1951                 return ret;
1952
1953         old_mode = intel_dp->psr.debug & I915_PSR_DEBUG_MODE_MASK;
1954         intel_dp->psr.debug = val;
1955
1956         /*
1957          * Do it right away if it's already enabled, otherwise it will be done
1958          * when enabling the source.
1959          */
1960         if (intel_dp->psr.enabled)
1961                 psr_irq_control(intel_dp);
1962
1963         mutex_unlock(&intel_dp->psr.lock);
1964
1965         if (old_mode != mode)
1966                 ret = intel_psr_fastset_force(dev_priv);
1967
1968         return ret;
1969 }
1970
1971 static void intel_psr_handle_irq(struct intel_dp *intel_dp)
1972 {
1973         struct intel_psr *psr = &intel_dp->psr;
1974
1975         intel_psr_disable_locked(intel_dp);
1976         psr->sink_not_reliable = true;
1977         /* let's make sure that sink is awaken */
1978         drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
1979 }
1980
1981 static void intel_psr_work(struct work_struct *work)
1982 {
1983         struct intel_dp *intel_dp =
1984                 container_of(work, typeof(*intel_dp), psr.work);
1985
1986         mutex_lock(&intel_dp->psr.lock);
1987
1988         if (!intel_dp->psr.enabled)
1989                 goto unlock;
1990
1991         if (READ_ONCE(intel_dp->psr.irq_aux_error))
1992                 intel_psr_handle_irq(intel_dp);
1993
1994         /*
1995          * We have to make sure PSR is ready for re-enable
1996          * otherwise it keeps disabled until next full enable/disable cycle.
1997          * PSR might take some time to get fully disabled
1998          * and be ready for re-enable.
1999          */
2000         if (!__psr_wait_for_idle_locked(intel_dp))
2001                 goto unlock;
2002
2003         /*
2004          * The delayed work can race with an invalidate hence we need to
2005          * recheck. Since psr_flush first clears this and then reschedules we
2006          * won't ever miss a flush when bailing out here.
2007          */
2008         if (intel_dp->psr.busy_frontbuffer_bits || intel_dp->psr.active)
2009                 goto unlock;
2010
2011         intel_psr_activate(intel_dp);
2012 unlock:
2013         mutex_unlock(&intel_dp->psr.lock);
2014 }
2015
2016 /**
2017  * intel_psr_invalidate - Invalidade PSR
2018  * @dev_priv: i915 device
2019  * @frontbuffer_bits: frontbuffer plane tracking bits
2020  * @origin: which operation caused the invalidate
2021  *
2022  * Since the hardware frontbuffer tracking has gaps we need to integrate
2023  * with the software frontbuffer tracking. This function gets called every
2024  * time frontbuffer rendering starts and a buffer gets dirtied. PSR must be
2025  * disabled if the frontbuffer mask contains a buffer relevant to PSR.
2026  *
2027  * Dirty frontbuffers relevant to PSR are tracked in busy_frontbuffer_bits."
2028  */
2029 void intel_psr_invalidate(struct drm_i915_private *dev_priv,
2030                           unsigned frontbuffer_bits, enum fb_op_origin origin)
2031 {
2032         struct intel_encoder *encoder;
2033
2034         if (origin == ORIGIN_FLIP)
2035                 return;
2036
2037         for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2038                 unsigned int pipe_frontbuffer_bits = frontbuffer_bits;
2039                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2040
2041                 mutex_lock(&intel_dp->psr.lock);
2042                 if (!intel_dp->psr.enabled) {
2043                         mutex_unlock(&intel_dp->psr.lock);
2044                         continue;
2045                 }
2046
2047                 pipe_frontbuffer_bits &=
2048                         INTEL_FRONTBUFFER_ALL_MASK(intel_dp->psr.pipe);
2049                 intel_dp->psr.busy_frontbuffer_bits |= pipe_frontbuffer_bits;
2050
2051                 if (pipe_frontbuffer_bits)
2052                         intel_psr_exit(intel_dp);
2053
2054                 mutex_unlock(&intel_dp->psr.lock);
2055         }
2056 }
2057 /*
2058  * When we will be completely rely on PSR2 S/W tracking in future,
2059  * intel_psr_flush() will invalidate and flush the PSR for ORIGIN_FLIP
2060  * event also therefore tgl_dc3co_flush() require to be changed
2061  * accordingly in future.
2062  */
2063 static void
2064 tgl_dc3co_flush(struct intel_dp *intel_dp, unsigned int frontbuffer_bits,
2065                 enum fb_op_origin origin)
2066 {
2067         mutex_lock(&intel_dp->psr.lock);
2068
2069         if (!intel_dp->psr.dc3co_exitline)
2070                 goto unlock;
2071
2072         if (!intel_dp->psr.psr2_enabled || !intel_dp->psr.active)
2073                 goto unlock;
2074
2075         /*
2076          * At every frontbuffer flush flip event modified delay of delayed work,
2077          * when delayed work schedules that means display has been idle.
2078          */
2079         if (!(frontbuffer_bits &
2080             INTEL_FRONTBUFFER_ALL_MASK(intel_dp->psr.pipe)))
2081                 goto unlock;
2082
2083         tgl_psr2_enable_dc3co(intel_dp);
2084         mod_delayed_work(system_wq, &intel_dp->psr.dc3co_work,
2085                          intel_dp->psr.dc3co_exit_delay);
2086
2087 unlock:
2088         mutex_unlock(&intel_dp->psr.lock);
2089 }
2090
2091 /**
2092  * intel_psr_flush - Flush PSR
2093  * @dev_priv: i915 device
2094  * @frontbuffer_bits: frontbuffer plane tracking bits
2095  * @origin: which operation caused the flush
2096  *
2097  * Since the hardware frontbuffer tracking has gaps we need to integrate
2098  * with the software frontbuffer tracking. This function gets called every
2099  * time frontbuffer rendering has completed and flushed out to memory. PSR
2100  * can be enabled again if no other frontbuffer relevant to PSR is dirty.
2101  *
2102  * Dirty frontbuffers relevant to PSR are tracked in busy_frontbuffer_bits.
2103  */
2104 void intel_psr_flush(struct drm_i915_private *dev_priv,
2105                      unsigned frontbuffer_bits, enum fb_op_origin origin)
2106 {
2107         struct intel_encoder *encoder;
2108
2109         for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2110                 unsigned int pipe_frontbuffer_bits = frontbuffer_bits;
2111                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2112
2113                 if (origin == ORIGIN_FLIP) {
2114                         tgl_dc3co_flush(intel_dp, frontbuffer_bits, origin);
2115                         continue;
2116                 }
2117
2118                 mutex_lock(&intel_dp->psr.lock);
2119                 if (!intel_dp->psr.enabled) {
2120                         mutex_unlock(&intel_dp->psr.lock);
2121                         continue;
2122                 }
2123
2124                 pipe_frontbuffer_bits &=
2125                         INTEL_FRONTBUFFER_ALL_MASK(intel_dp->psr.pipe);
2126                 intel_dp->psr.busy_frontbuffer_bits &= ~pipe_frontbuffer_bits;
2127
2128                 /*
2129                  * If the PSR is paused by an explicit intel_psr_paused() call,
2130                  * we have to ensure that the PSR is not activated until
2131                  * intel_psr_resume() is called.
2132                  */
2133                 if (intel_dp->psr.paused) {
2134                         mutex_unlock(&intel_dp->psr.lock);
2135                         continue;
2136                 }
2137
2138                 /* By definition flush = invalidate + flush */
2139                 if (pipe_frontbuffer_bits)
2140                         psr_force_hw_tracking_exit(intel_dp);
2141
2142                 if (!intel_dp->psr.active && !intel_dp->psr.busy_frontbuffer_bits)
2143                         schedule_work(&intel_dp->psr.work);
2144                 mutex_unlock(&intel_dp->psr.lock);
2145         }
2146 }
2147
2148 /**
2149  * intel_psr_init - Init basic PSR work and mutex.
2150  * @intel_dp: Intel DP
2151  *
2152  * This function is called after the initializing connector.
2153  * (the initializing of connector treats the handling of connector capabilities)
2154  * And it initializes basic PSR stuff for each DP Encoder.
2155  */
2156 void intel_psr_init(struct intel_dp *intel_dp)
2157 {
2158         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
2159         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2160
2161         if (!HAS_PSR(dev_priv))
2162                 return;
2163
2164         /*
2165          * HSW spec explicitly says PSR is tied to port A.
2166          * BDW+ platforms have a instance of PSR registers per transcoder but
2167          * BDW, GEN9 and GEN11 are not validated by HW team in other transcoder
2168          * than eDP one.
2169          * For now it only supports one instance of PSR for BDW, GEN9 and GEN11.
2170          * So lets keep it hardcoded to PORT_A for BDW, GEN9 and GEN11.
2171          * But GEN12 supports a instance of PSR registers per transcoder.
2172          */
2173         if (DISPLAY_VER(dev_priv) < 12 && dig_port->base.port != PORT_A) {
2174                 drm_dbg_kms(&dev_priv->drm,
2175                             "PSR condition failed: Port not supported\n");
2176                 return;
2177         }
2178
2179         intel_dp->psr.source_support = true;
2180
2181         if (IS_HASWELL(dev_priv))
2182                 /*
2183                  * HSW don't have PSR registers on the same space as transcoder
2184                  * so set this to a value that when subtract to the register
2185                  * in transcoder space results in the right offset for HSW
2186                  */
2187                 dev_priv->hsw_psr_mmio_adjust = _SRD_CTL_EDP - _HSW_EDP_PSR_BASE;
2188
2189         if (dev_priv->params.enable_psr == -1)
2190                 if (DISPLAY_VER(dev_priv) < 9 || !dev_priv->vbt.psr.enable)
2191                         dev_priv->params.enable_psr = 0;
2192
2193         /* Set link_standby x link_off defaults */
2194         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2195                 /* HSW and BDW require workarounds that we don't implement. */
2196                 intel_dp->psr.link_standby = false;
2197         else if (DISPLAY_VER(dev_priv) < 12)
2198                 /* For new platforms up to TGL let's respect VBT back again */
2199                 intel_dp->psr.link_standby = dev_priv->vbt.psr.full_link;
2200
2201         INIT_WORK(&intel_dp->psr.work, intel_psr_work);
2202         INIT_DELAYED_WORK(&intel_dp->psr.dc3co_work, tgl_dc3co_disable_work);
2203         mutex_init(&intel_dp->psr.lock);
2204 }
2205
2206 static int psr_get_status_and_error_status(struct intel_dp *intel_dp,
2207                                            u8 *status, u8 *error_status)
2208 {
2209         struct drm_dp_aux *aux = &intel_dp->aux;
2210         int ret;
2211
2212         ret = drm_dp_dpcd_readb(aux, DP_PSR_STATUS, status);
2213         if (ret != 1)
2214                 return ret;
2215
2216         ret = drm_dp_dpcd_readb(aux, DP_PSR_ERROR_STATUS, error_status);
2217         if (ret != 1)
2218                 return ret;
2219
2220         *status = *status & DP_PSR_SINK_STATE_MASK;
2221
2222         return 0;
2223 }
2224
2225 static void psr_alpm_check(struct intel_dp *intel_dp)
2226 {
2227         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2228         struct drm_dp_aux *aux = &intel_dp->aux;
2229         struct intel_psr *psr = &intel_dp->psr;
2230         u8 val;
2231         int r;
2232
2233         if (!psr->psr2_enabled)
2234                 return;
2235
2236         r = drm_dp_dpcd_readb(aux, DP_RECEIVER_ALPM_STATUS, &val);
2237         if (r != 1) {
2238                 drm_err(&dev_priv->drm, "Error reading ALPM status\n");
2239                 return;
2240         }
2241
2242         if (val & DP_ALPM_LOCK_TIMEOUT_ERROR) {
2243                 intel_psr_disable_locked(intel_dp);
2244                 psr->sink_not_reliable = true;
2245                 drm_dbg_kms(&dev_priv->drm,
2246                             "ALPM lock timeout error, disabling PSR\n");
2247
2248                 /* Clearing error */
2249                 drm_dp_dpcd_writeb(aux, DP_RECEIVER_ALPM_STATUS, val);
2250         }
2251 }
2252
2253 static void psr_capability_changed_check(struct intel_dp *intel_dp)
2254 {
2255         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2256         struct intel_psr *psr = &intel_dp->psr;
2257         u8 val;
2258         int r;
2259
2260         r = drm_dp_dpcd_readb(&intel_dp->aux, DP_PSR_ESI, &val);
2261         if (r != 1) {
2262                 drm_err(&dev_priv->drm, "Error reading DP_PSR_ESI\n");
2263                 return;
2264         }
2265
2266         if (val & DP_PSR_CAPS_CHANGE) {
2267                 intel_psr_disable_locked(intel_dp);
2268                 psr->sink_not_reliable = true;
2269                 drm_dbg_kms(&dev_priv->drm,
2270                             "Sink PSR capability changed, disabling PSR\n");
2271
2272                 /* Clearing it */
2273                 drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_ESI, val);
2274         }
2275 }
2276
2277 void intel_psr_short_pulse(struct intel_dp *intel_dp)
2278 {
2279         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2280         struct intel_psr *psr = &intel_dp->psr;
2281         u8 status, error_status;
2282         const u8 errors = DP_PSR_RFB_STORAGE_ERROR |
2283                           DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR |
2284                           DP_PSR_LINK_CRC_ERROR;
2285
2286         if (!CAN_PSR(intel_dp))
2287                 return;
2288
2289         mutex_lock(&psr->lock);
2290
2291         if (!psr->enabled)
2292                 goto exit;
2293
2294         if (psr_get_status_and_error_status(intel_dp, &status, &error_status)) {
2295                 drm_err(&dev_priv->drm,
2296                         "Error reading PSR status or error status\n");
2297                 goto exit;
2298         }
2299
2300         if (status == DP_PSR_SINK_INTERNAL_ERROR || (error_status & errors)) {
2301                 intel_psr_disable_locked(intel_dp);
2302                 psr->sink_not_reliable = true;
2303         }
2304
2305         if (status == DP_PSR_SINK_INTERNAL_ERROR && !error_status)
2306                 drm_dbg_kms(&dev_priv->drm,
2307                             "PSR sink internal error, disabling PSR\n");
2308         if (error_status & DP_PSR_RFB_STORAGE_ERROR)
2309                 drm_dbg_kms(&dev_priv->drm,
2310                             "PSR RFB storage error, disabling PSR\n");
2311         if (error_status & DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR)
2312                 drm_dbg_kms(&dev_priv->drm,
2313                             "PSR VSC SDP uncorrectable error, disabling PSR\n");
2314         if (error_status & DP_PSR_LINK_CRC_ERROR)
2315                 drm_dbg_kms(&dev_priv->drm,
2316                             "PSR Link CRC error, disabling PSR\n");
2317
2318         if (error_status & ~errors)
2319                 drm_err(&dev_priv->drm,
2320                         "PSR_ERROR_STATUS unhandled errors %x\n",
2321                         error_status & ~errors);
2322         /* clear status register */
2323         drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_ERROR_STATUS, error_status);
2324
2325         psr_alpm_check(intel_dp);
2326         psr_capability_changed_check(intel_dp);
2327
2328 exit:
2329         mutex_unlock(&psr->lock);
2330 }
2331
2332 bool intel_psr_enabled(struct intel_dp *intel_dp)
2333 {
2334         bool ret;
2335
2336         if (!CAN_PSR(intel_dp))
2337                 return false;
2338
2339         mutex_lock(&intel_dp->psr.lock);
2340         ret = intel_dp->psr.enabled;
2341         mutex_unlock(&intel_dp->psr.lock);
2342
2343         return ret;
2344 }