drivers/gpu/drm/i915/display/intel_crtc.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2020 Intel Corporation
   4  */
   5 #include <linux/kernel.h>
   6 #include <linux/pm_qos.h>
   7 #include <linux/slab.h>
   8
   9 #include <drm/drm_atomic_helper.h>
  10 #include <drm/drm_fourcc.h>
  11 #include <drm/drm_plane.h>
  12 #include <drm/drm_vblank_work.h>
  13
  14 #include "i915_vgpu.h"
  15 #include "i9xx_plane.h"
  16 #include "icl_dsi.h"
  17 #include "intel_atomic.h"
  18 #include "intel_atomic_plane.h"
  19 #include "intel_color.h"
  20 #include "intel_crtc.h"
  21 #include "intel_cursor.h"
  22 #include "intel_display_debugfs.h"
  23 #include "intel_display_irq.h"
  24 #include "intel_display_trace.h"
  25 #include "intel_display_types.h"
  26 #include "intel_drrs.h"
  27 #include "intel_dsi.h"
  28 #include "intel_fifo_underrun.h"
  29 #include "intel_pipe_crc.h"
  30 #include "intel_psr.h"
  31 #include "intel_sprite.h"
  32 #include "intel_vblank.h"
  33 #include "intel_vrr.h"
  34 #include "skl_universal_plane.h"
  35
  36 static void assert_vblank_disabled(struct drm_crtc *crtc)
  37 {
  38         struct drm_i915_private *i915 = to_i915(crtc->dev);
  39
  40         if (I915_STATE_WARN(i915, drm_crtc_vblank_get(crtc) == 0,
  41                             "[CRTC:%d:%s] vblank assertion failure (expected off, current on)\n",
  42                             crtc->base.id, crtc->name))
  43                 drm_crtc_vblank_put(crtc);
  44 }
  45
  46 struct intel_crtc *intel_first_crtc(struct drm_i915_private *i915)
  47 {
  48         return to_intel_crtc(drm_crtc_from_index(&i915->drm, 0));
  49 }
  50
  51 struct intel_crtc *intel_crtc_for_pipe(struct drm_i915_private *i915,
  52                                        enum pipe pipe)
  53 {
  54         struct intel_crtc *crtc;
  55
  56         for_each_intel_crtc(&i915->drm, crtc) {
  57                 if (crtc->pipe == pipe)
  58                         return crtc;
  59         }
  60
  61         return NULL;
  62 }
  63
  64 void intel_crtc_wait_for_next_vblank(struct intel_crtc *crtc)
  65 {
  66         drm_crtc_wait_one_vblank(&crtc->base);
  67 }
  68
  69 void intel_wait_for_vblank_if_active(struct drm_i915_private *i915,
  70                                      enum pipe pipe)
  71 {
  72         struct intel_crtc *crtc = intel_crtc_for_pipe(i915, pipe);
  73
  74         if (crtc->active)
  75                 intel_crtc_wait_for_next_vblank(crtc);
  76 }
  77
  78 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
  79 {
  80         struct drm_device *dev = crtc->base.dev;
  81         struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)];
  82
  83         if (!crtc->active)
  84                 return 0;
  85
  86         if (!vblank->max_vblank_count)
  87                 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
  88
  89         return crtc->base.funcs->get_vblank_counter(&crtc->base);
  90 }
  91
  92 u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
  93 {
  94         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
  95
  96         /*
  97          * From Gen 11, In case of dsi cmd mode, frame counter wouldnt
  98          * have updated at the beginning of TE, if we want to use
  99          * the hw counter, then we would find it updated in only
 100          * the next TE, hence switching to sw counter.
 101          */
 102         if (crtc_state->mode_flags & (I915_MODE_FLAG_DSI_USE_TE0 |
 103                                       I915_MODE_FLAG_DSI_USE_TE1))
 104                 return 0;
 105
 106         /*
 107          * On i965gm the hardware frame counter reads
 108          * zero when the TV encoder is enabled :(
 109          */
 110         if (IS_I965GM(dev_priv) &&
 111             (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
 112                 return 0;
 113
 114         if (DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv))
 115                 return 0xffffffff; /* full 32 bit counter */
 116         else if (DISPLAY_VER(dev_priv) >= 3)
 117                 return 0xffffff; /* only 24 bits of frame count */
 118         else
 119                 return 0; /* Gen2 doesn't have a hardware frame counter */
 120 }
 121
 122 void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
 123 {
 124         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 125
 126         assert_vblank_disabled(&crtc->base);
 127         drm_crtc_set_max_vblank_count(&crtc->base,
 128                                       intel_crtc_max_vblank_count(crtc_state));
 129         drm_crtc_vblank_on(&crtc->base);
 130
 131         /*
 132          * Should really happen exactly when we enable the pipe
 133          * but we want the frame counters in the trace, and that
 134          * requires vblank support on some platforms/outputs.
 135          */
 136         trace_intel_pipe_enable(crtc);
 137 }
 138
 139 void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
 140 {
 141         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 142
 143         /*
 144          * Should really happen exactly when we disable the pipe
 145          * but we want the frame counters in the trace, and that
 146          * requires vblank support on some platforms/outputs.
 147          */
 148         trace_intel_pipe_disable(crtc);
 149
 150         drm_crtc_vblank_off(&crtc->base);
 151         assert_vblank_disabled(&crtc->base);
 152 }
 153
 154 struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
 155 {
 156         struct intel_crtc_state *crtc_state;
 157
 158         crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
 159
 160         if (crtc_state)
 161                 intel_crtc_state_reset(crtc_state, crtc);
 162
 163         return crtc_state;
 164 }
 165
 166 void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
 167                             struct intel_crtc *crtc)
 168 {
 169         memset(crtc_state, 0, sizeof(*crtc_state));
 170
 171         __drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
 172
 173         crtc_state->cpu_transcoder = INVALID_TRANSCODER;
 174         crtc_state->master_transcoder = INVALID_TRANSCODER;
 175         crtc_state->hsw_workaround_pipe = INVALID_PIPE;
 176         crtc_state->scaler_state.scaler_id = -1;
 177         crtc_state->mst_master_transcoder = INVALID_TRANSCODER;
 178 }
 179
 180 static struct intel_crtc *intel_crtc_alloc(void)
 181 {
 182         struct intel_crtc_state *crtc_state;
 183         struct intel_crtc *crtc;
 184
 185         crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
 186         if (!crtc)
 187                 return ERR_PTR(-ENOMEM);
 188
 189         crtc_state = intel_crtc_state_alloc(crtc);
 190         if (!crtc_state) {
 191                 kfree(crtc);
 192                 return ERR_PTR(-ENOMEM);
 193         }
 194
 195         crtc->base.state = &crtc_state->uapi;
 196         crtc->config = crtc_state;
 197
 198         return crtc;
 199 }
 200
 201 static void intel_crtc_free(struct intel_crtc *crtc)
 202 {
 203         intel_crtc_destroy_state(&crtc->base, crtc->base.state);
 204         kfree(crtc);
 205 }
 206
 207 static void intel_crtc_destroy(struct drm_crtc *_crtc)
 208 {
 209         struct intel_crtc *crtc = to_intel_crtc(_crtc);
 210
 211         cpu_latency_qos_remove_request(&crtc->vblank_pm_qos);
 212
 213         drm_crtc_cleanup(&crtc->base);
 214         kfree(crtc);
 215 }
 216
 217 static int intel_crtc_late_register(struct drm_crtc *crtc)
 218 {
 219         intel_crtc_debugfs_add(to_intel_crtc(crtc));
 220         return 0;
 221 }
 222
 223 #define INTEL_CRTC_FUNCS \
 224         .set_config = drm_atomic_helper_set_config, \
 225         .destroy = intel_crtc_destroy, \
 226         .page_flip = drm_atomic_helper_page_flip, \
 227         .atomic_duplicate_state = intel_crtc_duplicate_state, \
 228         .atomic_destroy_state = intel_crtc_destroy_state, \
 229         .set_crc_source = intel_crtc_set_crc_source, \
 230         .verify_crc_source = intel_crtc_verify_crc_source, \
 231         .get_crc_sources = intel_crtc_get_crc_sources, \
 232         .late_register = intel_crtc_late_register
 233
 234 static const struct drm_crtc_funcs bdw_crtc_funcs = {
 235         INTEL_CRTC_FUNCS,
 236
 237         .get_vblank_counter = g4x_get_vblank_counter,
 238         .enable_vblank = bdw_enable_vblank,
 239         .disable_vblank = bdw_disable_vblank,
 240         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 241 };
 242
 243 static const struct drm_crtc_funcs ilk_crtc_funcs = {
 244         INTEL_CRTC_FUNCS,
 245
 246         .get_vblank_counter = g4x_get_vblank_counter,
 247         .enable_vblank = ilk_enable_vblank,
 248         .disable_vblank = ilk_disable_vblank,
 249         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 250 };
 251
 252 static const struct drm_crtc_funcs g4x_crtc_funcs = {
 253         INTEL_CRTC_FUNCS,
 254
 255         .get_vblank_counter = g4x_get_vblank_counter,
 256         .enable_vblank = i965_enable_vblank,
 257         .disable_vblank = i965_disable_vblank,
 258         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 259 };
 260
 261 static const struct drm_crtc_funcs i965_crtc_funcs = {
 262         INTEL_CRTC_FUNCS,
 263
 264         .get_vblank_counter = i915_get_vblank_counter,
 265         .enable_vblank = i965_enable_vblank,
 266         .disable_vblank = i965_disable_vblank,
 267         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 268 };
 269
 270 static const struct drm_crtc_funcs i915gm_crtc_funcs = {
 271         INTEL_CRTC_FUNCS,
 272
 273         .get_vblank_counter = i915_get_vblank_counter,
 274         .enable_vblank = i915gm_enable_vblank,
 275         .disable_vblank = i915gm_disable_vblank,
 276         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 277 };
 278
 279 static const struct drm_crtc_funcs i915_crtc_funcs = {
 280         INTEL_CRTC_FUNCS,
 281
 282         .get_vblank_counter = i915_get_vblank_counter,
 283         .enable_vblank = i8xx_enable_vblank,
 284         .disable_vblank = i8xx_disable_vblank,
 285         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 286 };
 287
 288 static const struct drm_crtc_funcs i8xx_crtc_funcs = {
 289         INTEL_CRTC_FUNCS,
 290
 291         /* no hw vblank counter */
 292         .enable_vblank = i8xx_enable_vblank,
 293         .disable_vblank = i8xx_disable_vblank,
 294         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
 295 };
 296
 297 int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
 298 {
 299         struct intel_plane *primary, *cursor;
 300         const struct drm_crtc_funcs *funcs;
 301         struct intel_crtc *crtc;
 302         int sprite, ret;
 303
 304         crtc = intel_crtc_alloc();
 305         if (IS_ERR(crtc))
 306                 return PTR_ERR(crtc);
 307
 308         crtc->pipe = pipe;
 309         crtc->num_scalers = DISPLAY_RUNTIME_INFO(dev_priv)->num_scalers[pipe];
 310
 311         if (DISPLAY_VER(dev_priv) >= 9)
 312                 primary = skl_universal_plane_create(dev_priv, pipe,
 313                                                      PLANE_PRIMARY);
 314         else
 315                 primary = intel_primary_plane_create(dev_priv, pipe);
 316         if (IS_ERR(primary)) {
 317                 ret = PTR_ERR(primary);
 318                 goto fail;
 319         }
 320         crtc->plane_ids_mask |= BIT(primary->id);
 321
 322         intel_init_fifo_underrun_reporting(dev_priv, crtc, false);
 323
 324         for_each_sprite(dev_priv, pipe, sprite) {
 325                 struct intel_plane *plane;
 326
 327                 if (DISPLAY_VER(dev_priv) >= 9)
 328                         plane = skl_universal_plane_create(dev_priv, pipe,
 329                                                            PLANE_SPRITE0 + sprite);
 330                 else
 331                         plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
 332                 if (IS_ERR(plane)) {
 333                         ret = PTR_ERR(plane);
 334                         goto fail;
 335                 }
 336                 crtc->plane_ids_mask |= BIT(plane->id);
 337         }
 338
 339         cursor = intel_cursor_plane_create(dev_priv, pipe);
 340         if (IS_ERR(cursor)) {
 341                 ret = PTR_ERR(cursor);
 342                 goto fail;
 343         }
 344         crtc->plane_ids_mask |= BIT(cursor->id);
 345
 346         if (HAS_GMCH(dev_priv)) {
 347                 if (IS_CHERRYVIEW(dev_priv) ||
 348                     IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
 349                         funcs = &g4x_crtc_funcs;
 350                 else if (DISPLAY_VER(dev_priv) == 4)
 351                         funcs = &i965_crtc_funcs;
 352                 else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv))
 353                         funcs = &i915gm_crtc_funcs;
 354                 else if (DISPLAY_VER(dev_priv) == 3)
 355                         funcs = &i915_crtc_funcs;
 356                 else
 357                         funcs = &i8xx_crtc_funcs;
 358         } else {
 359                 if (DISPLAY_VER(dev_priv) >= 8)
 360                         funcs = &bdw_crtc_funcs;
 361                 else
 362                         funcs = &ilk_crtc_funcs;
 363         }
 364
 365         ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base,
 366                                         &primary->base, &cursor->base,
 367                                         funcs, "pipe %c", pipe_name(pipe));
 368         if (ret)
 369                 goto fail;
 370
 371         if (DISPLAY_VER(dev_priv) >= 11)
 372                 drm_crtc_create_scaling_filter_property(&crtc->base,
 373                                                 BIT(DRM_SCALING_FILTER_DEFAULT) |
 374                                                 BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR));
 375
 376         intel_color_crtc_init(crtc);
 377         intel_drrs_crtc_init(crtc);
 378         intel_crtc_crc_init(crtc);
 379
 380         cpu_latency_qos_add_request(&crtc->vblank_pm_qos, PM_QOS_DEFAULT_VALUE);
 381
 382         drm_WARN_ON(&dev_priv->drm, drm_crtc_index(&crtc->base) != crtc->pipe);
 383
 384         return 0;
 385
 386 fail:
 387         intel_crtc_free(crtc);
 388
 389         return ret;
 390 }
 391
 392 static bool intel_crtc_needs_vblank_work(const struct intel_crtc_state *crtc_state)
 393 {
 394         return crtc_state->hw.active &&
 395                 !intel_crtc_needs_modeset(crtc_state) &&
 396                 !crtc_state->preload_luts &&
 397                 intel_crtc_needs_color_update(crtc_state);
 398 }
 399
 400 static void intel_crtc_vblank_work(struct kthread_work *base)
 401 {
 402         struct drm_vblank_work *work = to_drm_vblank_work(base);
 403         struct intel_crtc_state *crtc_state =
 404                 container_of(work, typeof(*crtc_state), vblank_work);
 405         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 406
 407         trace_intel_crtc_vblank_work_start(crtc);
 408
 409         intel_color_load_luts(crtc_state);
 410
 411         if (crtc_state->uapi.event) {
 412                 spin_lock_irq(&crtc->base.dev->event_lock);
 413                 drm_crtc_send_vblank_event(&crtc->base, crtc_state->uapi.event);
 414                 crtc_state->uapi.event = NULL;
 415                 spin_unlock_irq(&crtc->base.dev->event_lock);
 416         }
 417
 418         trace_intel_crtc_vblank_work_end(crtc);
 419 }
 420
 421 static void intel_crtc_vblank_work_init(struct intel_crtc_state *crtc_state)
 422 {
 423         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 424
 425         drm_vblank_work_init(&crtc_state->vblank_work, &crtc->base,
 426                              intel_crtc_vblank_work);
 427         /*
 428          * Interrupt latency is critical for getting the vblank
 429          * work executed as early as possible during the vblank.
 430          */
 431         cpu_latency_qos_update_request(&crtc->vblank_pm_qos, 0);
 432 }
 433
 434 void intel_wait_for_vblank_workers(struct intel_atomic_state *state)
 435 {
 436         struct intel_crtc_state *crtc_state;
 437         struct intel_crtc *crtc;
 438         int i;
 439
 440         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
 441                 if (!intel_crtc_needs_vblank_work(crtc_state))
 442                         continue;
 443
 444                 drm_vblank_work_flush(&crtc_state->vblank_work);
 445                 cpu_latency_qos_update_request(&crtc->vblank_pm_qos,
 446                                                PM_QOS_DEFAULT_VALUE);
 447         }
 448 }
 449
 450 int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
 451                              int usecs)
 452 {
 453         /* paranoia */
 454         if (!adjusted_mode->crtc_htotal)
 455                 return 1;
 456
 457         return DIV_ROUND_UP(usecs * adjusted_mode->crtc_clock,
 458                             1000 * adjusted_mode->crtc_htotal);
 459 }
 460
 461 static int intel_mode_vblank_start(const struct drm_display_mode *mode)
 462 {
 463         int vblank_start = mode->crtc_vblank_start;
 464
 465         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 466                 vblank_start = DIV_ROUND_UP(vblank_start, 2);
 467
 468         return vblank_start;
 469 }
 470
 471 /**
 472  * intel_pipe_update_start() - start update of a set of display registers
 473  * @new_crtc_state: the new crtc state
 474  *
 475  * Mark the start of an update to pipe registers that should be updated
 476  * atomically regarding vblank. If the next vblank will happens within
 477  * the next 100 us, this function waits until the vblank passes.
 478  *
 479  * After a successful call to this function, interrupts will be disabled
 480  * until a subsequent call to intel_pipe_update_end(). That is done to
 481  * avoid random delays.
 482  */
 483 void intel_pipe_update_start(struct intel_crtc_state *new_crtc_state)
 484 {
 485         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
 486         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 487         const struct drm_display_mode *adjusted_mode = &new_crtc_state->hw.adjusted_mode;
 488         long timeout = msecs_to_jiffies_timeout(1);
 489         int scanline, min, max, vblank_start;
 490         wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base);
 491         bool need_vlv_dsi_wa = (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
 492                 intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI);
 493         DEFINE_WAIT(wait);
 494
 495         intel_psr_lock(new_crtc_state);
 496
 497         if (new_crtc_state->do_async_flip)
 498                 return;
 499
 500         if (intel_crtc_needs_vblank_work(new_crtc_state))
 501                 intel_crtc_vblank_work_init(new_crtc_state);
 502
 503         if (new_crtc_state->vrr.enable) {
 504                 if (intel_vrr_is_push_sent(new_crtc_state))
 505                         vblank_start = intel_vrr_vmin_vblank_start(new_crtc_state);
 506                 else
 507                         vblank_start = intel_vrr_vmax_vblank_start(new_crtc_state);
 508         } else {
 509                 vblank_start = intel_mode_vblank_start(adjusted_mode);
 510         }
 511
 512         /* FIXME needs to be calibrated sensibly */
 513         min = vblank_start - intel_usecs_to_scanlines(adjusted_mode,
 514                                                       VBLANK_EVASION_TIME_US);
 515         max = vblank_start - 1;
 516
 517         /*
 518          * M/N is double buffered on the transcoder's undelayed vblank,
 519          * so with seamless M/N we must evade both vblanks.
 520          */
 521         if (new_crtc_state->seamless_m_n && intel_crtc_needs_fastset(new_crtc_state))
 522                 min -= adjusted_mode->crtc_vblank_start - adjusted_mode->crtc_vdisplay;
 523
 524         if (min <= 0 || max <= 0)
 525                 goto irq_disable;
 526
 527         if (drm_WARN_ON(&dev_priv->drm, drm_crtc_vblank_get(&crtc->base)))
 528                 goto irq_disable;
 529
 530         /*
 531          * Wait for psr to idle out after enabling the VBL interrupts
 532          * VBL interrupts will start the PSR exit and prevent a PSR
 533          * re-entry as well.
 534          */
 535         intel_psr_wait_for_idle_locked(new_crtc_state);
 536
 537         local_irq_disable();
 538
 539         crtc->debug.min_vbl = min;
 540         crtc->debug.max_vbl = max;
 541         trace_intel_pipe_update_start(crtc);
 542
 543         for (;;) {
 544                 /*
 545                  * prepare_to_wait() has a memory barrier, which guarantees
 546                  * other CPUs can see the task state update by the time we
 547                  * read the scanline.
 548                  */
 549                 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
 550
 551                 scanline = intel_get_crtc_scanline(crtc);
 552                 if (scanline < min || scanline > max)
 553                         break;
 554
 555                 if (!timeout) {
 556                         drm_err(&dev_priv->drm,
 557                                 "Potential atomic update failure on pipe %c\n",
 558                                 pipe_name(crtc->pipe));
 559                         break;
 560                 }
 561
 562                 local_irq_enable();
 563
 564                 timeout = schedule_timeout(timeout);
 565
 566                 local_irq_disable();
 567         }
 568
 569         finish_wait(wq, &wait);
 570
 571         drm_crtc_vblank_put(&crtc->base);
 572
 573         /*
 574          * On VLV/CHV DSI the scanline counter would appear to
 575          * increment approx. 1/3 of a scanline before start of vblank.
 576          * The registers still get latched at start of vblank however.
 577          * This means we must not write any registers on the first
 578          * line of vblank (since not the whole line is actually in
 579          * vblank). And unfortunately we can't use the interrupt to
 580          * wait here since it will fire too soon. We could use the
 581          * frame start interrupt instead since it will fire after the
 582          * critical scanline, but that would require more changes
 583          * in the interrupt code. So for now we'll just do the nasty
 584          * thing and poll for the bad scanline to pass us by.
 585          *
 586          * FIXME figure out if BXT+ DSI suffers from this as well
 587          */
 588         while (need_vlv_dsi_wa && scanline == vblank_start)
 589                 scanline = intel_get_crtc_scanline(crtc);
 590
 591         crtc->debug.scanline_start = scanline;
 592         crtc->debug.start_vbl_time = ktime_get();
 593         crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc);
 594
 595         trace_intel_pipe_update_vblank_evaded(crtc);
 596         return;
 597
 598 irq_disable:
 599         local_irq_disable();
 600 }
 601
 602 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)
 603 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end)
 604 {
 605         u64 delta = ktime_to_ns(ktime_sub(end, crtc->debug.start_vbl_time));
 606         unsigned int h;
 607
 608         h = ilog2(delta >> 9);
 609         if (h >= ARRAY_SIZE(crtc->debug.vbl.times))
 610                 h = ARRAY_SIZE(crtc->debug.vbl.times) - 1;
 611         crtc->debug.vbl.times[h]++;
 612
 613         crtc->debug.vbl.sum += delta;
 614         if (!crtc->debug.vbl.min || delta < crtc->debug.vbl.min)
 615                 crtc->debug.vbl.min = delta;
 616         if (delta > crtc->debug.vbl.max)
 617                 crtc->debug.vbl.max = delta;
 618
 619         if (delta > 1000 * VBLANK_EVASION_TIME_US) {
 620                 drm_dbg_kms(crtc->base.dev,
 621                             "Atomic update on pipe (%c) took %lld us, max time under evasion is %u us\n",
 622                             pipe_name(crtc->pipe),
 623                             div_u64(delta, 1000),
 624                             VBLANK_EVASION_TIME_US);
 625                 crtc->debug.vbl.over++;
 626         }
 627 }
 628 #else
 629 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) {}
 630 #endif
 631
 632 /**
 633  * intel_pipe_update_end() - end update of a set of display registers
 634  * @new_crtc_state: the new crtc state
 635  *
 636  * Mark the end of an update started with intel_pipe_update_start(). This
 637  * re-enables interrupts and verifies the update was actually completed
 638  * before a vblank.
 639  */
 640 void intel_pipe_update_end(struct intel_crtc_state *new_crtc_state)
 641 {
 642         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
 643         enum pipe pipe = crtc->pipe;
 644         int scanline_end = intel_get_crtc_scanline(crtc);
 645         u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc);
 646         ktime_t end_vbl_time = ktime_get();
 647         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 648
 649         intel_psr_unlock(new_crtc_state);
 650
 651         if (new_crtc_state->do_async_flip)
 652                 return;
 653
 654         trace_intel_pipe_update_end(crtc, end_vbl_count, scanline_end);
 655
 656         /*
 657          * Incase of mipi dsi command mode, we need to set frame update
 658          * request for every commit.
 659          */
 660         if (DISPLAY_VER(dev_priv) >= 11 &&
 661             intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
 662                 icl_dsi_frame_update(new_crtc_state);
 663
 664         /* We're still in the vblank-evade critical section, this can't race.
 665          * Would be slightly nice to just grab the vblank count and arm the
 666          * event outside of the critical section - the spinlock might spin for a
 667          * while ... */
 668         if (intel_crtc_needs_vblank_work(new_crtc_state)) {
 669                 drm_vblank_work_schedule(&new_crtc_state->vblank_work,
 670                                          drm_crtc_accurate_vblank_count(&crtc->base) + 1,
 671                                          false);
 672         } else if (new_crtc_state->uapi.event) {
 673                 drm_WARN_ON(&dev_priv->drm,
 674                             drm_crtc_vblank_get(&crtc->base) != 0);
 675
 676                 spin_lock(&crtc->base.dev->event_lock);
 677                 drm_crtc_arm_vblank_event(&crtc->base,
 678                                           new_crtc_state->uapi.event);
 679                 spin_unlock(&crtc->base.dev->event_lock);
 680
 681                 new_crtc_state->uapi.event = NULL;
 682         }
 683
 684         /*
 685          * Send VRR Push to terminate Vblank. If we are already in vblank
 686          * this has to be done _after_ sampling the frame counter, as
 687          * otherwise the push would immediately terminate the vblank and
 688          * the sampled frame counter would correspond to the next frame
 689          * instead of the current frame.
 690          *
 691          * There is a tiny race here (iff vblank evasion failed us) where
 692          * we might sample the frame counter just before vmax vblank start
 693          * but the push would be sent just after it. That would cause the
 694          * push to affect the next frame instead of the current frame,
 695          * which would cause the next frame to terminate already at vmin
 696          * vblank start instead of vmax vblank start.
 697          */
 698         intel_vrr_send_push(new_crtc_state);
 699
 700         /*
 701          * Seamless M/N update may need to update frame timings.
 702          *
 703          * FIXME Should be synchronized with the start of vblank somehow...
 704          */
 705         if (new_crtc_state->seamless_m_n && intel_crtc_needs_fastset(new_crtc_state))
 706                 intel_crtc_update_active_timings(new_crtc_state,
 707                                                  new_crtc_state->vrr.enable);
 708
 709         local_irq_enable();
 710
 711         if (intel_vgpu_active(dev_priv))
 712                 return;
 713
 714         if (crtc->debug.start_vbl_count &&
 715             crtc->debug.start_vbl_count != end_vbl_count) {
 716                 drm_err(&dev_priv->drm,
 717                         "Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n",
 718                         pipe_name(pipe), crtc->debug.start_vbl_count,
 719                         end_vbl_count,
 720                         ktime_us_delta(end_vbl_time,
 721                                        crtc->debug.start_vbl_time),
 722                         crtc->debug.min_vbl, crtc->debug.max_vbl,
 723                         crtc->debug.scanline_start, scanline_end);
 724         }
 725
 726         dbg_vblank_evade(crtc, end_vbl_time);
 727 }