drivers/gpu/drm/i915/display/intel_fbc.c

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 /**
  25  * DOC: Frame Buffer Compression (FBC)
  26  *
  27  * FBC tries to save memory bandwidth (and so power consumption) by
  28  * compressing the amount of memory used by the display. It is total
  29  * transparent to user space and completely handled in the kernel.
  30  *
  31  * The benefits of FBC are mostly visible with solid backgrounds and
  32  * variation-less patterns. It comes from keeping the memory footprint small
  33  * and having fewer memory pages opened and accessed for refreshing the display.
  34  *
  35  * i915 is responsible to reserve stolen memory for FBC and configure its
  36  * offset on proper registers. The hardware takes care of all
  37  * compress/decompress. However there are many known cases where we have to
  38  * forcibly disable it to allow proper screen updates.
  39  */
  40
  41 #include <drm/drm_fourcc.h>
  42
  43 #include "i915_drv.h"
  44 #include "i915_trace.h"
  45 #include "i915_vgpu.h"
  46 #include "intel_de.h"
  47 #include "intel_display_types.h"
  48 #include "intel_fbc.h"
  49 #include "intel_frontbuffer.h"
  50
  51 /*
  52  * For SKL+, the plane source size used by the hardware is based on the value we
  53  * write to the PLANE_SIZE register. For BDW-, the hardware looks at the value
  54  * we wrote to PIPESRC.
  55  */
  56 static void intel_fbc_get_plane_source_size(const struct intel_fbc_state_cache *cache,
  57                                             int *width, int *height)
  58 {
  59         if (width)
  60                 *width = cache->plane.src_w;
  61         if (height)
  62                 *height = cache->plane.src_h;
  63 }
  64
  65 /* plane stride in pixels */
  66 static unsigned int intel_fbc_plane_stride(const struct intel_plane_state *plane_state)
  67 {
  68         const struct drm_framebuffer *fb = plane_state->hw.fb;
  69         unsigned int stride;
  70
  71         stride = plane_state->view.color_plane[0].mapping_stride;
  72         if (!drm_rotation_90_or_270(plane_state->hw.rotation))
  73                 stride /= fb->format->cpp[0];
  74
  75         return stride;
  76 }
  77
  78 /* plane stride based cfb stride in bytes, assuming 1:1 compression limit */
  79 static unsigned int _intel_fbc_cfb_stride(const struct intel_fbc_state_cache *cache)
  80 {
  81         unsigned int cpp = 4; /* FBC always 4 bytes per pixel */
  82
  83         return cache->fb.stride * cpp;
  84 }
  85
  86 /* minimum acceptable cfb stride in bytes, assuming 1:1 compression limit */
  87 static unsigned int skl_fbc_min_cfb_stride(struct drm_i915_private *i915,
  88                                            const struct intel_fbc_state_cache *cache)
  89 {
  90         unsigned int limit = 4; /* 1:4 compression limit is the worst case */
  91         unsigned int cpp = 4; /* FBC always 4 bytes per pixel */
  92         unsigned int height = 4; /* FBC segment is 4 lines */
  93         unsigned int stride;
  94
  95         /* minimum segment stride we can use */
  96         stride = cache->plane.src_w * cpp * height / limit;
  97
  98         /*
  99          * Wa_16011863758: icl+
 100          * Avoid some hardware segment address miscalculation.
 101          */
 102         if (DISPLAY_VER(i915) >= 11)
 103                 stride += 64;
 104
 105         /*
 106          * At least some of the platforms require each 4 line segment to
 107          * be 512 byte aligned. Just do it always for simplicity.
 108          */
 109         stride = ALIGN(stride, 512);
 110
 111         /* convert back to single line equivalent with 1:1 compression limit */
 112         return stride * limit / height;
 113 }
 114
 115 /* properly aligned cfb stride in bytes, assuming 1:1 compression limit */
 116 static unsigned int intel_fbc_cfb_stride(struct drm_i915_private *i915,
 117                                          const struct intel_fbc_state_cache *cache)
 118 {
 119         unsigned int stride = _intel_fbc_cfb_stride(cache);
 120
 121         /*
 122          * At least some of the platforms require each 4 line segment to
 123          * be 512 byte aligned. Aligning each line to 512 bytes guarantees
 124          * that regardless of the compression limit we choose later.
 125          */
 126         if (DISPLAY_VER(i915) >= 9)
 127                 return max(ALIGN(stride, 512), skl_fbc_min_cfb_stride(i915, cache));
 128         else
 129                 return stride;
 130 }
 131
 132 static unsigned int intel_fbc_cfb_size(struct drm_i915_private *dev_priv,
 133                                        const struct intel_fbc_state_cache *cache)
 134 {
 135         int lines = cache->plane.src_h;
 136
 137         if (DISPLAY_VER(dev_priv) == 7)
 138                 lines = min(lines, 2048);
 139         else if (DISPLAY_VER(dev_priv) >= 8)
 140                 lines = min(lines, 2560);
 141
 142         return lines * intel_fbc_cfb_stride(dev_priv, cache);
 143 }
 144
 145 static void i8xx_fbc_deactivate(struct drm_i915_private *dev_priv)
 146 {
 147         u32 fbc_ctl;
 148
 149         /* Disable compression */
 150         fbc_ctl = intel_de_read(dev_priv, FBC_CONTROL);
 151         if ((fbc_ctl & FBC_CTL_EN) == 0)
 152                 return;
 153
 154         fbc_ctl &= ~FBC_CTL_EN;
 155         intel_de_write(dev_priv, FBC_CONTROL, fbc_ctl);
 156
 157         /* Wait for compressing bit to clear */
 158         if (intel_de_wait_for_clear(dev_priv, FBC_STATUS,
 159                                     FBC_STAT_COMPRESSING, 10)) {
 160                 drm_dbg_kms(&dev_priv->drm, "FBC idle timed out\n");
 161                 return;
 162         }
 163 }
 164
 165 static void i8xx_fbc_activate(struct drm_i915_private *dev_priv)
 166 {
 167         struct intel_fbc *fbc = &dev_priv->fbc;
 168         const struct intel_fbc_reg_params *params = &fbc->params;
 169         int cfb_pitch;
 170         int i;
 171         u32 fbc_ctl;
 172
 173         cfb_pitch = params->cfb_stride / fbc->limit;
 174
 175         /* FBC_CTL wants 32B or 64B units */
 176         if (DISPLAY_VER(dev_priv) == 2)
 177                 cfb_pitch = (cfb_pitch / 32) - 1;
 178         else
 179                 cfb_pitch = (cfb_pitch / 64) - 1;
 180
 181         /* Clear old tags */
 182         for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
 183                 intel_de_write(dev_priv, FBC_TAG(i), 0);
 184
 185         if (DISPLAY_VER(dev_priv) == 4) {
 186                 u32 fbc_ctl2;
 187
 188                 /* Set it up... */
 189                 fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM;
 190                 fbc_ctl2 |= FBC_CTL_PLANE(params->crtc.i9xx_plane);
 191                 if (params->fence_id >= 0)
 192                         fbc_ctl2 |= FBC_CTL_CPU_FENCE;
 193                 intel_de_write(dev_priv, FBC_CONTROL2, fbc_ctl2);
 194                 intel_de_write(dev_priv, FBC_FENCE_OFF,
 195                                params->fence_y_offset);
 196         }
 197
 198         /* enable it... */
 199         fbc_ctl = FBC_CTL_INTERVAL(params->interval);
 200         fbc_ctl |= FBC_CTL_EN | FBC_CTL_PERIODIC;
 201         if (IS_I945GM(dev_priv))
 202                 fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
 203         fbc_ctl |= FBC_CTL_STRIDE(cfb_pitch & 0xff);
 204         if (params->fence_id >= 0)
 205                 fbc_ctl |= FBC_CTL_FENCENO(params->fence_id);
 206         intel_de_write(dev_priv, FBC_CONTROL, fbc_ctl);
 207 }
 208
 209 static bool i8xx_fbc_is_active(struct drm_i915_private *dev_priv)
 210 {
 211         return intel_de_read(dev_priv, FBC_CONTROL) & FBC_CTL_EN;
 212 }
 213
 214 static bool i8xx_fbc_is_compressing(struct drm_i915_private *i915)
 215 {
 216         return intel_de_read(i915, FBC_STATUS) &
 217                 (FBC_STAT_COMPRESSING | FBC_STAT_COMPRESSED);
 218 }
 219
 220 static u32 g4x_dpfc_ctl_limit(struct drm_i915_private *i915)
 221 {
 222         switch (i915->fbc.limit) {
 223         default:
 224                 MISSING_CASE(i915->fbc.limit);
 225                 fallthrough;
 226         case 1:
 227                 return DPFC_CTL_LIMIT_1X;
 228         case 2:
 229                 return DPFC_CTL_LIMIT_2X;
 230         case 4:
 231                 return DPFC_CTL_LIMIT_4X;
 232         }
 233 }
 234
 235 static void g4x_fbc_activate(struct drm_i915_private *dev_priv)
 236 {
 237         struct intel_fbc_reg_params *params = &dev_priv->fbc.params;
 238         u32 dpfc_ctl;
 239
 240         dpfc_ctl = DPFC_CTL_PLANE(params->crtc.i9xx_plane) | DPFC_SR_EN;
 241
 242         dpfc_ctl |= g4x_dpfc_ctl_limit(dev_priv);
 243
 244         if (params->fence_id >= 0)
 245                 dpfc_ctl |= DPFC_CTL_FENCE_EN | params->fence_id;
 246
 247         intel_de_write(dev_priv, DPFC_FENCE_YOFF,
 248                        params->fence_y_offset);
 249
 250         /* enable it... */
 251         intel_de_write(dev_priv, DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
 252 }
 253
 254 static void g4x_fbc_deactivate(struct drm_i915_private *dev_priv)
 255 {
 256         u32 dpfc_ctl;
 257
 258         /* Disable compression */
 259         dpfc_ctl = intel_de_read(dev_priv, DPFC_CONTROL);
 260         if (dpfc_ctl & DPFC_CTL_EN) {
 261                 dpfc_ctl &= ~DPFC_CTL_EN;
 262                 intel_de_write(dev_priv, DPFC_CONTROL, dpfc_ctl);
 263         }
 264 }
 265
 266 static bool g4x_fbc_is_active(struct drm_i915_private *dev_priv)
 267 {
 268         return intel_de_read(dev_priv, DPFC_CONTROL) & DPFC_CTL_EN;
 269 }
 270
 271 static bool g4x_fbc_is_compressing(struct drm_i915_private *i915)
 272 {
 273         return intel_de_read(i915, DPFC_STATUS) & DPFC_COMP_SEG_MASK;
 274 }
 275
 276 static void i8xx_fbc_recompress(struct drm_i915_private *dev_priv)
 277 {
 278         struct intel_fbc_reg_params *params = &dev_priv->fbc.params;
 279         enum i9xx_plane_id i9xx_plane = params->crtc.i9xx_plane;
 280
 281         spin_lock_irq(&dev_priv->uncore.lock);
 282         intel_de_write_fw(dev_priv, DSPADDR(i9xx_plane),
 283                           intel_de_read_fw(dev_priv, DSPADDR(i9xx_plane)));
 284         spin_unlock_irq(&dev_priv->uncore.lock);
 285 }
 286
 287 static void i965_fbc_recompress(struct drm_i915_private *dev_priv)
 288 {
 289         struct intel_fbc_reg_params *params = &dev_priv->fbc.params;
 290         enum i9xx_plane_id i9xx_plane = params->crtc.i9xx_plane;
 291
 292         spin_lock_irq(&dev_priv->uncore.lock);
 293         intel_de_write_fw(dev_priv, DSPSURF(i9xx_plane),
 294                           intel_de_read_fw(dev_priv, DSPSURF(i9xx_plane)));
 295         spin_unlock_irq(&dev_priv->uncore.lock);
 296 }
 297
 298 /* This function forces a CFB recompression through the nuke operation. */
 299 static void snb_fbc_recompress(struct drm_i915_private *dev_priv)
 300 {
 301         intel_de_write(dev_priv, MSG_FBC_REND_STATE, FBC_REND_NUKE);
 302         intel_de_posting_read(dev_priv, MSG_FBC_REND_STATE);
 303 }
 304
 305 static void intel_fbc_recompress(struct drm_i915_private *dev_priv)
 306 {
 307         struct intel_fbc *fbc = &dev_priv->fbc;
 308
 309         trace_intel_fbc_nuke(fbc->crtc);
 310
 311         if (DISPLAY_VER(dev_priv) >= 6)
 312                 snb_fbc_recompress(dev_priv);
 313         else if (DISPLAY_VER(dev_priv) >= 4)
 314                 i965_fbc_recompress(dev_priv);
 315         else
 316                 i8xx_fbc_recompress(dev_priv);
 317 }
 318
 319 static void snb_fbc_program_fence(struct drm_i915_private *i915)
 320 {
 321         const struct intel_fbc_reg_params *params = &i915->fbc.params;
 322         u32 ctl = 0;
 323
 324         if (params->fence_id >= 0)
 325                 ctl = SNB_CPU_FENCE_ENABLE | params->fence_id;
 326
 327         intel_de_write(i915, SNB_DPFC_CTL_SA, ctl);
 328         intel_de_write(i915, DPFC_CPU_FENCE_OFFSET, params->fence_y_offset);
 329 }
 330
 331 static void ilk_fbc_activate(struct drm_i915_private *dev_priv)
 332 {
 333         struct intel_fbc_reg_params *params = &dev_priv->fbc.params;
 334         u32 dpfc_ctl;
 335
 336         dpfc_ctl = DPFC_CTL_PLANE(params->crtc.i9xx_plane);
 337
 338         dpfc_ctl |= g4x_dpfc_ctl_limit(dev_priv);
 339
 340         if (params->fence_id >= 0) {
 341                 dpfc_ctl |= DPFC_CTL_FENCE_EN;
 342                 if (IS_IRONLAKE(dev_priv))
 343                         dpfc_ctl |= params->fence_id;
 344         }
 345
 346         if (IS_SANDYBRIDGE(dev_priv))
 347                 snb_fbc_program_fence(dev_priv);
 348
 349         intel_de_write(dev_priv, ILK_DPFC_FENCE_YOFF,
 350                        params->fence_y_offset);
 351         /* enable it... */
 352         intel_de_write(dev_priv, ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
 353 }
 354
 355 static void ilk_fbc_deactivate(struct drm_i915_private *dev_priv)
 356 {
 357         u32 dpfc_ctl;
 358
 359         /* Disable compression */
 360         dpfc_ctl = intel_de_read(dev_priv, ILK_DPFC_CONTROL);
 361         if (dpfc_ctl & DPFC_CTL_EN) {
 362                 dpfc_ctl &= ~DPFC_CTL_EN;
 363                 intel_de_write(dev_priv, ILK_DPFC_CONTROL, dpfc_ctl);
 364         }
 365 }
 366
 367 static bool ilk_fbc_is_active(struct drm_i915_private *dev_priv)
 368 {
 369         return intel_de_read(dev_priv, ILK_DPFC_CONTROL) & DPFC_CTL_EN;
 370 }
 371
 372 static bool ilk_fbc_is_compressing(struct drm_i915_private *i915)
 373 {
 374         return intel_de_read(i915, ILK_DPFC_STATUS) & ILK_DPFC_COMP_SEG_MASK;
 375 }
 376
 377 static void glk_fbc_program_cfb_stride(struct drm_i915_private *i915)
 378 {
 379         struct intel_fbc *fbc = &i915->fbc;
 380         const struct intel_fbc_reg_params *params = &fbc->params;
 381         u32 val = 0;
 382
 383         if (params->override_cfb_stride)
 384                 val |= FBC_STRIDE_OVERRIDE |
 385                         FBC_STRIDE(params->override_cfb_stride / fbc->limit);
 386
 387         intel_de_write(i915, GLK_FBC_STRIDE, val);
 388 }
 389
 390 static void skl_fbc_program_cfb_stride(struct drm_i915_private *i915)
 391 {
 392         struct intel_fbc *fbc = &i915->fbc;
 393         const struct intel_fbc_reg_params *params = &fbc->params;
 394         u32 val = 0;
 395
 396         /* Display WA #0529: skl, kbl, bxt. */
 397         if (params->override_cfb_stride)
 398                 val |= CHICKEN_FBC_STRIDE_OVERRIDE |
 399                         CHICKEN_FBC_STRIDE(params->override_cfb_stride / fbc->limit);
 400
 401         intel_de_rmw(i915, CHICKEN_MISC_4,
 402                      CHICKEN_FBC_STRIDE_OVERRIDE |
 403                      CHICKEN_FBC_STRIDE_MASK, val);
 404 }
 405
 406 static void gen7_fbc_activate(struct drm_i915_private *dev_priv)
 407 {
 408         struct intel_fbc *fbc = &dev_priv->fbc;
 409         const struct intel_fbc_reg_params *params = &fbc->params;
 410         u32 dpfc_ctl;
 411
 412         if (DISPLAY_VER(dev_priv) >= 10)
 413                 glk_fbc_program_cfb_stride(dev_priv);
 414         else if (DISPLAY_VER(dev_priv) == 9)
 415                 skl_fbc_program_cfb_stride(dev_priv);
 416
 417         dpfc_ctl = 0;
 418         if (IS_IVYBRIDGE(dev_priv))
 419                 dpfc_ctl |= IVB_DPFC_CTL_PLANE(params->crtc.i9xx_plane);
 420
 421         dpfc_ctl |= g4x_dpfc_ctl_limit(dev_priv);
 422
 423         if (params->fence_id >= 0)
 424                 dpfc_ctl |= IVB_DPFC_CTL_FENCE_EN;
 425
 426         if (dev_priv->fbc.false_color)
 427                 dpfc_ctl |= FBC_CTL_FALSE_COLOR;
 428
 429         if (dev_priv->ggtt.num_fences)
 430                 snb_fbc_program_fence(dev_priv);
 431
 432         intel_de_write(dev_priv, ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
 433 }
 434
 435 static bool gen7_fbc_is_compressing(struct drm_i915_private *i915)
 436 {
 437         if (DISPLAY_VER(i915) >= 8)
 438                 return intel_de_read(i915, IVB_FBC_STATUS2) & BDW_FBC_COMP_SEG_MASK;
 439         else
 440                 return intel_de_read(i915, IVB_FBC_STATUS2) & IVB_FBC_COMP_SEG_MASK;
 441 }
 442
 443 static bool intel_fbc_hw_is_active(struct drm_i915_private *dev_priv)
 444 {
 445         if (DISPLAY_VER(dev_priv) >= 5)
 446                 return ilk_fbc_is_active(dev_priv);
 447         else if (IS_GM45(dev_priv))
 448                 return g4x_fbc_is_active(dev_priv);
 449         else
 450                 return i8xx_fbc_is_active(dev_priv);
 451 }
 452
 453 static void intel_fbc_hw_activate(struct drm_i915_private *dev_priv)
 454 {
 455         struct intel_fbc *fbc = &dev_priv->fbc;
 456
 457         trace_intel_fbc_activate(fbc->crtc);
 458
 459         fbc->active = true;
 460         fbc->activated = true;
 461
 462         if (DISPLAY_VER(dev_priv) >= 7)
 463                 gen7_fbc_activate(dev_priv);
 464         else if (DISPLAY_VER(dev_priv) >= 5)
 465                 ilk_fbc_activate(dev_priv);
 466         else if (IS_GM45(dev_priv))
 467                 g4x_fbc_activate(dev_priv);
 468         else
 469                 i8xx_fbc_activate(dev_priv);
 470 }
 471
 472 static void intel_fbc_hw_deactivate(struct drm_i915_private *dev_priv)
 473 {
 474         struct intel_fbc *fbc = &dev_priv->fbc;
 475
 476         trace_intel_fbc_deactivate(fbc->crtc);
 477
 478         fbc->active = false;
 479
 480         if (DISPLAY_VER(dev_priv) >= 5)
 481                 ilk_fbc_deactivate(dev_priv);
 482         else if (IS_GM45(dev_priv))
 483                 g4x_fbc_deactivate(dev_priv);
 484         else
 485                 i8xx_fbc_deactivate(dev_priv);
 486 }
 487
 488 bool intel_fbc_is_compressing(struct drm_i915_private *i915)
 489 {
 490         if (DISPLAY_VER(i915) >= 7)
 491                 return gen7_fbc_is_compressing(i915);
 492         else if (DISPLAY_VER(i915) >= 5)
 493                 return ilk_fbc_is_compressing(i915);
 494         else if (IS_G4X(i915))
 495                 return g4x_fbc_is_compressing(i915);
 496         else
 497                 return i8xx_fbc_is_compressing(i915);
 498 }
 499
 500 /**
 501  * intel_fbc_is_active - Is FBC active?
 502  * @dev_priv: i915 device instance
 503  *
 504  * This function is used to verify the current state of FBC.
 505  *
 506  * FIXME: This should be tracked in the plane config eventually
 507  * instead of queried at runtime for most callers.
 508  */
 509 bool intel_fbc_is_active(struct drm_i915_private *dev_priv)
 510 {
 511         return dev_priv->fbc.active;
 512 }
 513
 514 static void intel_fbc_activate(struct drm_i915_private *dev_priv)
 515 {
 516         intel_fbc_hw_activate(dev_priv);
 517         intel_fbc_recompress(dev_priv);
 518 }
 519
 520 static void intel_fbc_deactivate(struct drm_i915_private *dev_priv,
 521                                  const char *reason)
 522 {
 523         struct intel_fbc *fbc = &dev_priv->fbc;
 524
 525         drm_WARN_ON(&dev_priv->drm, !mutex_is_locked(&fbc->lock));
 526
 527         if (fbc->active)
 528                 intel_fbc_hw_deactivate(dev_priv);
 529
 530         fbc->no_fbc_reason = reason;
 531 }
 532
 533 static u64 intel_fbc_cfb_base_max(struct drm_i915_private *i915)
 534 {
 535         if (DISPLAY_VER(i915) >= 5 || IS_G4X(i915))
 536                 return BIT_ULL(28);
 537         else
 538                 return BIT_ULL(32);
 539 }
 540
 541 static u64 intel_fbc_stolen_end(struct drm_i915_private *dev_priv)
 542 {
 543         u64 end;
 544
 545         /* The FBC hardware for BDW/SKL doesn't have access to the stolen
 546          * reserved range size, so it always assumes the maximum (8mb) is used.
 547          * If we enable FBC using a CFB on that memory range we'll get FIFO
 548          * underruns, even if that range is not reserved by the BIOS. */
 549         if (IS_BROADWELL(dev_priv) || (DISPLAY_VER(dev_priv) == 9 &&
 550                                        !IS_BROXTON(dev_priv)))
 551                 end = resource_size(&dev_priv->dsm) - 8 * 1024 * 1024;
 552         else
 553                 end = U64_MAX;
 554
 555         return min(end, intel_fbc_cfb_base_max(dev_priv));
 556 }
 557
 558 static int intel_fbc_min_limit(int fb_cpp)
 559 {
 560         return fb_cpp == 2 ? 2 : 1;
 561 }
 562
 563 static int intel_fbc_max_limit(struct drm_i915_private *dev_priv)
 564 {
 565         /* WaFbcOnly1to1Ratio:ctg */
 566         if (IS_G4X(dev_priv))
 567                 return 1;
 568
 569         /*
 570          * FBC2 can only do 1:1, 1:2, 1:4, we limit
 571          * FBC1 to the same out of convenience.
 572          */
 573         return 4;
 574 }
 575
 576 static int find_compression_limit(struct drm_i915_private *dev_priv,
 577                                   unsigned int size, int min_limit)
 578 {
 579         struct intel_fbc *fbc = &dev_priv->fbc;
 580         u64 end = intel_fbc_stolen_end(dev_priv);
 581         int ret, limit = min_limit;
 582
 583         size /= limit;
 584
 585         /* Try to over-allocate to reduce reallocations and fragmentation. */
 586         ret = i915_gem_stolen_insert_node_in_range(dev_priv, &fbc->compressed_fb,
 587                                                    size <<= 1, 4096, 0, end);
 588         if (ret == 0)
 589                 return limit;
 590
 591         for (; limit <= intel_fbc_max_limit(dev_priv); limit <<= 1) {
 592                 ret = i915_gem_stolen_insert_node_in_range(dev_priv, &fbc->compressed_fb,
 593                                                            size >>= 1, 4096, 0, end);
 594                 if (ret == 0)
 595                         return limit;
 596         }
 597
 598         return 0;
 599 }
 600
 601 static int intel_fbc_alloc_cfb(struct drm_i915_private *dev_priv,
 602                                unsigned int size, int min_limit)
 603 {
 604         struct intel_fbc *fbc = &dev_priv->fbc;
 605         int ret;
 606
 607         drm_WARN_ON(&dev_priv->drm,
 608                     drm_mm_node_allocated(&fbc->compressed_fb));
 609         drm_WARN_ON(&dev_priv->drm,
 610                     drm_mm_node_allocated(&fbc->compressed_llb));
 611
 612         if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv)) {
 613                 ret = i915_gem_stolen_insert_node(dev_priv, &fbc->compressed_llb,
 614                                                   4096, 4096);
 615                 if (ret)
 616                         goto err;
 617         }
 618
 619         ret = find_compression_limit(dev_priv, size, min_limit);
 620         if (!ret)
 621                 goto err_llb;
 622         else if (ret > min_limit)
 623                 drm_info_once(&dev_priv->drm,
 624                               "Reducing the compressed framebuffer size. This may lead to less power savings than a non-reduced-size. Try to increase stolen memory size if available in BIOS.\n");
 625
 626         fbc->limit = ret;
 627
 628         drm_dbg_kms(&dev_priv->drm,
 629                     "reserved %llu bytes of contiguous stolen space for FBC, limit: %d\n",
 630                     fbc->compressed_fb.size, fbc->limit);
 631
 632         return 0;
 633
 634 err_llb:
 635         if (drm_mm_node_allocated(&fbc->compressed_llb))
 636                 i915_gem_stolen_remove_node(dev_priv, &fbc->compressed_llb);
 637 err:
 638         if (drm_mm_initialized(&dev_priv->mm.stolen))
 639                 drm_info_once(&dev_priv->drm, "not enough stolen space for compressed buffer (need %d more bytes), disabling. Hint: you may be able to increase stolen memory size in the BIOS to avoid this.\n", size);
 640         return -ENOSPC;
 641 }
 642
 643 static void intel_fbc_program_cfb(struct drm_i915_private *dev_priv)
 644 {
 645         struct intel_fbc *fbc = &dev_priv->fbc;
 646
 647         if (DISPLAY_VER(dev_priv) >= 5) {
 648                 intel_de_write(dev_priv, ILK_DPFC_CB_BASE,
 649                                fbc->compressed_fb.start);
 650         } else if (IS_GM45(dev_priv)) {
 651                 intel_de_write(dev_priv, DPFC_CB_BASE,
 652                                fbc->compressed_fb.start);
 653         } else {
 654                 GEM_BUG_ON(range_overflows_end_t(u64, dev_priv->dsm.start,
 655                                                  fbc->compressed_fb.start,
 656                                                  U32_MAX));
 657                 GEM_BUG_ON(range_overflows_end_t(u64, dev_priv->dsm.start,
 658                                                  fbc->compressed_llb.start,
 659                                                  U32_MAX));
 660
 661                 intel_de_write(dev_priv, FBC_CFB_BASE,
 662                                dev_priv->dsm.start + fbc->compressed_fb.start);
 663                 intel_de_write(dev_priv, FBC_LL_BASE,
 664                                dev_priv->dsm.start + fbc->compressed_llb.start);
 665         }
 666 }
 667
 668 static void __intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv)
 669 {
 670         struct intel_fbc *fbc = &dev_priv->fbc;
 671
 672         if (WARN_ON(intel_fbc_hw_is_active(dev_priv)))
 673                 return;
 674
 675         if (drm_mm_node_allocated(&fbc->compressed_llb))
 676                 i915_gem_stolen_remove_node(dev_priv, &fbc->compressed_llb);
 677         if (drm_mm_node_allocated(&fbc->compressed_fb))
 678                 i915_gem_stolen_remove_node(dev_priv, &fbc->compressed_fb);
 679 }
 680
 681 void intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv)
 682 {
 683         struct intel_fbc *fbc = &dev_priv->fbc;
 684
 685         if (!HAS_FBC(dev_priv))
 686                 return;
 687
 688         mutex_lock(&fbc->lock);
 689         __intel_fbc_cleanup_cfb(dev_priv);
 690         mutex_unlock(&fbc->lock);
 691 }
 692
 693 static bool stride_is_valid(struct drm_i915_private *dev_priv,
 694                             u64 modifier, unsigned int stride)
 695 {
 696         /* This should have been caught earlier. */
 697         if (drm_WARN_ON_ONCE(&dev_priv->drm, (stride & (64 - 1)) != 0))
 698                 return false;
 699
 700         /* Below are the additional FBC restrictions. */
 701         if (stride < 512)
 702                 return false;
 703
 704         if (DISPLAY_VER(dev_priv) == 2 || DISPLAY_VER(dev_priv) == 3)
 705                 return stride == 4096 || stride == 8192;
 706
 707         if (DISPLAY_VER(dev_priv) == 4 && !IS_G4X(dev_priv) && stride < 2048)
 708                 return false;
 709
 710         /* Display WA #1105: skl,bxt,kbl,cfl,glk */
 711         if ((DISPLAY_VER(dev_priv) == 9 || IS_GEMINILAKE(dev_priv)) &&
 712             modifier == DRM_FORMAT_MOD_LINEAR && stride & 511)
 713                 return false;
 714
 715         if (stride > 16384)
 716                 return false;
 717
 718         return true;
 719 }
 720
 721 static bool pixel_format_is_valid(struct drm_i915_private *dev_priv,
 722                                   u32 pixel_format)
 723 {
 724         switch (pixel_format) {
 725         case DRM_FORMAT_XRGB8888:
 726         case DRM_FORMAT_XBGR8888:
 727                 return true;
 728         case DRM_FORMAT_XRGB1555:
 729         case DRM_FORMAT_RGB565:
 730                 /* 16bpp not supported on gen2 */
 731                 if (DISPLAY_VER(dev_priv) == 2)
 732                         return false;
 733                 /* WaFbcOnly1to1Ratio:ctg */
 734                 if (IS_G4X(dev_priv))
 735                         return false;
 736                 return true;
 737         default:
 738                 return false;
 739         }
 740 }
 741
 742 static bool rotation_is_valid(struct drm_i915_private *dev_priv,
 743                               u32 pixel_format, unsigned int rotation)
 744 {
 745         if (DISPLAY_VER(dev_priv) >= 9 && pixel_format == DRM_FORMAT_RGB565 &&
 746             drm_rotation_90_or_270(rotation))
 747                 return false;
 748         else if (DISPLAY_VER(dev_priv) <= 4 && !IS_G4X(dev_priv) &&
 749                  rotation != DRM_MODE_ROTATE_0)
 750                 return false;
 751
 752         return true;
 753 }
 754
 755 /*
 756  * For some reason, the hardware tracking starts looking at whatever we
 757  * programmed as the display plane base address register. It does not look at
 758  * the X and Y offset registers. That's why we include the src x/y offsets
 759  * instead of just looking at the plane size.
 760  */
 761 static bool intel_fbc_hw_tracking_covers_screen(struct intel_crtc *crtc)
 762 {
 763         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 764         struct intel_fbc *fbc = &dev_priv->fbc;
 765         unsigned int effective_w, effective_h, max_w, max_h;
 766
 767         if (DISPLAY_VER(dev_priv) >= 10) {
 768                 max_w = 5120;
 769                 max_h = 4096;
 770         } else if (DISPLAY_VER(dev_priv) >= 8 || IS_HASWELL(dev_priv)) {
 771                 max_w = 4096;
 772                 max_h = 4096;
 773         } else if (IS_G4X(dev_priv) || DISPLAY_VER(dev_priv) >= 5) {
 774                 max_w = 4096;
 775                 max_h = 2048;
 776         } else {
 777                 max_w = 2048;
 778                 max_h = 1536;
 779         }
 780
 781         intel_fbc_get_plane_source_size(&fbc->state_cache, &effective_w,
 782                                         &effective_h);
 783         effective_w += fbc->state_cache.plane.adjusted_x;
 784         effective_h += fbc->state_cache.plane.adjusted_y;
 785
 786         return effective_w <= max_w && effective_h <= max_h;
 787 }
 788
 789 static bool tiling_is_valid(struct drm_i915_private *dev_priv,
 790                             u64 modifier)
 791 {
 792         switch (modifier) {
 793         case DRM_FORMAT_MOD_LINEAR:
 794         case I915_FORMAT_MOD_Y_TILED:
 795         case I915_FORMAT_MOD_Yf_TILED:
 796                 return DISPLAY_VER(dev_priv) >= 9;
 797         case I915_FORMAT_MOD_X_TILED:
 798                 return true;
 799         default:
 800                 return false;
 801         }
 802 }
 803
 804 static void intel_fbc_update_state_cache(struct intel_crtc *crtc,
 805                                          const struct intel_crtc_state *crtc_state,
 806                                          const struct intel_plane_state *plane_state)
 807 {
 808         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 809         struct intel_fbc *fbc = &dev_priv->fbc;
 810         struct intel_fbc_state_cache *cache = &fbc->state_cache;
 811         struct drm_framebuffer *fb = plane_state->hw.fb;
 812
 813         cache->plane.visible = plane_state->uapi.visible;
 814         if (!cache->plane.visible)
 815                 return;
 816
 817         cache->crtc.mode_flags = crtc_state->hw.adjusted_mode.flags;
 818         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 819                 cache->crtc.hsw_bdw_pixel_rate = crtc_state->pixel_rate;
 820
 821         cache->plane.rotation = plane_state->hw.rotation;
 822         /*
 823          * Src coordinates are already rotated by 270 degrees for
 824          * the 90/270 degree plane rotation cases (to match the
 825          * GTT mapping), hence no need to account for rotation here.
 826          */
 827         cache->plane.src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
 828         cache->plane.src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
 829         cache->plane.adjusted_x = plane_state->view.color_plane[0].x;
 830         cache->plane.adjusted_y = plane_state->view.color_plane[0].y;
 831
 832         cache->plane.pixel_blend_mode = plane_state->hw.pixel_blend_mode;
 833
 834         cache->fb.format = fb->format;
 835         cache->fb.modifier = fb->modifier;
 836         cache->fb.stride = intel_fbc_plane_stride(plane_state);
 837
 838         /* FBC1 compression interval: arbitrary choice of 1 second */
 839         cache->interval = drm_mode_vrefresh(&crtc_state->hw.adjusted_mode);
 840
 841         cache->fence_y_offset = intel_plane_fence_y_offset(plane_state);
 842
 843         drm_WARN_ON(&dev_priv->drm, plane_state->flags & PLANE_HAS_FENCE &&
 844                     !plane_state->ggtt_vma->fence);
 845
 846         if (plane_state->flags & PLANE_HAS_FENCE &&
 847             plane_state->ggtt_vma->fence)
 848                 cache->fence_id = plane_state->ggtt_vma->fence->id;
 849         else
 850                 cache->fence_id = -1;
 851
 852         cache->psr2_active = crtc_state->has_psr2;
 853 }
 854
 855 static bool intel_fbc_cfb_size_changed(struct drm_i915_private *dev_priv)
 856 {
 857         struct intel_fbc *fbc = &dev_priv->fbc;
 858
 859         return intel_fbc_cfb_size(dev_priv, &fbc->state_cache) >
 860                 fbc->compressed_fb.size * fbc->limit;
 861 }
 862
 863 static u16 intel_fbc_override_cfb_stride(struct drm_i915_private *dev_priv,
 864                                          const struct intel_fbc_state_cache *cache)
 865 {
 866         unsigned int stride = _intel_fbc_cfb_stride(cache);
 867         unsigned int stride_aligned = intel_fbc_cfb_stride(dev_priv, cache);
 868
 869         /*
 870          * Override stride in 64 byte units per 4 line segment.
 871          *
 872          * Gen9 hw miscalculates cfb stride for linear as
 873          * PLANE_STRIDE*512 instead of PLANE_STRIDE*64, so
 874          * we always need to use the override there.
 875          */
 876         if (stride != stride_aligned ||
 877             (DISPLAY_VER(dev_priv) == 9 &&
 878              cache->fb.modifier == DRM_FORMAT_MOD_LINEAR))
 879                 return stride_aligned * 4 / 64;
 880
 881         return 0;
 882 }
 883
 884 static bool intel_fbc_can_enable(struct drm_i915_private *dev_priv)
 885 {
 886         struct intel_fbc *fbc = &dev_priv->fbc;
 887
 888         if (intel_vgpu_active(dev_priv)) {
 889                 fbc->no_fbc_reason = "VGPU is active";
 890                 return false;
 891         }
 892
 893         if (!dev_priv->params.enable_fbc) {
 894                 fbc->no_fbc_reason = "disabled per module param or by default";
 895                 return false;
 896         }
 897
 898         if (fbc->underrun_detected) {
 899                 fbc->no_fbc_reason = "underrun detected";
 900                 return false;
 901         }
 902
 903         return true;
 904 }
 905
 906 static bool intel_fbc_can_activate(struct intel_crtc *crtc)
 907 {
 908         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 909         struct intel_fbc *fbc = &dev_priv->fbc;
 910         struct intel_fbc_state_cache *cache = &fbc->state_cache;
 911
 912         if (!intel_fbc_can_enable(dev_priv))
 913                 return false;
 914
 915         if (!cache->plane.visible) {
 916                 fbc->no_fbc_reason = "primary plane not visible";
 917                 return false;
 918         }
 919
 920         /* We don't need to use a state cache here since this information is
 921          * global for all CRTC.
 922          */
 923         if (fbc->underrun_detected) {
 924                 fbc->no_fbc_reason = "underrun detected";
 925                 return false;
 926         }
 927
 928         if (cache->crtc.mode_flags & DRM_MODE_FLAG_INTERLACE) {
 929                 fbc->no_fbc_reason = "incompatible mode";
 930                 return false;
 931         }
 932
 933         if (!intel_fbc_hw_tracking_covers_screen(crtc)) {
 934                 fbc->no_fbc_reason = "mode too large for compression";
 935                 return false;
 936         }
 937
 938         /* The use of a CPU fence is one of two ways to detect writes by the
 939          * CPU to the scanout and trigger updates to the FBC.
 940          *
 941          * The other method is by software tracking (see
 942          * intel_fbc_invalidate/flush()), it will manually notify FBC and nuke
 943          * the current compressed buffer and recompress it.
 944          *
 945          * Note that is possible for a tiled surface to be unmappable (and
 946          * so have no fence associated with it) due to aperture constraints
 947          * at the time of pinning.
 948          *
 949          * FIXME with 90/270 degree rotation we should use the fence on
 950          * the normal GTT view (the rotated view doesn't even have a
 951          * fence). Would need changes to the FBC fence Y offset as well.
 952          * For now this will effectively disable FBC with 90/270 degree
 953          * rotation.
 954          */
 955         if (DISPLAY_VER(dev_priv) < 9 && cache->fence_id < 0) {
 956                 fbc->no_fbc_reason = "framebuffer not tiled or fenced";
 957                 return false;
 958         }
 959
 960         if (!pixel_format_is_valid(dev_priv, cache->fb.format->format)) {
 961                 fbc->no_fbc_reason = "pixel format is invalid";
 962                 return false;
 963         }
 964
 965         if (!rotation_is_valid(dev_priv, cache->fb.format->format,
 966                                cache->plane.rotation)) {
 967                 fbc->no_fbc_reason = "rotation unsupported";
 968                 return false;
 969         }
 970
 971         if (!tiling_is_valid(dev_priv, cache->fb.modifier)) {
 972                 fbc->no_fbc_reason = "tiling unsupported";
 973                 return false;
 974         }
 975
 976         if (!stride_is_valid(dev_priv, cache->fb.modifier,
 977                              cache->fb.stride * cache->fb.format->cpp[0])) {
 978                 fbc->no_fbc_reason = "framebuffer stride not supported";
 979                 return false;
 980         }
 981
 982         if (cache->plane.pixel_blend_mode != DRM_MODE_BLEND_PIXEL_NONE &&
 983             cache->fb.format->has_alpha) {
 984                 fbc->no_fbc_reason = "per-pixel alpha blending is incompatible with FBC";
 985                 return false;
 986         }
 987
 988         /* WaFbcExceedCdClockThreshold:hsw,bdw */
 989         if ((IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) &&
 990             cache->crtc.hsw_bdw_pixel_rate >= dev_priv->cdclk.hw.cdclk * 95 / 100) {
 991                 fbc->no_fbc_reason = "pixel rate is too big";
 992                 return false;
 993         }
 994
 995         /* It is possible for the required CFB size change without a
 996          * crtc->disable + crtc->enable since it is possible to change the
 997          * stride without triggering a full modeset. Since we try to
 998          * over-allocate the CFB, there's a chance we may keep FBC enabled even
 999          * if this happens, but if we exceed the current CFB size we'll have to
1000          * disable FBC. Notice that it would be possible to disable FBC, wait
1001          * for a frame, free the stolen node, then try to reenable FBC in case
1002          * we didn't get any invalidate/deactivate calls, but this would require
1003          * a lot of tracking just for a specific case. If we conclude it's an
1004          * important case, we can implement it later. */
1005         if (intel_fbc_cfb_size_changed(dev_priv)) {
1006                 fbc->no_fbc_reason = "CFB requirements changed";
1007                 return false;
1008         }
1009
1010         /*
1011          * Work around a problem on GEN9+ HW, where enabling FBC on a plane
1012          * having a Y offset that isn't divisible by 4 causes FIFO underrun
1013          * and screen flicker.
1014          */
1015         if (DISPLAY_VER(dev_priv) >= 9 &&
1016             (fbc->state_cache.plane.adjusted_y & 3)) {
1017                 fbc->no_fbc_reason = "plane Y offset is misaligned";
1018                 return false;
1019         }
1020
1021         /* Wa_22010751166: icl, ehl, tgl, dg1, rkl */
1022         if (DISPLAY_VER(dev_priv) >= 11 &&
1023             (cache->plane.src_h + cache->plane.adjusted_y) % 4) {
1024                 fbc->no_fbc_reason = "plane height + offset is non-modulo of 4";
1025                 return false;
1026         }
1027
1028         /*
1029          * Display 12+ is not supporting FBC with PSR2.
1030          * Recommendation is to keep this combination disabled
1031          * Bspec: 50422 HSD: 14010260002
1032          */
1033         if (fbc->state_cache.psr2_active && DISPLAY_VER(dev_priv) >= 12) {
1034                 fbc->no_fbc_reason = "not supported with PSR2";
1035                 return false;
1036         }
1037
1038         return true;
1039 }
1040
1041 static void intel_fbc_get_reg_params(struct intel_crtc *crtc,
1042                                      struct intel_fbc_reg_params *params)
1043 {
1044         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1045         struct intel_fbc *fbc = &dev_priv->fbc;
1046         struct intel_fbc_state_cache *cache = &fbc->state_cache;
1047
1048         /* Since all our fields are integer types, use memset here so the
1049          * comparison function can rely on memcmp because the padding will be
1050          * zero. */
1051         memset(params, 0, sizeof(*params));
1052
1053         params->fence_id = cache->fence_id;
1054         params->fence_y_offset = cache->fence_y_offset;
1055
1056         params->interval = cache->interval;
1057
1058         params->crtc.pipe = crtc->pipe;
1059         params->crtc.i9xx_plane = to_intel_plane(crtc->base.primary)->i9xx_plane;
1060
1061         params->fb.format = cache->fb.format;
1062         params->fb.modifier = cache->fb.modifier;
1063         params->fb.stride = cache->fb.stride;
1064
1065         params->cfb_stride = intel_fbc_cfb_stride(dev_priv, cache);
1066         params->cfb_size = intel_fbc_cfb_size(dev_priv, cache);
1067         params->override_cfb_stride = intel_fbc_override_cfb_stride(dev_priv, cache);
1068
1069         params->plane_visible = cache->plane.visible;
1070 }
1071
1072 static bool intel_fbc_can_flip_nuke(const struct intel_crtc_state *crtc_state)
1073 {
1074         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1075         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1076         const struct intel_fbc *fbc = &dev_priv->fbc;
1077         const struct intel_fbc_state_cache *cache = &fbc->state_cache;
1078         const struct intel_fbc_reg_params *params = &fbc->params;
1079
1080         if (drm_atomic_crtc_needs_modeset(&crtc_state->uapi))
1081                 return false;
1082
1083         if (!params->plane_visible)
1084                 return false;
1085
1086         if (!intel_fbc_can_activate(crtc))
1087                 return false;
1088
1089         if (params->fb.format != cache->fb.format)
1090                 return false;
1091
1092         if (params->fb.modifier != cache->fb.modifier)
1093                 return false;
1094
1095         if (params->fb.stride != cache->fb.stride)
1096                 return false;
1097
1098         if (params->cfb_stride != intel_fbc_cfb_stride(dev_priv, cache))
1099                 return false;
1100
1101         if (params->cfb_size != intel_fbc_cfb_size(dev_priv, cache))
1102                 return false;
1103
1104         if (params->override_cfb_stride != intel_fbc_override_cfb_stride(dev_priv, cache))
1105                 return false;
1106
1107         return true;
1108 }
1109
1110 bool intel_fbc_pre_update(struct intel_atomic_state *state,
1111                           struct intel_crtc *crtc)
1112 {
1113         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
1114         const struct intel_crtc_state *crtc_state =
1115                 intel_atomic_get_new_crtc_state(state, crtc);
1116         const struct intel_plane_state *plane_state =
1117                 intel_atomic_get_new_plane_state(state, plane);
1118         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1119         struct intel_fbc *fbc = &dev_priv->fbc;
1120         const char *reason = "update pending";
1121         bool need_vblank_wait = false;
1122
1123         if (!plane->has_fbc || !plane_state)
1124                 return need_vblank_wait;
1125
1126         mutex_lock(&fbc->lock);
1127
1128         if (fbc->crtc != crtc)
1129                 goto unlock;
1130
1131         intel_fbc_update_state_cache(crtc, crtc_state, plane_state);
1132         fbc->flip_pending = true;
1133
1134         if (!intel_fbc_can_flip_nuke(crtc_state)) {
1135                 intel_fbc_deactivate(dev_priv, reason);
1136
1137                 /*
1138                  * Display WA #1198: glk+
1139                  * Need an extra vblank wait between FBC disable and most plane
1140                  * updates. Bspec says this is only needed for plane disable, but
1141                  * that is not true. Touching most plane registers will cause the
1142                  * corruption to appear. Also SKL/derivatives do not seem to be
1143                  * affected.
1144                  *
1145                  * TODO: could optimize this a bit by sampling the frame
1146                  * counter when we disable FBC (if it was already done earlier)
1147                  * and skipping the extra vblank wait before the plane update
1148                  * if at least one frame has already passed.
1149                  */
1150                 if (fbc->activated &&
1151                     DISPLAY_VER(dev_priv) >= 10)
1152                         need_vblank_wait = true;
1153                 fbc->activated = false;
1154         }
1155 unlock:
1156         mutex_unlock(&fbc->lock);
1157
1158         return need_vblank_wait;
1159 }
1160
1161 /**
1162  * __intel_fbc_disable - disable FBC
1163  * @dev_priv: i915 device instance
1164  *
1165  * This is the low level function that actually disables FBC. Callers should
1166  * grab the FBC lock.
1167  */
1168 static void __intel_fbc_disable(struct drm_i915_private *dev_priv)
1169 {
1170         struct intel_fbc *fbc = &dev_priv->fbc;
1171         struct intel_crtc *crtc = fbc->crtc;
1172
1173         drm_WARN_ON(&dev_priv->drm, !mutex_is_locked(&fbc->lock));
1174         drm_WARN_ON(&dev_priv->drm, !fbc->crtc);
1175         drm_WARN_ON(&dev_priv->drm, fbc->active);
1176
1177         drm_dbg_kms(&dev_priv->drm, "Disabling FBC on pipe %c\n",
1178                     pipe_name(crtc->pipe));
1179
1180         __intel_fbc_cleanup_cfb(dev_priv);
1181
1182         fbc->crtc = NULL;
1183 }
1184
1185 static void __intel_fbc_post_update(struct intel_crtc *crtc)
1186 {
1187         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1188         struct intel_fbc *fbc = &dev_priv->fbc;
1189
1190         drm_WARN_ON(&dev_priv->drm, !mutex_is_locked(&fbc->lock));
1191
1192         if (fbc->crtc != crtc)
1193                 return;
1194
1195         fbc->flip_pending = false;
1196
1197         if (!dev_priv->params.enable_fbc) {
1198                 intel_fbc_deactivate(dev_priv, "disabled at runtime per module param");
1199                 __intel_fbc_disable(dev_priv);
1200
1201                 return;
1202         }
1203
1204         intel_fbc_get_reg_params(crtc, &fbc->params);
1205
1206         if (!intel_fbc_can_activate(crtc))
1207                 return;
1208
1209         if (!fbc->busy_bits)
1210                 intel_fbc_activate(dev_priv);
1211         else
1212                 intel_fbc_deactivate(dev_priv, "frontbuffer write");
1213 }
1214
1215 void intel_fbc_post_update(struct intel_atomic_state *state,
1216                            struct intel_crtc *crtc)
1217 {
1218         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1219         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
1220         const struct intel_plane_state *plane_state =
1221                 intel_atomic_get_new_plane_state(state, plane);
1222         struct intel_fbc *fbc = &dev_priv->fbc;
1223
1224         if (!plane->has_fbc || !plane_state)
1225                 return;
1226
1227         mutex_lock(&fbc->lock);
1228         __intel_fbc_post_update(crtc);
1229         mutex_unlock(&fbc->lock);
1230 }
1231
1232 static unsigned int intel_fbc_get_frontbuffer_bit(struct intel_fbc *fbc)
1233 {
1234         if (fbc->crtc)
1235                 return to_intel_plane(fbc->crtc->base.primary)->frontbuffer_bit;
1236         else
1237                 return fbc->possible_framebuffer_bits;
1238 }
1239
1240 void intel_fbc_invalidate(struct drm_i915_private *dev_priv,
1241                           unsigned int frontbuffer_bits,
1242                           enum fb_op_origin origin)
1243 {
1244         struct intel_fbc *fbc = &dev_priv->fbc;
1245
1246         if (!HAS_FBC(dev_priv))
1247                 return;
1248
1249         if (origin == ORIGIN_FLIP || origin == ORIGIN_CURSOR_UPDATE)
1250                 return;
1251
1252         mutex_lock(&fbc->lock);
1253
1254         fbc->busy_bits |= intel_fbc_get_frontbuffer_bit(fbc) & frontbuffer_bits;
1255
1256         if (fbc->crtc && fbc->busy_bits)
1257                 intel_fbc_deactivate(dev_priv, "frontbuffer write");
1258
1259         mutex_unlock(&fbc->lock);
1260 }
1261
1262 void intel_fbc_flush(struct drm_i915_private *dev_priv,
1263                      unsigned int frontbuffer_bits, enum fb_op_origin origin)
1264 {
1265         struct intel_fbc *fbc = &dev_priv->fbc;
1266
1267         if (!HAS_FBC(dev_priv))
1268                 return;
1269
1270         mutex_lock(&fbc->lock);
1271
1272         fbc->busy_bits &= ~frontbuffer_bits;
1273
1274         if (origin == ORIGIN_FLIP || origin == ORIGIN_CURSOR_UPDATE)
1275                 goto out;
1276
1277         if (!fbc->busy_bits && fbc->crtc &&
1278             (frontbuffer_bits & intel_fbc_get_frontbuffer_bit(fbc))) {
1279                 if (fbc->active)
1280                         intel_fbc_recompress(dev_priv);
1281                 else if (!fbc->flip_pending)
1282                         __intel_fbc_post_update(fbc->crtc);
1283         }
1284
1285 out:
1286         mutex_unlock(&fbc->lock);
1287 }
1288
1289 /**
1290  * intel_fbc_choose_crtc - select a CRTC to enable FBC on
1291  * @dev_priv: i915 device instance
1292  * @state: the atomic state structure
1293  *
1294  * This function looks at the proposed state for CRTCs and planes, then chooses
1295  * which pipe is going to have FBC by setting intel_crtc_state->enable_fbc to
1296  * true.
1297  *
1298  * Later, intel_fbc_enable is going to look for state->enable_fbc and then maybe
1299  * enable FBC for the chosen CRTC. If it does, it will set dev_priv->fbc.crtc.
1300  */
1301 void intel_fbc_choose_crtc(struct drm_i915_private *dev_priv,
1302                            struct intel_atomic_state *state)
1303 {
1304         struct intel_fbc *fbc = &dev_priv->fbc;
1305         struct intel_plane *plane;
1306         struct intel_plane_state *plane_state;
1307         bool crtc_chosen = false;
1308         int i;
1309
1310         mutex_lock(&fbc->lock);
1311
1312         /* Does this atomic commit involve the CRTC currently tied to FBC? */
1313         if (fbc->crtc &&
1314             !intel_atomic_get_new_crtc_state(state, fbc->crtc))
1315                 goto out;
1316
1317         if (!intel_fbc_can_enable(dev_priv))
1318                 goto out;
1319
1320         /* Simply choose the first CRTC that is compatible and has a visible
1321          * plane. We could go for fancier schemes such as checking the plane
1322          * size, but this would just affect the few platforms that don't tie FBC
1323          * to pipe or plane A. */
1324         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1325                 struct intel_crtc_state *crtc_state;
1326                 struct intel_crtc *crtc = to_intel_crtc(plane_state->hw.crtc);
1327
1328                 if (!plane->has_fbc)
1329                         continue;
1330
1331                 if (!plane_state->uapi.visible)
1332                         continue;
1333
1334                 crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
1335
1336                 crtc_state->enable_fbc = true;
1337                 crtc_chosen = true;
1338                 break;
1339         }
1340
1341         if (!crtc_chosen)
1342                 fbc->no_fbc_reason = "no suitable CRTC for FBC";
1343
1344 out:
1345         mutex_unlock(&fbc->lock);
1346 }
1347
1348 /**
1349  * intel_fbc_enable: tries to enable FBC on the CRTC
1350  * @crtc: the CRTC
1351  * @state: corresponding &drm_crtc_state for @crtc
1352  *
1353  * This function checks if the given CRTC was chosen for FBC, then enables it if
1354  * possible. Notice that it doesn't activate FBC. It is valid to call
1355  * intel_fbc_enable multiple times for the same pipe without an
1356  * intel_fbc_disable in the middle, as long as it is deactivated.
1357  */
1358 static void intel_fbc_enable(struct intel_atomic_state *state,
1359                              struct intel_crtc *crtc)
1360 {
1361         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1362         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
1363         const struct intel_crtc_state *crtc_state =
1364                 intel_atomic_get_new_crtc_state(state, crtc);
1365         const struct intel_plane_state *plane_state =
1366                 intel_atomic_get_new_plane_state(state, plane);
1367         struct intel_fbc *fbc = &dev_priv->fbc;
1368         struct intel_fbc_state_cache *cache = &fbc->state_cache;
1369         int min_limit;
1370
1371         if (!plane->has_fbc || !plane_state)
1372                 return;
1373
1374         min_limit = intel_fbc_min_limit(plane_state->hw.fb ?
1375                                         plane_state->hw.fb->format->cpp[0] : 0);
1376
1377         mutex_lock(&fbc->lock);
1378
1379         if (fbc->crtc) {
1380                 if (fbc->crtc != crtc)
1381                         goto out;
1382
1383                 if (fbc->limit >= min_limit &&
1384                     !intel_fbc_cfb_size_changed(dev_priv))
1385                         goto out;
1386
1387                 __intel_fbc_disable(dev_priv);
1388         }
1389
1390         drm_WARN_ON(&dev_priv->drm, fbc->active);
1391
1392         intel_fbc_update_state_cache(crtc, crtc_state, plane_state);
1393
1394         /* FIXME crtc_state->enable_fbc lies :( */
1395         if (!cache->plane.visible)
1396                 goto out;
1397
1398         if (intel_fbc_alloc_cfb(dev_priv,
1399                                 intel_fbc_cfb_size(dev_priv, cache), min_limit)) {
1400                 cache->plane.visible = false;
1401                 fbc->no_fbc_reason = "not enough stolen memory";
1402                 goto out;
1403         }
1404
1405         drm_dbg_kms(&dev_priv->drm, "Enabling FBC on pipe %c\n",
1406                     pipe_name(crtc->pipe));
1407         fbc->no_fbc_reason = "FBC enabled but not active yet\n";
1408
1409         fbc->crtc = crtc;
1410
1411         intel_fbc_program_cfb(dev_priv);
1412 out:
1413         mutex_unlock(&fbc->lock);
1414 }
1415
1416 /**
1417  * intel_fbc_disable - disable FBC if it's associated with crtc
1418  * @crtc: the CRTC
1419  *
1420  * This function disables FBC if it's associated with the provided CRTC.
1421  */
1422 void intel_fbc_disable(struct intel_crtc *crtc)
1423 {
1424         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1425         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
1426         struct intel_fbc *fbc = &dev_priv->fbc;
1427
1428         if (!plane->has_fbc)
1429                 return;
1430
1431         mutex_lock(&fbc->lock);
1432         if (fbc->crtc == crtc)
1433                 __intel_fbc_disable(dev_priv);
1434         mutex_unlock(&fbc->lock);
1435 }
1436
1437 /**
1438  * intel_fbc_update: enable/disable FBC on the CRTC
1439  * @state: atomic state
1440  * @crtc: the CRTC
1441  *
1442  * This function checks if the given CRTC was chosen for FBC, then enables it if
1443  * possible. Notice that it doesn't activate FBC. It is valid to call
1444  * intel_fbc_update multiple times for the same pipe without an
1445  * intel_fbc_disable in the middle.
1446  */
1447 void intel_fbc_update(struct intel_atomic_state *state,
1448                       struct intel_crtc *crtc)
1449 {
1450         const struct intel_crtc_state *crtc_state =
1451                 intel_atomic_get_new_crtc_state(state, crtc);
1452
1453         if (crtc_state->update_pipe && !crtc_state->enable_fbc)
1454                 intel_fbc_disable(crtc);
1455         else
1456                 intel_fbc_enable(state, crtc);
1457 }
1458
1459 /**
1460  * intel_fbc_global_disable - globally disable FBC
1461  * @dev_priv: i915 device instance
1462  *
1463  * This function disables FBC regardless of which CRTC is associated with it.
1464  */
1465 void intel_fbc_global_disable(struct drm_i915_private *dev_priv)
1466 {
1467         struct intel_fbc *fbc = &dev_priv->fbc;
1468
1469         if (!HAS_FBC(dev_priv))
1470                 return;
1471
1472         mutex_lock(&fbc->lock);
1473         if (fbc->crtc) {
1474                 drm_WARN_ON(&dev_priv->drm, fbc->crtc->active);
1475                 __intel_fbc_disable(dev_priv);
1476         }
1477         mutex_unlock(&fbc->lock);
1478 }
1479
1480 static void intel_fbc_underrun_work_fn(struct work_struct *work)
1481 {
1482         struct drm_i915_private *dev_priv =
1483                 container_of(work, struct drm_i915_private, fbc.underrun_work);
1484         struct intel_fbc *fbc = &dev_priv->fbc;
1485
1486         mutex_lock(&fbc->lock);
1487
1488         /* Maybe we were scheduled twice. */
1489         if (fbc->underrun_detected || !fbc->crtc)
1490                 goto out;
1491
1492         drm_dbg_kms(&dev_priv->drm, "Disabling FBC due to FIFO underrun.\n");
1493         fbc->underrun_detected = true;
1494
1495         intel_fbc_deactivate(dev_priv, "FIFO underrun");
1496 out:
1497         mutex_unlock(&fbc->lock);
1498 }
1499
1500 /*
1501  * intel_fbc_reset_underrun - reset FBC fifo underrun status.
1502  * @dev_priv: i915 device instance
1503  *
1504  * See intel_fbc_handle_fifo_underrun_irq(). For automated testing we
1505  * want to re-enable FBC after an underrun to increase test coverage.
1506  */
1507 int intel_fbc_reset_underrun(struct drm_i915_private *dev_priv)
1508 {
1509         int ret;
1510
1511         cancel_work_sync(&dev_priv->fbc.underrun_work);
1512
1513         ret = mutex_lock_interruptible(&dev_priv->fbc.lock);
1514         if (ret)
1515                 return ret;
1516
1517         if (dev_priv->fbc.underrun_detected) {
1518                 drm_dbg_kms(&dev_priv->drm,
1519                             "Re-allowing FBC after fifo underrun\n");
1520                 dev_priv->fbc.no_fbc_reason = "FIFO underrun cleared";
1521         }
1522
1523         dev_priv->fbc.underrun_detected = false;
1524         mutex_unlock(&dev_priv->fbc.lock);
1525
1526         return 0;
1527 }
1528
1529 /**
1530  * intel_fbc_handle_fifo_underrun_irq - disable FBC when we get a FIFO underrun
1531  * @dev_priv: i915 device instance
1532  *
1533  * Without FBC, most underruns are harmless and don't really cause too many
1534  * problems, except for an annoying message on dmesg. With FBC, underruns can
1535  * become black screens or even worse, especially when paired with bad
1536  * watermarks. So in order for us to be on the safe side, completely disable FBC
1537  * in case we ever detect a FIFO underrun on any pipe. An underrun on any pipe
1538  * already suggests that watermarks may be bad, so try to be as safe as
1539  * possible.
1540  *
1541  * This function is called from the IRQ handler.
1542  */
1543 void intel_fbc_handle_fifo_underrun_irq(struct drm_i915_private *dev_priv)
1544 {
1545         struct intel_fbc *fbc = &dev_priv->fbc;
1546
1547         if (!HAS_FBC(dev_priv))
1548                 return;
1549
1550         /* There's no guarantee that underrun_detected won't be set to true
1551          * right after this check and before the work is scheduled, but that's
1552          * not a problem since we'll check it again under the work function
1553          * while FBC is locked. This check here is just to prevent us from
1554          * unnecessarily scheduling the work, and it relies on the fact that we
1555          * never switch underrun_detect back to false after it's true. */
1556         if (READ_ONCE(fbc->underrun_detected))
1557                 return;
1558
1559         schedule_work(&fbc->underrun_work);
1560 }
1561
1562 /*
1563  * The DDX driver changes its behavior depending on the value it reads from
1564  * i915.enable_fbc, so sanitize it by translating the default value into either
1565  * 0 or 1 in order to allow it to know what's going on.
1566  *
1567  * Notice that this is done at driver initialization and we still allow user
1568  * space to change the value during runtime without sanitizing it again. IGT
1569  * relies on being able to change i915.enable_fbc at runtime.
1570  */
1571 static int intel_sanitize_fbc_option(struct drm_i915_private *dev_priv)
1572 {
1573         if (dev_priv->params.enable_fbc >= 0)
1574                 return !!dev_priv->params.enable_fbc;
1575
1576         if (!HAS_FBC(dev_priv))
1577                 return 0;
1578
1579         if (IS_BROADWELL(dev_priv) || DISPLAY_VER(dev_priv) >= 9)
1580                 return 1;
1581
1582         return 0;
1583 }
1584
1585 static bool need_fbc_vtd_wa(struct drm_i915_private *dev_priv)
1586 {
1587         /* WaFbcTurnOffFbcWhenHyperVisorIsUsed:skl,bxt */
1588         if (intel_vtd_active() &&
1589             (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))) {
1590                 drm_info(&dev_priv->drm,
1591                          "Disabling framebuffer compression (FBC) to prevent screen flicker with VT-d enabled\n");
1592                 return true;
1593         }
1594
1595         return false;
1596 }
1597
1598 /**
1599  * intel_fbc_init - Initialize FBC
1600  * @dev_priv: the i915 device
1601  *
1602  * This function might be called during PM init process.
1603  */
1604 void intel_fbc_init(struct drm_i915_private *dev_priv)
1605 {
1606         struct intel_fbc *fbc = &dev_priv->fbc;
1607
1608         INIT_WORK(&fbc->underrun_work, intel_fbc_underrun_work_fn);
1609         mutex_init(&fbc->lock);
1610         fbc->active = false;
1611
1612         if (!drm_mm_initialized(&dev_priv->mm.stolen))
1613                 mkwrite_device_info(dev_priv)->display.has_fbc = false;
1614
1615         if (need_fbc_vtd_wa(dev_priv))
1616                 mkwrite_device_info(dev_priv)->display.has_fbc = false;
1617
1618         dev_priv->params.enable_fbc = intel_sanitize_fbc_option(dev_priv);
1619         drm_dbg_kms(&dev_priv->drm, "Sanitized enable_fbc value: %d\n",
1620                     dev_priv->params.enable_fbc);
1621
1622         if (!HAS_FBC(dev_priv)) {
1623                 fbc->no_fbc_reason = "unsupported by this chipset";
1624                 return;
1625         }
1626
1627         /* We still don't have any sort of hardware state readout for FBC, so
1628          * deactivate it in case the BIOS activated it to make sure software
1629          * matches the hardware state. */
1630         if (intel_fbc_hw_is_active(dev_priv))
1631                 intel_fbc_hw_deactivate(dev_priv);
1632 }