drivers/gpu/drm/i915/display/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/i2c.h>
  28 #include <linux/input.h>
  29 #include <linux/intel-iommu.h>
  30 #include <linux/kernel.h>
  31 #include <linux/module.h>
  32 #include <linux/dma-resv.h>
  33 #include <linux/slab.h>
  34
  35 #include <drm/drm_atomic.h>
  36 #include <drm/drm_atomic_helper.h>
  37 #include <drm/drm_atomic_uapi.h>
  38 #include <drm/drm_damage_helper.h>
  39 #include <drm/drm_dp_helper.h>
  40 #include <drm/drm_edid.h>
  41 #include <drm/drm_fourcc.h>
  42 #include <drm/drm_plane_helper.h>
  43 #include <drm/drm_probe_helper.h>
  44 #include <drm/drm_rect.h>
  45
  46 #include "display/intel_crt.h"
  47 #include "display/intel_ddi.h"
  48 #include "display/intel_dp.h"
  49 #include "display/intel_dp_mst.h"
  50 #include "display/intel_dpll_mgr.h"
  51 #include "display/intel_dsi.h"
  52 #include "display/intel_dvo.h"
  53 #include "display/intel_gmbus.h"
  54 #include "display/intel_hdmi.h"
  55 #include "display/intel_lvds.h"
  56 #include "display/intel_sdvo.h"
  57 #include "display/intel_tv.h"
  58 #include "display/intel_vdsc.h"
  59
  60 #include "gt/intel_rps.h"
  61
  62 #include "i915_drv.h"
  63 #include "i915_trace.h"
  64 #include "intel_acpi.h"
  65 #include "intel_atomic.h"
  66 #include "intel_atomic_plane.h"
  67 #include "intel_bw.h"
  68 #include "intel_cdclk.h"
  69 #include "intel_color.h"
  70 #include "intel_display_types.h"
  71 #include "intel_dp_link_training.h"
  72 #include "intel_fbc.h"
  73 #include "intel_fbdev.h"
  74 #include "intel_fifo_underrun.h"
  75 #include "intel_frontbuffer.h"
  76 #include "intel_hdcp.h"
  77 #include "intel_hotplug.h"
  78 #include "intel_overlay.h"
  79 #include "intel_pipe_crc.h"
  80 #include "intel_pm.h"
  81 #include "intel_psr.h"
  82 #include "intel_quirks.h"
  83 #include "intel_sideband.h"
  84 #include "intel_sprite.h"
  85 #include "intel_tc.h"
  86 #include "intel_vga.h"
  87
  88 /* Primary plane formats for gen <= 3 */
  89 static const u32 i8xx_primary_formats[] = {
  90         DRM_FORMAT_C8,
  91         DRM_FORMAT_XRGB1555,
  92         DRM_FORMAT_RGB565,
  93         DRM_FORMAT_XRGB8888,
  94 };
  95
  96 /* Primary plane formats for ivb (no fp16 due to hw issue) */
  97 static const u32 ivb_primary_formats[] = {
  98         DRM_FORMAT_C8,
  99         DRM_FORMAT_RGB565,
 100         DRM_FORMAT_XRGB8888,
 101         DRM_FORMAT_XBGR8888,
 102         DRM_FORMAT_XRGB2101010,
 103         DRM_FORMAT_XBGR2101010,
 104 };
 105
 106 /* Primary plane formats for gen >= 4, except ivb */
 107 static const u32 i965_primary_formats[] = {
 108         DRM_FORMAT_C8,
 109         DRM_FORMAT_RGB565,
 110         DRM_FORMAT_XRGB8888,
 111         DRM_FORMAT_XBGR8888,
 112         DRM_FORMAT_XRGB2101010,
 113         DRM_FORMAT_XBGR2101010,
 114         DRM_FORMAT_XBGR16161616F,
 115 };
 116
 117 /* Primary plane formats for vlv/chv */
 118 static const u32 vlv_primary_formats[] = {
 119         DRM_FORMAT_C8,
 120         DRM_FORMAT_RGB565,
 121         DRM_FORMAT_XRGB8888,
 122         DRM_FORMAT_XBGR8888,
 123         DRM_FORMAT_ARGB8888,
 124         DRM_FORMAT_ABGR8888,
 125         DRM_FORMAT_XRGB2101010,
 126         DRM_FORMAT_XBGR2101010,
 127         DRM_FORMAT_ARGB2101010,
 128         DRM_FORMAT_ABGR2101010,
 129         DRM_FORMAT_XBGR16161616F,
 130 };
 131
 132 static const u64 i9xx_format_modifiers[] = {
 133         I915_FORMAT_MOD_X_TILED,
 134         DRM_FORMAT_MOD_LINEAR,
 135         DRM_FORMAT_MOD_INVALID
 136 };
 137
 138 /* Cursor formats */
 139 static const u32 intel_cursor_formats[] = {
 140         DRM_FORMAT_ARGB8888,
 141 };
 142
 143 static const u64 cursor_format_modifiers[] = {
 144         DRM_FORMAT_MOD_LINEAR,
 145         DRM_FORMAT_MOD_INVALID
 146 };
 147
 148 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
 149                                 struct intel_crtc_state *pipe_config);
 150 static void ilk_pch_clock_get(struct intel_crtc *crtc,
 151                               struct intel_crtc_state *pipe_config);
 152
 153 static int intel_framebuffer_init(struct intel_framebuffer *ifb,
 154                                   struct drm_i915_gem_object *obj,
 155                                   struct drm_mode_fb_cmd2 *mode_cmd);
 156 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state);
 157 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state);
 158 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
 159                                          const struct intel_link_m_n *m_n,
 160                                          const struct intel_link_m_n *m2_n2);
 161 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state);
 162 static void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state);
 163 static void hsw_set_pipeconf(const struct intel_crtc_state *crtc_state);
 164 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state);
 165 static void vlv_prepare_pll(struct intel_crtc *crtc,
 166                             const struct intel_crtc_state *pipe_config);
 167 static void chv_prepare_pll(struct intel_crtc *crtc,
 168                             const struct intel_crtc_state *pipe_config);
 169 static void skl_pfit_enable(const struct intel_crtc_state *crtc_state);
 170 static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state);
 171 static void intel_modeset_setup_hw_state(struct drm_device *dev,
 172                                          struct drm_modeset_acquire_ctx *ctx);
 173 static struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc);
 174
 175 struct intel_limit {
 176         struct {
 177                 int min, max;
 178         } dot, vco, n, m, m1, m2, p, p1;
 179
 180         struct {
 181                 int dot_limit;
 182                 int p2_slow, p2_fast;
 183         } p2;
 184 };
 185
 186 /* returns HPLL frequency in kHz */
 187 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv)
 188 {
 189         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 190
 191         /* Obtain SKU information */
 192         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
 193                 CCK_FUSE_HPLL_FREQ_MASK;
 194
 195         return vco_freq[hpll_freq] * 1000;
 196 }
 197
 198 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
 199                       const char *name, u32 reg, int ref_freq)
 200 {
 201         u32 val;
 202         int divider;
 203
 204         val = vlv_cck_read(dev_priv, reg);
 205         divider = val & CCK_FREQUENCY_VALUES;
 206
 207         drm_WARN(&dev_priv->drm, (val & CCK_FREQUENCY_STATUS) !=
 208                  (divider << CCK_FREQUENCY_STATUS_SHIFT),
 209                  "%s change in progress\n", name);
 210
 211         return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1);
 212 }
 213
 214 int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 215                            const char *name, u32 reg)
 216 {
 217         int hpll;
 218
 219         vlv_cck_get(dev_priv);
 220
 221         if (dev_priv->hpll_freq == 0)
 222                 dev_priv->hpll_freq = vlv_get_hpll_vco(dev_priv);
 223
 224         hpll = vlv_get_cck_clock(dev_priv, name, reg, dev_priv->hpll_freq);
 225
 226         vlv_cck_put(dev_priv);
 227
 228         return hpll;
 229 }
 230
 231 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 232 {
 233         if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
 234                 return;
 235
 236         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
 237                                                       CCK_CZ_CLOCK_CONTROL);
 238
 239         drm_dbg(&dev_priv->drm, "CZ clock rate: %d kHz\n",
 240                 dev_priv->czclk_freq);
 241 }
 242
 243 /* units of 100MHz */
 244 static u32 intel_fdi_link_freq(struct drm_i915_private *dev_priv,
 245                                const struct intel_crtc_state *pipe_config)
 246 {
 247         if (HAS_DDI(dev_priv))
 248                 return pipe_config->port_clock; /* SPLL */
 249         else
 250                 return dev_priv->fdi_pll_freq;
 251 }
 252
 253 static const struct intel_limit intel_limits_i8xx_dac = {
 254         .dot = { .min = 25000, .max = 350000 },
 255         .vco = { .min = 908000, .max = 1512000 },
 256         .n = { .min = 2, .max = 16 },
 257         .m = { .min = 96, .max = 140 },
 258         .m1 = { .min = 18, .max = 26 },
 259         .m2 = { .min = 6, .max = 16 },
 260         .p = { .min = 4, .max = 128 },
 261         .p1 = { .min = 2, .max = 33 },
 262         .p2 = { .dot_limit = 165000,
 263                 .p2_slow = 4, .p2_fast = 2 },
 264 };
 265
 266 static const struct intel_limit intel_limits_i8xx_dvo = {
 267         .dot = { .min = 25000, .max = 350000 },
 268         .vco = { .min = 908000, .max = 1512000 },
 269         .n = { .min = 2, .max = 16 },
 270         .m = { .min = 96, .max = 140 },
 271         .m1 = { .min = 18, .max = 26 },
 272         .m2 = { .min = 6, .max = 16 },
 273         .p = { .min = 4, .max = 128 },
 274         .p1 = { .min = 2, .max = 33 },
 275         .p2 = { .dot_limit = 165000,
 276                 .p2_slow = 4, .p2_fast = 4 },
 277 };
 278
 279 static const struct intel_limit intel_limits_i8xx_lvds = {
 280         .dot = { .min = 25000, .max = 350000 },
 281         .vco = { .min = 908000, .max = 1512000 },
 282         .n = { .min = 2, .max = 16 },
 283         .m = { .min = 96, .max = 140 },
 284         .m1 = { .min = 18, .max = 26 },
 285         .m2 = { .min = 6, .max = 16 },
 286         .p = { .min = 4, .max = 128 },
 287         .p1 = { .min = 1, .max = 6 },
 288         .p2 = { .dot_limit = 165000,
 289                 .p2_slow = 14, .p2_fast = 7 },
 290 };
 291
 292 static const struct intel_limit intel_limits_i9xx_sdvo = {
 293         .dot = { .min = 20000, .max = 400000 },
 294         .vco = { .min = 1400000, .max = 2800000 },
 295         .n = { .min = 1, .max = 6 },
 296         .m = { .min = 70, .max = 120 },
 297         .m1 = { .min = 8, .max = 18 },
 298         .m2 = { .min = 3, .max = 7 },
 299         .p = { .min = 5, .max = 80 },
 300         .p1 = { .min = 1, .max = 8 },
 301         .p2 = { .dot_limit = 200000,
 302                 .p2_slow = 10, .p2_fast = 5 },
 303 };
 304
 305 static const struct intel_limit intel_limits_i9xx_lvds = {
 306         .dot = { .min = 20000, .max = 400000 },
 307         .vco = { .min = 1400000, .max = 2800000 },
 308         .n = { .min = 1, .max = 6 },
 309         .m = { .min = 70, .max = 120 },
 310         .m1 = { .min = 8, .max = 18 },
 311         .m2 = { .min = 3, .max = 7 },
 312         .p = { .min = 7, .max = 98 },
 313         .p1 = { .min = 1, .max = 8 },
 314         .p2 = { .dot_limit = 112000,
 315                 .p2_slow = 14, .p2_fast = 7 },
 316 };
 317
 318
 319 static const struct intel_limit intel_limits_g4x_sdvo = {
 320         .dot = { .min = 25000, .max = 270000 },
 321         .vco = { .min = 1750000, .max = 3500000},
 322         .n = { .min = 1, .max = 4 },
 323         .m = { .min = 104, .max = 138 },
 324         .m1 = { .min = 17, .max = 23 },
 325         .m2 = { .min = 5, .max = 11 },
 326         .p = { .min = 10, .max = 30 },
 327         .p1 = { .min = 1, .max = 3},
 328         .p2 = { .dot_limit = 270000,
 329                 .p2_slow = 10,
 330                 .p2_fast = 10
 331         },
 332 };
 333
 334 static const struct intel_limit intel_limits_g4x_hdmi = {
 335         .dot = { .min = 22000, .max = 400000 },
 336         .vco = { .min = 1750000, .max = 3500000},
 337         .n = { .min = 1, .max = 4 },
 338         .m = { .min = 104, .max = 138 },
 339         .m1 = { .min = 16, .max = 23 },
 340         .m2 = { .min = 5, .max = 11 },
 341         .p = { .min = 5, .max = 80 },
 342         .p1 = { .min = 1, .max = 8},
 343         .p2 = { .dot_limit = 165000,
 344                 .p2_slow = 10, .p2_fast = 5 },
 345 };
 346
 347 static const struct intel_limit intel_limits_g4x_single_channel_lvds = {
 348         .dot = { .min = 20000, .max = 115000 },
 349         .vco = { .min = 1750000, .max = 3500000 },
 350         .n = { .min = 1, .max = 3 },
 351         .m = { .min = 104, .max = 138 },
 352         .m1 = { .min = 17, .max = 23 },
 353         .m2 = { .min = 5, .max = 11 },
 354         .p = { .min = 28, .max = 112 },
 355         .p1 = { .min = 2, .max = 8 },
 356         .p2 = { .dot_limit = 0,
 357                 .p2_slow = 14, .p2_fast = 14
 358         },
 359 };
 360
 361 static const struct intel_limit intel_limits_g4x_dual_channel_lvds = {
 362         .dot = { .min = 80000, .max = 224000 },
 363         .vco = { .min = 1750000, .max = 3500000 },
 364         .n = { .min = 1, .max = 3 },
 365         .m = { .min = 104, .max = 138 },
 366         .m1 = { .min = 17, .max = 23 },
 367         .m2 = { .min = 5, .max = 11 },
 368         .p = { .min = 14, .max = 42 },
 369         .p1 = { .min = 2, .max = 6 },
 370         .p2 = { .dot_limit = 0,
 371                 .p2_slow = 7, .p2_fast = 7
 372         },
 373 };
 374
 375 static const struct intel_limit pnv_limits_sdvo = {
 376         .dot = { .min = 20000, .max = 400000},
 377         .vco = { .min = 1700000, .max = 3500000 },
 378         /* Pineview's Ncounter is a ring counter */
 379         .n = { .min = 3, .max = 6 },
 380         .m = { .min = 2, .max = 256 },
 381         /* Pineview only has one combined m divider, which we treat as m2. */
 382         .m1 = { .min = 0, .max = 0 },
 383         .m2 = { .min = 0, .max = 254 },
 384         .p = { .min = 5, .max = 80 },
 385         .p1 = { .min = 1, .max = 8 },
 386         .p2 = { .dot_limit = 200000,
 387                 .p2_slow = 10, .p2_fast = 5 },
 388 };
 389
 390 static const struct intel_limit pnv_limits_lvds = {
 391         .dot = { .min = 20000, .max = 400000 },
 392         .vco = { .min = 1700000, .max = 3500000 },
 393         .n = { .min = 3, .max = 6 },
 394         .m = { .min = 2, .max = 256 },
 395         .m1 = { .min = 0, .max = 0 },
 396         .m2 = { .min = 0, .max = 254 },
 397         .p = { .min = 7, .max = 112 },
 398         .p1 = { .min = 1, .max = 8 },
 399         .p2 = { .dot_limit = 112000,
 400                 .p2_slow = 14, .p2_fast = 14 },
 401 };
 402
 403 /* Ironlake / Sandybridge
 404  *
 405  * We calculate clock using (register_value + 2) for N/M1/M2, so here
 406  * the range value for them is (actual_value - 2).
 407  */
 408 static const struct intel_limit ilk_limits_dac = {
 409         .dot = { .min = 25000, .max = 350000 },
 410         .vco = { .min = 1760000, .max = 3510000 },
 411         .n = { .min = 1, .max = 5 },
 412         .m = { .min = 79, .max = 127 },
 413         .m1 = { .min = 12, .max = 22 },
 414         .m2 = { .min = 5, .max = 9 },
 415         .p = { .min = 5, .max = 80 },
 416         .p1 = { .min = 1, .max = 8 },
 417         .p2 = { .dot_limit = 225000,
 418                 .p2_slow = 10, .p2_fast = 5 },
 419 };
 420
 421 static const struct intel_limit ilk_limits_single_lvds = {
 422         .dot = { .min = 25000, .max = 350000 },
 423         .vco = { .min = 1760000, .max = 3510000 },
 424         .n = { .min = 1, .max = 3 },
 425         .m = { .min = 79, .max = 118 },
 426         .m1 = { .min = 12, .max = 22 },
 427         .m2 = { .min = 5, .max = 9 },
 428         .p = { .min = 28, .max = 112 },
 429         .p1 = { .min = 2, .max = 8 },
 430         .p2 = { .dot_limit = 225000,
 431                 .p2_slow = 14, .p2_fast = 14 },
 432 };
 433
 434 static const struct intel_limit ilk_limits_dual_lvds = {
 435         .dot = { .min = 25000, .max = 350000 },
 436         .vco = { .min = 1760000, .max = 3510000 },
 437         .n = { .min = 1, .max = 3 },
 438         .m = { .min = 79, .max = 127 },
 439         .m1 = { .min = 12, .max = 22 },
 440         .m2 = { .min = 5, .max = 9 },
 441         .p = { .min = 14, .max = 56 },
 442         .p1 = { .min = 2, .max = 8 },
 443         .p2 = { .dot_limit = 225000,
 444                 .p2_slow = 7, .p2_fast = 7 },
 445 };
 446
 447 /* LVDS 100mhz refclk limits. */
 448 static const struct intel_limit ilk_limits_single_lvds_100m = {
 449         .dot = { .min = 25000, .max = 350000 },
 450         .vco = { .min = 1760000, .max = 3510000 },
 451         .n = { .min = 1, .max = 2 },
 452         .m = { .min = 79, .max = 126 },
 453         .m1 = { .min = 12, .max = 22 },
 454         .m2 = { .min = 5, .max = 9 },
 455         .p = { .min = 28, .max = 112 },
 456         .p1 = { .min = 2, .max = 8 },
 457         .p2 = { .dot_limit = 225000,
 458                 .p2_slow = 14, .p2_fast = 14 },
 459 };
 460
 461 static const struct intel_limit ilk_limits_dual_lvds_100m = {
 462         .dot = { .min = 25000, .max = 350000 },
 463         .vco = { .min = 1760000, .max = 3510000 },
 464         .n = { .min = 1, .max = 3 },
 465         .m = { .min = 79, .max = 126 },
 466         .m1 = { .min = 12, .max = 22 },
 467         .m2 = { .min = 5, .max = 9 },
 468         .p = { .min = 14, .max = 42 },
 469         .p1 = { .min = 2, .max = 6 },
 470         .p2 = { .dot_limit = 225000,
 471                 .p2_slow = 7, .p2_fast = 7 },
 472 };
 473
 474 static const struct intel_limit intel_limits_vlv = {
 475          /*
 476           * These are the data rate limits (measured in fast clocks)
 477           * since those are the strictest limits we have. The fast
 478           * clock and actual rate limits are more relaxed, so checking
 479           * them would make no difference.
 480           */
 481         .dot = { .min = 25000 * 5, .max = 270000 * 5 },
 482         .vco = { .min = 4000000, .max = 6000000 },
 483         .n = { .min = 1, .max = 7 },
 484         .m1 = { .min = 2, .max = 3 },
 485         .m2 = { .min = 11, .max = 156 },
 486         .p1 = { .min = 2, .max = 3 },
 487         .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */
 488 };
 489
 490 static const struct intel_limit intel_limits_chv = {
 491         /*
 492          * These are the data rate limits (measured in fast clocks)
 493          * since those are the strictest limits we have.  The fast
 494          * clock and actual rate limits are more relaxed, so checking
 495          * them would make no difference.
 496          */
 497         .dot = { .min = 25000 * 5, .max = 540000 * 5},
 498         .vco = { .min = 4800000, .max = 6480000 },
 499         .n = { .min = 1, .max = 1 },
 500         .m1 = { .min = 2, .max = 2 },
 501         .m2 = { .min = 24 << 22, .max = 175 << 22 },
 502         .p1 = { .min = 2, .max = 4 },
 503         .p2 = { .p2_slow = 1, .p2_fast = 14 },
 504 };
 505
 506 static const struct intel_limit intel_limits_bxt = {
 507         /* FIXME: find real dot limits */
 508         .dot = { .min = 0, .max = INT_MAX },
 509         .vco = { .min = 4800000, .max = 6700000 },
 510         .n = { .min = 1, .max = 1 },
 511         .m1 = { .min = 2, .max = 2 },
 512         /* FIXME: find real m2 limits */
 513         .m2 = { .min = 2 << 22, .max = 255 << 22 },
 514         .p1 = { .min = 2, .max = 4 },
 515         .p2 = { .p2_slow = 1, .p2_fast = 20 },
 516 };
 517
 518 /* WA Display #0827: Gen9:all */
 519 static void
 520 skl_wa_827(struct drm_i915_private *dev_priv, enum pipe pipe, bool enable)
 521 {
 522         if (enable)
 523                 intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe),
 524                                intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) | DUPS1_GATING_DIS | DUPS2_GATING_DIS);
 525         else
 526                 intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe),
 527                                intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) & ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS));
 528 }
 529
 530 /* Wa_2006604312:icl,ehl */
 531 static void
 532 icl_wa_scalerclkgating(struct drm_i915_private *dev_priv, enum pipe pipe,
 533                        bool enable)
 534 {
 535         if (enable)
 536                 intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe),
 537                                intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) | DPFR_GATING_DIS);
 538         else
 539                 intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe),
 540                                intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) & ~DPFR_GATING_DIS);
 541 }
 542
 543 static bool
 544 needs_modeset(const struct intel_crtc_state *state)
 545 {
 546         return drm_atomic_crtc_needs_modeset(&state->uapi);
 547 }
 548
 549 static bool
 550 is_trans_port_sync_slave(const struct intel_crtc_state *crtc_state)
 551 {
 552         return crtc_state->master_transcoder != INVALID_TRANSCODER;
 553 }
 554
 555 static bool
 556 is_trans_port_sync_master(const struct intel_crtc_state *crtc_state)
 557 {
 558         return crtc_state->sync_mode_slaves_mask != 0;
 559 }
 560
 561 bool
 562 is_trans_port_sync_mode(const struct intel_crtc_state *crtc_state)
 563 {
 564         return is_trans_port_sync_master(crtc_state) ||
 565                 is_trans_port_sync_slave(crtc_state);
 566 }
 567
 568 /*
 569  * Platform specific helpers to calculate the port PLL loopback- (clock.m),
 570  * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast
 571  * (clock.dot) clock rates. This fast dot clock is fed to the port's IO logic.
 572  * The helpers' return value is the rate of the clock that is fed to the
 573  * display engine's pipe which can be the above fast dot clock rate or a
 574  * divided-down version of it.
 575  */
 576 /* m1 is reserved as 0 in Pineview, n is a ring counter */
 577 static int pnv_calc_dpll_params(int refclk, struct dpll *clock)
 578 {
 579         clock->m = clock->m2 + 2;
 580         clock->p = clock->p1 * clock->p2;
 581         if (WARN_ON(clock->n == 0 || clock->p == 0))
 582                 return 0;
 583         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 584         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 585
 586         return clock->dot;
 587 }
 588
 589 static u32 i9xx_dpll_compute_m(struct dpll *dpll)
 590 {
 591         return 5 * (dpll->m1 + 2) + (dpll->m2 + 2);
 592 }
 593
 594 static int i9xx_calc_dpll_params(int refclk, struct dpll *clock)
 595 {
 596         clock->m = i9xx_dpll_compute_m(clock);
 597         clock->p = clock->p1 * clock->p2;
 598         if (WARN_ON(clock->n + 2 == 0 || clock->p == 0))
 599                 return 0;
 600         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n + 2);
 601         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 602
 603         return clock->dot;
 604 }
 605
 606 static int vlv_calc_dpll_params(int refclk, struct dpll *clock)
 607 {
 608         clock->m = clock->m1 * clock->m2;
 609         clock->p = clock->p1 * clock->p2;
 610         if (WARN_ON(clock->n == 0 || clock->p == 0))
 611                 return 0;
 612         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 613         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 614
 615         return clock->dot / 5;
 616 }
 617
 618 int chv_calc_dpll_params(int refclk, struct dpll *clock)
 619 {
 620         clock->m = clock->m1 * clock->m2;
 621         clock->p = clock->p1 * clock->p2;
 622         if (WARN_ON(clock->n == 0 || clock->p == 0))
 623                 return 0;
 624         clock->vco = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(refclk, clock->m),
 625                                            clock->n << 22);
 626         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 627
 628         return clock->dot / 5;
 629 }
 630
 631 /*
 632  * Returns whether the given set of divisors are valid for a given refclk with
 633  * the given connectors.
 634  */
 635 static bool intel_pll_is_valid(struct drm_i915_private *dev_priv,
 636                                const struct intel_limit *limit,
 637                                const struct dpll *clock)
 638 {
 639         if (clock->n < limit->n.min || limit->n.max < clock->n)
 640                 return false;
 641         if (clock->p1 < limit->p1.min || limit->p1.max < clock->p1)
 642                 return false;
 643         if (clock->m2 < limit->m2.min || limit->m2.max < clock->m2)
 644                 return false;
 645         if (clock->m1 < limit->m1.min || limit->m1.max < clock->m1)
 646                 return false;
 647
 648         if (!IS_PINEVIEW(dev_priv) && !IS_VALLEYVIEW(dev_priv) &&
 649             !IS_CHERRYVIEW(dev_priv) && !IS_GEN9_LP(dev_priv))
 650                 if (clock->m1 <= clock->m2)
 651                         return false;
 652
 653         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
 654             !IS_GEN9_LP(dev_priv)) {
 655                 if (clock->p < limit->p.min || limit->p.max < clock->p)
 656                         return false;
 657                 if (clock->m < limit->m.min || limit->m.max < clock->m)
 658                         return false;
 659         }
 660
 661         if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
 662                 return false;
 663         /* XXX: We may need to be checking "Dot clock" depending on the multiplier,
 664          * connector, etc., rather than just a single range.
 665          */
 666         if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
 667                 return false;
 668
 669         return true;
 670 }
 671
 672 static int
 673 i9xx_select_p2_div(const struct intel_limit *limit,
 674                    const struct intel_crtc_state *crtc_state,
 675                    int target)
 676 {
 677         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 678
 679         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 680                 /*
 681                  * For LVDS just rely on its current settings for dual-channel.
 682                  * We haven't figured out how to reliably set up different
 683                  * single/dual channel state, if we even can.
 684                  */
 685                 if (intel_is_dual_link_lvds(dev_priv))
 686                         return limit->p2.p2_fast;
 687                 else
 688                         return limit->p2.p2_slow;
 689         } else {
 690                 if (target < limit->p2.dot_limit)
 691                         return limit->p2.p2_slow;
 692                 else
 693                         return limit->p2.p2_fast;
 694         }
 695 }
 696
 697 /*
 698  * Returns a set of divisors for the desired target clock with the given
 699  * refclk, or FALSE.  The returned values represent the clock equation:
 700  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 701  *
 702  * Target and reference clocks are specified in kHz.
 703  *
 704  * If match_clock is provided, then best_clock P divider must match the P
 705  * divider from @match_clock used for LVDS downclocking.
 706  */
 707 static bool
 708 i9xx_find_best_dpll(const struct intel_limit *limit,
 709                     struct intel_crtc_state *crtc_state,
 710                     int target, int refclk, struct dpll *match_clock,
 711                     struct dpll *best_clock)
 712 {
 713         struct drm_device *dev = crtc_state->uapi.crtc->dev;
 714         struct dpll clock;
 715         int err = target;
 716
 717         memset(best_clock, 0, sizeof(*best_clock));
 718
 719         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 720
 721         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 722              clock.m1++) {
 723                 for (clock.m2 = limit->m2.min;
 724                      clock.m2 <= limit->m2.max; clock.m2++) {
 725                         if (clock.m2 >= clock.m1)
 726                                 break;
 727                         for (clock.n = limit->n.min;
 728                              clock.n <= limit->n.max; clock.n++) {
 729                                 for (clock.p1 = limit->p1.min;
 730                                         clock.p1 <= limit->p1.max; clock.p1++) {
 731                                         int this_err;
 732
 733                                         i9xx_calc_dpll_params(refclk, &clock);
 734                                         if (!intel_pll_is_valid(to_i915(dev),
 735                                                                 limit,
 736                                                                 &clock))
 737                                                 continue;
 738                                         if (match_clock &&
 739                                             clock.p != match_clock->p)
 740                                                 continue;
 741
 742                                         this_err = abs(clock.dot - target);
 743                                         if (this_err < err) {
 744                                                 *best_clock = clock;
 745                                                 err = this_err;
 746                                         }
 747                                 }
 748                         }
 749                 }
 750         }
 751
 752         return (err != target);
 753 }
 754
 755 /*
 756  * Returns a set of divisors for the desired target clock with the given
 757  * refclk, or FALSE.  The returned values represent the clock equation:
 758  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 759  *
 760  * Target and reference clocks are specified in kHz.
 761  *
 762  * If match_clock is provided, then best_clock P divider must match the P
 763  * divider from @match_clock used for LVDS downclocking.
 764  */
 765 static bool
 766 pnv_find_best_dpll(const struct intel_limit *limit,
 767                    struct intel_crtc_state *crtc_state,
 768                    int target, int refclk, struct dpll *match_clock,
 769                    struct dpll *best_clock)
 770 {
 771         struct drm_device *dev = crtc_state->uapi.crtc->dev;
 772         struct dpll clock;
 773         int err = target;
 774
 775         memset(best_clock, 0, sizeof(*best_clock));
 776
 777         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 778
 779         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 780              clock.m1++) {
 781                 for (clock.m2 = limit->m2.min;
 782                      clock.m2 <= limit->m2.max; clock.m2++) {
 783                         for (clock.n = limit->n.min;
 784                              clock.n <= limit->n.max; clock.n++) {
 785                                 for (clock.p1 = limit->p1.min;
 786                                         clock.p1 <= limit->p1.max; clock.p1++) {
 787                                         int this_err;
 788
 789                                         pnv_calc_dpll_params(refclk, &clock);
 790                                         if (!intel_pll_is_valid(to_i915(dev),
 791                                                                 limit,
 792                                                                 &clock))
 793                                                 continue;
 794                                         if (match_clock &&
 795                                             clock.p != match_clock->p)
 796                                                 continue;
 797
 798                                         this_err = abs(clock.dot - target);
 799                                         if (this_err < err) {
 800                                                 *best_clock = clock;
 801                                                 err = this_err;
 802                                         }
 803                                 }
 804                         }
 805                 }
 806         }
 807
 808         return (err != target);
 809 }
 810
 811 /*
 812  * Returns a set of divisors for the desired target clock with the given
 813  * refclk, or FALSE.  The returned values represent the clock equation:
 814  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 815  *
 816  * Target and reference clocks are specified in kHz.
 817  *
 818  * If match_clock is provided, then best_clock P divider must match the P
 819  * divider from @match_clock used for LVDS downclocking.
 820  */
 821 static bool
 822 g4x_find_best_dpll(const struct intel_limit *limit,
 823                    struct intel_crtc_state *crtc_state,
 824                    int target, int refclk, struct dpll *match_clock,
 825                    struct dpll *best_clock)
 826 {
 827         struct drm_device *dev = crtc_state->uapi.crtc->dev;
 828         struct dpll clock;
 829         int max_n;
 830         bool found = false;
 831         /* approximately equals target * 0.00585 */
 832         int err_most = (target >> 8) + (target >> 9);
 833
 834         memset(best_clock, 0, sizeof(*best_clock));
 835
 836         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 837
 838         max_n = limit->n.max;
 839         /* based on hardware requirement, prefer smaller n to precision */
 840         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 841                 /* based on hardware requirement, prefere larger m1,m2 */
 842                 for (clock.m1 = limit->m1.max;
 843                      clock.m1 >= limit->m1.min; clock.m1--) {
 844                         for (clock.m2 = limit->m2.max;
 845                              clock.m2 >= limit->m2.min; clock.m2--) {
 846                                 for (clock.p1 = limit->p1.max;
 847                                      clock.p1 >= limit->p1.min; clock.p1--) {
 848                                         int this_err;
 849
 850                                         i9xx_calc_dpll_params(refclk, &clock);
 851                                         if (!intel_pll_is_valid(to_i915(dev),
 852                                                                 limit,
 853                                                                 &clock))
 854                                                 continue;
 855
 856                                         this_err = abs(clock.dot - target);
 857                                         if (this_err < err_most) {
 858                                                 *best_clock = clock;
 859                                                 err_most = this_err;
 860                                                 max_n = clock.n;
 861                                                 found = true;
 862                                         }
 863                                 }
 864                         }
 865                 }
 866         }
 867         return found;
 868 }
 869
 870 /*
 871  * Check if the calculated PLL configuration is more optimal compared to the
 872  * best configuration and error found so far. Return the calculated error.
 873  */
 874 static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
 875                                const struct dpll *calculated_clock,
 876                                const struct dpll *best_clock,
 877                                unsigned int best_error_ppm,
 878                                unsigned int *error_ppm)
 879 {
 880         /*
 881          * For CHV ignore the error and consider only the P value.
 882          * Prefer a bigger P value based on HW requirements.
 883          */
 884         if (IS_CHERRYVIEW(to_i915(dev))) {
 885                 *error_ppm = 0;
 886
 887                 return calculated_clock->p > best_clock->p;
 888         }
 889
 890         if (drm_WARN_ON_ONCE(dev, !target_freq))
 891                 return false;
 892
 893         *error_ppm = div_u64(1000000ULL *
 894                                 abs(target_freq - calculated_clock->dot),
 895                              target_freq);
 896         /*
 897          * Prefer a better P value over a better (smaller) error if the error
 898          * is small. Ensure this preference for future configurations too by
 899          * setting the error to 0.
 900          */
 901         if (*error_ppm < 100 && calculated_clock->p > best_clock->p) {
 902                 *error_ppm = 0;
 903
 904                 return true;
 905         }
 906
 907         return *error_ppm + 10 < best_error_ppm;
 908 }
 909
 910 /*
 911  * Returns a set of divisors for the desired target clock with the given
 912  * refclk, or FALSE.  The returned values represent the clock equation:
 913  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 914  */
 915 static bool
 916 vlv_find_best_dpll(const struct intel_limit *limit,
 917                    struct intel_crtc_state *crtc_state,
 918                    int target, int refclk, struct dpll *match_clock,
 919                    struct dpll *best_clock)
 920 {
 921         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 922         struct drm_device *dev = crtc->base.dev;
 923         struct dpll clock;
 924         unsigned int bestppm = 1000000;
 925         /* min update 19.2 MHz */
 926         int max_n = min(limit->n.max, refclk / 19200);
 927         bool found = false;
 928
 929         target *= 5; /* fast clock */
 930
 931         memset(best_clock, 0, sizeof(*best_clock));
 932
 933         /* based on hardware requirement, prefer smaller n to precision */
 934         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 935                 for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 936                         for (clock.p2 = limit->p2.p2_fast; clock.p2 >= limit->p2.p2_slow;
 937                              clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 938                                 clock.p = clock.p1 * clock.p2;
 939                                 /* based on hardware requirement, prefer bigger m1,m2 values */
 940                                 for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) {
 941                                         unsigned int ppm;
 942
 943                                         clock.m2 = DIV_ROUND_CLOSEST(target * clock.p * clock.n,
 944                                                                      refclk * clock.m1);
 945
 946                                         vlv_calc_dpll_params(refclk, &clock);
 947
 948                                         if (!intel_pll_is_valid(to_i915(dev),
 949                                                                 limit,
 950                                                                 &clock))
 951                                                 continue;
 952
 953                                         if (!vlv_PLL_is_optimal(dev, target,
 954                                                                 &clock,
 955                                                                 best_clock,
 956                                                                 bestppm, &ppm))
 957                                                 continue;
 958
 959                                         *best_clock = clock;
 960                                         bestppm = ppm;
 961                                         found = true;
 962                                 }
 963                         }
 964                 }
 965         }
 966
 967         return found;
 968 }
 969
 970 /*
 971  * Returns a set of divisors for the desired target clock with the given
 972  * refclk, or FALSE.  The returned values represent the clock equation:
 973  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 974  */
 975 static bool
 976 chv_find_best_dpll(const struct intel_limit *limit,
 977                    struct intel_crtc_state *crtc_state,
 978                    int target, int refclk, struct dpll *match_clock,
 979                    struct dpll *best_clock)
 980 {
 981         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 982         struct drm_device *dev = crtc->base.dev;
 983         unsigned int best_error_ppm;
 984         struct dpll clock;
 985         u64 m2;
 986         int found = false;
 987
 988         memset(best_clock, 0, sizeof(*best_clock));
 989         best_error_ppm = 1000000;
 990
 991         /*
 992          * Based on hardware doc, the n always set to 1, and m1 always
 993          * set to 2.  If requires to support 200Mhz refclk, we need to
 994          * revisit this because n may not 1 anymore.
 995          */
 996         clock.n = 1, clock.m1 = 2;
 997         target *= 5;    /* fast clock */
 998
 999         for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
1000                 for (clock.p2 = limit->p2.p2_fast;
1001                                 clock.p2 >= limit->p2.p2_slow;
1002                                 clock.p2 -= clock.p2 > 10 ? 2 : 1) {
1003                         unsigned int error_ppm;
1004
1005                         clock.p = clock.p1 * clock.p2;
1006
1007                         m2 = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(target, clock.p * clock.n) << 22,
1008                                                    refclk * clock.m1);
1009
1010                         if (m2 > INT_MAX/clock.m1)
1011                                 continue;
1012
1013                         clock.m2 = m2;
1014
1015                         chv_calc_dpll_params(refclk, &clock);
1016
1017                         if (!intel_pll_is_valid(to_i915(dev), limit, &clock))
1018                                 continue;
1019
1020                         if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock,
1021                                                 best_error_ppm, &error_ppm))
1022                                 continue;
1023
1024                         *best_clock = clock;
1025                         best_error_ppm = error_ppm;
1026                         found = true;
1027                 }
1028         }
1029
1030         return found;
1031 }
1032
1033 bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state,
1034                         struct dpll *best_clock)
1035 {
1036         int refclk = 100000;
1037         const struct intel_limit *limit = &intel_limits_bxt;
1038
1039         return chv_find_best_dpll(limit, crtc_state,
1040                                   crtc_state->port_clock, refclk,
1041                                   NULL, best_clock);
1042 }
1043
1044 static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv,
1045                                     enum pipe pipe)
1046 {
1047         i915_reg_t reg = PIPEDSL(pipe);
1048         u32 line1, line2;
1049         u32 line_mask;
1050
1051         if (IS_GEN(dev_priv, 2))
1052                 line_mask = DSL_LINEMASK_GEN2;
1053         else
1054                 line_mask = DSL_LINEMASK_GEN3;
1055
1056         line1 = intel_de_read(dev_priv, reg) & line_mask;
1057         msleep(5);
1058         line2 = intel_de_read(dev_priv, reg) & line_mask;
1059
1060         return line1 != line2;
1061 }
1062
1063 static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state)
1064 {
1065         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1066         enum pipe pipe = crtc->pipe;
1067
1068         /* Wait for the display line to settle/start moving */
1069         if (wait_for(pipe_scanline_is_moving(dev_priv, pipe) == state, 100))
1070                 drm_err(&dev_priv->drm,
1071                         "pipe %c scanline %s wait timed out\n",
1072                         pipe_name(pipe), onoff(state));
1073 }
1074
1075 static void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc)
1076 {
1077         wait_for_pipe_scanline_moving(crtc, false);
1078 }
1079
1080 static void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
1081 {
1082         wait_for_pipe_scanline_moving(crtc, true);
1083 }
1084
1085 static void
1086 intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
1087 {
1088         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
1089         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1090
1091         if (INTEL_GEN(dev_priv) >= 4) {
1092                 enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1093                 i915_reg_t reg = PIPECONF(cpu_transcoder);
1094
1095                 /* Wait for the Pipe State to go off */
1096                 if (intel_de_wait_for_clear(dev_priv, reg,
1097                                             I965_PIPECONF_ACTIVE, 100))
1098                         drm_WARN(&dev_priv->drm, 1,
1099                                  "pipe_off wait timed out\n");
1100         } else {
1101                 intel_wait_for_pipe_scanline_stopped(crtc);
1102         }
1103 }
1104
1105 /* Only for pre-ILK configs */
1106 void assert_pll(struct drm_i915_private *dev_priv,
1107                 enum pipe pipe, bool state)
1108 {
1109         u32 val;
1110         bool cur_state;
1111
1112         val = intel_de_read(dev_priv, DPLL(pipe));
1113         cur_state = !!(val & DPLL_VCO_ENABLE);
1114         I915_STATE_WARN(cur_state != state,
1115              "PLL state assertion failure (expected %s, current %s)\n",
1116                         onoff(state), onoff(cur_state));
1117 }
1118
1119 /* XXX: the dsi pll is shared between MIPI DSI ports */
1120 void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state)
1121 {
1122         u32 val;
1123         bool cur_state;
1124
1125         vlv_cck_get(dev_priv);
1126         val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL);
1127         vlv_cck_put(dev_priv);
1128
1129         cur_state = val & DSI_PLL_VCO_EN;
1130         I915_STATE_WARN(cur_state != state,
1131              "DSI PLL state assertion failure (expected %s, current %s)\n",
1132                         onoff(state), onoff(cur_state));
1133 }
1134
1135 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
1136                           enum pipe pipe, bool state)
1137 {
1138         bool cur_state;
1139
1140         if (HAS_DDI(dev_priv)) {
1141                 /*
1142                  * DDI does not have a specific FDI_TX register.
1143                  *
1144                  * FDI is never fed from EDP transcoder
1145                  * so pipe->transcoder cast is fine here.
1146                  */
1147                 enum transcoder cpu_transcoder = (enum transcoder)pipe;
1148                 u32 val = intel_de_read(dev_priv,
1149                                         TRANS_DDI_FUNC_CTL(cpu_transcoder));
1150                 cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
1151         } else {
1152                 u32 val = intel_de_read(dev_priv, FDI_TX_CTL(pipe));
1153                 cur_state = !!(val & FDI_TX_ENABLE);
1154         }
1155         I915_STATE_WARN(cur_state != state,
1156              "FDI TX state assertion failure (expected %s, current %s)\n",
1157                         onoff(state), onoff(cur_state));
1158 }
1159 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
1160 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
1161
1162 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
1163                           enum pipe pipe, bool state)
1164 {
1165         u32 val;
1166         bool cur_state;
1167
1168         val = intel_de_read(dev_priv, FDI_RX_CTL(pipe));
1169         cur_state = !!(val & FDI_RX_ENABLE);
1170         I915_STATE_WARN(cur_state != state,
1171              "FDI RX state assertion failure (expected %s, current %s)\n",
1172                         onoff(state), onoff(cur_state));
1173 }
1174 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
1175 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
1176
1177 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
1178                                       enum pipe pipe)
1179 {
1180         u32 val;
1181
1182         /* ILK FDI PLL is always enabled */
1183         if (IS_GEN(dev_priv, 5))
1184                 return;
1185
1186         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
1187         if (HAS_DDI(dev_priv))
1188                 return;
1189
1190         val = intel_de_read(dev_priv, FDI_TX_CTL(pipe));
1191         I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
1192 }
1193
1194 void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
1195                        enum pipe pipe, bool state)
1196 {
1197         u32 val;
1198         bool cur_state;
1199
1200         val = intel_de_read(dev_priv, FDI_RX_CTL(pipe));
1201         cur_state = !!(val & FDI_RX_PLL_ENABLE);
1202         I915_STATE_WARN(cur_state != state,
1203              "FDI RX PLL assertion failure (expected %s, current %s)\n",
1204                         onoff(state), onoff(cur_state));
1205 }
1206
1207 void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe)
1208 {
1209         i915_reg_t pp_reg;
1210         u32 val;
1211         enum pipe panel_pipe = INVALID_PIPE;
1212         bool locked = true;
1213
1214         if (drm_WARN_ON(&dev_priv->drm, HAS_DDI(dev_priv)))
1215                 return;
1216
1217         if (HAS_PCH_SPLIT(dev_priv)) {
1218                 u32 port_sel;
1219
1220                 pp_reg = PP_CONTROL(0);
1221                 port_sel = intel_de_read(dev_priv, PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1222
1223                 switch (port_sel) {
1224                 case PANEL_PORT_SELECT_LVDS:
1225                         intel_lvds_port_enabled(dev_priv, PCH_LVDS, &panel_pipe);
1226                         break;
1227                 case PANEL_PORT_SELECT_DPA:
1228                         intel_dp_port_enabled(dev_priv, DP_A, PORT_A, &panel_pipe);
1229                         break;
1230                 case PANEL_PORT_SELECT_DPC:
1231                         intel_dp_port_enabled(dev_priv, PCH_DP_C, PORT_C, &panel_pipe);
1232                         break;
1233                 case PANEL_PORT_SELECT_DPD:
1234                         intel_dp_port_enabled(dev_priv, PCH_DP_D, PORT_D, &panel_pipe);
1235                         break;
1236                 default:
1237                         MISSING_CASE(port_sel);
1238                         break;
1239                 }
1240         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1241                 /* presumably write lock depends on pipe, not port select */
1242                 pp_reg = PP_CONTROL(pipe);
1243                 panel_pipe = pipe;
1244         } else {
1245                 u32 port_sel;
1246
1247                 pp_reg = PP_CONTROL(0);
1248                 port_sel = intel_de_read(dev_priv, PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1249
1250                 drm_WARN_ON(&dev_priv->drm,
1251                             port_sel != PANEL_PORT_SELECT_LVDS);
1252                 intel_lvds_port_enabled(dev_priv, LVDS, &panel_pipe);
1253         }
1254
1255         val = intel_de_read(dev_priv, pp_reg);
1256         if (!(val & PANEL_POWER_ON) ||
1257             ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
1258                 locked = false;
1259
1260         I915_STATE_WARN(panel_pipe == pipe && locked,
1261              "panel assertion failure, pipe %c regs locked\n",
1262              pipe_name(pipe));
1263 }
1264
1265 void assert_pipe(struct drm_i915_private *dev_priv,
1266                  enum transcoder cpu_transcoder, bool state)
1267 {
1268         bool cur_state;
1269         enum intel_display_power_domain power_domain;
1270         intel_wakeref_t wakeref;
1271
1272         /* we keep both pipes enabled on 830 */
1273         if (IS_I830(dev_priv))
1274                 state = true;
1275
1276         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
1277         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
1278         if (wakeref) {
1279                 u32 val = intel_de_read(dev_priv, PIPECONF(cpu_transcoder));
1280                 cur_state = !!(val & PIPECONF_ENABLE);
1281
1282                 intel_display_power_put(dev_priv, power_domain, wakeref);
1283         } else {
1284                 cur_state = false;
1285         }
1286
1287         I915_STATE_WARN(cur_state != state,
1288                         "transcoder %s assertion failure (expected %s, current %s)\n",
1289                         transcoder_name(cpu_transcoder),
1290                         onoff(state), onoff(cur_state));
1291 }
1292
1293 static void assert_plane(struct intel_plane *plane, bool state)
1294 {
1295         enum pipe pipe;
1296         bool cur_state;
1297
1298         cur_state = plane->get_hw_state(plane, &pipe);
1299
1300         I915_STATE_WARN(cur_state != state,
1301                         "%s assertion failure (expected %s, current %s)\n",
1302                         plane->base.name, onoff(state), onoff(cur_state));
1303 }
1304
1305 #define assert_plane_enabled(p) assert_plane(p, true)
1306 #define assert_plane_disabled(p) assert_plane(p, false)
1307
1308 static void assert_planes_disabled(struct intel_crtc *crtc)
1309 {
1310         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1311         struct intel_plane *plane;
1312
1313         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
1314                 assert_plane_disabled(plane);
1315 }
1316
1317 static void assert_vblank_disabled(struct drm_crtc *crtc)
1318 {
1319         if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
1320                 drm_crtc_vblank_put(crtc);
1321 }
1322
1323 void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
1324                                     enum pipe pipe)
1325 {
1326         u32 val;
1327         bool enabled;
1328
1329         val = intel_de_read(dev_priv, PCH_TRANSCONF(pipe));
1330         enabled = !!(val & TRANS_ENABLE);
1331         I915_STATE_WARN(enabled,
1332              "transcoder assertion failed, should be off on pipe %c but is still active\n",
1333              pipe_name(pipe));
1334 }
1335
1336 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
1337                                    enum pipe pipe, enum port port,
1338                                    i915_reg_t dp_reg)
1339 {
1340         enum pipe port_pipe;
1341         bool state;
1342
1343         state = intel_dp_port_enabled(dev_priv, dp_reg, port, &port_pipe);
1344
1345         I915_STATE_WARN(state && port_pipe == pipe,
1346                         "PCH DP %c enabled on transcoder %c, should be disabled\n",
1347                         port_name(port), pipe_name(pipe));
1348
1349         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1350                         "IBX PCH DP %c still using transcoder B\n",
1351                         port_name(port));
1352 }
1353
1354 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
1355                                      enum pipe pipe, enum port port,
1356                                      i915_reg_t hdmi_reg)
1357 {
1358         enum pipe port_pipe;
1359         bool state;
1360
1361         state = intel_sdvo_port_enabled(dev_priv, hdmi_reg, &port_pipe);
1362
1363         I915_STATE_WARN(state && port_pipe == pipe,
1364                         "PCH HDMI %c enabled on transcoder %c, should be disabled\n",
1365                         port_name(port), pipe_name(pipe));
1366
1367         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1368                         "IBX PCH HDMI %c still using transcoder B\n",
1369                         port_name(port));
1370 }
1371
1372 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
1373                                       enum pipe pipe)
1374 {
1375         enum pipe port_pipe;
1376
1377         assert_pch_dp_disabled(dev_priv, pipe, PORT_B, PCH_DP_B);
1378         assert_pch_dp_disabled(dev_priv, pipe, PORT_C, PCH_DP_C);
1379         assert_pch_dp_disabled(dev_priv, pipe, PORT_D, PCH_DP_D);
1380
1381         I915_STATE_WARN(intel_crt_port_enabled(dev_priv, PCH_ADPA, &port_pipe) &&
1382                         port_pipe == pipe,
1383                         "PCH VGA enabled on transcoder %c, should be disabled\n",
1384                         pipe_name(pipe));
1385
1386         I915_STATE_WARN(intel_lvds_port_enabled(dev_priv, PCH_LVDS, &port_pipe) &&
1387                         port_pipe == pipe,
1388                         "PCH LVDS enabled on transcoder %c, should be disabled\n",
1389                         pipe_name(pipe));
1390
1391         /* PCH SDVOB multiplex with HDMIB */
1392         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_B, PCH_HDMIB);
1393         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_C, PCH_HDMIC);
1394         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_D, PCH_HDMID);
1395 }
1396
1397 static void _vlv_enable_pll(struct intel_crtc *crtc,
1398                             const struct intel_crtc_state *pipe_config)
1399 {
1400         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1401         enum pipe pipe = crtc->pipe;
1402
1403         intel_de_write(dev_priv, DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1404         intel_de_posting_read(dev_priv, DPLL(pipe));
1405         udelay(150);
1406
1407         if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1))
1408                 drm_err(&dev_priv->drm, "DPLL %d failed to lock\n", pipe);
1409 }
1410
1411 static void vlv_enable_pll(struct intel_crtc *crtc,
1412                            const struct intel_crtc_state *pipe_config)
1413 {
1414         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1415         enum pipe pipe = crtc->pipe;
1416
1417         assert_pipe_disabled(dev_priv, pipe_config->cpu_transcoder);
1418
1419         /* PLL is protected by panel, make sure we can write it */
1420         assert_panel_unlocked(dev_priv, pipe);
1421
1422         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1423                 _vlv_enable_pll(crtc, pipe_config);
1424
1425         intel_de_write(dev_priv, DPLL_MD(pipe),
1426                        pipe_config->dpll_hw_state.dpll_md);
1427         intel_de_posting_read(dev_priv, DPLL_MD(pipe));
1428 }
1429
1430
1431 static void _chv_enable_pll(struct intel_crtc *crtc,
1432                             const struct intel_crtc_state *pipe_config)
1433 {
1434         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1435         enum pipe pipe = crtc->pipe;
1436         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1437         u32 tmp;
1438
1439         vlv_dpio_get(dev_priv);
1440
1441         /* Enable back the 10bit clock to display controller */
1442         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1443         tmp |= DPIO_DCLKP_EN;
1444         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp);
1445
1446         vlv_dpio_put(dev_priv);
1447
1448         /*
1449          * Need to wait > 100ns between dclkp clock enable bit and PLL enable.
1450          */
1451         udelay(1);
1452
1453         /* Enable PLL */
1454         intel_de_write(dev_priv, DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1455
1456         /* Check PLL is locked */
1457         if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1))
1458                 drm_err(&dev_priv->drm, "PLL %d failed to lock\n", pipe);
1459 }
1460
1461 static void chv_enable_pll(struct intel_crtc *crtc,
1462                            const struct intel_crtc_state *pipe_config)
1463 {
1464         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1465         enum pipe pipe = crtc->pipe;
1466
1467         assert_pipe_disabled(dev_priv, pipe_config->cpu_transcoder);
1468
1469         /* PLL is protected by panel, make sure we can write it */
1470         assert_panel_unlocked(dev_priv, pipe);
1471
1472         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1473                 _chv_enable_pll(crtc, pipe_config);
1474
1475         if (pipe != PIPE_A) {
1476                 /*
1477                  * WaPixelRepeatModeFixForC0:chv
1478                  *
1479                  * DPLLCMD is AWOL. Use chicken bits to propagate
1480                  * the value from DPLLBMD to either pipe B or C.
1481                  */
1482                 intel_de_write(dev_priv, CBR4_VLV, CBR_DPLLBMD_PIPE(pipe));
1483                 intel_de_write(dev_priv, DPLL_MD(PIPE_B),
1484                                pipe_config->dpll_hw_state.dpll_md);
1485                 intel_de_write(dev_priv, CBR4_VLV, 0);
1486                 dev_priv->chv_dpll_md[pipe] = pipe_config->dpll_hw_state.dpll_md;
1487
1488                 /*
1489                  * DPLLB VGA mode also seems to cause problems.
1490                  * We should always have it disabled.
1491                  */
1492                 drm_WARN_ON(&dev_priv->drm,
1493                             (intel_de_read(dev_priv, DPLL(PIPE_B)) &
1494                              DPLL_VGA_MODE_DIS) == 0);
1495         } else {
1496                 intel_de_write(dev_priv, DPLL_MD(pipe),
1497                                pipe_config->dpll_hw_state.dpll_md);
1498                 intel_de_posting_read(dev_priv, DPLL_MD(pipe));
1499         }
1500 }
1501
1502 static bool i9xx_has_pps(struct drm_i915_private *dev_priv)
1503 {
1504         if (IS_I830(dev_priv))
1505                 return false;
1506
1507         return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
1508 }
1509
1510 static void i9xx_enable_pll(struct intel_crtc *crtc,
1511                             const struct intel_crtc_state *crtc_state)
1512 {
1513         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1514         i915_reg_t reg = DPLL(crtc->pipe);
1515         u32 dpll = crtc_state->dpll_hw_state.dpll;
1516         int i;
1517
1518         assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder);
1519
1520         /* PLL is protected by panel, make sure we can write it */
1521         if (i9xx_has_pps(dev_priv))
1522                 assert_panel_unlocked(dev_priv, crtc->pipe);
1523
1524         /*
1525          * Apparently we need to have VGA mode enabled prior to changing
1526          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
1527          * dividers, even though the register value does change.
1528          */
1529         intel_de_write(dev_priv, reg, dpll & ~DPLL_VGA_MODE_DIS);
1530         intel_de_write(dev_priv, reg, dpll);
1531
1532         /* Wait for the clocks to stabilize. */
1533         intel_de_posting_read(dev_priv, reg);
1534         udelay(150);
1535
1536         if (INTEL_GEN(dev_priv) >= 4) {
1537                 intel_de_write(dev_priv, DPLL_MD(crtc->pipe),
1538                                crtc_state->dpll_hw_state.dpll_md);
1539         } else {
1540                 /* The pixel multiplier can only be updated once the
1541                  * DPLL is enabled and the clocks are stable.
1542                  *
1543                  * So write it again.
1544                  */
1545                 intel_de_write(dev_priv, reg, dpll);
1546         }
1547
1548         /* We do this three times for luck */
1549         for (i = 0; i < 3; i++) {
1550                 intel_de_write(dev_priv, reg, dpll);
1551                 intel_de_posting_read(dev_priv, reg);
1552                 udelay(150); /* wait for warmup */
1553         }
1554 }
1555
1556 static void i9xx_disable_pll(const struct intel_crtc_state *crtc_state)
1557 {
1558         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1559         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1560         enum pipe pipe = crtc->pipe;
1561
1562         /* Don't disable pipe or pipe PLLs if needed */
1563         if (IS_I830(dev_priv))
1564                 return;
1565
1566         /* Make sure the pipe isn't still relying on us */
1567         assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder);
1568
1569         intel_de_write(dev_priv, DPLL(pipe), DPLL_VGA_MODE_DIS);
1570         intel_de_posting_read(dev_priv, DPLL(pipe));
1571 }
1572
1573 static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1574 {
1575         u32 val;
1576
1577         /* Make sure the pipe isn't still relying on us */
1578         assert_pipe_disabled(dev_priv, (enum transcoder)pipe);
1579
1580         val = DPLL_INTEGRATED_REF_CLK_VLV |
1581                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1582         if (pipe != PIPE_A)
1583                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1584
1585         intel_de_write(dev_priv, DPLL(pipe), val);
1586         intel_de_posting_read(dev_priv, DPLL(pipe));
1587 }
1588
1589 static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1590 {
1591         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1592         u32 val;
1593
1594         /* Make sure the pipe isn't still relying on us */
1595         assert_pipe_disabled(dev_priv, (enum transcoder)pipe);
1596
1597         val = DPLL_SSC_REF_CLK_CHV |
1598                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1599         if (pipe != PIPE_A)
1600                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1601
1602         intel_de_write(dev_priv, DPLL(pipe), val);
1603         intel_de_posting_read(dev_priv, DPLL(pipe));
1604
1605         vlv_dpio_get(dev_priv);
1606
1607         /* Disable 10bit clock to display controller */
1608         val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1609         val &= ~DPIO_DCLKP_EN;
1610         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
1611
1612         vlv_dpio_put(dev_priv);
1613 }
1614
1615 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
1616                          struct intel_digital_port *dig_port,
1617                          unsigned int expected_mask)
1618 {
1619         u32 port_mask;
1620         i915_reg_t dpll_reg;
1621
1622         switch (dig_port->base.port) {
1623         case PORT_B:
1624                 port_mask = DPLL_PORTB_READY_MASK;
1625                 dpll_reg = DPLL(0);
1626                 break;
1627         case PORT_C:
1628                 port_mask = DPLL_PORTC_READY_MASK;
1629                 dpll_reg = DPLL(0);
1630                 expected_mask <<= 4;
1631                 break;
1632         case PORT_D:
1633                 port_mask = DPLL_PORTD_READY_MASK;
1634                 dpll_reg = DPIO_PHY_STATUS;
1635                 break;
1636         default:
1637                 BUG();
1638         }
1639
1640         if (intel_de_wait_for_register(dev_priv, dpll_reg,
1641                                        port_mask, expected_mask, 1000))
1642                 drm_WARN(&dev_priv->drm, 1,
1643                          "timed out waiting for [ENCODER:%d:%s] port ready: got 0x%x, expected 0x%x\n",
1644                          dig_port->base.base.base.id, dig_port->base.base.name,
1645                          intel_de_read(dev_priv, dpll_reg) & port_mask,
1646                          expected_mask);
1647 }
1648
1649 static void ilk_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
1650 {
1651         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1652         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1653         enum pipe pipe = crtc->pipe;
1654         i915_reg_t reg;
1655         u32 val, pipeconf_val;
1656
1657         /* Make sure PCH DPLL is enabled */
1658         assert_shared_dpll_enabled(dev_priv, crtc_state->shared_dpll);
1659
1660         /* FDI must be feeding us bits for PCH ports */
1661         assert_fdi_tx_enabled(dev_priv, pipe);
1662         assert_fdi_rx_enabled(dev_priv, pipe);
1663
1664         if (HAS_PCH_CPT(dev_priv)) {
1665                 reg = TRANS_CHICKEN2(pipe);
1666                 val = intel_de_read(dev_priv, reg);
1667                 /*
1668                  * Workaround: Set the timing override bit
1669                  * before enabling the pch transcoder.
1670                  */
1671                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1672                 /* Configure frame start delay to match the CPU */
1673                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
1674                 val |= TRANS_CHICKEN2_FRAME_START_DELAY(0);
1675                 intel_de_write(dev_priv, reg, val);
1676         }
1677
1678         reg = PCH_TRANSCONF(pipe);
1679         val = intel_de_read(dev_priv, reg);
1680         pipeconf_val = intel_de_read(dev_priv, PIPECONF(pipe));
1681
1682         if (HAS_PCH_IBX(dev_priv)) {
1683                 /* Configure frame start delay to match the CPU */
1684                 val &= ~TRANS_FRAME_START_DELAY_MASK;
1685                 val |= TRANS_FRAME_START_DELAY(0);
1686
1687                 /*
1688                  * Make the BPC in transcoder be consistent with
1689                  * that in pipeconf reg. For HDMI we must use 8bpc
1690                  * here for both 8bpc and 12bpc.
1691                  */
1692                 val &= ~PIPECONF_BPC_MASK;
1693                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
1694                         val |= PIPECONF_8BPC;
1695                 else
1696                         val |= pipeconf_val & PIPECONF_BPC_MASK;
1697         }
1698
1699         val &= ~TRANS_INTERLACE_MASK;
1700         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK) {
1701                 if (HAS_PCH_IBX(dev_priv) &&
1702                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
1703                         val |= TRANS_LEGACY_INTERLACED_ILK;
1704                 else
1705                         val |= TRANS_INTERLACED;
1706         } else {
1707                 val |= TRANS_PROGRESSIVE;
1708         }
1709
1710         intel_de_write(dev_priv, reg, val | TRANS_ENABLE);
1711         if (intel_de_wait_for_set(dev_priv, reg, TRANS_STATE_ENABLE, 100))
1712                 drm_err(&dev_priv->drm, "failed to enable transcoder %c\n",
1713                         pipe_name(pipe));
1714 }
1715
1716 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1717                                       enum transcoder cpu_transcoder)
1718 {
1719         u32 val, pipeconf_val;
1720
1721         /* FDI must be feeding us bits for PCH ports */
1722         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
1723         assert_fdi_rx_enabled(dev_priv, PIPE_A);
1724
1725         val = intel_de_read(dev_priv, TRANS_CHICKEN2(PIPE_A));
1726         /* Workaround: set timing override bit. */
1727         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1728         /* Configure frame start delay to match the CPU */
1729         val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
1730         val |= TRANS_CHICKEN2_FRAME_START_DELAY(0);
1731         intel_de_write(dev_priv, TRANS_CHICKEN2(PIPE_A), val);
1732
1733         val = TRANS_ENABLE;
1734         pipeconf_val = intel_de_read(dev_priv, PIPECONF(cpu_transcoder));
1735
1736         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
1737             PIPECONF_INTERLACED_ILK)
1738                 val |= TRANS_INTERLACED;
1739         else
1740                 val |= TRANS_PROGRESSIVE;
1741
1742         intel_de_write(dev_priv, LPT_TRANSCONF, val);
1743         if (intel_de_wait_for_set(dev_priv, LPT_TRANSCONF,
1744                                   TRANS_STATE_ENABLE, 100))
1745                 drm_err(&dev_priv->drm, "Failed to enable PCH transcoder\n");
1746 }
1747
1748 static void ilk_disable_pch_transcoder(struct drm_i915_private *dev_priv,
1749                                        enum pipe pipe)
1750 {
1751         i915_reg_t reg;
1752         u32 val;
1753
1754         /* FDI relies on the transcoder */
1755         assert_fdi_tx_disabled(dev_priv, pipe);
1756         assert_fdi_rx_disabled(dev_priv, pipe);
1757
1758         /* Ports must be off as well */
1759         assert_pch_ports_disabled(dev_priv, pipe);
1760
1761         reg = PCH_TRANSCONF(pipe);
1762         val = intel_de_read(dev_priv, reg);
1763         val &= ~TRANS_ENABLE;
1764         intel_de_write(dev_priv, reg, val);
1765         /* wait for PCH transcoder off, transcoder state */
1766         if (intel_de_wait_for_clear(dev_priv, reg, TRANS_STATE_ENABLE, 50))
1767                 drm_err(&dev_priv->drm, "failed to disable transcoder %c\n",
1768                         pipe_name(pipe));
1769
1770         if (HAS_PCH_CPT(dev_priv)) {
1771                 /* Workaround: Clear the timing override chicken bit again. */
1772                 reg = TRANS_CHICKEN2(pipe);
1773                 val = intel_de_read(dev_priv, reg);
1774                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1775                 intel_de_write(dev_priv, reg, val);
1776         }
1777 }
1778
1779 void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
1780 {
1781         u32 val;
1782
1783         val = intel_de_read(dev_priv, LPT_TRANSCONF);
1784         val &= ~TRANS_ENABLE;
1785         intel_de_write(dev_priv, LPT_TRANSCONF, val);
1786         /* wait for PCH transcoder off, transcoder state */
1787         if (intel_de_wait_for_clear(dev_priv, LPT_TRANSCONF,
1788                                     TRANS_STATE_ENABLE, 50))
1789                 drm_err(&dev_priv->drm, "Failed to disable PCH transcoder\n");
1790
1791         /* Workaround: clear timing override bit. */
1792         val = intel_de_read(dev_priv, TRANS_CHICKEN2(PIPE_A));
1793         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1794         intel_de_write(dev_priv, TRANS_CHICKEN2(PIPE_A), val);
1795 }
1796
1797 enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
1798 {
1799         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1800
1801         if (HAS_PCH_LPT(dev_priv))
1802                 return PIPE_A;
1803         else
1804                 return crtc->pipe;
1805 }
1806
1807 static u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
1808 {
1809         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1810
1811         /*
1812          * On i965gm the hardware frame counter reads
1813          * zero when the TV encoder is enabled :(
1814          */
1815         if (IS_I965GM(dev_priv) &&
1816             (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
1817                 return 0;
1818
1819         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1820                 return 0xffffffff; /* full 32 bit counter */
1821         else if (INTEL_GEN(dev_priv) >= 3)
1822                 return 0xffffff; /* only 24 bits of frame count */
1823         else
1824                 return 0; /* Gen2 doesn't have a hardware frame counter */
1825 }
1826
1827 void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
1828 {
1829         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1830
1831         assert_vblank_disabled(&crtc->base);
1832         drm_crtc_set_max_vblank_count(&crtc->base,
1833                                       intel_crtc_max_vblank_count(crtc_state));
1834         drm_crtc_vblank_on(&crtc->base);
1835 }
1836
1837 void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
1838 {
1839         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1840
1841         drm_crtc_vblank_off(&crtc->base);
1842         assert_vblank_disabled(&crtc->base);
1843 }
1844
1845 void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state)
1846 {
1847         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
1848         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1849         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
1850         enum pipe pipe = crtc->pipe;
1851         i915_reg_t reg;
1852         u32 val;
1853
1854         drm_dbg_kms(&dev_priv->drm, "enabling pipe %c\n", pipe_name(pipe));
1855
1856         assert_planes_disabled(crtc);
1857
1858         /*
1859          * A pipe without a PLL won't actually be able to drive bits from
1860          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
1861          * need the check.
1862          */
1863         if (HAS_GMCH(dev_priv)) {
1864                 if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
1865                         assert_dsi_pll_enabled(dev_priv);
1866                 else
1867                         assert_pll_enabled(dev_priv, pipe);
1868         } else {
1869                 if (new_crtc_state->has_pch_encoder) {
1870                         /* if driving the PCH, we need FDI enabled */
1871                         assert_fdi_rx_pll_enabled(dev_priv,
1872                                                   intel_crtc_pch_transcoder(crtc));
1873                         assert_fdi_tx_pll_enabled(dev_priv,
1874                                                   (enum pipe) cpu_transcoder);
1875                 }
1876                 /* FIXME: assert CPU port conditions for SNB+ */
1877         }
1878
1879         trace_intel_pipe_enable(crtc);
1880
1881         reg = PIPECONF(cpu_transcoder);
1882         val = intel_de_read(dev_priv, reg);
1883         if (val & PIPECONF_ENABLE) {
1884                 /* we keep both pipes enabled on 830 */
1885                 drm_WARN_ON(&dev_priv->drm, !IS_I830(dev_priv));
1886                 return;
1887         }
1888
1889         intel_de_write(dev_priv, reg, val | PIPECONF_ENABLE);
1890         intel_de_posting_read(dev_priv, reg);
1891
1892         /*
1893          * Until the pipe starts PIPEDSL reads will return a stale value,
1894          * which causes an apparent vblank timestamp jump when PIPEDSL
1895          * resets to its proper value. That also messes up the frame count
1896          * when it's derived from the timestamps. So let's wait for the
1897          * pipe to start properly before we call drm_crtc_vblank_on()
1898          */
1899         if (intel_crtc_max_vblank_count(new_crtc_state) == 0)
1900                 intel_wait_for_pipe_scanline_moving(crtc);
1901 }
1902
1903 void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state)
1904 {
1905         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
1906         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1907         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1908         enum pipe pipe = crtc->pipe;
1909         i915_reg_t reg;
1910         u32 val;
1911
1912         drm_dbg_kms(&dev_priv->drm, "disabling pipe %c\n", pipe_name(pipe));
1913
1914         /*
1915          * Make sure planes won't keep trying to pump pixels to us,
1916          * or we might hang the display.
1917          */
1918         assert_planes_disabled(crtc);
1919
1920         trace_intel_pipe_disable(crtc);
1921
1922         reg = PIPECONF(cpu_transcoder);
1923         val = intel_de_read(dev_priv, reg);
1924         if ((val & PIPECONF_ENABLE) == 0)
1925                 return;
1926
1927         /*
1928          * Double wide has implications for planes
1929          * so best keep it disabled when not needed.
1930          */
1931         if (old_crtc_state->double_wide)
1932                 val &= ~PIPECONF_DOUBLE_WIDE;
1933
1934         /* Don't disable pipe or pipe PLLs if needed */
1935         if (!IS_I830(dev_priv))
1936                 val &= ~PIPECONF_ENABLE;
1937
1938         intel_de_write(dev_priv, reg, val);
1939         if ((val & PIPECONF_ENABLE) == 0)
1940                 intel_wait_for_pipe_off(old_crtc_state);
1941 }
1942
1943 static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv)
1944 {
1945         return IS_GEN(dev_priv, 2) ? 2048 : 4096;
1946 }
1947
1948 static bool is_ccs_plane(const struct drm_framebuffer *fb, int plane)
1949 {
1950         if (!is_ccs_modifier(fb->modifier))
1951                 return false;
1952
1953         return plane >= fb->format->num_planes / 2;
1954 }
1955
1956 static bool is_gen12_ccs_modifier(u64 modifier)
1957 {
1958         return modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS ||
1959                modifier == I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS;
1960
1961 }
1962
1963 static bool is_gen12_ccs_plane(const struct drm_framebuffer *fb, int plane)
1964 {
1965         return is_gen12_ccs_modifier(fb->modifier) && is_ccs_plane(fb, plane);
1966 }
1967
1968 static bool is_aux_plane(const struct drm_framebuffer *fb, int plane)
1969 {
1970         if (is_ccs_modifier(fb->modifier))
1971                 return is_ccs_plane(fb, plane);
1972
1973         return plane == 1;
1974 }
1975
1976 static int main_to_ccs_plane(const struct drm_framebuffer *fb, int main_plane)
1977 {
1978         drm_WARN_ON(fb->dev, !is_ccs_modifier(fb->modifier) ||
1979                     (main_plane && main_plane >= fb->format->num_planes / 2));
1980
1981         return fb->format->num_planes / 2 + main_plane;
1982 }
1983
1984 static int ccs_to_main_plane(const struct drm_framebuffer *fb, int ccs_plane)
1985 {
1986         drm_WARN_ON(fb->dev, !is_ccs_modifier(fb->modifier) ||
1987                     ccs_plane < fb->format->num_planes / 2);
1988
1989         return ccs_plane - fb->format->num_planes / 2;
1990 }
1991
1992 /* Return either the main plane's CCS or - if not a CCS FB - UV plane */
1993 int intel_main_to_aux_plane(const struct drm_framebuffer *fb, int main_plane)
1994 {
1995         if (is_ccs_modifier(fb->modifier))
1996                 return main_to_ccs_plane(fb, main_plane);
1997
1998         return 1;
1999 }
2000
2001 bool
2002 intel_format_info_is_yuv_semiplanar(const struct drm_format_info *info,
2003                                     uint64_t modifier)
2004 {
2005         return info->is_yuv &&
2006                info->num_planes == (is_ccs_modifier(modifier) ? 4 : 2);
2007 }
2008
2009 static bool is_semiplanar_uv_plane(const struct drm_framebuffer *fb,
2010                                    int color_plane)
2011 {
2012         return intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier) &&
2013                color_plane == 1;
2014 }
2015
2016 static unsigned int
2017 intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane)
2018 {
2019         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2020         unsigned int cpp = fb->format->cpp[color_plane];
2021
2022         switch (fb->modifier) {
2023         case DRM_FORMAT_MOD_LINEAR:
2024                 return intel_tile_size(dev_priv);
2025         case I915_FORMAT_MOD_X_TILED:
2026                 if (IS_GEN(dev_priv, 2))
2027                         return 128;
2028                 else
2029                         return 512;
2030         case I915_FORMAT_MOD_Y_TILED_CCS:
2031                 if (is_ccs_plane(fb, color_plane))
2032                         return 128;
2033                 fallthrough;
2034         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
2035         case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS:
2036                 if (is_ccs_plane(fb, color_plane))
2037                         return 64;
2038                 fallthrough;
2039         case I915_FORMAT_MOD_Y_TILED:
2040                 if (IS_GEN(dev_priv, 2) || HAS_128_BYTE_Y_TILING(dev_priv))
2041                         return 128;
2042                 else
2043                         return 512;
2044         case I915_FORMAT_MOD_Yf_TILED_CCS:
2045                 if (is_ccs_plane(fb, color_plane))
2046                         return 128;
2047                 fallthrough;
2048         case I915_FORMAT_MOD_Yf_TILED:
2049                 switch (cpp) {
2050                 case 1:
2051                         return 64;
2052                 case 2:
2053                 case 4:
2054                         return 128;
2055                 case 8:
2056                 case 16:
2057                         return 256;
2058                 default:
2059                         MISSING_CASE(cpp);
2060                         return cpp;
2061                 }
2062                 break;
2063         default:
2064                 MISSING_CASE(fb->modifier);
2065                 return cpp;
2066         }
2067 }
2068
2069 static unsigned int
2070 intel_tile_height(const struct drm_framebuffer *fb, int color_plane)
2071 {
2072         if (is_gen12_ccs_plane(fb, color_plane))
2073                 return 1;
2074
2075         return intel_tile_size(to_i915(fb->dev)) /
2076                 intel_tile_width_bytes(fb, color_plane);
2077 }
2078
2079 /* Return the tile dimensions in pixel units */
2080 static void intel_tile_dims(const struct drm_framebuffer *fb, int color_plane,
2081                             unsigned int *tile_width,
2082                             unsigned int *tile_height)
2083 {
2084         unsigned int tile_width_bytes = intel_tile_width_bytes(fb, color_plane);
2085         unsigned int cpp = fb->format->cpp[color_plane];
2086
2087         *tile_width = tile_width_bytes / cpp;
2088         *tile_height = intel_tile_height(fb, color_plane);
2089 }
2090
2091 static unsigned int intel_tile_row_size(const struct drm_framebuffer *fb,
2092                                         int color_plane)
2093 {
2094         unsigned int tile_width, tile_height;
2095
2096         intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2097
2098         return fb->pitches[color_plane] * tile_height;
2099 }
2100
2101 unsigned int
2102 intel_fb_align_height(const struct drm_framebuffer *fb,
2103                       int color_plane, unsigned int height)
2104 {
2105         unsigned int tile_height = intel_tile_height(fb, color_plane);
2106
2107         return ALIGN(height, tile_height);
2108 }
2109
2110 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info)
2111 {
2112         unsigned int size = 0;
2113         int i;
2114
2115         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
2116                 size += rot_info->plane[i].width * rot_info->plane[i].height;
2117
2118         return size;
2119 }
2120
2121 unsigned int intel_remapped_info_size(const struct intel_remapped_info *rem_info)
2122 {
2123         unsigned int size = 0;
2124         int i;
2125
2126         for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++)
2127                 size += rem_info->plane[i].width * rem_info->plane[i].height;
2128
2129         return size;
2130 }
2131
2132 static void
2133 intel_fill_fb_ggtt_view(struct i915_ggtt_view *view,
2134                         const struct drm_framebuffer *fb,
2135                         unsigned int rotation)
2136 {
2137         view->type = I915_GGTT_VIEW_NORMAL;
2138         if (drm_rotation_90_or_270(rotation)) {
2139                 view->type = I915_GGTT_VIEW_ROTATED;
2140                 view->rotated = to_intel_framebuffer(fb)->rot_info;
2141         }
2142 }
2143
2144 static unsigned int intel_cursor_alignment(const struct drm_i915_private *dev_priv)
2145 {
2146         if (IS_I830(dev_priv))
2147                 return 16 * 1024;
2148         else if (IS_I85X(dev_priv))
2149                 return 256;
2150         else if (IS_I845G(dev_priv) || IS_I865G(dev_priv))
2151                 return 32;
2152         else
2153                 return 4 * 1024;
2154 }
2155
2156 static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv)
2157 {
2158         if (INTEL_GEN(dev_priv) >= 9)
2159                 return 256 * 1024;
2160         else if (IS_I965G(dev_priv) || IS_I965GM(dev_priv) ||
2161                  IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2162                 return 128 * 1024;
2163         else if (INTEL_GEN(dev_priv) >= 4)
2164                 return 4 * 1024;
2165         else
2166                 return 0;
2167 }
2168
2169 static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb,
2170                                          int color_plane)
2171 {
2172         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2173
2174         /* AUX_DIST needs only 4K alignment */
2175         if ((INTEL_GEN(dev_priv) < 12 && is_aux_plane(fb, color_plane)) ||
2176             is_ccs_plane(fb, color_plane))
2177                 return 4096;
2178
2179         switch (fb->modifier) {
2180         case DRM_FORMAT_MOD_LINEAR:
2181                 return intel_linear_alignment(dev_priv);
2182         case I915_FORMAT_MOD_X_TILED:
2183                 if (INTEL_GEN(dev_priv) >= 9)
2184                         return 256 * 1024;
2185                 return 0;
2186         case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS:
2187                 if (is_semiplanar_uv_plane(fb, color_plane))
2188                         return intel_tile_row_size(fb, color_plane);
2189                 fallthrough;
2190         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
2191                 return 16 * 1024;
2192         case I915_FORMAT_MOD_Y_TILED_CCS:
2193         case I915_FORMAT_MOD_Yf_TILED_CCS:
2194         case I915_FORMAT_MOD_Y_TILED:
2195                 if (INTEL_GEN(dev_priv) >= 12 &&
2196                     is_semiplanar_uv_plane(fb, color_plane))
2197                         return intel_tile_row_size(fb, color_plane);
2198                 fallthrough;
2199         case I915_FORMAT_MOD_Yf_TILED:
2200                 return 1 * 1024 * 1024;
2201         default:
2202                 MISSING_CASE(fb->modifier);
2203                 return 0;
2204         }
2205 }
2206
2207 static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
2208 {
2209         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2210         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2211
2212         return INTEL_GEN(dev_priv) < 4 ||
2213                 (plane->has_fbc &&
2214                  plane_state->view.type == I915_GGTT_VIEW_NORMAL);
2215 }
2216
2217 struct i915_vma *
2218 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
2219                            const struct i915_ggtt_view *view,
2220                            bool uses_fence,
2221                            unsigned long *out_flags)
2222 {
2223         struct drm_device *dev = fb->dev;
2224         struct drm_i915_private *dev_priv = to_i915(dev);
2225         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2226         intel_wakeref_t wakeref;
2227         struct i915_vma *vma;
2228         unsigned int pinctl;
2229         u32 alignment;
2230
2231         if (drm_WARN_ON(dev, !i915_gem_object_is_framebuffer(obj)))
2232                 return ERR_PTR(-EINVAL);
2233
2234         alignment = intel_surf_alignment(fb, 0);
2235         if (drm_WARN_ON(dev, alignment && !is_power_of_2(alignment)))
2236                 return ERR_PTR(-EINVAL);
2237
2238         /* Note that the w/a also requires 64 PTE of padding following the
2239          * bo. We currently fill all unused PTE with the shadow page and so
2240          * we should always have valid PTE following the scanout preventing
2241          * the VT-d warning.
2242          */
2243         if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024)
2244                 alignment = 256 * 1024;
2245
2246         /*
2247          * Global gtt pte registers are special registers which actually forward
2248          * writes to a chunk of system memory. Which means that there is no risk
2249          * that the register values disappear as soon as we call
2250          * intel_runtime_pm_put(), so it is correct to wrap only the
2251          * pin/unpin/fence and not more.
2252          */
2253         wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
2254
2255         atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
2256
2257         /*
2258          * Valleyview is definitely limited to scanning out the first
2259          * 512MiB. Lets presume this behaviour was inherited from the
2260          * g4x display engine and that all earlier gen are similarly
2261          * limited. Testing suggests that it is a little more
2262          * complicated than this. For example, Cherryview appears quite
2263          * happy to scanout from anywhere within its global aperture.
2264          */
2265         pinctl = 0;
2266         if (HAS_GMCH(dev_priv))
2267                 pinctl |= PIN_MAPPABLE;
2268
2269         vma = i915_gem_object_pin_to_display_plane(obj,
2270                                                    alignment, view, pinctl);
2271         if (IS_ERR(vma))
2272                 goto err;
2273
2274         if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
2275                 int ret;
2276
2277                 /*
2278                  * Install a fence for tiled scan-out. Pre-i965 always needs a
2279                  * fence, whereas 965+ only requires a fence if using
2280                  * framebuffer compression.  For simplicity, we always, when
2281                  * possible, install a fence as the cost is not that onerous.
2282                  *
2283                  * If we fail to fence the tiled scanout, then either the
2284                  * modeset will reject the change (which is highly unlikely as
2285                  * the affected systems, all but one, do not have unmappable
2286                  * space) or we will not be able to enable full powersaving
2287                  * techniques (also likely not to apply due to various limits
2288                  * FBC and the like impose on the size of the buffer, which
2289                  * presumably we violated anyway with this unmappable buffer).
2290                  * Anyway, it is presumably better to stumble onwards with
2291                  * something and try to run the system in a "less than optimal"
2292                  * mode that matches the user configuration.
2293                  */
2294                 ret = i915_vma_pin_fence(vma);
2295                 if (ret != 0 && INTEL_GEN(dev_priv) < 4) {
2296                         i915_gem_object_unpin_from_display_plane(vma);
2297                         vma = ERR_PTR(ret);
2298                         goto err;
2299                 }
2300
2301                 if (ret == 0 && vma->fence)
2302                         *out_flags |= PLANE_HAS_FENCE;
2303         }
2304
2305         i915_vma_get(vma);
2306 err:
2307         atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
2308         intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
2309         return vma;
2310 }
2311
2312 void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
2313 {
2314         i915_gem_object_lock(vma->obj, NULL);
2315         if (flags & PLANE_HAS_FENCE)
2316                 i915_vma_unpin_fence(vma);
2317         i915_gem_object_unpin_from_display_plane(vma);
2318         i915_gem_object_unlock(vma->obj);
2319
2320         i915_vma_put(vma);
2321 }
2322
2323 static int intel_fb_pitch(const struct drm_framebuffer *fb, int color_plane,
2324                           unsigned int rotation)
2325 {
2326         if (drm_rotation_90_or_270(rotation))
2327                 return to_intel_framebuffer(fb)->rotated[color_plane].pitch;
2328         else
2329                 return fb->pitches[color_plane];
2330 }
2331
2332 /*
2333  * Convert the x/y offsets into a linear offset.
2334  * Only valid with 0/180 degree rotation, which is fine since linear
2335  * offset is only used with linear buffers on pre-hsw and tiled buffers
2336  * with gen2/3, and 90/270 degree rotations isn't supported on any of them.
2337  */
2338 u32 intel_fb_xy_to_linear(int x, int y,
2339                           const struct intel_plane_state *state,
2340                           int color_plane)
2341 {
2342         const struct drm_framebuffer *fb = state->hw.fb;
2343         unsigned int cpp = fb->format->cpp[color_plane];
2344         unsigned int pitch = state->color_plane[color_plane].stride;
2345
2346         return y * pitch + x * cpp;
2347 }
2348
2349 /*
2350  * Add the x/y offsets derived from fb->offsets[] to the user
2351  * specified plane src x/y offsets. The resulting x/y offsets
2352  * specify the start of scanout from the beginning of the gtt mapping.
2353  */
2354 void intel_add_fb_offsets(int *x, int *y,
2355                           const struct intel_plane_state *state,
2356                           int color_plane)
2357
2358 {
2359         *x += state->color_plane[color_plane].x;
2360         *y += state->color_plane[color_plane].y;
2361 }
2362
2363 static u32 intel_adjust_tile_offset(int *x, int *y,
2364                                     unsigned int tile_width,
2365                                     unsigned int tile_height,
2366                                     unsigned int tile_size,
2367                                     unsigned int pitch_tiles,
2368                                     u32 old_offset,
2369                                     u32 new_offset)
2370 {
2371         unsigned int pitch_pixels = pitch_tiles * tile_width;
2372         unsigned int tiles;
2373
2374         WARN_ON(old_offset & (tile_size - 1));
2375         WARN_ON(new_offset & (tile_size - 1));
2376         WARN_ON(new_offset > old_offset);
2377
2378         tiles = (old_offset - new_offset) / tile_size;
2379
2380         *y += tiles / pitch_tiles * tile_height;
2381         *x += tiles % pitch_tiles * tile_width;
2382
2383         /* minimize x in case it got needlessly big */
2384         *y += *x / pitch_pixels * tile_height;
2385         *x %= pitch_pixels;
2386
2387         return new_offset;
2388 }
2389
2390 static bool is_surface_linear(const struct drm_framebuffer *fb, int color_plane)
2391 {
2392         return fb->modifier == DRM_FORMAT_MOD_LINEAR ||
2393                is_gen12_ccs_plane(fb, color_plane);
2394 }
2395
2396 static u32 intel_adjust_aligned_offset(int *x, int *y,
2397                                        const struct drm_framebuffer *fb,
2398                                        int color_plane,
2399                                        unsigned int rotation,
2400                                        unsigned int pitch,
2401                                        u32 old_offset, u32 new_offset)
2402 {
2403         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2404         unsigned int cpp = fb->format->cpp[color_plane];
2405
2406         drm_WARN_ON(&dev_priv->drm, new_offset > old_offset);
2407
2408         if (!is_surface_linear(fb, color_plane)) {
2409                 unsigned int tile_size, tile_width, tile_height;
2410                 unsigned int pitch_tiles;
2411
2412                 tile_size = intel_tile_size(dev_priv);
2413                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2414
2415                 if (drm_rotation_90_or_270(rotation)) {
2416                         pitch_tiles = pitch / tile_height;
2417                         swap(tile_width, tile_height);
2418                 } else {
2419                         pitch_tiles = pitch / (tile_width * cpp);
2420                 }
2421
2422                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2423                                          tile_size, pitch_tiles,
2424                                          old_offset, new_offset);
2425         } else {
2426                 old_offset += *y * pitch + *x * cpp;
2427
2428                 *y = (old_offset - new_offset) / pitch;
2429                 *x = ((old_offset - new_offset) - *y * pitch) / cpp;
2430         }
2431
2432         return new_offset;
2433 }
2434
2435 /*
2436  * Adjust the tile offset by moving the difference into
2437  * the x/y offsets.
2438  */
2439 static u32 intel_plane_adjust_aligned_offset(int *x, int *y,
2440                                              const struct intel_plane_state *state,
2441                                              int color_plane,
2442                                              u32 old_offset, u32 new_offset)
2443 {
2444         return intel_adjust_aligned_offset(x, y, state->hw.fb, color_plane,
2445                                            state->hw.rotation,
2446                                            state->color_plane[color_plane].stride,
2447                                            old_offset, new_offset);
2448 }
2449
2450 /*
2451  * Computes the aligned offset to the base tile and adjusts
2452  * x, y. bytes per pixel is assumed to be a power-of-two.
2453  *
2454  * In the 90/270 rotated case, x and y are assumed
2455  * to be already rotated to match the rotated GTT view, and
2456  * pitch is the tile_height aligned framebuffer height.
2457  *
2458  * This function is used when computing the derived information
2459  * under intel_framebuffer, so using any of that information
2460  * here is not allowed. Anything under drm_framebuffer can be
2461  * used. This is why the user has to pass in the pitch since it
2462  * is specified in the rotated orientation.
2463  */
2464 static u32 intel_compute_aligned_offset(struct drm_i915_private *dev_priv,
2465                                         int *x, int *y,
2466                                         const struct drm_framebuffer *fb,
2467                                         int color_plane,
2468                                         unsigned int pitch,
2469                                         unsigned int rotation,
2470                                         u32 alignment)
2471 {
2472         unsigned int cpp = fb->format->cpp[color_plane];
2473         u32 offset, offset_aligned;
2474
2475         if (!is_surface_linear(fb, color_plane)) {
2476                 unsigned int tile_size, tile_width, tile_height;
2477                 unsigned int tile_rows, tiles, pitch_tiles;
2478
2479                 tile_size = intel_tile_size(dev_priv);
2480                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2481
2482                 if (drm_rotation_90_or_270(rotation)) {
2483                         pitch_tiles = pitch / tile_height;
2484                         swap(tile_width, tile_height);
2485                 } else {
2486                         pitch_tiles = pitch / (tile_width * cpp);
2487                 }
2488
2489                 tile_rows = *y / tile_height;
2490                 *y %= tile_height;
2491
2492                 tiles = *x / tile_width;
2493                 *x %= tile_width;
2494
2495                 offset = (tile_rows * pitch_tiles + tiles) * tile_size;
2496
2497                 offset_aligned = offset;
2498                 if (alignment)
2499                         offset_aligned = rounddown(offset_aligned, alignment);
2500
2501                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2502                                          tile_size, pitch_tiles,
2503                                          offset, offset_aligned);
2504         } else {
2505                 offset = *y * pitch + *x * cpp;
2506                 offset_aligned = offset;
2507                 if (alignment) {
2508                         offset_aligned = rounddown(offset_aligned, alignment);
2509                         *y = (offset % alignment) / pitch;
2510                         *x = ((offset % alignment) - *y * pitch) / cpp;
2511                 } else {
2512                         *y = *x = 0;
2513                 }
2514         }
2515
2516         return offset_aligned;
2517 }
2518
2519 static u32 intel_plane_compute_aligned_offset(int *x, int *y,
2520                                               const struct intel_plane_state *state,
2521                                               int color_plane)
2522 {
2523         struct intel_plane *intel_plane = to_intel_plane(state->uapi.plane);
2524         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
2525         const struct drm_framebuffer *fb = state->hw.fb;
2526         unsigned int rotation = state->hw.rotation;
2527         int pitch = state->color_plane[color_plane].stride;
2528         u32 alignment;
2529
2530         if (intel_plane->id == PLANE_CURSOR)
2531                 alignment = intel_cursor_alignment(dev_priv);
2532         else
2533                 alignment = intel_surf_alignment(fb, color_plane);
2534
2535         return intel_compute_aligned_offset(dev_priv, x, y, fb, color_plane,
2536                                             pitch, rotation, alignment);
2537 }
2538
2539 /* Convert the fb->offset[] into x/y offsets */
2540 static int intel_fb_offset_to_xy(int *x, int *y,
2541                                  const struct drm_framebuffer *fb,
2542                                  int color_plane)
2543 {
2544         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2545         unsigned int height;
2546         u32 alignment;
2547
2548         if (INTEL_GEN(dev_priv) >= 12 &&
2549             is_semiplanar_uv_plane(fb, color_plane))
2550                 alignment = intel_tile_row_size(fb, color_plane);
2551         else if (fb->modifier != DRM_FORMAT_MOD_LINEAR)
2552                 alignment = intel_tile_size(dev_priv);
2553         else
2554                 alignment = 0;
2555
2556         if (alignment != 0 && fb->offsets[color_plane] % alignment) {
2557                 drm_dbg_kms(&dev_priv->drm,
2558                             "Misaligned offset 0x%08x for color plane %d\n",
2559                             fb->offsets[color_plane], color_plane);
2560                 return -EINVAL;
2561         }
2562
2563         height = drm_framebuffer_plane_height(fb->height, fb, color_plane);
2564         height = ALIGN(height, intel_tile_height(fb, color_plane));
2565
2566         /* Catch potential overflows early */
2567         if (add_overflows_t(u32, mul_u32_u32(height, fb->pitches[color_plane]),
2568                             fb->offsets[color_plane])) {
2569                 drm_dbg_kms(&dev_priv->drm,
2570                             "Bad offset 0x%08x or pitch %d for color plane %d\n",
2571                             fb->offsets[color_plane], fb->pitches[color_plane],
2572                             color_plane);
2573                 return -ERANGE;
2574         }
2575
2576         *x = 0;
2577         *y = 0;
2578
2579         intel_adjust_aligned_offset(x, y,
2580                                     fb, color_plane, DRM_MODE_ROTATE_0,
2581                                     fb->pitches[color_plane],
2582                                     fb->offsets[color_plane], 0);
2583
2584         return 0;
2585 }
2586
2587 static unsigned int intel_fb_modifier_to_tiling(u64 fb_modifier)
2588 {
2589         switch (fb_modifier) {
2590         case I915_FORMAT_MOD_X_TILED:
2591                 return I915_TILING_X;
2592         case I915_FORMAT_MOD_Y_TILED:
2593         case I915_FORMAT_MOD_Y_TILED_CCS:
2594         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
2595         case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS:
2596                 return I915_TILING_Y;
2597         default:
2598                 return I915_TILING_NONE;
2599         }
2600 }
2601
2602 /*
2603  * From the Sky Lake PRM:
2604  * "The Color Control Surface (CCS) contains the compression status of
2605  *  the cache-line pairs. The compression state of the cache-line pair
2606  *  is specified by 2 bits in the CCS. Each CCS cache-line represents
2607  *  an area on the main surface of 16 x16 sets of 128 byte Y-tiled
2608  *  cache-line-pairs. CCS is always Y tiled."
2609  *
2610  * Since cache line pairs refers to horizontally adjacent cache lines,
2611  * each cache line in the CCS corresponds to an area of 32x16 cache
2612  * lines on the main surface. Since each pixel is 4 bytes, this gives
2613  * us a ratio of one byte in the CCS for each 8x16 pixels in the
2614  * main surface.
2615  */
2616 static const struct drm_format_info skl_ccs_formats[] = {
2617         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
2618           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2619         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
2620           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2621         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
2622           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
2623         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
2624           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
2625 };
2626
2627 /*
2628  * Gen-12 compression uses 4 bits of CCS data for each cache line pair in the
2629  * main surface. And each 64B CCS cache line represents an area of 4x1 Y-tiles
2630  * in the main surface. With 4 byte pixels and each Y-tile having dimensions of
2631  * 32x32 pixels, the ratio turns out to 1B in the CCS for every 2x32 pixels in
2632  * the main surface.
2633  */
2634 static const struct drm_format_info gen12_ccs_formats[] = {
2635         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
2636           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2637           .hsub = 1, .vsub = 1, },
2638         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
2639           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2640           .hsub = 1, .vsub = 1, },
2641         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
2642           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2643           .hsub = 1, .vsub = 1, .has_alpha = true },
2644         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
2645           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2646           .hsub = 1, .vsub = 1, .has_alpha = true },
2647         { .format = DRM_FORMAT_YUYV, .num_planes = 2,
2648           .char_per_block = { 2, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2649           .hsub = 2, .vsub = 1, .is_yuv = true },
2650         { .format = DRM_FORMAT_YVYU, .num_planes = 2,
2651           .char_per_block = { 2, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2652           .hsub = 2, .vsub = 1, .is_yuv = true },
2653         { .format = DRM_FORMAT_UYVY, .num_planes = 2,
2654           .char_per_block = { 2, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2655           .hsub = 2, .vsub = 1, .is_yuv = true },
2656         { .format = DRM_FORMAT_VYUY, .num_planes = 2,
2657           .char_per_block = { 2, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2658           .hsub = 2, .vsub = 1, .is_yuv = true },
2659         { .format = DRM_FORMAT_NV12, .num_planes = 4,
2660           .char_per_block = { 1, 2, 1, 1 }, .block_w = { 1, 1, 4, 4 }, .block_h = { 1, 1, 1, 1 },
2661           .hsub = 2, .vsub = 2, .is_yuv = true },
2662         { .format = DRM_FORMAT_P010, .num_planes = 4,
2663           .char_per_block = { 2, 4, 1, 1 }, .block_w = { 1, 1, 2, 2 }, .block_h = { 1, 1, 1, 1 },
2664           .hsub = 2, .vsub = 2, .is_yuv = true },
2665         { .format = DRM_FORMAT_P012, .num_planes = 4,
2666           .char_per_block = { 2, 4, 1, 1 }, .block_w = { 1, 1, 2, 2 }, .block_h = { 1, 1, 1, 1 },
2667           .hsub = 2, .vsub = 2, .is_yuv = true },
2668         { .format = DRM_FORMAT_P016, .num_planes = 4,
2669           .char_per_block = { 2, 4, 1, 1 }, .block_w = { 1, 1, 2, 2 }, .block_h = { 1, 1, 1, 1 },
2670           .hsub = 2, .vsub = 2, .is_yuv = true },
2671 };
2672
2673 static const struct drm_format_info *
2674 lookup_format_info(const struct drm_format_info formats[],
2675                    int num_formats, u32 format)
2676 {
2677         int i;
2678
2679         for (i = 0; i < num_formats; i++) {
2680                 if (formats[i].format == format)
2681                         return &formats[i];
2682         }
2683
2684         return NULL;
2685 }
2686
2687 static const struct drm_format_info *
2688 intel_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
2689 {
2690         switch (cmd->modifier[0]) {
2691         case I915_FORMAT_MOD_Y_TILED_CCS:
2692         case I915_FORMAT_MOD_Yf_TILED_CCS:
2693                 return lookup_format_info(skl_ccs_formats,
2694                                           ARRAY_SIZE(skl_ccs_formats),
2695                                           cmd->pixel_format);
2696         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
2697         case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS:
2698                 return lookup_format_info(gen12_ccs_formats,
2699                                           ARRAY_SIZE(gen12_ccs_formats),
2700                                           cmd->pixel_format);
2701         default:
2702                 return NULL;
2703         }
2704 }
2705
2706 bool is_ccs_modifier(u64 modifier)
2707 {
2708         return modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS ||
2709                modifier == I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS ||
2710                modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
2711                modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
2712 }
2713
2714 static int gen12_ccs_aux_stride(struct drm_framebuffer *fb, int ccs_plane)
2715 {
2716         return DIV_ROUND_UP(fb->pitches[ccs_to_main_plane(fb, ccs_plane)],
2717                             512) * 64;
2718 }
2719
2720 u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv,
2721                               u32 pixel_format, u64 modifier)
2722 {
2723         struct intel_crtc *crtc;
2724         struct intel_plane *plane;
2725
2726         /*
2727          * We assume the primary plane for pipe A has
2728          * the highest stride limits of them all,
2729          * if in case pipe A is disabled, use the first pipe from pipe_mask.
2730          */
2731         crtc = intel_get_first_crtc(dev_priv);
2732         if (!crtc)
2733                 return 0;
2734
2735         plane = to_intel_plane(crtc->base.primary);
2736
2737         return plane->max_stride(plane, pixel_format, modifier,
2738                                  DRM_MODE_ROTATE_0);
2739 }
2740
2741 static
2742 u32 intel_fb_max_stride(struct drm_i915_private *dev_priv,
2743                         u32 pixel_format, u64 modifier)
2744 {
2745         /*
2746          * Arbitrary limit for gen4+ chosen to match the
2747          * render engine max stride.
2748          *
2749          * The new CCS hash mode makes remapping impossible
2750          */
2751         if (!is_ccs_modifier(modifier)) {
2752                 if (INTEL_GEN(dev_priv) >= 7)
2753                         return 256*1024;
2754                 else if (INTEL_GEN(dev_priv) >= 4)
2755                         return 128*1024;
2756         }
2757
2758         return intel_plane_fb_max_stride(dev_priv, pixel_format, modifier);
2759 }
2760
2761 static u32
2762 intel_fb_stride_alignment(const struct drm_framebuffer *fb, int color_plane)
2763 {
2764         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2765         u32 tile_width;
2766
2767         if (is_surface_linear(fb, color_plane)) {
2768                 u32 max_stride = intel_plane_fb_max_stride(dev_priv,
2769                                                            fb->format->format,
2770                                                            fb->modifier);
2771
2772                 /*
2773                  * To make remapping with linear generally feasible
2774                  * we need the stride to be page aligned.
2775                  */
2776                 if (fb->pitches[color_plane] > max_stride &&
2777                     !is_ccs_modifier(fb->modifier))
2778                         return intel_tile_size(dev_priv);
2779                 else
2780                         return 64;
2781         }
2782
2783         tile_width = intel_tile_width_bytes(fb, color_plane);
2784         if (is_ccs_modifier(fb->modifier)) {
2785                 /*
2786                  * Display WA #0531: skl,bxt,kbl,glk
2787                  *
2788                  * Render decompression and plane width > 3840
2789                  * combined with horizontal panning requires the
2790                  * plane stride to be a multiple of 4. We'll just
2791                  * require the entire fb to accommodate that to avoid
2792                  * potential runtime errors at plane configuration time.
2793                  */
2794                 if (IS_GEN(dev_priv, 9) && color_plane == 0 && fb->width > 3840)
2795                         tile_width *= 4;
2796                 /*
2797                  * The main surface pitch must be padded to a multiple of four
2798                  * tile widths.
2799                  */
2800                 else if (INTEL_GEN(dev_priv) >= 12)
2801                         tile_width *= 4;
2802         }
2803         return tile_width;
2804 }
2805
2806 bool intel_plane_can_remap(const struct intel_plane_state *plane_state)
2807 {
2808         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2809         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2810         const struct drm_framebuffer *fb = plane_state->hw.fb;
2811         int i;
2812
2813         /* We don't want to deal with remapping with cursors */
2814         if (plane->id == PLANE_CURSOR)
2815                 return false;
2816
2817         /*
2818          * The display engine limits already match/exceed the
2819          * render engine limits, so not much point in remapping.
2820          * Would also need to deal with the fence POT alignment
2821          * and gen2 2KiB GTT tile size.
2822          */
2823         if (INTEL_GEN(dev_priv) < 4)
2824                 return false;
2825
2826         /*
2827          * The new CCS hash mode isn't compatible with remapping as
2828          * the virtual address of the pages affects the compressed data.
2829          */
2830         if (is_ccs_modifier(fb->modifier))
2831                 return false;
2832
2833         /* Linear needs a page aligned stride for remapping */
2834         if (fb->modifier == DRM_FORMAT_MOD_LINEAR) {
2835                 unsigned int alignment = intel_tile_size(dev_priv) - 1;
2836
2837                 for (i = 0; i < fb->format->num_planes; i++) {
2838                         if (fb->pitches[i] & alignment)
2839                                 return false;
2840                 }
2841         }
2842
2843         return true;
2844 }
2845
2846 static bool intel_plane_needs_remap(const struct intel_plane_state *plane_state)
2847 {
2848         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2849         const struct drm_framebuffer *fb = plane_state->hw.fb;
2850         unsigned int rotation = plane_state->hw.rotation;
2851         u32 stride, max_stride;
2852
2853         /*
2854          * No remapping for invisible planes since we don't have
2855          * an actual source viewport to remap.
2856          */
2857         if (!plane_state->uapi.visible)
2858                 return false;
2859
2860         if (!intel_plane_can_remap(plane_state))
2861                 return false;
2862
2863         /*
2864          * FIXME: aux plane limits on gen9+ are
2865          * unclear in Bspec, for now no checking.
2866          */
2867         stride = intel_fb_pitch(fb, 0, rotation);
2868         max_stride = plane->max_stride(plane, fb->format->format,
2869                                        fb->modifier, rotation);
2870
2871         return stride > max_stride;
2872 }
2873
2874 static void
2875 intel_fb_plane_get_subsampling(int *hsub, int *vsub,
2876                                const struct drm_framebuffer *fb,
2877                                int color_plane)
2878 {
2879         int main_plane;
2880
2881         if (color_plane == 0) {
2882                 *hsub = 1;
2883                 *vsub = 1;
2884
2885                 return;
2886         }
2887
2888         /*
2889          * TODO: Deduct the subsampling from the char block for all CCS
2890          * formats and planes.
2891          */
2892         if (!is_gen12_ccs_plane(fb, color_plane)) {
2893                 *hsub = fb->format->hsub;
2894                 *vsub = fb->format->vsub;
2895
2896                 return;
2897         }
2898
2899         main_plane = ccs_to_main_plane(fb, color_plane);
2900         *hsub = drm_format_info_block_width(fb->format, color_plane) /
2901                 drm_format_info_block_width(fb->format, main_plane);
2902
2903         /*
2904          * The min stride check in the core framebuffer_check() function
2905          * assumes that format->hsub applies to every plane except for the
2906          * first plane. That's incorrect for the CCS AUX plane of the first
2907          * plane, but for the above check to pass we must define the block
2908          * width with that subsampling applied to it. Adjust the width here
2909          * accordingly, so we can calculate the actual subsampling factor.
2910          */
2911         if (main_plane == 0)
2912                 *hsub *= fb->format->hsub;
2913
2914         *vsub = 32;
2915 }
2916 static int
2917 intel_fb_check_ccs_xy(struct drm_framebuffer *fb, int ccs_plane, int x, int y)
2918 {
2919         struct drm_i915_private *i915 = to_i915(fb->dev);
2920         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2921         int main_plane;
2922         int hsub, vsub;
2923         int tile_width, tile_height;
2924         int ccs_x, ccs_y;
2925         int main_x, main_y;
2926
2927         if (!is_ccs_plane(fb, ccs_plane))
2928                 return 0;
2929
2930         intel_tile_dims(fb, ccs_plane, &tile_width, &tile_height);
2931         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, ccs_plane);
2932
2933         tile_width *= hsub;
2934         tile_height *= vsub;
2935
2936         ccs_x = (x * hsub) % tile_width;
2937         ccs_y = (y * vsub) % tile_height;
2938
2939         main_plane = ccs_to_main_plane(fb, ccs_plane);
2940         main_x = intel_fb->normal[main_plane].x % tile_width;
2941         main_y = intel_fb->normal[main_plane].y % tile_height;
2942
2943         /*
2944          * CCS doesn't have its own x/y offset register, so the intra CCS tile
2945          * x/y offsets must match between CCS and the main surface.
2946          */
2947         if (main_x != ccs_x || main_y != ccs_y) {
2948                 drm_dbg_kms(&i915->drm,
2949                               "Bad CCS x/y (main %d,%d ccs %d,%d) full (main %d,%d ccs %d,%d)\n",
2950                               main_x, main_y,
2951                               ccs_x, ccs_y,
2952                               intel_fb->normal[main_plane].x,
2953                               intel_fb->normal[main_plane].y,
2954                               x, y);
2955                 return -EINVAL;
2956         }
2957
2958         return 0;
2959 }
2960
2961 static void
2962 intel_fb_plane_dims(int *w, int *h, struct drm_framebuffer *fb, int color_plane)
2963 {
2964         int main_plane = is_ccs_plane(fb, color_plane) ?
2965                          ccs_to_main_plane(fb, color_plane) : 0;
2966         int main_hsub, main_vsub;
2967         int hsub, vsub;
2968
2969         intel_fb_plane_get_subsampling(&main_hsub, &main_vsub, fb, main_plane);
2970         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, color_plane);
2971         *w = fb->width / main_hsub / hsub;
2972         *h = fb->height / main_vsub / vsub;
2973 }
2974
2975 /*
2976  * Setup the rotated view for an FB plane and return the size the GTT mapping
2977  * requires for this view.
2978  */
2979 static u32
2980 setup_fb_rotation(int plane, const struct intel_remapped_plane_info *plane_info,
2981                   u32 gtt_offset_rotated, int x, int y,
2982                   unsigned int width, unsigned int height,
2983                   unsigned int tile_size,
2984                   unsigned int tile_width, unsigned int tile_height,
2985                   struct drm_framebuffer *fb)
2986 {
2987         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2988         struct intel_rotation_info *rot_info = &intel_fb->rot_info;
2989         unsigned int pitch_tiles;
2990         struct drm_rect r;
2991
2992         /* Y or Yf modifiers required for 90/270 rotation */
2993         if (fb->modifier != I915_FORMAT_MOD_Y_TILED &&
2994             fb->modifier != I915_FORMAT_MOD_Yf_TILED)
2995                 return 0;
2996
2997         if (drm_WARN_ON(fb->dev, plane >= ARRAY_SIZE(rot_info->plane)))
2998                 return 0;
2999
3000         rot_info->plane[plane] = *plane_info;
3001
3002         intel_fb->rotated[plane].pitch = plane_info->height * tile_height;
3003
3004         /* rotate the x/y offsets to match the GTT view */
3005         drm_rect_init(&r, x, y, width, height);
3006         drm_rect_rotate(&r,
3007                         plane_info->width * tile_width,
3008                         plane_info->height * tile_height,
3009                         DRM_MODE_ROTATE_270);
3010         x = r.x1;
3011         y = r.y1;
3012
3013         /* rotate the tile dimensions to match the GTT view */
3014         pitch_tiles = intel_fb->rotated[plane].pitch / tile_height;
3015         swap(tile_width, tile_height);
3016
3017         /*
3018          * We only keep the x/y offsets, so push all of the
3019          * gtt offset into the x/y offsets.
3020          */
3021         intel_adjust_tile_offset(&x, &y,
3022                                  tile_width, tile_height,
3023                                  tile_size, pitch_tiles,
3024                                  gtt_offset_rotated * tile_size, 0);
3025
3026         /*
3027          * First pixel of the framebuffer from
3028          * the start of the rotated gtt mapping.
3029          */
3030         intel_fb->rotated[plane].x = x;
3031         intel_fb->rotated[plane].y = y;
3032
3033         return plane_info->width * plane_info->height;
3034 }
3035
3036 static int
3037 intel_fill_fb_info(struct drm_i915_private *dev_priv,
3038                    struct drm_framebuffer *fb)
3039 {
3040         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
3041         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
3042         u32 gtt_offset_rotated = 0;
3043         unsigned int max_size = 0;
3044         int i, num_planes = fb->format->num_planes;
3045         unsigned int tile_size = intel_tile_size(dev_priv);
3046
3047         for (i = 0; i < num_planes; i++) {
3048                 unsigned int width, height;
3049                 unsigned int cpp, size;
3050                 u32 offset;
3051                 int x, y;
3052                 int ret;
3053
3054                 cpp = fb->format->cpp[i];
3055                 intel_fb_plane_dims(&width, &height, fb, i);
3056
3057                 ret = intel_fb_offset_to_xy(&x, &y, fb, i);
3058                 if (ret) {
3059                         drm_dbg_kms(&dev_priv->drm,
3060                                     "bad fb plane %d offset: 0x%x\n",
3061                                     i, fb->offsets[i]);
3062                         return ret;
3063                 }
3064
3065                 ret = intel_fb_check_ccs_xy(fb, i, x, y);
3066                 if (ret)
3067                         return ret;
3068
3069                 /*
3070                  * The fence (if used) is aligned to the start of the object
3071                  * so having the framebuffer wrap around across the edge of the
3072                  * fenced region doesn't really work. We have no API to configure
3073                  * the fence start offset within the object (nor could we probably
3074                  * on gen2/3). So it's just easier if we just require that the
3075                  * fb layout agrees with the fence layout. We already check that the
3076                  * fb stride matches the fence stride elsewhere.
3077                  */
3078                 if (i == 0 && i915_gem_object_is_tiled(obj) &&
3079                     (x + width) * cpp > fb->pitches[i]) {
3080                         drm_dbg_kms(&dev_priv->drm,
3081                                     "bad fb plane %d offset: 0x%x\n",
3082                                      i, fb->offsets[i]);
3083                         return -EINVAL;
3084                 }
3085
3086                 /*
3087                  * First pixel of the framebuffer from
3088                  * the start of the normal gtt mapping.
3089                  */
3090                 intel_fb->normal[i].x = x;
3091                 intel_fb->normal[i].y = y;
3092
3093                 offset = intel_compute_aligned_offset(dev_priv, &x, &y, fb, i,
3094                                                       fb->pitches[i],
3095                                                       DRM_MODE_ROTATE_0,
3096                                                       tile_size);
3097                 offset /= tile_size;
3098
3099                 if (!is_surface_linear(fb, i)) {
3100                         struct intel_remapped_plane_info plane_info;
3101                         unsigned int tile_width, tile_height;
3102
3103                         intel_tile_dims(fb, i, &tile_width, &tile_height);
3104
3105                         plane_info.offset = offset;
3106                         plane_info.stride = DIV_ROUND_UP(fb->pitches[i],
3107                                                          tile_width * cpp);
3108                         plane_info.width = DIV_ROUND_UP(x + width, tile_width);
3109                         plane_info.height = DIV_ROUND_UP(y + height,
3110                                                          tile_height);
3111
3112                         /* how many tiles does this plane need */
3113                         size = plane_info.stride * plane_info.height;
3114                         /*
3115                          * If the plane isn't horizontally tile aligned,
3116                          * we need one more tile.
3117                          */
3118                         if (x != 0)
3119                                 size++;
3120
3121                         gtt_offset_rotated +=
3122                                 setup_fb_rotation(i, &plane_info,
3123                                                   gtt_offset_rotated,
3124                                                   x, y, width, height,
3125                                                   tile_size,
3126                                                   tile_width, tile_height,
3127                                                   fb);
3128                 } else {
3129                         size = DIV_ROUND_UP((y + height) * fb->pitches[i] +
3130                                             x * cpp, tile_size);
3131                 }
3132
3133                 /* how many tiles in total needed in the bo */
3134                 max_size = max(max_size, offset + size);
3135         }
3136
3137         if (mul_u32_u32(max_size, tile_size) > obj->base.size) {
3138                 drm_dbg_kms(&dev_priv->drm,
3139                             "fb too big for bo (need %llu bytes, have %zu bytes)\n",
3140                             mul_u32_u32(max_size, tile_size), obj->base.size);
3141                 return -EINVAL;
3142         }
3143
3144         return 0;
3145 }
3146
3147 static void
3148 intel_plane_remap_gtt(struct intel_plane_state *plane_state)
3149 {
3150         struct drm_i915_private *dev_priv =
3151                 to_i915(plane_state->uapi.plane->dev);
3152         struct drm_framebuffer *fb = plane_state->hw.fb;
3153         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
3154         struct intel_rotation_info *info = &plane_state->view.rotated;
3155         unsigned int rotation = plane_state->hw.rotation;
3156         int i, num_planes = fb->format->num_planes;
3157         unsigned int tile_size = intel_tile_size(dev_priv);
3158         unsigned int src_x, src_y;
3159         unsigned int src_w, src_h;
3160         u32 gtt_offset = 0;
3161
3162         memset(&plane_state->view, 0, sizeof(plane_state->view));
3163         plane_state->view.type = drm_rotation_90_or_270(rotation) ?
3164                 I915_GGTT_VIEW_ROTATED : I915_GGTT_VIEW_REMAPPED;
3165
3166         src_x = plane_state->uapi.src.x1 >> 16;
3167         src_y = plane_state->uapi.src.y1 >> 16;
3168         src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
3169         src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
3170
3171         drm_WARN_ON(&dev_priv->drm, is_ccs_modifier(fb->modifier));
3172
3173         /* Make src coordinates relative to the viewport */
3174         drm_rect_translate(&plane_state->uapi.src,
3175                            -(src_x << 16), -(src_y << 16));
3176
3177         /* Rotate src coordinates to match rotated GTT view */
3178         if (drm_rotation_90_or_270(rotation))
3179                 drm_rect_rotate(&plane_state->uapi.src,
3180                                 src_w << 16, src_h << 16,
3181                                 DRM_MODE_ROTATE_270);
3182
3183         for (i = 0; i < num_planes; i++) {
3184                 unsigned int hsub = i ? fb->format->hsub : 1;
3185                 unsigned int vsub = i ? fb->format->vsub : 1;
3186                 unsigned int cpp = fb->format->cpp[i];
3187                 unsigned int tile_width, tile_height;
3188                 unsigned int width, height;
3189                 unsigned int pitch_tiles;
3190                 unsigned int x, y;
3191                 u32 offset;
3192
3193                 intel_tile_dims(fb, i, &tile_width, &tile_height);
3194
3195                 x = src_x / hsub;
3196                 y = src_y / vsub;
3197                 width = src_w / hsub;
3198                 height = src_h / vsub;
3199
3200                 /*
3201                  * First pixel of the src viewport from the
3202                  * start of the normal gtt mapping.
3203                  */
3204                 x += intel_fb->normal[i].x;
3205                 y += intel_fb->normal[i].y;
3206
3207                 offset = intel_compute_aligned_offset(dev_priv, &x, &y,
3208                                                       fb, i, fb->pitches[i],
3209                                                       DRM_MODE_ROTATE_0, tile_size);
3210                 offset /= tile_size;
3211
3212                 drm_WARN_ON(&dev_priv->drm, i >= ARRAY_SIZE(info->plane));
3213                 info->plane[i].offset = offset;
3214                 info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i],
3215                                                      tile_width * cpp);
3216                 info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
3217                 info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
3218
3219                 if (drm_rotation_90_or_270(rotation)) {
3220                         struct drm_rect r;
3221
3222                         /* rotate the x/y offsets to match the GTT view */
3223                         drm_rect_init(&r, x, y, width, height);
3224                         drm_rect_rotate(&r,
3225                                         info->plane[i].width * tile_width,
3226                                         info->plane[i].height * tile_height,
3227                                         DRM_MODE_ROTATE_270);
3228                         x = r.x1;
3229                         y = r.y1;
3230
3231                         pitch_tiles = info->plane[i].height;
3232                         plane_state->color_plane[i].stride = pitch_tiles * tile_height;
3233
3234                         /* rotate the tile dimensions to match the GTT view */
3235                         swap(tile_width, tile_height);
3236                 } else {
3237                         pitch_tiles = info->plane[i].width;
3238                         plane_state->color_plane[i].stride = pitch_tiles * tile_width * cpp;
3239                 }
3240
3241                 /*
3242                  * We only keep the x/y offsets, so push all of the
3243                  * gtt offset into the x/y offsets.
3244                  */
3245                 intel_adjust_tile_offset(&x, &y,
3246                                          tile_width, tile_height,
3247                                          tile_size, pitch_tiles,
3248                                          gtt_offset * tile_size, 0);
3249
3250                 gtt_offset += info->plane[i].width * info->plane[i].height;
3251
3252                 plane_state->color_plane[i].offset = 0;
3253                 plane_state->color_plane[i].x = x;
3254                 plane_state->color_plane[i].y = y;
3255         }
3256 }
3257
3258 static int
3259 intel_plane_compute_gtt(struct intel_plane_state *plane_state)
3260 {
3261         const struct intel_framebuffer *fb =
3262                 to_intel_framebuffer(plane_state->hw.fb);
3263         unsigned int rotation = plane_state->hw.rotation;
3264         int i, num_planes;
3265
3266         if (!fb)
3267                 return 0;
3268
3269         num_planes = fb->base.format->num_planes;
3270
3271         if (intel_plane_needs_remap(plane_state)) {
3272                 intel_plane_remap_gtt(plane_state);
3273
3274                 /*
3275                  * Sometimes even remapping can't overcome
3276                  * the stride limitations :( Can happen with
3277                  * big plane sizes and suitably misaligned
3278                  * offsets.
3279                  */
3280                 return intel_plane_check_stride(plane_state);
3281         }
3282
3283         intel_fill_fb_ggtt_view(&plane_state->view, &fb->base, rotation);
3284
3285         for (i = 0; i < num_planes; i++) {
3286                 plane_state->color_plane[i].stride = intel_fb_pitch(&fb->base, i, rotation);
3287                 plane_state->color_plane[i].offset = 0;
3288
3289                 if (drm_rotation_90_or_270(rotation)) {
3290                         plane_state->color_plane[i].x = fb->rotated[i].x;
3291                         plane_state->color_plane[i].y = fb->rotated[i].y;
3292                 } else {
3293                         plane_state->color_plane[i].x = fb->normal[i].x;
3294                         plane_state->color_plane[i].y = fb->normal[i].y;
3295                 }
3296         }
3297
3298         /* Rotate src coordinates to match rotated GTT view */
3299         if (drm_rotation_90_or_270(rotation))
3300                 drm_rect_rotate(&plane_state->uapi.src,
3301                                 fb->base.width << 16, fb->base.height << 16,
3302                                 DRM_MODE_ROTATE_270);
3303
3304         return intel_plane_check_stride(plane_state);
3305 }
3306
3307 static int i9xx_format_to_fourcc(int format)
3308 {
3309         switch (format) {
3310         case DISPPLANE_8BPP:
3311                 return DRM_FORMAT_C8;
3312         case DISPPLANE_BGRA555:
3313                 return DRM_FORMAT_ARGB1555;
3314         case DISPPLANE_BGRX555:
3315                 return DRM_FORMAT_XRGB1555;
3316         case DISPPLANE_BGRX565:
3317                 return DRM_FORMAT_RGB565;
3318         default:
3319         case DISPPLANE_BGRX888:
3320                 return DRM_FORMAT_XRGB8888;
3321         case DISPPLANE_RGBX888:
3322                 return DRM_FORMAT_XBGR8888;
3323         case DISPPLANE_BGRA888:
3324                 return DRM_FORMAT_ARGB8888;
3325         case DISPPLANE_RGBA888:
3326                 return DRM_FORMAT_ABGR8888;
3327         case DISPPLANE_BGRX101010:
3328                 return DRM_FORMAT_XRGB2101010;
3329         case DISPPLANE_RGBX101010:
3330                 return DRM_FORMAT_XBGR2101010;
3331         case DISPPLANE_BGRA101010:
3332                 return DRM_FORMAT_ARGB2101010;
3333         case DISPPLANE_RGBA101010:
3334                 return DRM_FORMAT_ABGR2101010;
3335         case DISPPLANE_RGBX161616:
3336                 return DRM_FORMAT_XBGR16161616F;
3337         }
3338 }
3339
3340 int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
3341 {
3342         switch (format) {
3343         case PLANE_CTL_FORMAT_RGB_565:
3344                 return DRM_FORMAT_RGB565;
3345         case PLANE_CTL_FORMAT_NV12:
3346                 return DRM_FORMAT_NV12;
3347         case PLANE_CTL_FORMAT_XYUV:
3348                 return DRM_FORMAT_XYUV8888;
3349         case PLANE_CTL_FORMAT_P010:
3350                 return DRM_FORMAT_P010;
3351         case PLANE_CTL_FORMAT_P012:
3352                 return DRM_FORMAT_P012;
3353         case PLANE_CTL_FORMAT_P016:
3354                 return DRM_FORMAT_P016;
3355         case PLANE_CTL_FORMAT_Y210:
3356                 return DRM_FORMAT_Y210;
3357         case PLANE_CTL_FORMAT_Y212:
3358                 return DRM_FORMAT_Y212;
3359         case PLANE_CTL_FORMAT_Y216:
3360                 return DRM_FORMAT_Y216;
3361         case PLANE_CTL_FORMAT_Y410:
3362                 return DRM_FORMAT_XVYU2101010;
3363         case PLANE_CTL_FORMAT_Y412:
3364                 return DRM_FORMAT_XVYU12_16161616;
3365         case PLANE_CTL_FORMAT_Y416:
3366                 return DRM_FORMAT_XVYU16161616;
3367         default:
3368         case PLANE_CTL_FORMAT_XRGB_8888:
3369                 if (rgb_order) {
3370                         if (alpha)
3371                                 return DRM_FORMAT_ABGR8888;
3372                         else
3373                                 return DRM_FORMAT_XBGR8888;
3374                 } else {
3375                         if (alpha)
3376                                 return DRM_FORMAT_ARGB8888;
3377                         else
3378                                 return DRM_FORMAT_XRGB8888;
3379                 }
3380         case PLANE_CTL_FORMAT_XRGB_2101010:
3381                 if (rgb_order) {
3382                         if (alpha)
3383                                 return DRM_FORMAT_ABGR2101010;
3384                         else
3385                                 return DRM_FORMAT_XBGR2101010;
3386                 } else {
3387                         if (alpha)
3388                                 return DRM_FORMAT_ARGB2101010;
3389                         else
3390                                 return DRM_FORMAT_XRGB2101010;
3391                 }
3392         case PLANE_CTL_FORMAT_XRGB_16161616F:
3393                 if (rgb_order) {
3394                         if (alpha)
3395                                 return DRM_FORMAT_ABGR16161616F;
3396                         else
3397                                 return DRM_FORMAT_XBGR16161616F;
3398                 } else {
3399                         if (alpha)
3400                                 return DRM_FORMAT_ARGB16161616F;
3401                         else
3402                                 return DRM_FORMAT_XRGB16161616F;
3403                 }
3404         }
3405 }
3406
3407 static struct i915_vma *
3408 initial_plane_vma(struct drm_i915_private *i915,
3409                   struct intel_initial_plane_config *plane_config)
3410 {
3411         struct drm_i915_gem_object *obj;
3412         struct i915_vma *vma;
3413         u32 base, size;
3414
3415         if (plane_config->size == 0)
3416                 return NULL;
3417
3418         base = round_down(plane_config->base,
3419                           I915_GTT_MIN_ALIGNMENT);
3420         size = round_up(plane_config->base + plane_config->size,
3421                         I915_GTT_MIN_ALIGNMENT);
3422         size -= base;
3423
3424         /*
3425          * If the FB is too big, just don't use it since fbdev is not very
3426          * important and we should probably use that space with FBC or other
3427          * features.
3428          */
3429         if (size * 2 > i915->stolen_usable_size)
3430                 return NULL;
3431
3432         obj = i915_gem_object_create_stolen_for_preallocated(i915, base, size);
3433         if (IS_ERR(obj))
3434                 return NULL;
3435
3436         switch (plane_config->tiling) {
3437         case I915_TILING_NONE:
3438                 break;
3439         case I915_TILING_X:
3440         case I915_TILING_Y:
3441                 obj->tiling_and_stride =
3442                         plane_config->fb->base.pitches[0] |
3443                         plane_config->tiling;
3444                 break;
3445         default:
3446                 MISSING_CASE(plane_config->tiling);
3447                 goto err_obj;
3448         }
3449
3450         vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
3451         if (IS_ERR(vma))
3452                 goto err_obj;
3453
3454         if (i915_ggtt_pin(vma, NULL, 0, PIN_MAPPABLE | PIN_OFFSET_FIXED | base))
3455                 goto err_obj;
3456
3457         if (i915_gem_object_is_tiled(obj) &&
3458             !i915_vma_is_map_and_fenceable(vma))
3459                 goto err_obj;
3460
3461         return vma;
3462
3463 err_obj:
3464         i915_gem_object_put(obj);
3465         return NULL;
3466 }
3467
3468 static bool
3469 intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
3470                               struct intel_initial_plane_config *plane_config)
3471 {
3472         struct drm_device *dev = crtc->base.dev;
3473         struct drm_i915_private *dev_priv = to_i915(dev);
3474         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
3475         struct drm_framebuffer *fb = &plane_config->fb->base;
3476         struct i915_vma *vma;
3477
3478         switch (fb->modifier) {
3479         case DRM_FORMAT_MOD_LINEAR:
3480         case I915_FORMAT_MOD_X_TILED:
3481         case I915_FORMAT_MOD_Y_TILED:
3482                 break;
3483         default:
3484                 drm_dbg(&dev_priv->drm,
3485                         "Unsupported modifier for initial FB: 0x%llx\n",
3486                         fb->modifier);
3487                 return false;
3488         }
3489
3490         vma = initial_plane_vma(dev_priv, plane_config);
3491         if (!vma)
3492                 return false;
3493
3494         mode_cmd.pixel_format = fb->format->format;
3495         mode_cmd.width = fb->width;
3496         mode_cmd.height = fb->height;
3497         mode_cmd.pitches[0] = fb->pitches[0];
3498         mode_cmd.modifier[0] = fb->modifier;
3499         mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
3500
3501         if (intel_framebuffer_init(to_intel_framebuffer(fb),
3502                                    vma->obj, &mode_cmd)) {
3503                 drm_dbg_kms(&dev_priv->drm, "intel fb init failed\n");
3504                 goto err_vma;
3505         }
3506
3507         plane_config->vma = vma;
3508         return true;
3509
3510 err_vma:
3511         i915_vma_put(vma);
3512         return false;
3513 }
3514
3515 static void
3516 intel_set_plane_visible(struct intel_crtc_state *crtc_state,
3517                         struct intel_plane_state *plane_state,
3518                         bool visible)
3519 {
3520         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
3521
3522         plane_state->uapi.visible = visible;
3523
3524         if (visible)
3525                 crtc_state->uapi.plane_mask |= drm_plane_mask(&plane->base);
3526         else
3527                 crtc_state->uapi.plane_mask &= ~drm_plane_mask(&plane->base);
3528 }
3529
3530 static void fixup_active_planes(struct intel_crtc_state *crtc_state)
3531 {
3532         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3533         struct drm_plane *plane;
3534
3535         /*
3536          * Active_planes aliases if multiple "primary" or cursor planes
3537          * have been used on the same (or wrong) pipe. plane_mask uses
3538          * unique ids, hence we can use that to reconstruct active_planes.
3539          */
3540         crtc_state->active_planes = 0;
3541
3542         drm_for_each_plane_mask(plane, &dev_priv->drm,
3543                                 crtc_state->uapi.plane_mask)
3544                 crtc_state->active_planes |= BIT(to_intel_plane(plane)->id);
3545 }
3546
3547 static void intel_plane_disable_noatomic(struct intel_crtc *crtc,
3548                                          struct intel_plane *plane)
3549 {
3550         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3551         struct intel_crtc_state *crtc_state =
3552                 to_intel_crtc_state(crtc->base.state);
3553         struct intel_plane_state *plane_state =
3554                 to_intel_plane_state(plane->base.state);
3555
3556         drm_dbg_kms(&dev_priv->drm,
3557                     "Disabling [PLANE:%d:%s] on [CRTC:%d:%s]\n",
3558                     plane->base.base.id, plane->base.name,
3559                     crtc->base.base.id, crtc->base.name);
3560
3561         intel_set_plane_visible(crtc_state, plane_state, false);
3562         fixup_active_planes(crtc_state);
3563         crtc_state->data_rate[plane->id] = 0;
3564         crtc_state->min_cdclk[plane->id] = 0;
3565
3566         if (plane->id == PLANE_PRIMARY)
3567                 hsw_disable_ips(crtc_state);
3568
3569         /*
3570          * Vblank time updates from the shadow to live plane control register
3571          * are blocked if the memory self-refresh mode is active at that
3572          * moment. So to make sure the plane gets truly disabled, disable
3573          * first the self-refresh mode. The self-refresh enable bit in turn
3574          * will be checked/applied by the HW only at the next frame start
3575          * event which is after the vblank start event, so we need to have a
3576          * wait-for-vblank between disabling the plane and the pipe.
3577          */
3578         if (HAS_GMCH(dev_priv) &&
3579             intel_set_memory_cxsr(dev_priv, false))
3580                 intel_wait_for_vblank(dev_priv, crtc->pipe);
3581
3582         /*
3583          * Gen2 reports pipe underruns whenever all planes are disabled.
3584          * So disable underrun reporting before all the planes get disabled.
3585          */
3586         if (IS_GEN(dev_priv, 2) && !crtc_state->active_planes)
3587                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
3588
3589         intel_disable_plane(plane, crtc_state);
3590 }
3591
3592 static struct intel_frontbuffer *
3593 to_intel_frontbuffer(struct drm_framebuffer *fb)
3594 {
3595         return fb ? to_intel_framebuffer(fb)->frontbuffer : NULL;
3596 }
3597
3598 static void
3599 intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
3600                              struct intel_initial_plane_config *plane_config)
3601 {
3602         struct drm_device *dev = intel_crtc->base.dev;
3603         struct drm_i915_private *dev_priv = to_i915(dev);
3604         struct drm_crtc *c;
3605         struct drm_plane *primary = intel_crtc->base.primary;
3606         struct drm_plane_state *plane_state = primary->state;
3607         struct intel_plane *intel_plane = to_intel_plane(primary);
3608         struct intel_plane_state *intel_state =
3609                 to_intel_plane_state(plane_state);
3610         struct drm_framebuffer *fb;
3611         struct i915_vma *vma;
3612
3613         if (!plane_config->fb)
3614                 return;
3615
3616         if (intel_alloc_initial_plane_obj(intel_crtc, plane_config)) {
3617                 fb = &plane_config->fb->base;
3618                 vma = plane_config->vma;
3619                 goto valid_fb;
3620         }
3621
3622         /*
3623          * Failed to alloc the obj, check to see if we should share
3624          * an fb with another CRTC instead
3625          */
3626         for_each_crtc(dev, c) {
3627                 struct intel_plane_state *state;
3628
3629                 if (c == &intel_crtc->base)
3630                         continue;
3631
3632                 if (!to_intel_crtc(c)->active)
3633                         continue;
3634
3635                 state = to_intel_plane_state(c->primary->state);
3636                 if (!state->vma)
3637                         continue;
3638
3639                 if (intel_plane_ggtt_offset(state) == plane_config->base) {
3640                         fb = state->hw.fb;
3641                         vma = state->vma;
3642                         goto valid_fb;
3643                 }
3644         }
3645
3646         /*
3647          * We've failed to reconstruct the BIOS FB.  Current display state
3648          * indicates that the primary plane is visible, but has a NULL FB,
3649          * which will lead to problems later if we don't fix it up.  The
3650          * simplest solution is to just disable the primary plane now and
3651          * pretend the BIOS never had it enabled.
3652          */
3653         intel_plane_disable_noatomic(intel_crtc, intel_plane);
3654
3655         return;
3656
3657 valid_fb:
3658         intel_state->hw.rotation = plane_config->rotation;
3659         intel_fill_fb_ggtt_view(&intel_state->view, fb,
3660                                 intel_state->hw.rotation);
3661         intel_state->color_plane[0].stride =
3662                 intel_fb_pitch(fb, 0, intel_state->hw.rotation);
3663
3664         __i915_vma_pin(vma);
3665         intel_state->vma = i915_vma_get(vma);
3666         if (intel_plane_uses_fence(intel_state) && i915_vma_pin_fence(vma) == 0)
3667                 if (vma->fence)
3668                         intel_state->flags |= PLANE_HAS_FENCE;
3669
3670         plane_state->src_x = 0;
3671         plane_state->src_y = 0;
3672         plane_state->src_w = fb->width << 16;
3673         plane_state->src_h = fb->height << 16;
3674
3675         plane_state->crtc_x = 0;
3676         plane_state->crtc_y = 0;
3677         plane_state->crtc_w = fb->width;
3678         plane_state->crtc_h = fb->height;
3679
3680         intel_state->uapi.src = drm_plane_state_src(plane_state);
3681         intel_state->uapi.dst = drm_plane_state_dest(plane_state);
3682
3683         if (plane_config->tiling)
3684                 dev_priv->preserve_bios_swizzle = true;
3685
3686         plane_state->fb = fb;
3687         drm_framebuffer_get(fb);
3688
3689         plane_state->crtc = &intel_crtc->base;
3690         intel_plane_copy_uapi_to_hw_state(intel_state, intel_state);
3691
3692         intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB);
3693
3694         atomic_or(to_intel_plane(primary)->frontbuffer_bit,
3695                   &to_intel_frontbuffer(fb)->bits);
3696 }
3697
3698 static int skl_max_plane_width(const struct drm_framebuffer *fb,
3699                                int color_plane,
3700                                unsigned int rotation)
3701 {
3702         int cpp = fb->format->cpp[color_plane];
3703
3704         switch (fb->modifier) {
3705         case DRM_FORMAT_MOD_LINEAR:
3706         case I915_FORMAT_MOD_X_TILED:
3707                 /*
3708                  * Validated limit is 4k, but has 5k should
3709                  * work apart from the following features:
3710                  * - Ytile (already limited to 4k)
3711                  * - FP16 (already limited to 4k)
3712                  * - render compression (already limited to 4k)
3713                  * - KVMR sprite and cursor (don't care)
3714                  * - horizontal panning (TODO verify this)
3715                  * - pipe and plane scaling (TODO verify this)
3716                  */
3717                 if (cpp == 8)
3718                         return 4096;
3719                 else
3720                         return 5120;
3721         case I915_FORMAT_MOD_Y_TILED_CCS:
3722         case I915_FORMAT_MOD_Yf_TILED_CCS:
3723         case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS:
3724                 /* FIXME AUX plane? */
3725         case I915_FORMAT_MOD_Y_TILED:
3726         case I915_FORMAT_MOD_Yf_TILED:
3727                 if (cpp == 8)
3728                         return 2048;
3729                 else
3730                         return 4096;
3731         default:
3732                 MISSING_CASE(fb->modifier);
3733                 return 2048;
3734         }
3735 }
3736
3737 static int glk_max_plane_width(const struct drm_framebuffer *fb,
3738                                int color_plane,
3739                                unsigned int rotation)
3740 {
3741         int cpp = fb->format->cpp[color_plane];
3742
3743         switch (fb->modifier) {
3744         case DRM_FORMAT_MOD_LINEAR:
3745         case I915_FORMAT_MOD_X_TILED:
3746                 if (cpp == 8)
3747                         return 4096;
3748                 else
3749                         return 5120;
3750         case I915_FORMAT_MOD_Y_TILED_CCS:
3751         case I915_FORMAT_MOD_Yf_TILED_CCS:
3752                 /* FIXME AUX plane? */
3753         case I915_FORMAT_MOD_Y_TILED:
3754         case I915_FORMAT_MOD_Yf_TILED:
3755                 if (cpp == 8)
3756                         return 2048;
3757                 else
3758                         return 5120;
3759         default:
3760                 MISSING_CASE(fb->modifier);
3761                 return 2048;
3762         }
3763 }
3764
3765 static int icl_min_plane_width(const struct drm_framebuffer *fb)
3766 {
3767         /* Wa_14011264657, Wa_14011050563: gen11+ */
3768         switch (fb->format->format) {
3769         case DRM_FORMAT_C8:
3770                 return 18;
3771         case DRM_FORMAT_RGB565:
3772                 return 10;
3773         case DRM_FORMAT_XRGB8888:
3774         case DRM_FORMAT_XBGR8888:
3775         case DRM_FORMAT_ARGB8888:
3776         case DRM_FORMAT_ABGR8888:
3777         case DRM_FORMAT_XRGB2101010:
3778         case DRM_FORMAT_XBGR2101010:
3779         case DRM_FORMAT_ARGB2101010:
3780         case DRM_FORMAT_ABGR2101010:
3781         case DRM_FORMAT_XVYU2101010:
3782         case DRM_FORMAT_Y212:
3783         case DRM_FORMAT_Y216:
3784                 return 6;
3785         case DRM_FORMAT_NV12:
3786                 return 20;
3787         case DRM_FORMAT_P010:
3788         case DRM_FORMAT_P012:
3789         case DRM_FORMAT_P016:
3790                 return 12;
3791         case DRM_FORMAT_XRGB16161616F:
3792         case DRM_FORMAT_XBGR16161616F:
3793         case DRM_FORMAT_ARGB16161616F:
3794         case DRM_FORMAT_ABGR16161616F:
3795         case DRM_FORMAT_XVYU12_16161616:
3796         case DRM_FORMAT_XVYU16161616:
3797                 return 4;
3798         default:
3799                 return 1;
3800         }
3801 }
3802
3803 static int icl_max_plane_width(const struct drm_framebuffer *fb,
3804                                int color_plane,
3805                                unsigned int rotation)
3806 {
3807         return 5120;
3808 }
3809
3810 static int skl_max_plane_height(void)
3811 {
3812         return 4096;
3813 }
3814
3815 static int icl_max_plane_height(void)
3816 {
3817         return 4320;
3818 }
3819
3820 static bool
3821 skl_check_main_ccs_coordinates(struct intel_plane_state *plane_state,
3822                                int main_x, int main_y, u32 main_offset,
3823                                int ccs_plane)
3824 {
3825         const struct drm_framebuffer *fb = plane_state->hw.fb;
3826         int aux_x = plane_state->color_plane[ccs_plane].x;
3827         int aux_y = plane_state->color_plane[ccs_plane].y;
3828         u32 aux_offset = plane_state->color_plane[ccs_plane].offset;
3829         u32 alignment = intel_surf_alignment(fb, ccs_plane);
3830         int hsub;
3831         int vsub;
3832
3833         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, ccs_plane);
3834         while (aux_offset >= main_offset && aux_y <= main_y) {
3835                 int x, y;
3836
3837                 if (aux_x == main_x && aux_y == main_y)
3838                         break;
3839
3840                 if (aux_offset == 0)
3841                         break;
3842
3843                 x = aux_x / hsub;
3844                 y = aux_y / vsub;
3845                 aux_offset = intel_plane_adjust_aligned_offset(&x, &y,
3846                                                                plane_state,
3847                                                                ccs_plane,
3848                                                                aux_offset,
3849                                                                aux_offset -
3850                                                                 alignment);
3851                 aux_x = x * hsub + aux_x % hsub;
3852                 aux_y = y * vsub + aux_y % vsub;
3853         }
3854
3855         if (aux_x != main_x || aux_y != main_y)
3856                 return false;
3857
3858         plane_state->color_plane[ccs_plane].offset = aux_offset;
3859         plane_state->color_plane[ccs_plane].x = aux_x;
3860         plane_state->color_plane[ccs_plane].y = aux_y;
3861
3862         return true;
3863 }
3864
3865 unsigned int
3866 intel_plane_fence_y_offset(const struct intel_plane_state *plane_state)
3867 {
3868         int x = 0, y = 0;
3869
3870         intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3871                                           plane_state->color_plane[0].offset, 0);
3872
3873         return y;
3874 }
3875
3876 static int skl_check_main_surface(struct intel_plane_state *plane_state)
3877 {
3878         struct drm_i915_private *dev_priv = to_i915(plane_state->uapi.plane->dev);
3879         const struct drm_framebuffer *fb = plane_state->hw.fb;
3880         unsigned int rotation = plane_state->hw.rotation;
3881         int x = plane_state->uapi.src.x1 >> 16;
3882         int y = plane_state->uapi.src.y1 >> 16;
3883         int w = drm_rect_width(&plane_state->uapi.src) >> 16;
3884         int h = drm_rect_height(&plane_state->uapi.src) >> 16;
3885         int max_width, min_width, max_height;
3886         u32 alignment, offset;
3887         int aux_plane = intel_main_to_aux_plane(fb, 0);
3888         u32 aux_offset = plane_state->color_plane[aux_plane].offset;
3889
3890         if (INTEL_GEN(dev_priv) >= 11) {
3891                 max_width = icl_max_plane_width(fb, 0, rotation);
3892                 min_width = icl_min_plane_width(fb);
3893         } else if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
3894                 max_width = glk_max_plane_width(fb, 0, rotation);
3895                 min_width = 1;
3896         } else {
3897                 max_width = skl_max_plane_width(fb, 0, rotation);
3898                 min_width = 1;
3899         }
3900
3901         if (INTEL_GEN(dev_priv) >= 11)
3902                 max_height = icl_max_plane_height();
3903         else
3904                 max_height = skl_max_plane_height();
3905
3906         if (w > max_width || w < min_width || h > max_height) {
3907                 drm_dbg_kms(&dev_priv->drm,
3908                             "requested Y/RGB source size %dx%d outside limits (min: %dx1 max: %dx%d)\n",
3909                             w, h, min_width, max_width, max_height);
3910                 return -EINVAL;
3911         }
3912
3913         intel_add_fb_offsets(&x, &y, plane_state, 0);
3914         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 0);
3915         alignment = intel_surf_alignment(fb, 0);
3916         if (drm_WARN_ON(&dev_priv->drm, alignment && !is_power_of_2(alignment)))
3917                 return -EINVAL;
3918
3919         /*
3920          * AUX surface offset is specified as the distance from the
3921          * main surface offset, and it must be non-negative. Make
3922          * sure that is what we will get.
3923          */
3924         if (offset > aux_offset)
3925                 offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3926                                                            offset, aux_offset & ~(alignment - 1));
3927
3928         /*
3929          * When using an X-tiled surface, the plane blows up
3930          * if the x offset + width exceed the stride.
3931          *
3932          * TODO: linear and Y-tiled seem fine, Yf untested,
3933          */
3934         if (fb->modifier == I915_FORMAT_MOD_X_TILED) {
3935                 int cpp = fb->format->cpp[0];
3936
3937                 while ((x + w) * cpp > plane_state->color_plane[0].stride) {
3938                         if (offset == 0) {
3939                                 drm_dbg_kms(&dev_priv->drm,
3940                                             "Unable to find suitable display surface offset due to X-tiling\n");
3941                                 return -EINVAL;
3942                         }
3943
3944                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3945                                                                    offset, offset - alignment);
3946                 }
3947         }
3948
3949         /*
3950          * CCS AUX surface doesn't have its own x/y offsets, we must make sure
3951          * they match with the main surface x/y offsets.
3952          */
3953         if (is_ccs_modifier(fb->modifier)) {
3954                 while (!skl_check_main_ccs_coordinates(plane_state, x, y,
3955                                                        offset, aux_plane)) {
3956                         if (offset == 0)
3957                                 break;
3958
3959                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3960                                                                    offset, offset - alignment);
3961                 }
3962
3963                 if (x != plane_state->color_plane[aux_plane].x ||
3964                     y != plane_state->color_plane[aux_plane].y) {
3965                         drm_dbg_kms(&dev_priv->drm,
3966                                     "Unable to find suitable display surface offset due to CCS\n");
3967                         return -EINVAL;
3968                 }
3969         }
3970
3971         plane_state->color_plane[0].offset = offset;
3972         plane_state->color_plane[0].x = x;
3973         plane_state->color_plane[0].y = y;
3974
3975         /*
3976          * Put the final coordinates back so that the src
3977          * coordinate checks will see the right values.
3978          */
3979         drm_rect_translate_to(&plane_state->uapi.src,
3980                               x << 16, y << 16);
3981
3982         return 0;
3983 }
3984
3985 static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
3986 {
3987         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
3988         const struct drm_framebuffer *fb = plane_state->hw.fb;
3989         unsigned int rotation = plane_state->hw.rotation;
3990         int uv_plane = 1;
3991         int max_width = skl_max_plane_width(fb, uv_plane, rotation);
3992         int max_height = 4096;
3993         int x = plane_state->uapi.src.x1 >> 17;
3994         int y = plane_state->uapi.src.y1 >> 17;
3995         int w = drm_rect_width(&plane_state->uapi.src) >> 17;
3996         int h = drm_rect_height(&plane_state->uapi.src) >> 17;
3997         u32 offset;
3998
3999         intel_add_fb_offsets(&x, &y, plane_state, uv_plane);
4000         offset = intel_plane_compute_aligned_offset(&x, &y,
4001                                                     plane_state, uv_plane);
4002
4003         /* FIXME not quite sure how/if these apply to the chroma plane */
4004         if (w > max_width || h > max_height) {
4005                 drm_dbg_kms(&i915->drm,
4006                             "CbCr source size %dx%d too big (limit %dx%d)\n",
4007                             w, h, max_width, max_height);
4008                 return -EINVAL;
4009         }
4010
4011         if (is_ccs_modifier(fb->modifier)) {
4012                 int ccs_plane = main_to_ccs_plane(fb, uv_plane);
4013                 int aux_offset = plane_state->color_plane[ccs_plane].offset;
4014                 int alignment = intel_surf_alignment(fb, uv_plane);
4015
4016                 if (offset > aux_offset)
4017                         offset = intel_plane_adjust_aligned_offset(&x, &y,
4018                                                                    plane_state,
4019                                                                    uv_plane,
4020                                                                    offset,
4021                                                                    aux_offset & ~(alignment - 1));
4022
4023                 while (!skl_check_main_ccs_coordinates(plane_state, x, y,
4024                                                        offset, ccs_plane)) {
4025                         if (offset == 0)
4026                                 break;
4027
4028                         offset = intel_plane_adjust_aligned_offset(&x, &y,
4029                                                                    plane_state,
4030                                                                    uv_plane,
4031                                                                    offset, offset - alignment);
4032                 }
4033
4034                 if (x != plane_state->color_plane[ccs_plane].x ||
4035                     y != plane_state->color_plane[ccs_plane].y) {
4036                         drm_dbg_kms(&i915->drm,
4037                                     "Unable to find suitable display surface offset due to CCS\n");
4038                         return -EINVAL;
4039                 }
4040         }
4041
4042         plane_state->color_plane[uv_plane].offset = offset;
4043         plane_state->color_plane[uv_plane].x = x;
4044         plane_state->color_plane[uv_plane].y = y;
4045
4046         return 0;
4047 }
4048
4049 static int skl_check_ccs_aux_surface(struct intel_plane_state *plane_state)
4050 {
4051         const struct drm_framebuffer *fb = plane_state->hw.fb;
4052         int src_x = plane_state->uapi.src.x1 >> 16;
4053         int src_y = plane_state->uapi.src.y1 >> 16;
4054         u32 offset;
4055         int ccs_plane;
4056
4057         for (ccs_plane = 0; ccs_plane < fb->format->num_planes; ccs_plane++) {
4058                 int main_hsub, main_vsub;
4059                 int hsub, vsub;
4060                 int x, y;
4061
4062                 if (!is_ccs_plane(fb, ccs_plane))
4063                         continue;
4064
4065                 intel_fb_plane_get_subsampling(&main_hsub, &main_vsub, fb,
4066                                                ccs_to_main_plane(fb, ccs_plane));
4067                 intel_fb_plane_get_subsampling(&hsub, &vsub, fb, ccs_plane);
4068
4069                 hsub *= main_hsub;
4070                 vsub *= main_vsub;
4071                 x = src_x / hsub;
4072                 y = src_y / vsub;
4073
4074                 intel_add_fb_offsets(&x, &y, plane_state, ccs_plane);
4075
4076                 offset = intel_plane_compute_aligned_offset(&x, &y,
4077                                                             plane_state,
4078                                                             ccs_plane);
4079
4080                 plane_state->color_plane[ccs_plane].offset = offset;
4081                 plane_state->color_plane[ccs_plane].x = (x * hsub +
4082                                                          src_x % hsub) /
4083                                                         main_hsub;
4084                 plane_state->color_plane[ccs_plane].y = (y * vsub +
4085                                                          src_y % vsub) /
4086                                                         main_vsub;
4087         }
4088
4089         return 0;
4090 }
4091
4092 int skl_check_plane_surface(struct intel_plane_state *plane_state)
4093 {
4094         const struct drm_framebuffer *fb = plane_state->hw.fb;
4095         int ret;
4096         bool needs_aux = false;
4097
4098         ret = intel_plane_compute_gtt(plane_state);
4099         if (ret)
4100                 return ret;
4101
4102         if (!plane_state->uapi.visible)
4103                 return 0;
4104
4105         /*
4106          * Handle the AUX surface first since the main surface setup depends on
4107          * it.
4108          */
4109         if (is_ccs_modifier(fb->modifier)) {
4110                 needs_aux = true;
4111                 ret = skl_check_ccs_aux_surface(plane_state);
4112                 if (ret)
4113                         return ret;
4114         }
4115
4116         if (intel_format_info_is_yuv_semiplanar(fb->format,
4117                                                 fb->modifier)) {
4118                 needs_aux = true;
4119                 ret = skl_check_nv12_aux_surface(plane_state);
4120                 if (ret)
4121                         return ret;
4122         }
4123
4124         if (!needs_aux) {
4125                 int i;
4126
4127                 for (i = 1; i < fb->format->num_planes; i++) {
4128                         plane_state->color_plane[i].offset = ~0xfff;
4129                         plane_state->color_plane[i].x = 0;
4130                         plane_state->color_plane[i].y = 0;
4131                 }
4132         }
4133
4134         ret = skl_check_main_surface(plane_state);
4135         if (ret)
4136                 return ret;
4137
4138         return 0;
4139 }
4140
4141 static void i9xx_plane_ratio(const struct intel_crtc_state *crtc_state,
4142                              const struct intel_plane_state *plane_state,
4143                              unsigned int *num, unsigned int *den)
4144 {
4145         const struct drm_framebuffer *fb = plane_state->hw.fb;
4146         unsigned int cpp = fb->format->cpp[0];
4147
4148         /*
4149          * g4x bspec says 64bpp pixel rate can't exceed 80%
4150          * of cdclk when the sprite plane is enabled on the
4151          * same pipe. ilk/snb bspec says 64bpp pixel rate is
4152          * never allowed to exceed 80% of cdclk. Let's just go
4153          * with the ilk/snb limit always.
4154          */
4155         if (cpp == 8) {
4156                 *num = 10;
4157                 *den = 8;
4158         } else {
4159                 *num = 1;
4160                 *den = 1;
4161         }
4162 }
4163
4164 static int i9xx_plane_min_cdclk(const struct intel_crtc_state *crtc_state,
4165                                 const struct intel_plane_state *plane_state)
4166 {
4167         unsigned int pixel_rate;
4168         unsigned int num, den;
4169
4170         /*
4171          * Note that crtc_state->pixel_rate accounts for both
4172          * horizontal and vertical panel fitter downscaling factors.
4173          * Pre-HSW bspec tells us to only consider the horizontal
4174          * downscaling factor here. We ignore that and just consider
4175          * both for simplicity.
4176          */
4177         pixel_rate = crtc_state->pixel_rate;
4178
4179         i9xx_plane_ratio(crtc_state, plane_state, &num, &den);
4180
4181         /* two pixels per clock with double wide pipe */
4182         if (crtc_state->double_wide)
4183                 den *= 2;
4184
4185         return DIV_ROUND_UP(pixel_rate * num, den);
4186 }
4187
4188 unsigned int
4189 i9xx_plane_max_stride(struct intel_plane *plane,
4190                       u32 pixel_format, u64 modifier,
4191                       unsigned int rotation)
4192 {
4193         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4194
4195         if (!HAS_GMCH(dev_priv)) {
4196                 return 32*1024;
4197         } else if (INTEL_GEN(dev_priv) >= 4) {
4198                 if (modifier == I915_FORMAT_MOD_X_TILED)
4199                         return 16*1024;
4200                 else
4201                         return 32*1024;
4202         } else if (INTEL_GEN(dev_priv) >= 3) {
4203                 if (modifier == I915_FORMAT_MOD_X_TILED)
4204                         return 8*1024;
4205                 else
4206                         return 16*1024;
4207         } else {
4208                 if (plane->i9xx_plane == PLANE_C)
4209                         return 4*1024;
4210                 else
4211                         return 8*1024;
4212         }
4213 }
4214
4215 static u32 i9xx_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
4216 {
4217         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4218         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4219         u32 dspcntr = 0;
4220
4221         if (crtc_state->gamma_enable)
4222                 dspcntr |= DISPPLANE_GAMMA_ENABLE;
4223
4224         if (crtc_state->csc_enable)
4225                 dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
4226
4227         if (INTEL_GEN(dev_priv) < 5)
4228                 dspcntr |= DISPPLANE_SEL_PIPE(crtc->pipe);
4229
4230         return dspcntr;
4231 }
4232
4233 static u32 i9xx_plane_ctl(const struct intel_crtc_state *crtc_state,
4234                           const struct intel_plane_state *plane_state)
4235 {
4236         struct drm_i915_private *dev_priv =
4237                 to_i915(plane_state->uapi.plane->dev);
4238         const struct drm_framebuffer *fb = plane_state->hw.fb;
4239         unsigned int rotation = plane_state->hw.rotation;
4240         u32 dspcntr;
4241
4242         dspcntr = DISPLAY_PLANE_ENABLE;
4243
4244         if (IS_G4X(dev_priv) || IS_GEN(dev_priv, 5) ||
4245             IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
4246                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
4247
4248         switch (fb->format->format) {
4249         case DRM_FORMAT_C8:
4250                 dspcntr |= DISPPLANE_8BPP;
4251                 break;
4252         case DRM_FORMAT_XRGB1555:
4253                 dspcntr |= DISPPLANE_BGRX555;
4254                 break;
4255         case DRM_FORMAT_ARGB1555:
4256                 dspcntr |= DISPPLANE_BGRA555;
4257                 break;
4258         case DRM_FORMAT_RGB565:
4259                 dspcntr |= DISPPLANE_BGRX565;
4260                 break;
4261         case DRM_FORMAT_XRGB8888:
4262                 dspcntr |= DISPPLANE_BGRX888;
4263                 break;
4264         case DRM_FORMAT_XBGR8888:
4265                 dspcntr |= DISPPLANE_RGBX888;
4266                 break;
4267         case DRM_FORMAT_ARGB8888:
4268                 dspcntr |= DISPPLANE_BGRA888;
4269                 break;
4270         case DRM_FORMAT_ABGR8888:
4271                 dspcntr |= DISPPLANE_RGBA888;
4272                 break;
4273         case DRM_FORMAT_XRGB2101010:
4274                 dspcntr |= DISPPLANE_BGRX101010;
4275                 break;
4276         case DRM_FORMAT_XBGR2101010:
4277                 dspcntr |= DISPPLANE_RGBX101010;
4278                 break;
4279         case DRM_FORMAT_ARGB2101010:
4280                 dspcntr |= DISPPLANE_BGRA101010;
4281                 break;
4282         case DRM_FORMAT_ABGR2101010:
4283                 dspcntr |= DISPPLANE_RGBA101010;
4284                 break;
4285         case DRM_FORMAT_XBGR16161616F:
4286                 dspcntr |= DISPPLANE_RGBX161616;
4287                 break;
4288         default:
4289                 MISSING_CASE(fb->format->format);
4290                 return 0;
4291         }
4292
4293         if (INTEL_GEN(dev_priv) >= 4 &&
4294             fb->modifier == I915_FORMAT_MOD_X_TILED)
4295                 dspcntr |= DISPPLANE_TILED;
4296
4297         if (rotation & DRM_MODE_ROTATE_180)
4298                 dspcntr |= DISPPLANE_ROTATE_180;
4299
4300         if (rotation & DRM_MODE_REFLECT_X)
4301                 dspcntr |= DISPPLANE_MIRROR;
4302
4303         return dspcntr;
4304 }
4305
4306 int i9xx_check_plane_surface(struct intel_plane_state *plane_state)
4307 {
4308         struct drm_i915_private *dev_priv =
4309                 to_i915(plane_state->uapi.plane->dev);
4310         const struct drm_framebuffer *fb = plane_state->hw.fb;
4311         int src_x, src_y, src_w;
4312         u32 offset;
4313         int ret;
4314
4315         ret = intel_plane_compute_gtt(plane_state);
4316         if (ret)
4317                 return ret;
4318
4319         if (!plane_state->uapi.visible)
4320                 return 0;
4321
4322         src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4323         src_x = plane_state->uapi.src.x1 >> 16;
4324         src_y = plane_state->uapi.src.y1 >> 16;
4325
4326         /* Undocumented hardware limit on i965/g4x/vlv/chv */
4327         if (HAS_GMCH(dev_priv) && fb->format->cpp[0] == 8 && src_w > 2048)
4328                 return -EINVAL;
4329
4330         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
4331
4332         if (INTEL_GEN(dev_priv) >= 4)
4333                 offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
4334                                                             plane_state, 0);
4335         else
4336                 offset = 0;
4337
4338         /*
4339          * Put the final coordinates back so that the src
4340          * coordinate checks will see the right values.
4341          */
4342         drm_rect_translate_to(&plane_state->uapi.src,
4343                               src_x << 16, src_y << 16);
4344
4345         /* HSW/BDW do this automagically in hardware */
4346         if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) {
4347                 unsigned int rotation = plane_state->hw.rotation;
4348                 int src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4349                 int src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
4350
4351                 if (rotation & DRM_MODE_ROTATE_180) {
4352                         src_x += src_w - 1;
4353                         src_y += src_h - 1;
4354                 } else if (rotation & DRM_MODE_REFLECT_X) {
4355                         src_x += src_w - 1;
4356                 }
4357         }
4358
4359         plane_state->color_plane[0].offset = offset;
4360         plane_state->color_plane[0].x = src_x;
4361         plane_state->color_plane[0].y = src_y;
4362
4363         return 0;
4364 }
4365
4366 static bool i9xx_plane_has_windowing(struct intel_plane *plane)
4367 {
4368         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4369         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4370
4371         if (IS_CHERRYVIEW(dev_priv))
4372                 return i9xx_plane == PLANE_B;
4373         else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
4374                 return false;
4375         else if (IS_GEN(dev_priv, 4))
4376                 return i9xx_plane == PLANE_C;
4377         else
4378                 return i9xx_plane == PLANE_B ||
4379                         i9xx_plane == PLANE_C;
4380 }
4381
4382 static int
4383 i9xx_plane_check(struct intel_crtc_state *crtc_state,
4384                  struct intel_plane_state *plane_state)
4385 {
4386         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4387         int ret;
4388
4389         ret = chv_plane_check_rotation(plane_state);
4390         if (ret)
4391                 return ret;
4392
4393         ret = drm_atomic_helper_check_plane_state(&plane_state->uapi,
4394                                                   &crtc_state->uapi,
4395                                                   DRM_PLANE_HELPER_NO_SCALING,
4396                                                   DRM_PLANE_HELPER_NO_SCALING,
4397                                                   i9xx_plane_has_windowing(plane),
4398                                                   true);
4399         if (ret)
4400                 return ret;
4401
4402         ret = i9xx_check_plane_surface(plane_state);
4403         if (ret)
4404                 return ret;
4405
4406         if (!plane_state->uapi.visible)
4407                 return 0;
4408
4409         ret = intel_plane_check_src_coordinates(plane_state);
4410         if (ret)
4411                 return ret;
4412
4413         plane_state->ctl = i9xx_plane_ctl(crtc_state, plane_state);
4414
4415         return 0;
4416 }
4417
4418 static void i9xx_update_plane(struct intel_plane *plane,
4419                               const struct intel_crtc_state *crtc_state,
4420                               const struct intel_plane_state *plane_state)
4421 {
4422         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4423         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4424         u32 linear_offset;
4425         int x = plane_state->color_plane[0].x;
4426         int y = plane_state->color_plane[0].y;
4427         int crtc_x = plane_state->uapi.dst.x1;
4428         int crtc_y = plane_state->uapi.dst.y1;
4429         int crtc_w = drm_rect_width(&plane_state->uapi.dst);
4430         int crtc_h = drm_rect_height(&plane_state->uapi.dst);
4431         unsigned long irqflags;
4432         u32 dspaddr_offset;
4433         u32 dspcntr;
4434
4435         dspcntr = plane_state->ctl | i9xx_plane_ctl_crtc(crtc_state);
4436
4437         linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
4438
4439         if (INTEL_GEN(dev_priv) >= 4)
4440                 dspaddr_offset = plane_state->color_plane[0].offset;
4441         else
4442                 dspaddr_offset = linear_offset;
4443
4444         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
4445
4446         intel_de_write_fw(dev_priv, DSPSTRIDE(i9xx_plane),
4447                           plane_state->color_plane[0].stride);
4448
4449         if (INTEL_GEN(dev_priv) < 4) {
4450                 /*
4451                  * PLANE_A doesn't actually have a full window
4452                  * generator but let's assume we still need to
4453                  * program whatever is there.
4454                  */
4455                 intel_de_write_fw(dev_priv, DSPPOS(i9xx_plane),
4456                                   (crtc_y << 16) | crtc_x);
4457                 intel_de_write_fw(dev_priv, DSPSIZE(i9xx_plane),
4458                                   ((crtc_h - 1) << 16) | (crtc_w - 1));
4459         } else if (IS_CHERRYVIEW(dev_priv) && i9xx_plane == PLANE_B) {
4460                 intel_de_write_fw(dev_priv, PRIMPOS(i9xx_plane),
4461                                   (crtc_y << 16) | crtc_x);
4462                 intel_de_write_fw(dev_priv, PRIMSIZE(i9xx_plane),
4463                                   ((crtc_h - 1) << 16) | (crtc_w - 1));
4464                 intel_de_write_fw(dev_priv, PRIMCNSTALPHA(i9xx_plane), 0);
4465         }
4466
4467         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
4468                 intel_de_write_fw(dev_priv, DSPOFFSET(i9xx_plane),
4469                                   (y << 16) | x);
4470         } else if (INTEL_GEN(dev_priv) >= 4) {
4471                 intel_de_write_fw(dev_priv, DSPLINOFF(i9xx_plane),
4472                                   linear_offset);
4473                 intel_de_write_fw(dev_priv, DSPTILEOFF(i9xx_plane),
4474                                   (y << 16) | x);
4475         }
4476
4477         /*
4478          * The control register self-arms if the plane was previously
4479          * disabled. Try to make the plane enable atomic by writing
4480          * the control register just before the surface register.
4481          */
4482         intel_de_write_fw(dev_priv, DSPCNTR(i9xx_plane), dspcntr);
4483         if (INTEL_GEN(dev_priv) >= 4)
4484                 intel_de_write_fw(dev_priv, DSPSURF(i9xx_plane),
4485                                   intel_plane_ggtt_offset(plane_state) + dspaddr_offset);
4486         else
4487                 intel_de_write_fw(dev_priv, DSPADDR(i9xx_plane),
4488                                   intel_plane_ggtt_offset(plane_state) + dspaddr_offset);
4489
4490         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
4491 }
4492
4493 static void i9xx_disable_plane(struct intel_plane *plane,
4494                                const struct intel_crtc_state *crtc_state)
4495 {
4496         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4497         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4498         unsigned long irqflags;
4499         u32 dspcntr;
4500
4501         /*
4502          * DSPCNTR pipe gamma enable on g4x+ and pipe csc
4503          * enable on ilk+ affect the pipe bottom color as
4504          * well, so we must configure them even if the plane
4505          * is disabled.
4506          *
4507          * On pre-g4x there is no way to gamma correct the
4508          * pipe bottom color but we'll keep on doing this
4509          * anyway so that the crtc state readout works correctly.
4510          */
4511         dspcntr = i9xx_plane_ctl_crtc(crtc_state);
4512
4513         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
4514
4515         intel_de_write_fw(dev_priv, DSPCNTR(i9xx_plane), dspcntr);
4516         if (INTEL_GEN(dev_priv) >= 4)
4517                 intel_de_write_fw(dev_priv, DSPSURF(i9xx_plane), 0);
4518         else
4519                 intel_de_write_fw(dev_priv, DSPADDR(i9xx_plane), 0);
4520
4521         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
4522 }
4523
4524 static bool i9xx_plane_get_hw_state(struct intel_plane *plane,
4525                                     enum pipe *pipe)
4526 {
4527         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4528         enum intel_display_power_domain power_domain;
4529         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4530         intel_wakeref_t wakeref;
4531         bool ret;
4532         u32 val;
4533
4534         /*
4535          * Not 100% correct for planes that can move between pipes,
4536          * but that's only the case for gen2-4 which don't have any
4537          * display power wells.
4538          */
4539         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
4540         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4541         if (!wakeref)
4542                 return false;
4543
4544         val = intel_de_read(dev_priv, DSPCNTR(i9xx_plane));
4545
4546         ret = val & DISPLAY_PLANE_ENABLE;
4547
4548         if (INTEL_GEN(dev_priv) >= 5)
4549                 *pipe = plane->pipe;
4550         else
4551                 *pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
4552                         DISPPLANE_SEL_PIPE_SHIFT;
4553
4554         intel_display_power_put(dev_priv, power_domain, wakeref);
4555
4556         return ret;
4557 }
4558
4559 static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
4560 {
4561         struct drm_device *dev = intel_crtc->base.dev;
4562         struct drm_i915_private *dev_priv = to_i915(dev);
4563         unsigned long irqflags;
4564
4565         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
4566
4567         intel_de_write_fw(dev_priv, SKL_PS_CTRL(intel_crtc->pipe, id), 0);
4568         intel_de_write_fw(dev_priv, SKL_PS_WIN_POS(intel_crtc->pipe, id), 0);
4569         intel_de_write_fw(dev_priv, SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0);
4570
4571         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
4572 }
4573
4574 /*
4575  * This function detaches (aka. unbinds) unused scalers in hardware
4576  */
4577 static void skl_detach_scalers(const struct intel_crtc_state *crtc_state)
4578 {
4579         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->uapi.crtc);
4580         const struct intel_crtc_scaler_state *scaler_state =
4581                 &crtc_state->scaler_state;
4582         int i;
4583
4584         /* loop through and disable scalers that aren't in use */
4585         for (i = 0; i < intel_crtc->num_scalers; i++) {
4586                 if (!scaler_state->scalers[i].in_use)
4587                         skl_detach_scaler(intel_crtc, i);
4588         }
4589 }
4590
4591 static unsigned int skl_plane_stride_mult(const struct drm_framebuffer *fb,
4592                                           int color_plane, unsigned int rotation)
4593 {
4594         /*
4595          * The stride is either expressed as a multiple of 64 bytes chunks for
4596          * linear buffers or in number of tiles for tiled buffers.
4597          */
4598         if (is_surface_linear(fb, color_plane))
4599                 return 64;
4600         else if (drm_rotation_90_or_270(rotation))
4601                 return intel_tile_height(fb, color_plane);
4602         else
4603                 return intel_tile_width_bytes(fb, color_plane);
4604 }
4605
4606 u32 skl_plane_stride(const struct intel_plane_state *plane_state,
4607                      int color_plane)
4608 {
4609         const struct drm_framebuffer *fb = plane_state->hw.fb;
4610         unsigned int rotation = plane_state->hw.rotation;
4611         u32 stride = plane_state->color_plane[color_plane].stride;
4612
4613         if (color_plane >= fb->format->num_planes)
4614                 return 0;
4615
4616         return stride / skl_plane_stride_mult(fb, color_plane, rotation);
4617 }
4618
4619 static u32 skl_plane_ctl_format(u32 pixel_format)
4620 {
4621         switch (pixel_format) {
4622         case DRM_FORMAT_C8:
4623                 return PLANE_CTL_FORMAT_INDEXED;
4624         case DRM_FORMAT_RGB565:
4625                 return PLANE_CTL_FORMAT_RGB_565;
4626         case DRM_FORMAT_XBGR8888:
4627         case DRM_FORMAT_ABGR8888:
4628                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX;
4629         case DRM_FORMAT_XRGB8888:
4630         case DRM_FORMAT_ARGB8888:
4631                 return PLANE_CTL_FORMAT_XRGB_8888;
4632         case DRM_FORMAT_XBGR2101010:
4633         case DRM_FORMAT_ABGR2101010:
4634                 return PLANE_CTL_FORMAT_XRGB_2101010 | PLANE_CTL_ORDER_RGBX;
4635         case DRM_FORMAT_XRGB2101010:
4636         case DRM_FORMAT_ARGB2101010:
4637                 return PLANE_CTL_FORMAT_XRGB_2101010;
4638         case DRM_FORMAT_XBGR16161616F:
4639         case DRM_FORMAT_ABGR16161616F:
4640                 return PLANE_CTL_FORMAT_XRGB_16161616F | PLANE_CTL_ORDER_RGBX;
4641         case DRM_FORMAT_XRGB16161616F:
4642         case DRM_FORMAT_ARGB16161616F:
4643                 return PLANE_CTL_FORMAT_XRGB_16161616F;
4644         case DRM_FORMAT_XYUV8888:
4645                 return PLANE_CTL_FORMAT_XYUV;
4646         case DRM_FORMAT_YUYV:
4647                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV;
4648         case DRM_FORMAT_YVYU:
4649                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YVYU;
4650         case DRM_FORMAT_UYVY:
4651                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_UYVY;
4652         case DRM_FORMAT_VYUY:
4653                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY;
4654         case DRM_FORMAT_NV12:
4655                 return PLANE_CTL_FORMAT_NV12;
4656         case DRM_FORMAT_P010:
4657                 return PLANE_CTL_FORMAT_P010;
4658         case DRM_FORMAT_P012:
4659                 return PLANE_CTL_FORMAT_P012;
4660         case DRM_FORMAT_P016:
4661                 return PLANE_CTL_FORMAT_P016;
4662         case DRM_FORMAT_Y210:
4663                 return PLANE_CTL_FORMAT_Y210;
4664         case DRM_FORMAT_Y212:
4665                 return PLANE_CTL_FORMAT_Y212;
4666         case DRM_FORMAT_Y216:
4667                 return PLANE_CTL_FORMAT_Y216;
4668         case DRM_FORMAT_XVYU2101010:
4669                 return PLANE_CTL_FORMAT_Y410;
4670         case DRM_FORMAT_XVYU12_16161616:
4671                 return PLANE_CTL_FORMAT_Y412;
4672         case DRM_FORMAT_XVYU16161616:
4673                 return PLANE_CTL_FORMAT_Y416;
4674         default:
4675                 MISSING_CASE(pixel_format);
4676         }
4677
4678         return 0;
4679 }
4680
4681 static u32 skl_plane_ctl_alpha(const struct intel_plane_state *plane_state)
4682 {
4683         if (!plane_state->hw.fb->format->has_alpha)
4684                 return PLANE_CTL_ALPHA_DISABLE;
4685
4686         switch (plane_state->hw.pixel_blend_mode) {
4687         case DRM_MODE_BLEND_PIXEL_NONE:
4688                 return PLANE_CTL_ALPHA_DISABLE;
4689         case DRM_MODE_BLEND_PREMULTI:
4690                 return PLANE_CTL_ALPHA_SW_PREMULTIPLY;
4691         case DRM_MODE_BLEND_COVERAGE:
4692                 return PLANE_CTL_ALPHA_HW_PREMULTIPLY;
4693         default:
4694                 MISSING_CASE(plane_state->hw.pixel_blend_mode);
4695                 return PLANE_CTL_ALPHA_DISABLE;
4696         }
4697 }
4698
4699 static u32 glk_plane_color_ctl_alpha(const struct intel_plane_state *plane_state)
4700 {
4701         if (!plane_state->hw.fb->format->has_alpha)
4702                 return PLANE_COLOR_ALPHA_DISABLE;
4703
4704         switch (plane_state->hw.pixel_blend_mode) {
4705         case DRM_MODE_BLEND_PIXEL_NONE:
4706                 return PLANE_COLOR_ALPHA_DISABLE;
4707         case DRM_MODE_BLEND_PREMULTI:
4708                 return PLANE_COLOR_ALPHA_SW_PREMULTIPLY;
4709         case DRM_MODE_BLEND_COVERAGE:
4710                 return PLANE_COLOR_ALPHA_HW_PREMULTIPLY;
4711         default:
4712                 MISSING_CASE(plane_state->hw.pixel_blend_mode);
4713                 return PLANE_COLOR_ALPHA_DISABLE;
4714         }
4715 }
4716
4717 static u32 skl_plane_ctl_tiling(u64 fb_modifier)
4718 {
4719         switch (fb_modifier) {
4720         case DRM_FORMAT_MOD_LINEAR:
4721                 break;
4722         case I915_FORMAT_MOD_X_TILED:
4723                 return PLANE_CTL_TILED_X;
4724         case I915_FORMAT_MOD_Y_TILED:
4725                 return PLANE_CTL_TILED_Y;
4726         case I915_FORMAT_MOD_Y_TILED_CCS:
4727                 return PLANE_CTL_TILED_Y | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
4728         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
4729                 return PLANE_CTL_TILED_Y |
4730                        PLANE_CTL_RENDER_DECOMPRESSION_ENABLE |
4731                        PLANE_CTL_CLEAR_COLOR_DISABLE;
4732         case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS:
4733                 return PLANE_CTL_TILED_Y | PLANE_CTL_MEDIA_DECOMPRESSION_ENABLE;
4734         case I915_FORMAT_MOD_Yf_TILED:
4735                 return PLANE_CTL_TILED_YF;
4736         case I915_FORMAT_MOD_Yf_TILED_CCS:
4737                 return PLANE_CTL_TILED_YF | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
4738         default:
4739                 MISSING_CASE(fb_modifier);
4740         }
4741
4742         return 0;
4743 }
4744
4745 static u32 skl_plane_ctl_rotate(unsigned int rotate)
4746 {
4747         switch (rotate) {
4748         case DRM_MODE_ROTATE_0:
4749                 break;
4750         /*
4751          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
4752          * while i915 HW rotation is clockwise, thats why this swapping.
4753          */
4754         case DRM_MODE_ROTATE_90:
4755                 return PLANE_CTL_ROTATE_270;
4756         case DRM_MODE_ROTATE_180:
4757                 return PLANE_CTL_ROTATE_180;
4758         case DRM_MODE_ROTATE_270:
4759                 return PLANE_CTL_ROTATE_90;
4760         default:
4761                 MISSING_CASE(rotate);
4762         }
4763
4764         return 0;
4765 }
4766
4767 static u32 cnl_plane_ctl_flip(unsigned int reflect)
4768 {
4769         switch (reflect) {
4770         case 0:
4771                 break;
4772         case DRM_MODE_REFLECT_X:
4773                 return PLANE_CTL_FLIP_HORIZONTAL;
4774         case DRM_MODE_REFLECT_Y:
4775         default:
4776                 MISSING_CASE(reflect);
4777         }
4778
4779         return 0;
4780 }
4781
4782 u32 skl_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
4783 {
4784         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4785         u32 plane_ctl = 0;
4786
4787         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
4788                 return plane_ctl;
4789
4790         if (crtc_state->gamma_enable)
4791                 plane_ctl |= PLANE_CTL_PIPE_GAMMA_ENABLE;
4792
4793         if (crtc_state->csc_enable)
4794                 plane_ctl |= PLANE_CTL_PIPE_CSC_ENABLE;
4795
4796         return plane_ctl;
4797 }
4798
4799 u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
4800                   const struct intel_plane_state *plane_state)
4801 {
4802         struct drm_i915_private *dev_priv =
4803                 to_i915(plane_state->uapi.plane->dev);
4804         const struct drm_framebuffer *fb = plane_state->hw.fb;
4805         unsigned int rotation = plane_state->hw.rotation;
4806         const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
4807         u32 plane_ctl;
4808
4809         plane_ctl = PLANE_CTL_ENABLE;
4810
4811         if (INTEL_GEN(dev_priv) < 10 && !IS_GEMINILAKE(dev_priv)) {
4812                 plane_ctl |= skl_plane_ctl_alpha(plane_state);
4813                 plane_ctl |= PLANE_CTL_PLANE_GAMMA_DISABLE;
4814
4815                 if (plane_state->hw.color_encoding == DRM_COLOR_YCBCR_BT709)
4816                         plane_ctl |= PLANE_CTL_YUV_TO_RGB_CSC_FORMAT_BT709;
4817
4818                 if (plane_state->hw.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
4819                         plane_ctl |= PLANE_CTL_YUV_RANGE_CORRECTION_DISABLE;
4820         }
4821
4822         plane_ctl |= skl_plane_ctl_format(fb->format->format);
4823         plane_ctl |= skl_plane_ctl_tiling(fb->modifier);
4824         plane_ctl |= skl_plane_ctl_rotate(rotation & DRM_MODE_ROTATE_MASK);
4825
4826         if (INTEL_GEN(dev_priv) >= 10)
4827                 plane_ctl |= cnl_plane_ctl_flip(rotation &
4828                                                 DRM_MODE_REFLECT_MASK);
4829
4830         if (key->flags & I915_SET_COLORKEY_DESTINATION)
4831                 plane_ctl |= PLANE_CTL_KEY_ENABLE_DESTINATION;
4832         else if (key->flags & I915_SET_COLORKEY_SOURCE)
4833                 plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE;
4834
4835         return plane_ctl;
4836 }
4837
4838 u32 glk_plane_color_ctl_crtc(const struct intel_crtc_state *crtc_state)
4839 {
4840         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4841         u32 plane_color_ctl = 0;
4842
4843         if (INTEL_GEN(dev_priv) >= 11)
4844                 return plane_color_ctl;
4845
4846         if (crtc_state->gamma_enable)
4847                 plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE;
4848
4849         if (crtc_state->csc_enable)
4850                 plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE;
4851
4852         return plane_color_ctl;
4853 }
4854
4855 u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
4856                         const struct intel_plane_state *plane_state)
4857 {
4858         struct drm_i915_private *dev_priv =
4859                 to_i915(plane_state->uapi.plane->dev);
4860         const struct drm_framebuffer *fb = plane_state->hw.fb;
4861         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4862         u32 plane_color_ctl = 0;
4863
4864         plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE;
4865         plane_color_ctl |= glk_plane_color_ctl_alpha(plane_state);
4866
4867         if (fb->format->is_yuv && !icl_is_hdr_plane(dev_priv, plane->id)) {
4868                 switch (plane_state->hw.color_encoding) {
4869                 case DRM_COLOR_YCBCR_BT709:
4870                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709;
4871                         break;
4872                 case DRM_COLOR_YCBCR_BT2020:
4873                         plane_color_ctl |=
4874                                 PLANE_COLOR_CSC_MODE_YUV2020_TO_RGB2020;
4875                         break;
4876                 default:
4877                         plane_color_ctl |=
4878                                 PLANE_COLOR_CSC_MODE_YUV601_TO_RGB601;
4879                 }
4880                 if (plane_state->hw.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
4881                         plane_color_ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE;
4882         } else if (fb->format->is_yuv) {
4883                 plane_color_ctl |= PLANE_COLOR_INPUT_CSC_ENABLE;
4884         }
4885
4886         return plane_color_ctl;
4887 }
4888
4889 static int
4890 __intel_display_resume(struct drm_device *dev,
4891                        struct drm_atomic_state *state,
4892                        struct drm_modeset_acquire_ctx *ctx)
4893 {
4894         struct drm_crtc_state *crtc_state;
4895         struct drm_crtc *crtc;
4896         int i, ret;
4897
4898         intel_modeset_setup_hw_state(dev, ctx);
4899         intel_vga_redisable(to_i915(dev));
4900
4901         if (!state)
4902                 return 0;
4903
4904         /*
4905          * We've duplicated the state, pointers to the old state are invalid.
4906          *
4907          * Don't attempt to use the old state until we commit the duplicated state.
4908          */
4909         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
4910                 /*
4911                  * Force recalculation even if we restore
4912                  * current state. With fast modeset this may not result
4913                  * in a modeset when the state is compatible.
4914                  */
4915                 crtc_state->mode_changed = true;
4916         }
4917
4918         /* ignore any reset values/BIOS leftovers in the WM registers */
4919         if (!HAS_GMCH(to_i915(dev)))
4920                 to_intel_atomic_state(state)->skip_intermediate_wm = true;
4921
4922         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
4923
4924         drm_WARN_ON(dev, ret == -EDEADLK);
4925         return ret;
4926 }
4927
4928 static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv)
4929 {
4930         return (INTEL_INFO(dev_priv)->gpu_reset_clobbers_display &&
4931                 intel_has_gpu_reset(&dev_priv->gt));
4932 }
4933
4934 void intel_prepare_reset(struct drm_i915_private *dev_priv)
4935 {
4936         struct drm_device *dev = &dev_priv->drm;
4937         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
4938         struct drm_atomic_state *state;
4939         int ret;
4940
4941         /* reset doesn't touch the display */
4942         if (!dev_priv->params.force_reset_modeset_test &&
4943             !gpu_reset_clobbers_display(dev_priv))
4944                 return;
4945
4946         /* We have a modeset vs reset deadlock, defensively unbreak it. */
4947         set_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags);
4948         smp_mb__after_atomic();
4949         wake_up_bit(&dev_priv->gt.reset.flags, I915_RESET_MODESET);
4950
4951         if (atomic_read(&dev_priv->gpu_error.pending_fb_pin)) {
4952                 drm_dbg_kms(&dev_priv->drm,
4953                             "Modeset potentially stuck, unbreaking through wedging\n");
4954                 intel_gt_set_wedged(&dev_priv->gt);
4955         }
4956
4957         /*
4958          * Need mode_config.mutex so that we don't
4959          * trample ongoing ->detect() and whatnot.
4960          */
4961         mutex_lock(&dev->mode_config.mutex);
4962         drm_modeset_acquire_init(ctx, 0);
4963         while (1) {
4964                 ret = drm_modeset_lock_all_ctx(dev, ctx);
4965                 if (ret != -EDEADLK)
4966                         break;
4967
4968                 drm_modeset_backoff(ctx);
4969         }
4970         /*
4971          * Disabling the crtcs gracefully seems nicer. Also the
4972          * g33 docs say we should at least disable all the planes.
4973          */
4974         state = drm_atomic_helper_duplicate_state(dev, ctx);
4975         if (IS_ERR(state)) {
4976                 ret = PTR_ERR(state);
4977                 drm_err(&dev_priv->drm, "Duplicating state failed with %i\n",
4978                         ret);
4979                 return;
4980         }
4981
4982         ret = drm_atomic_helper_disable_all(dev, ctx);
4983         if (ret) {
4984                 drm_err(&dev_priv->drm, "Suspending crtc's failed with %i\n",
4985                         ret);
4986                 drm_atomic_state_put(state);
4987                 return;
4988         }
4989
4990         dev_priv->modeset_restore_state = state;
4991         state->acquire_ctx = ctx;
4992 }
4993
4994 void intel_finish_reset(struct drm_i915_private *dev_priv)
4995 {
4996         struct drm_device *dev = &dev_priv->drm;
4997         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
4998         struct drm_atomic_state *state;
4999         int ret;
5000
5001         /* reset doesn't touch the display */
5002         if (!test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags))
5003                 return;
5004
5005         state = fetch_and_zero(&dev_priv->modeset_restore_state);
5006         if (!state)
5007                 goto unlock;
5008
5009         /* reset doesn't touch the display */
5010         if (!gpu_reset_clobbers_display(dev_priv)) {
5011                 /* for testing only restore the display */
5012                 ret = __intel_display_resume(dev, state, ctx);
5013                 if (ret)
5014                         drm_err(&dev_priv->drm,
5015                                 "Restoring old state failed with %i\n", ret);
5016         } else {
5017                 /*
5018                  * The display has been reset as well,
5019                  * so need a full re-initialization.
5020                  */
5021                 intel_pps_unlock_regs_wa(dev_priv);
5022                 intel_modeset_init_hw(dev_priv);
5023                 intel_init_clock_gating(dev_priv);
5024
5025                 spin_lock_irq(&dev_priv->irq_lock);
5026                 if (dev_priv->display.hpd_irq_setup)
5027                         dev_priv->display.hpd_irq_setup(dev_priv);
5028                 spin_unlock_irq(&dev_priv->irq_lock);
5029
5030                 ret = __intel_display_resume(dev, state, ctx);
5031                 if (ret)
5032                         drm_err(&dev_priv->drm,
5033                                 "Restoring old state failed with %i\n", ret);
5034
5035                 intel_hpd_init(dev_priv);
5036         }
5037
5038         drm_atomic_state_put(state);
5039 unlock:
5040         drm_modeset_drop_locks(ctx);
5041         drm_modeset_acquire_fini(ctx);
5042         mutex_unlock(&dev->mode_config.mutex);
5043
5044         clear_bit_unlock(I915_RESET_MODESET, &dev_priv->gt.reset.flags);
5045 }
5046
5047 static void icl_set_pipe_chicken(struct intel_crtc *crtc)
5048 {
5049         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5050         enum pipe pipe = crtc->pipe;
5051         u32 tmp;
5052
5053         tmp = intel_de_read(dev_priv, PIPE_CHICKEN(pipe));
5054
5055         /*
5056          * Display WA #1153: icl
5057          * enable hardware to bypass the alpha math
5058          * and rounding for per-pixel values 00 and 0xff
5059          */
5060         tmp |= PER_PIXEL_ALPHA_BYPASS_EN;
5061         /*
5062          * Display WA # 1605353570: icl
5063          * Set the pixel rounding bit to 1 for allowing
5064          * passthrough of Frame buffer pixels unmodified
5065          * across pipe
5066          */
5067         tmp |= PIXEL_ROUNDING_TRUNC_FB_PASSTHRU;
5068         intel_de_write(dev_priv, PIPE_CHICKEN(pipe), tmp);
5069 }
5070
5071 static void intel_fdi_normal_train(struct intel_crtc *crtc)
5072 {
5073         struct drm_device *dev = crtc->base.dev;
5074         struct drm_i915_private *dev_priv = to_i915(dev);
5075         enum pipe pipe = crtc->pipe;
5076         i915_reg_t reg;
5077         u32 temp;
5078
5079         /* enable normal train */
5080         reg = FDI_TX_CTL(pipe);
5081         temp = intel_de_read(dev_priv, reg);
5082         if (IS_IVYBRIDGE(dev_priv)) {
5083                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
5084                 temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
5085         } else {
5086                 temp &= ~FDI_LINK_TRAIN_NONE;
5087                 temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
5088         }
5089         intel_de_write(dev_priv, reg, temp);
5090
5091         reg = FDI_RX_CTL(pipe);
5092         temp = intel_de_read(dev_priv, reg);
5093         if (HAS_PCH_CPT(dev_priv)) {
5094                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5095                 temp |= FDI_LINK_TRAIN_NORMAL_CPT;
5096         } else {
5097                 temp &= ~FDI_LINK_TRAIN_NONE;
5098                 temp |= FDI_LINK_TRAIN_NONE;
5099         }
5100         intel_de_write(dev_priv, reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
5101
5102         /* wait one idle pattern time */
5103         intel_de_posting_read(dev_priv, reg);
5104         udelay(1000);
5105
5106         /* IVB wants error correction enabled */
5107         if (IS_IVYBRIDGE(dev_priv))
5108                 intel_de_write(dev_priv, reg,
5109                                intel_de_read(dev_priv, reg) | FDI_FS_ERRC_ENABLE | FDI_FE_ERRC_ENABLE);
5110 }
5111
5112 /* The FDI link training functions for ILK/Ibexpeak. */
5113 static void ilk_fdi_link_train(struct intel_crtc *crtc,
5114                                const struct intel_crtc_state *crtc_state)
5115 {
5116         struct drm_device *dev = crtc->base.dev;
5117         struct drm_i915_private *dev_priv = to_i915(dev);
5118         enum pipe pipe = crtc->pipe;
5119         i915_reg_t reg;
5120         u32 temp, tries;
5121
5122         /* FDI needs bits from pipe first */
5123         assert_pipe_enabled(dev_priv, crtc_state->cpu_transcoder);
5124
5125         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
5126            for train result */
5127         reg = FDI_RX_IMR(pipe);
5128         temp = intel_de_read(dev_priv, reg);
5129         temp &= ~FDI_RX_SYMBOL_LOCK;
5130         temp &= ~FDI_RX_BIT_LOCK;
5131         intel_de_write(dev_priv, reg, temp);
5132         intel_de_read(dev_priv, reg);
5133         udelay(150);
5134
5135         /* enable CPU FDI TX and PCH FDI RX */
5136         reg = FDI_TX_CTL(pipe);
5137         temp = intel_de_read(dev_priv, reg);
5138         temp &= ~FDI_DP_PORT_WIDTH_MASK;
5139         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
5140         temp &= ~FDI_LINK_TRAIN_NONE;
5141         temp |= FDI_LINK_TRAIN_PATTERN_1;
5142         intel_de_write(dev_priv, reg, temp | FDI_TX_ENABLE);
5143
5144         reg = FDI_RX_CTL(pipe);
5145         temp = intel_de_read(dev_priv, reg);
5146         temp &= ~FDI_LINK_TRAIN_NONE;
5147         temp |= FDI_LINK_TRAIN_PATTERN_1;
5148         intel_de_write(dev_priv, reg, temp | FDI_RX_ENABLE);
5149
5150         intel_de_posting_read(dev_priv, reg);
5151         udelay(150);
5152
5153         /* Ironlake workaround, enable clock pointer after FDI enable*/
5154         intel_de_write(dev_priv, FDI_RX_CHICKEN(pipe),
5155                        FDI_RX_PHASE_SYNC_POINTER_OVR);
5156         intel_de_write(dev_priv, FDI_RX_CHICKEN(pipe),
5157                        FDI_RX_PHASE_SYNC_POINTER_OVR | FDI_RX_PHASE_SYNC_POINTER_EN);
5158
5159         reg = FDI_RX_IIR(pipe);
5160         for (tries = 0; tries < 5; tries++) {
5161                 temp = intel_de_read(dev_priv, reg);
5162                 drm_dbg_kms(&dev_priv->drm, "FDI_RX_IIR 0x%x\n", temp);
5163
5164                 if ((temp & FDI_RX_BIT_LOCK)) {
5165                         drm_dbg_kms(&dev_priv->drm, "FDI train 1 done.\n");
5166                         intel_de_write(dev_priv, reg, temp | FDI_RX_BIT_LOCK);
5167                         break;
5168                 }
5169         }
5170         if (tries == 5)
5171                 drm_err(&dev_priv->drm, "FDI train 1 fail!\n");
5172
5173         /* Train 2 */
5174         reg = FDI_TX_CTL(pipe);
5175         temp = intel_de_read(dev_priv, reg);
5176         temp &= ~FDI_LINK_TRAIN_NONE;
5177         temp |= FDI_LINK_TRAIN_PATTERN_2;
5178         intel_de_write(dev_priv, reg, temp);
5179
5180         reg = FDI_RX_CTL(pipe);
5181         temp = intel_de_read(dev_priv, reg);
5182         temp &= ~FDI_LINK_TRAIN_NONE;
5183         temp |= FDI_LINK_TRAIN_PATTERN_2;
5184         intel_de_write(dev_priv, reg, temp);
5185
5186         intel_de_posting_read(dev_priv, reg);
5187         udelay(150);
5188
5189         reg = FDI_RX_IIR(pipe);
5190         for (tries = 0; tries < 5; tries++) {
5191                 temp = intel_de_read(dev_priv, reg);
5192                 drm_dbg_kms(&dev_priv->drm, "FDI_RX_IIR 0x%x\n", temp);
5193
5194                 if (temp & FDI_RX_SYMBOL_LOCK) {
5195                         intel_de_write(dev_priv, reg,
5196                                        temp | FDI_RX_SYMBOL_LOCK);
5197                         drm_dbg_kms(&dev_priv->drm, "FDI train 2 done.\n");
5198                         break;
5199                 }
5200         }
5201         if (tries == 5)
5202                 drm_err(&dev_priv->drm, "FDI train 2 fail!\n");
5203
5204         drm_dbg_kms(&dev_priv->drm, "FDI train done\n");
5205
5206 }
5207
5208 static const int snb_b_fdi_train_param[] = {
5209         FDI_LINK_TRAIN_400MV_0DB_SNB_B,
5210         FDI_LINK_TRAIN_400MV_6DB_SNB_B,
5211         FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
5212         FDI_LINK_TRAIN_800MV_0DB_SNB_B,
5213 };
5214
5215 /* The FDI link training functions for SNB/Cougarpoint. */
5216 static void gen6_fdi_link_train(struct intel_crtc *crtc,
5217                                 const struct intel_crtc_state *crtc_state)
5218 {
5219         struct drm_device *dev = crtc->base.dev;
5220         struct drm_i915_private *dev_priv = to_i915(dev);
5221         enum pipe pipe = crtc->pipe;
5222         i915_reg_t reg;
5223         u32 temp, i, retry;
5224
5225         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
5226            for train result */
5227         reg = FDI_RX_IMR(pipe);
5228         temp = intel_de_read(dev_priv, reg);
5229         temp &= ~FDI_RX_SYMBOL_LOCK;
5230         temp &= ~FDI_RX_BIT_LOCK;
5231         intel_de_write(dev_priv, reg, temp);
5232
5233         intel_de_posting_read(dev_priv, reg);
5234         udelay(150);
5235
5236         /* enable CPU FDI TX and PCH FDI RX */
5237         reg = FDI_TX_CTL(pipe);
5238         temp = intel_de_read(dev_priv, reg);
5239         temp &= ~FDI_DP_PORT_WIDTH_MASK;
5240         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
5241         temp &= ~FDI_LINK_TRAIN_NONE;
5242         temp |= FDI_LINK_TRAIN_PATTERN_1;
5243         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5244         /* SNB-B */
5245         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
5246         intel_de_write(dev_priv, reg, temp | FDI_TX_ENABLE);
5247
5248         intel_de_write(dev_priv, FDI_RX_MISC(pipe),
5249                        FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
5250
5251         reg = FDI_RX_CTL(pipe);
5252         temp = intel_de_read(dev_priv, reg);
5253         if (HAS_PCH_CPT(dev_priv)) {
5254                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5255                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
5256         } else {
5257                 temp &= ~FDI_LINK_TRAIN_NONE;
5258                 temp |= FDI_LINK_TRAIN_PATTERN_1;
5259         }
5260         intel_de_write(dev_priv, reg, temp | FDI_RX_ENABLE);
5261
5262         intel_de_posting_read(dev_priv, reg);
5263         udelay(150);
5264
5265         for (i = 0; i < 4; i++) {
5266                 reg = FDI_TX_CTL(pipe);
5267                 temp = intel_de_read(dev_priv, reg);
5268                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5269                 temp |= snb_b_fdi_train_param[i];
5270                 intel_de_write(dev_priv, reg, temp);
5271
5272                 intel_de_posting_read(dev_priv, reg);
5273                 udelay(500);
5274
5275                 for (retry = 0; retry < 5; retry++) {
5276                         reg = FDI_RX_IIR(pipe);
5277                         temp = intel_de_read(dev_priv, reg);
5278                         drm_dbg_kms(&dev_priv->drm, "FDI_RX_IIR 0x%x\n", temp);
5279                         if (temp & FDI_RX_BIT_LOCK) {
5280                                 intel_de_write(dev_priv, reg,
5281                                                temp | FDI_RX_BIT_LOCK);
5282                                 drm_dbg_kms(&dev_priv->drm,
5283                                             "FDI train 1 done.\n");
5284                                 break;
5285                         }
5286                         udelay(50);
5287                 }
5288                 if (retry < 5)
5289                         break;
5290         }
5291         if (i == 4)
5292                 drm_err(&dev_priv->drm, "FDI train 1 fail!\n");
5293
5294         /* Train 2 */
5295         reg = FDI_TX_CTL(pipe);
5296         temp = intel_de_read(dev_priv, reg);
5297         temp &= ~FDI_LINK_TRAIN_NONE;
5298         temp |= FDI_LINK_TRAIN_PATTERN_2;
5299         if (IS_GEN(dev_priv, 6)) {
5300                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5301                 /* SNB-B */
5302                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
5303         }
5304         intel_de_write(dev_priv, reg, temp);
5305
5306         reg = FDI_RX_CTL(pipe);
5307         temp = intel_de_read(dev_priv, reg);
5308         if (HAS_PCH_CPT(dev_priv)) {
5309                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5310                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
5311         } else {
5312                 temp &= ~FDI_LINK_TRAIN_NONE;
5313                 temp |= FDI_LINK_TRAIN_PATTERN_2;
5314         }
5315         intel_de_write(dev_priv, reg, temp);
5316
5317         intel_de_posting_read(dev_priv, reg);
5318         udelay(150);
5319
5320         for (i = 0; i < 4; i++) {
5321                 reg = FDI_TX_CTL(pipe);
5322                 temp = intel_de_read(dev_priv, reg);
5323                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5324                 temp |= snb_b_fdi_train_param[i];
5325                 intel_de_write(dev_priv, reg, temp);
5326
5327                 intel_de_posting_read(dev_priv, reg);
5328                 udelay(500);
5329
5330                 for (retry = 0; retry < 5; retry++) {
5331                         reg = FDI_RX_IIR(pipe);
5332                         temp = intel_de_read(dev_priv, reg);
5333                         drm_dbg_kms(&dev_priv->drm, "FDI_RX_IIR 0x%x\n", temp);
5334                         if (temp & FDI_RX_SYMBOL_LOCK) {
5335                                 intel_de_write(dev_priv, reg,
5336                                                temp | FDI_RX_SYMBOL_LOCK);
5337                                 drm_dbg_kms(&dev_priv->drm,
5338                                             "FDI train 2 done.\n");
5339                                 break;
5340                         }
5341                         udelay(50);
5342                 }
5343                 if (retry < 5)
5344                         break;
5345         }
5346         if (i == 4)
5347                 drm_err(&dev_priv->drm, "FDI train 2 fail!\n");
5348
5349         drm_dbg_kms(&dev_priv->drm, "FDI train done.\n");
5350 }
5351
5352 /* Manual link training for Ivy Bridge A0 parts */
5353 static void ivb_manual_fdi_link_train(struct intel_crtc *crtc,
5354                                       const struct intel_crtc_state *crtc_state)
5355 {
5356         struct drm_device *dev = crtc->base.dev;
5357         struct drm_i915_private *dev_priv = to_i915(dev);
5358         enum pipe pipe = crtc->pipe;
5359         i915_reg_t reg;
5360         u32 temp, i, j;
5361
5362         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
5363            for train result */
5364         reg = FDI_RX_IMR(pipe);
5365         temp = intel_de_read(dev_priv, reg);
5366         temp &= ~FDI_RX_SYMBOL_LOCK;
5367         temp &= ~FDI_RX_BIT_LOCK;
5368         intel_de_write(dev_priv, reg, temp);
5369
5370         intel_de_posting_read(dev_priv, reg);
5371         udelay(150);
5372
5373         drm_dbg_kms(&dev_priv->drm, "FDI_RX_IIR before link train 0x%x\n",
5374                     intel_de_read(dev_priv, FDI_RX_IIR(pipe)));
5375
5376         /* Try each vswing and preemphasis setting twice before moving on */
5377         for (j = 0; j < ARRAY_SIZE(snb_b_fdi_train_param) * 2; j++) {
5378                 /* disable first in case we need to retry */
5379                 reg = FDI_TX_CTL(pipe);
5380                 temp = intel_de_read(dev_priv, reg);
5381                 temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
5382                 temp &= ~FDI_TX_ENABLE;
5383                 intel_de_write(dev_priv, reg, temp);
5384
5385                 reg = FDI_RX_CTL(pipe);
5386                 temp = intel_de_read(dev_priv, reg);
5387                 temp &= ~FDI_LINK_TRAIN_AUTO;
5388                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5389                 temp &= ~FDI_RX_ENABLE;
5390                 intel_de_write(dev_priv, reg, temp);
5391
5392                 /* enable CPU FDI TX and PCH FDI RX */
5393                 reg = FDI_TX_CTL(pipe);
5394                 temp = intel_de_read(dev_priv, reg);
5395                 temp &= ~FDI_DP_PORT_WIDTH_MASK;
5396                 temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
5397                 temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
5398                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5399                 temp |= snb_b_fdi_train_param[j/2];
5400                 temp |= FDI_COMPOSITE_SYNC;
5401                 intel_de_write(dev_priv, reg, temp | FDI_TX_ENABLE);
5402
5403                 intel_de_write(dev_priv, FDI_RX_MISC(pipe),
5404                                FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
5405
5406                 reg = FDI_RX_CTL(pipe);
5407                 temp = intel_de_read(dev_priv, reg);
5408                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
5409                 temp |= FDI_COMPOSITE_SYNC;
5410                 intel_de_write(dev_priv, reg, temp | FDI_RX_ENABLE);
5411
5412                 intel_de_posting_read(dev_priv, reg);
5413                 udelay(1); /* should be 0.5us */
5414
5415                 for (i = 0; i < 4; i++) {
5416                         reg = FDI_RX_IIR(pipe);
5417                         temp = intel_de_read(dev_priv, reg);
5418                         drm_dbg_kms(&dev_priv->drm, "FDI_RX_IIR 0x%x\n", temp);
5419
5420                         if (temp & FDI_RX_BIT_LOCK ||
5421                             (intel_de_read(dev_priv, reg) & FDI_RX_BIT_LOCK)) {
5422                                 intel_de_write(dev_priv, reg,
5423                                                temp | FDI_RX_BIT_LOCK);
5424                                 drm_dbg_kms(&dev_priv->drm,
5425                                             "FDI train 1 done, level %i.\n",
5426                                             i);
5427                                 break;
5428                         }
5429                         udelay(1); /* should be 0.5us */
5430                 }
5431                 if (i == 4) {
5432                         drm_dbg_kms(&dev_priv->drm,
5433                                     "FDI train 1 fail on vswing %d\n", j / 2);
5434                         continue;
5435                 }
5436
5437                 /* Train 2 */
5438                 reg = FDI_TX_CTL(pipe);
5439                 temp = intel_de_read(dev_priv, reg);
5440                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
5441                 temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
5442                 intel_de_write(dev_priv, reg, temp);
5443
5444                 reg = FDI_RX_CTL(pipe);
5445                 temp = intel_de_read(dev_priv, reg);
5446                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5447                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
5448                 intel_de_write(dev_priv, reg, temp);
5449
5450                 intel_de_posting_read(dev_priv, reg);
5451                 udelay(2); /* should be 1.5us */
5452
5453                 for (i = 0; i < 4; i++) {
5454                         reg = FDI_RX_IIR(pipe);
5455                         temp = intel_de_read(dev_priv, reg);
5456                         drm_dbg_kms(&dev_priv->drm, "FDI_RX_IIR 0x%x\n", temp);
5457
5458                         if (temp & FDI_RX_SYMBOL_LOCK ||
5459                             (intel_de_read(dev_priv, reg) & FDI_RX_SYMBOL_LOCK)) {
5460                                 intel_de_write(dev_priv, reg,
5461                                                temp | FDI_RX_SYMBOL_LOCK);
5462                                 drm_dbg_kms(&dev_priv->drm,
5463                                             "FDI train 2 done, level %i.\n",
5464                                             i);
5465                                 goto train_done;
5466                         }
5467                         udelay(2); /* should be 1.5us */
5468                 }
5469                 if (i == 4)
5470                         drm_dbg_kms(&dev_priv->drm,
5471                                     "FDI train 2 fail on vswing %d\n", j / 2);
5472         }
5473
5474 train_done:
5475         drm_dbg_kms(&dev_priv->drm, "FDI train done.\n");
5476 }
5477
5478 static void ilk_fdi_pll_enable(const struct intel_crtc_state *crtc_state)
5479 {
5480         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->uapi.crtc);
5481         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
5482         enum pipe pipe = intel_crtc->pipe;
5483         i915_reg_t reg;
5484         u32 temp;
5485
5486         /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
5487         reg = FDI_RX_CTL(pipe);
5488         temp = intel_de_read(dev_priv, reg);
5489         temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16));
5490         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
5491         temp |= (intel_de_read(dev_priv, PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
5492         intel_de_write(dev_priv, reg, temp | FDI_RX_PLL_ENABLE);
5493
5494         intel_de_posting_read(dev_priv, reg);
5495         udelay(200);
5496
5497         /* Switch from Rawclk to PCDclk */
5498         temp = intel_de_read(dev_priv, reg);
5499         intel_de_write(dev_priv, reg, temp | FDI_PCDCLK);
5500
5501         intel_de_posting_read(dev_priv, reg);
5502         udelay(200);
5503
5504         /* Enable CPU FDI TX PLL, always on for Ironlake */
5505         reg = FDI_TX_CTL(pipe);
5506         temp = intel_de_read(dev_priv, reg);
5507         if ((temp & FDI_TX_PLL_ENABLE) == 0) {
5508                 intel_de_write(dev_priv, reg, temp | FDI_TX_PLL_ENABLE);
5509
5510                 intel_de_posting_read(dev_priv, reg);
5511                 udelay(100);
5512         }
5513 }
5514
5515 static void ilk_fdi_pll_disable(struct intel_crtc *intel_crtc)
5516 {
5517         struct drm_device *dev = intel_crtc->base.dev;
5518         struct drm_i915_private *dev_priv = to_i915(dev);
5519         enum pipe pipe = intel_crtc->pipe;
5520         i915_reg_t reg;
5521         u32 temp;
5522
5523         /* Switch from PCDclk to Rawclk */
5524         reg = FDI_RX_CTL(pipe);
5525         temp = intel_de_read(dev_priv, reg);
5526         intel_de_write(dev_priv, reg, temp & ~FDI_PCDCLK);
5527
5528         /* Disable CPU FDI TX PLL */
5529         reg = FDI_TX_CTL(pipe);
5530         temp = intel_de_read(dev_priv, reg);
5531         intel_de_write(dev_priv, reg, temp & ~FDI_TX_PLL_ENABLE);
5532
5533         intel_de_posting_read(dev_priv, reg);
5534         udelay(100);
5535
5536         reg = FDI_RX_CTL(pipe);
5537         temp = intel_de_read(dev_priv, reg);
5538         intel_de_write(dev_priv, reg, temp & ~FDI_RX_PLL_ENABLE);
5539
5540         /* Wait for the clocks to turn off. */
5541         intel_de_posting_read(dev_priv, reg);
5542         udelay(100);
5543 }
5544
5545 static void ilk_fdi_disable(struct intel_crtc *crtc)
5546 {
5547         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5548         enum pipe pipe = crtc->pipe;
5549         i915_reg_t reg;
5550         u32 temp;
5551
5552         /* disable CPU FDI tx and PCH FDI rx */
5553         reg = FDI_TX_CTL(pipe);
5554         temp = intel_de_read(dev_priv, reg);
5555         intel_de_write(dev_priv, reg, temp & ~FDI_TX_ENABLE);
5556         intel_de_posting_read(dev_priv, reg);
5557
5558         reg = FDI_RX_CTL(pipe);
5559         temp = intel_de_read(dev_priv, reg);
5560         temp &= ~(0x7 << 16);
5561         temp |= (intel_de_read(dev_priv, PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
5562         intel_de_write(dev_priv, reg, temp & ~FDI_RX_ENABLE);
5563
5564         intel_de_posting_read(dev_priv, reg);
5565         udelay(100);
5566
5567         /* Ironlake workaround, disable clock pointer after downing FDI */
5568         if (HAS_PCH_IBX(dev_priv))
5569                 intel_de_write(dev_priv, FDI_RX_CHICKEN(pipe),
5570                                FDI_RX_PHASE_SYNC_POINTER_OVR);
5571
5572         /* still set train pattern 1 */
5573         reg = FDI_TX_CTL(pipe);
5574         temp = intel_de_read(dev_priv, reg);
5575         temp &= ~FDI_LINK_TRAIN_NONE;
5576         temp |= FDI_LINK_TRAIN_PATTERN_1;
5577         intel_de_write(dev_priv, reg, temp);
5578
5579         reg = FDI_RX_CTL(pipe);
5580         temp = intel_de_read(dev_priv, reg);
5581         if (HAS_PCH_CPT(dev_priv)) {
5582                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5583                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
5584         } else {
5585                 temp &= ~FDI_LINK_TRAIN_NONE;
5586                 temp |= FDI_LINK_TRAIN_PATTERN_1;
5587         }
5588         /* BPC in FDI rx is consistent with that in PIPECONF */
5589         temp &= ~(0x07 << 16);
5590         temp |= (intel_de_read(dev_priv, PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
5591         intel_de_write(dev_priv, reg, temp);
5592
5593         intel_de_posting_read(dev_priv, reg);
5594         udelay(100);
5595 }
5596
5597 bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv)
5598 {
5599         struct drm_crtc *crtc;
5600         bool cleanup_done;
5601
5602         drm_for_each_crtc(crtc, &dev_priv->drm) {
5603                 struct drm_crtc_commit *commit;
5604                 spin_lock(&crtc->commit_lock);
5605                 commit = list_first_entry_or_null(&crtc->commit_list,
5606                                                   struct drm_crtc_commit, commit_entry);
5607                 cleanup_done = commit ?
5608                         try_wait_for_completion(&commit->cleanup_done) : true;
5609                 spin_unlock(&crtc->commit_lock);
5610
5611                 if (cleanup_done)
5612                         continue;
5613
5614                 drm_crtc_wait_one_vblank(crtc);
5615
5616                 return true;
5617         }
5618
5619         return false;
5620 }
5621
5622 void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
5623 {
5624         u32 temp;
5625
5626         intel_de_write(dev_priv, PIXCLK_GATE, PIXCLK_GATE_GATE);
5627
5628         mutex_lock(&dev_priv->sb_lock);
5629
5630         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5631         temp |= SBI_SSCCTL_DISABLE;
5632         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
5633
5634         mutex_unlock(&dev_priv->sb_lock);
5635 }
5636
5637 /* Program iCLKIP clock to the desired frequency */
5638 static void lpt_program_iclkip(const struct intel_crtc_state *crtc_state)
5639 {
5640         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5641         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5642         int clock = crtc_state->hw.adjusted_mode.crtc_clock;
5643         u32 divsel, phaseinc, auxdiv, phasedir = 0;
5644         u32 temp;
5645
5646         lpt_disable_iclkip(dev_priv);
5647
5648         /* The iCLK virtual clock root frequency is in MHz,
5649          * but the adjusted_mode->crtc_clock in in KHz. To get the
5650          * divisors, it is necessary to divide one by another, so we
5651          * convert the virtual clock precision to KHz here for higher
5652          * precision.
5653          */
5654         for (auxdiv = 0; auxdiv < 2; auxdiv++) {
5655                 u32 iclk_virtual_root_freq = 172800 * 1000;
5656                 u32 iclk_pi_range = 64;
5657                 u32 desired_divisor;
5658
5659                 desired_divisor = DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
5660                                                     clock << auxdiv);
5661                 divsel = (desired_divisor / iclk_pi_range) - 2;
5662                 phaseinc = desired_divisor % iclk_pi_range;
5663
5664                 /*
5665                  * Near 20MHz is a corner case which is
5666                  * out of range for the 7-bit divisor
5667                  */
5668                 if (divsel <= 0x7f)
5669                         break;
5670         }
5671
5672         /* This should not happen with any sane values */
5673         drm_WARN_ON(&dev_priv->drm, SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
5674                     ~SBI_SSCDIVINTPHASE_DIVSEL_MASK);
5675         drm_WARN_ON(&dev_priv->drm, SBI_SSCDIVINTPHASE_DIR(phasedir) &
5676                     ~SBI_SSCDIVINTPHASE_INCVAL_MASK);
5677
5678         drm_dbg_kms(&dev_priv->drm,
5679                     "iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
5680                     clock, auxdiv, divsel, phasedir, phaseinc);
5681
5682         mutex_lock(&dev_priv->sb_lock);
5683
5684         /* Program SSCDIVINTPHASE6 */
5685         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
5686         temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
5687         temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
5688         temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
5689         temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
5690         temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
5691         temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
5692         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK);
5693
5694         /* Program SSCAUXDIV */
5695         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
5696         temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
5697         temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
5698         intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK);
5699
5700         /* Enable modulator and associated divider */
5701         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5702         temp &= ~SBI_SSCCTL_DISABLE;
5703         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
5704
5705         mutex_unlock(&dev_priv->sb_lock);
5706
5707         /* Wait for initialization time */
5708         udelay(24);
5709
5710         intel_de_write(dev_priv, PIXCLK_GATE, PIXCLK_GATE_UNGATE);
5711 }
5712
5713 int lpt_get_iclkip(struct drm_i915_private *dev_priv)
5714 {
5715         u32 divsel, phaseinc, auxdiv;
5716         u32 iclk_virtual_root_freq = 172800 * 1000;
5717         u32 iclk_pi_range = 64;
5718         u32 desired_divisor;
5719         u32 temp;
5720
5721         if ((intel_de_read(dev_priv, PIXCLK_GATE) & PIXCLK_GATE_UNGATE) == 0)
5722                 return 0;
5723
5724         mutex_lock(&dev_priv->sb_lock);
5725
5726         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5727         if (temp & SBI_SSCCTL_DISABLE) {
5728                 mutex_unlock(&dev_priv->sb_lock);
5729                 return 0;
5730         }
5731
5732         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
5733         divsel = (temp & SBI_SSCDIVINTPHASE_DIVSEL_MASK) >>
5734                 SBI_SSCDIVINTPHASE_DIVSEL_SHIFT;
5735         phaseinc = (temp & SBI_SSCDIVINTPHASE_INCVAL_MASK) >>
5736                 SBI_SSCDIVINTPHASE_INCVAL_SHIFT;
5737
5738         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
5739         auxdiv = (temp & SBI_SSCAUXDIV_FINALDIV2SEL_MASK) >>
5740                 SBI_SSCAUXDIV_FINALDIV2SEL_SHIFT;
5741
5742         mutex_unlock(&dev_priv->sb_lock);
5743
5744         desired_divisor = (divsel + 2) * iclk_pi_range + phaseinc;
5745
5746         return DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
5747                                  desired_divisor << auxdiv);
5748 }
5749
5750 static void ilk_pch_transcoder_set_timings(const struct intel_crtc_state *crtc_state,
5751                                            enum pipe pch_transcoder)
5752 {
5753         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5754         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5755         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
5756
5757         intel_de_write(dev_priv, PCH_TRANS_HTOTAL(pch_transcoder),
5758                        intel_de_read(dev_priv, HTOTAL(cpu_transcoder)));
5759         intel_de_write(dev_priv, PCH_TRANS_HBLANK(pch_transcoder),
5760                        intel_de_read(dev_priv, HBLANK(cpu_transcoder)));
5761         intel_de_write(dev_priv, PCH_TRANS_HSYNC(pch_transcoder),
5762                        intel_de_read(dev_priv, HSYNC(cpu_transcoder)));
5763
5764         intel_de_write(dev_priv, PCH_TRANS_VTOTAL(pch_transcoder),
5765                        intel_de_read(dev_priv, VTOTAL(cpu_transcoder)));
5766         intel_de_write(dev_priv, PCH_TRANS_VBLANK(pch_transcoder),
5767                        intel_de_read(dev_priv, VBLANK(cpu_transcoder)));
5768         intel_de_write(dev_priv, PCH_TRANS_VSYNC(pch_transcoder),
5769                        intel_de_read(dev_priv, VSYNC(cpu_transcoder)));
5770         intel_de_write(dev_priv, PCH_TRANS_VSYNCSHIFT(pch_transcoder),
5771                        intel_de_read(dev_priv, VSYNCSHIFT(cpu_transcoder)));
5772 }
5773
5774 static void cpt_set_fdi_bc_bifurcation(struct drm_i915_private *dev_priv, bool enable)
5775 {
5776         u32 temp;
5777
5778         temp = intel_de_read(dev_priv, SOUTH_CHICKEN1);
5779         if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable)
5780                 return;
5781
5782         drm_WARN_ON(&dev_priv->drm,
5783                     intel_de_read(dev_priv, FDI_RX_CTL(PIPE_B)) &
5784                     FDI_RX_ENABLE);
5785         drm_WARN_ON(&dev_priv->drm,
5786                     intel_de_read(dev_priv, FDI_RX_CTL(PIPE_C)) &
5787                     FDI_RX_ENABLE);
5788
5789         temp &= ~FDI_BC_BIFURCATION_SELECT;
5790         if (enable)
5791                 temp |= FDI_BC_BIFURCATION_SELECT;
5792
5793         drm_dbg_kms(&dev_priv->drm, "%sabling fdi C rx\n",
5794                     enable ? "en" : "dis");
5795         intel_de_write(dev_priv, SOUTH_CHICKEN1, temp);
5796         intel_de_posting_read(dev_priv, SOUTH_CHICKEN1);
5797 }
5798
5799 static void ivb_update_fdi_bc_bifurcation(const struct intel_crtc_state *crtc_state)
5800 {
5801         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5802         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5803
5804         switch (crtc->pipe) {
5805         case PIPE_A:
5806                 break;
5807         case PIPE_B:
5808                 if (crtc_state->fdi_lanes > 2)
5809                         cpt_set_fdi_bc_bifurcation(dev_priv, false);
5810                 else
5811                         cpt_set_fdi_bc_bifurcation(dev_priv, true);
5812
5813                 break;
5814         case PIPE_C:
5815                 cpt_set_fdi_bc_bifurcation(dev_priv, true);
5816
5817                 break;
5818         default:
5819                 BUG();
5820         }
5821 }
5822
5823 /*
5824  * Finds the encoder associated with the given CRTC. This can only be
5825  * used when we know that the CRTC isn't feeding multiple encoders!
5826  */
5827 static struct intel_encoder *
5828 intel_get_crtc_new_encoder(const struct intel_atomic_state *state,
5829                            const struct intel_crtc_state *crtc_state)
5830 {
5831         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5832         const struct drm_connector_state *connector_state;
5833         const struct drm_connector *connector;
5834         struct intel_encoder *encoder = NULL;
5835         int num_encoders = 0;
5836         int i;
5837
5838         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
5839                 if (connector_state->crtc != &crtc->base)
5840                         continue;
5841
5842                 encoder = to_intel_encoder(connector_state->best_encoder);
5843                 num_encoders++;
5844         }
5845
5846         drm_WARN(encoder->base.dev, num_encoders != 1,
5847                  "%d encoders for pipe %c\n",
5848                  num_encoders, pipe_name(crtc->pipe));
5849
5850         return encoder;
5851 }
5852
5853 /*
5854  * Enable PCH resources required for PCH ports:
5855  *   - PCH PLLs
5856  *   - FDI training & RX/TX
5857  *   - update transcoder timings
5858  *   - DP transcoding bits
5859  *   - transcoder
5860  */
5861 static void ilk_pch_enable(const struct intel_atomic_state *state,
5862                            const struct intel_crtc_state *crtc_state)
5863 {
5864         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5865         struct drm_device *dev = crtc->base.dev;
5866         struct drm_i915_private *dev_priv = to_i915(dev);
5867         enum pipe pipe = crtc->pipe;
5868         u32 temp;
5869
5870         assert_pch_transcoder_disabled(dev_priv, pipe);
5871
5872         if (IS_IVYBRIDGE(dev_priv))
5873                 ivb_update_fdi_bc_bifurcation(crtc_state);
5874
5875         /* Write the TU size bits before fdi link training, so that error
5876          * detection works. */
5877         intel_de_write(dev_priv, FDI_RX_TUSIZE1(pipe),
5878                        intel_de_read(dev_priv, PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
5879
5880         /* For PCH output, training FDI link */
5881         dev_priv->display.fdi_link_train(crtc, crtc_state);
5882
5883         /* We need to program the right clock selection before writing the pixel
5884          * mutliplier into the DPLL. */
5885         if (HAS_PCH_CPT(dev_priv)) {
5886                 u32 sel;
5887
5888                 temp = intel_de_read(dev_priv, PCH_DPLL_SEL);
5889                 temp |= TRANS_DPLL_ENABLE(pipe);
5890                 sel = TRANS_DPLLB_SEL(pipe);
5891                 if (crtc_state->shared_dpll ==
5892                     intel_get_shared_dpll_by_id(dev_priv, DPLL_ID_PCH_PLL_B))
5893                         temp |= sel;
5894                 else
5895                         temp &= ~sel;
5896                 intel_de_write(dev_priv, PCH_DPLL_SEL, temp);
5897         }
5898
5899         /* XXX: pch pll's can be enabled any time before we enable the PCH
5900          * transcoder, and we actually should do this to not upset any PCH
5901          * transcoder that already use the clock when we share it.
5902          *
5903          * Note that enable_shared_dpll tries to do the right thing, but
5904          * get_shared_dpll unconditionally resets the pll - we need that to have
5905          * the right LVDS enable sequence. */
5906         intel_enable_shared_dpll(crtc_state);
5907
5908         /* set transcoder timing, panel must allow it */
5909         assert_panel_unlocked(dev_priv, pipe);
5910         ilk_pch_transcoder_set_timings(crtc_state, pipe);
5911
5912         intel_fdi_normal_train(crtc);
5913
5914         /* For PCH DP, enable TRANS_DP_CTL */
5915         if (HAS_PCH_CPT(dev_priv) &&
5916             intel_crtc_has_dp_encoder(crtc_state)) {
5917                 const struct drm_display_mode *adjusted_mode =
5918                         &crtc_state->hw.adjusted_mode;
5919                 u32 bpc = (intel_de_read(dev_priv, PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
5920                 i915_reg_t reg = TRANS_DP_CTL(pipe);
5921                 enum port port;
5922
5923                 temp = intel_de_read(dev_priv, reg);
5924                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
5925                           TRANS_DP_SYNC_MASK |
5926                           TRANS_DP_BPC_MASK);
5927                 temp |= TRANS_DP_OUTPUT_ENABLE;
5928                 temp |= bpc << 9; /* same format but at 11:9 */
5929
5930                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
5931                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
5932                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
5933                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
5934
5935                 port = intel_get_crtc_new_encoder(state, crtc_state)->port;
5936                 drm_WARN_ON(dev, port < PORT_B || port > PORT_D);
5937                 temp |= TRANS_DP_PORT_SEL(port);
5938
5939                 intel_de_write(dev_priv, reg, temp);
5940         }
5941
5942         ilk_enable_pch_transcoder(crtc_state);
5943 }
5944
5945 void lpt_pch_enable(const struct intel_crtc_state *crtc_state)
5946 {
5947         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5948         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5949         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
5950
5951         assert_pch_transcoder_disabled(dev_priv, PIPE_A);
5952
5953         lpt_program_iclkip(crtc_state);
5954
5955         /* Set transcoder timing. */
5956         ilk_pch_transcoder_set_timings(crtc_state, PIPE_A);
5957
5958         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
5959 }
5960
5961 static void cpt_verify_modeset(struct drm_i915_private *dev_priv,
5962                                enum pipe pipe)
5963 {
5964         i915_reg_t dslreg = PIPEDSL(pipe);
5965         u32 temp;
5966
5967         temp = intel_de_read(dev_priv, dslreg);
5968         udelay(500);
5969         if (wait_for(intel_de_read(dev_priv, dslreg) != temp, 5)) {
5970                 if (wait_for(intel_de_read(dev_priv, dslreg) != temp, 5))
5971                         drm_err(&dev_priv->drm,
5972                                 "mode set failed: pipe %c stuck\n",
5973                                 pipe_name(pipe));
5974         }
5975 }
5976
5977 /*
5978  * The hardware phase 0.0 refers to the center of the pixel.
5979  * We want to start from the top/left edge which is phase
5980  * -0.5. That matches how the hardware calculates the scaling
5981  * factors (from top-left of the first pixel to bottom-right
5982  * of the last pixel, as opposed to the pixel centers).
5983  *
5984  * For 4:2:0 subsampled chroma planes we obviously have to
5985  * adjust that so that the chroma sample position lands in
5986  * the right spot.
5987  *
5988  * Note that for packed YCbCr 4:2:2 formats there is no way to
5989  * control chroma siting. The hardware simply replicates the
5990  * chroma samples for both of the luma samples, and thus we don't
5991  * actually get the expected MPEG2 chroma siting convention :(
5992  * The same behaviour is observed on pre-SKL platforms as well.
5993  *
5994  * Theory behind the formula (note that we ignore sub-pixel
5995  * source coordinates):
5996  * s = source sample position
5997  * d = destination sample position
5998  *
5999  * Downscaling 4:1:
6000  * -0.5
6001  * | 0.0
6002  * | |     1.5 (initial phase)
6003  * | |     |
6004  * v v     v
6005  * | s | s | s | s |
6006  * |       d       |
6007  *
6008  * Upscaling 1:4:
6009  * -0.5
6010  * | -0.375 (initial phase)
6011  * | |     0.0
6012  * | |     |
6013  * v v     v
6014  * |       s       |
6015  * | d | d | d | d |
6016  */
6017 u16 skl_scaler_calc_phase(int sub, int scale, bool chroma_cosited)
6018 {
6019         int phase = -0x8000;
6020         u16 trip = 0;
6021
6022         if (chroma_cosited)
6023                 phase += (sub - 1) * 0x8000 / sub;
6024
6025         phase += scale / (2 * sub);
6026
6027         /*
6028          * Hardware initial phase limited to [-0.5:1.5].
6029          * Since the max hardware scale factor is 3.0, we
6030          * should never actually excdeed 1.0 here.
6031          */
6032         WARN_ON(phase < -0x8000 || phase > 0x18000);
6033
6034         if (phase < 0)
6035                 phase = 0x10000 + phase;
6036         else
6037                 trip = PS_PHASE_TRIP;
6038
6039         return ((phase >> 2) & PS_PHASE_MASK) | trip;
6040 }
6041
6042 #define SKL_MIN_SRC_W 8
6043 #define SKL_MAX_SRC_W 4096
6044 #define SKL_MIN_SRC_H 8
6045 #define SKL_MAX_SRC_H 4096
6046 #define SKL_MIN_DST_W 8
6047 #define SKL_MAX_DST_W 4096
6048 #define SKL_MIN_DST_H 8
6049 #define SKL_MAX_DST_H 4096
6050 #define ICL_MAX_SRC_W 5120
6051 #define ICL_MAX_SRC_H 4096
6052 #define ICL_MAX_DST_W 5120
6053 #define ICL_MAX_DST_H 4096
6054 #define SKL_MIN_YUV_420_SRC_W 16
6055 #define SKL_MIN_YUV_420_SRC_H 16
6056
6057 static int
6058 skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
6059                   unsigned int scaler_user, int *scaler_id,
6060                   int src_w, int src_h, int dst_w, int dst_h,
6061                   const struct drm_format_info *format,
6062                   u64 modifier, bool need_scaler)
6063 {
6064         struct intel_crtc_scaler_state *scaler_state =
6065                 &crtc_state->scaler_state;
6066         struct intel_crtc *intel_crtc =
6067                 to_intel_crtc(crtc_state->uapi.crtc);
6068         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
6069         const struct drm_display_mode *adjusted_mode =
6070                 &crtc_state->hw.adjusted_mode;
6071
6072         /*
6073          * Src coordinates are already rotated by 270 degrees for
6074          * the 90/270 degree plane rotation cases (to match the
6075          * GTT mapping), hence no need to account for rotation here.
6076          */
6077         if (src_w != dst_w || src_h != dst_h)
6078                 need_scaler = true;
6079
6080         /*
6081          * Scaling/fitting not supported in IF-ID mode in GEN9+
6082          * TODO: Interlace fetch mode doesn't support YUV420 planar formats.
6083          * Once NV12 is enabled, handle it here while allocating scaler
6084          * for NV12.
6085          */
6086         if (INTEL_GEN(dev_priv) >= 9 && crtc_state->hw.enable &&
6087             need_scaler && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
6088                 drm_dbg_kms(&dev_priv->drm,
6089                             "Pipe/Plane scaling not supported with IF-ID mode\n");
6090                 return -EINVAL;
6091         }
6092
6093         /*
6094          * if plane is being disabled or scaler is no more required or force detach
6095          *  - free scaler binded to this plane/crtc
6096          *  - in order to do this, update crtc->scaler_usage
6097          *
6098          * Here scaler state in crtc_state is set free so that
6099          * scaler can be assigned to other user. Actual register
6100          * update to free the scaler is done in plane/panel-fit programming.
6101          * For this purpose crtc/plane_state->scaler_id isn't reset here.
6102          */
6103         if (force_detach || !need_scaler) {
6104                 if (*scaler_id >= 0) {
6105                         scaler_state->scaler_users &= ~(1 << scaler_user);
6106                         scaler_state->scalers[*scaler_id].in_use = 0;
6107
6108                         drm_dbg_kms(&dev_priv->drm,
6109                                     "scaler_user index %u.%u: "
6110                                     "Staged freeing scaler id %d scaler_users = 0x%x\n",
6111                                     intel_crtc->pipe, scaler_user, *scaler_id,
6112                                     scaler_state->scaler_users);
6113                         *scaler_id = -1;
6114                 }
6115                 return 0;
6116         }
6117
6118         if (format && intel_format_info_is_yuv_semiplanar(format, modifier) &&
6119             (src_h < SKL_MIN_YUV_420_SRC_H || src_w < SKL_MIN_YUV_420_SRC_W)) {
6120                 drm_dbg_kms(&dev_priv->drm,
6121                             "Planar YUV: src dimensions not met\n");
6122                 return -EINVAL;
6123         }
6124
6125         /* range checks */
6126         if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H ||
6127             dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
6128             (INTEL_GEN(dev_priv) >= 11 &&
6129              (src_w > ICL_MAX_SRC_W || src_h > ICL_MAX_SRC_H ||
6130               dst_w > ICL_MAX_DST_W || dst_h > ICL_MAX_DST_H)) ||
6131             (INTEL_GEN(dev_priv) < 11 &&
6132              (src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
6133               dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H))) {
6134                 drm_dbg_kms(&dev_priv->drm,
6135                             "scaler_user index %u.%u: src %ux%u dst %ux%u "
6136                             "size is out of scaler range\n",
6137                             intel_crtc->pipe, scaler_user, src_w, src_h,
6138                             dst_w, dst_h);
6139                 return -EINVAL;
6140         }
6141
6142         /* mark this plane as a scaler user in crtc_state */
6143         scaler_state->scaler_users |= (1 << scaler_user);
6144         drm_dbg_kms(&dev_priv->drm, "scaler_user index %u.%u: "
6145                     "staged scaling request for %ux%u->%ux%u scaler_users = 0x%x\n",
6146                     intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h,
6147                     scaler_state->scaler_users);
6148
6149         return 0;
6150 }
6151
6152 static int skl_update_scaler_crtc(struct intel_crtc_state *crtc_state)
6153 {
6154         const struct drm_display_mode *adjusted_mode =
6155                 &crtc_state->hw.adjusted_mode;
6156         int width, height;
6157
6158         if (crtc_state->pch_pfit.enabled) {
6159                 width = drm_rect_width(&crtc_state->pch_pfit.dst);
6160                 height = drm_rect_height(&crtc_state->pch_pfit.dst);
6161         } else {
6162                 width = adjusted_mode->crtc_hdisplay;
6163                 height = adjusted_mode->crtc_vdisplay;
6164         }
6165
6166         return skl_update_scaler(crtc_state, !crtc_state->hw.active,
6167                                  SKL_CRTC_INDEX,
6168                                  &crtc_state->scaler_state.scaler_id,
6169                                  crtc_state->pipe_src_w, crtc_state->pipe_src_h,
6170                                  width, height, NULL, 0,
6171                                  crtc_state->pch_pfit.enabled);
6172 }
6173
6174 /**
6175  * skl_update_scaler_plane - Stages update to scaler state for a given plane.
6176  * @crtc_state: crtc's scaler state
6177  * @plane_state: atomic plane state to update
6178  *
6179  * Return
6180  *     0 - scaler_usage updated successfully
6181  *    error - requested scaling cannot be supported or other error condition
6182  */
6183 static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
6184                                    struct intel_plane_state *plane_state)
6185 {
6186         struct intel_plane *intel_plane =
6187                 to_intel_plane(plane_state->uapi.plane);
6188         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
6189         struct drm_framebuffer *fb = plane_state->hw.fb;
6190         int ret;
6191         bool force_detach = !fb || !plane_state->uapi.visible;
6192         bool need_scaler = false;
6193
6194         /* Pre-gen11 and SDR planes always need a scaler for planar formats. */
6195         if (!icl_is_hdr_plane(dev_priv, intel_plane->id) &&
6196             fb && intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
6197                 need_scaler = true;
6198
6199         ret = skl_update_scaler(crtc_state, force_detach,
6200                                 drm_plane_index(&intel_plane->base),
6201                                 &plane_state->scaler_id,
6202                                 drm_rect_width(&plane_state->uapi.src) >> 16,
6203                                 drm_rect_height(&plane_state->uapi.src) >> 16,
6204                                 drm_rect_width(&plane_state->uapi.dst),
6205                                 drm_rect_height(&plane_state->uapi.dst),
6206                                 fb ? fb->format : NULL,
6207                                 fb ? fb->modifier : 0,
6208                                 need_scaler);
6209
6210         if (ret || plane_state->scaler_id < 0)
6211                 return ret;
6212
6213         /* check colorkey */
6214         if (plane_state->ckey.flags) {
6215                 drm_dbg_kms(&dev_priv->drm,
6216                             "[PLANE:%d:%s] scaling with color key not allowed",
6217                             intel_plane->base.base.id,
6218                             intel_plane->base.name);
6219                 return -EINVAL;
6220         }
6221
6222         /* Check src format */
6223         switch (fb->format->format) {
6224         case DRM_FORMAT_RGB565:
6225         case DRM_FORMAT_XBGR8888:
6226         case DRM_FORMAT_XRGB8888:
6227         case DRM_FORMAT_ABGR8888:
6228         case DRM_FORMAT_ARGB8888:
6229         case DRM_FORMAT_XRGB2101010:
6230         case DRM_FORMAT_XBGR2101010:
6231         case DRM_FORMAT_ARGB2101010:
6232         case DRM_FORMAT_ABGR2101010:
6233         case DRM_FORMAT_YUYV:
6234         case DRM_FORMAT_YVYU:
6235         case DRM_FORMAT_UYVY:
6236         case DRM_FORMAT_VYUY:
6237         case DRM_FORMAT_NV12:
6238         case DRM_FORMAT_XYUV8888:
6239         case DRM_FORMAT_P010:
6240         case DRM_FORMAT_P012:
6241         case DRM_FORMAT_P016:
6242         case DRM_FORMAT_Y210:
6243         case DRM_FORMAT_Y212:
6244         case DRM_FORMAT_Y216:
6245         case DRM_FORMAT_XVYU2101010:
6246         case DRM_FORMAT_XVYU12_16161616:
6247         case DRM_FORMAT_XVYU16161616:
6248                 break;
6249         case DRM_FORMAT_XBGR16161616F:
6250         case DRM_FORMAT_ABGR16161616F:
6251         case DRM_FORMAT_XRGB16161616F:
6252         case DRM_FORMAT_ARGB16161616F:
6253                 if (INTEL_GEN(dev_priv) >= 11)
6254                         break;
6255                 fallthrough;
6256         default:
6257                 drm_dbg_kms(&dev_priv->drm,
6258                             "[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n",
6259                             intel_plane->base.base.id, intel_plane->base.name,
6260                             fb->base.id, fb->format->format);
6261                 return -EINVAL;
6262         }
6263
6264         return 0;
6265 }
6266
6267 void skl_scaler_disable(const struct intel_crtc_state *old_crtc_state)
6268 {
6269         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
6270         int i;
6271
6272         for (i = 0; i < crtc->num_scalers; i++)
6273                 skl_detach_scaler(crtc, i);
6274 }
6275
6276 static void skl_pfit_enable(const struct intel_crtc_state *crtc_state)
6277 {
6278         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6279         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6280         const struct intel_crtc_scaler_state *scaler_state =
6281                 &crtc_state->scaler_state;
6282         struct drm_rect src = {
6283                 .x2 = crtc_state->pipe_src_w << 16,
6284                 .y2 = crtc_state->pipe_src_h << 16,
6285         };
6286         const struct drm_rect *dst = &crtc_state->pch_pfit.dst;
6287         u16 uv_rgb_hphase, uv_rgb_vphase;
6288         enum pipe pipe = crtc->pipe;
6289         int width = drm_rect_width(dst);
6290         int height = drm_rect_height(dst);
6291         int x = dst->x1;
6292         int y = dst->y1;
6293         int hscale, vscale;
6294         unsigned long irqflags;
6295         int id;
6296
6297         if (!crtc_state->pch_pfit.enabled)
6298                 return;
6299
6300         if (drm_WARN_ON(&dev_priv->drm,
6301                         crtc_state->scaler_state.scaler_id < 0))
6302                 return;
6303
6304         hscale = drm_rect_calc_hscale(&src, dst, 0, INT_MAX);
6305         vscale = drm_rect_calc_vscale(&src, dst, 0, INT_MAX);
6306
6307         uv_rgb_hphase = skl_scaler_calc_phase(1, hscale, false);
6308         uv_rgb_vphase = skl_scaler_calc_phase(1, vscale, false);
6309
6310         id = scaler_state->scaler_id;
6311
6312         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
6313
6314         intel_de_write_fw(dev_priv, SKL_PS_CTRL(pipe, id), PS_SCALER_EN |
6315                           PS_FILTER_MEDIUM | scaler_state->scalers[id].mode);
6316         intel_de_write_fw(dev_priv, SKL_PS_VPHASE(pipe, id),
6317                           PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_vphase));
6318         intel_de_write_fw(dev_priv, SKL_PS_HPHASE(pipe, id),
6319                           PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_hphase));
6320         intel_de_write_fw(dev_priv, SKL_PS_WIN_POS(pipe, id),
6321                           x << 16 | y);
6322         intel_de_write_fw(dev_priv, SKL_PS_WIN_SZ(pipe, id),
6323                           width << 16 | height);
6324
6325         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
6326 }
6327
6328 static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state)
6329 {
6330         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6331         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6332         const struct drm_rect *dst = &crtc_state->pch_pfit.dst;
6333         enum pipe pipe = crtc->pipe;
6334         int width = drm_rect_width(dst);
6335         int height = drm_rect_height(dst);
6336         int x = dst->x1;
6337         int y = dst->y1;
6338
6339         if (!crtc_state->pch_pfit.enabled)
6340                 return;
6341
6342         /* Force use of hard-coded filter coefficients
6343          * as some pre-programmed values are broken,
6344          * e.g. x201.
6345          */
6346         if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv))
6347                 intel_de_write(dev_priv, PF_CTL(pipe), PF_ENABLE |
6348                                PF_FILTER_MED_3x3 | PF_PIPE_SEL_IVB(pipe));
6349         else
6350                 intel_de_write(dev_priv, PF_CTL(pipe), PF_ENABLE |
6351                                PF_FILTER_MED_3x3);
6352         intel_de_write(dev_priv, PF_WIN_POS(pipe), x << 16 | y);
6353         intel_de_write(dev_priv, PF_WIN_SZ(pipe), width << 16 | height);
6354 }
6355
6356 void hsw_enable_ips(const struct intel_crtc_state *crtc_state)
6357 {
6358         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6359         struct drm_device *dev = crtc->base.dev;
6360         struct drm_i915_private *dev_priv = to_i915(dev);
6361
6362         if (!crtc_state->ips_enabled)
6363                 return;
6364
6365         /*
6366          * We can only enable IPS after we enable a plane and wait for a vblank
6367          * This function is called from post_plane_update, which is run after
6368          * a vblank wait.
6369          */
6370         drm_WARN_ON(dev, !(crtc_state->active_planes & ~BIT(PLANE_CURSOR)));
6371
6372         if (IS_BROADWELL(dev_priv)) {
6373                 drm_WARN_ON(dev, sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL,
6374                                                          IPS_ENABLE | IPS_PCODE_CONTROL));
6375                 /* Quoting Art Runyan: "its not safe to expect any particular
6376                  * value in IPS_CTL bit 31 after enabling IPS through the
6377                  * mailbox." Moreover, the mailbox may return a bogus state,
6378                  * so we need to just enable it and continue on.
6379                  */
6380         } else {
6381                 intel_de_write(dev_priv, IPS_CTL, IPS_ENABLE);
6382                 /* The bit only becomes 1 in the next vblank, so this wait here
6383                  * is essentially intel_wait_for_vblank. If we don't have this
6384                  * and don't wait for vblanks until the end of crtc_enable, then
6385                  * the HW state readout code will complain that the expected
6386                  * IPS_CTL value is not the one we read. */
6387                 if (intel_de_wait_for_set(dev_priv, IPS_CTL, IPS_ENABLE, 50))
6388                         drm_err(&dev_priv->drm,
6389                                 "Timed out waiting for IPS enable\n");
6390         }
6391 }
6392
6393 void hsw_disable_ips(const struct intel_crtc_state *crtc_state)
6394 {
6395         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6396         struct drm_device *dev = crtc->base.dev;
6397         struct drm_i915_private *dev_priv = to_i915(dev);
6398
6399         if (!crtc_state->ips_enabled)
6400                 return;
6401
6402         if (IS_BROADWELL(dev_priv)) {
6403                 drm_WARN_ON(dev,
6404                             sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
6405                 /*
6406                  * Wait for PCODE to finish disabling IPS. The BSpec specified
6407                  * 42ms timeout value leads to occasional timeouts so use 100ms
6408                  * instead.
6409                  */
6410                 if (intel_de_wait_for_clear(dev_priv, IPS_CTL, IPS_ENABLE, 100))
6411                         drm_err(&dev_priv->drm,
6412                                 "Timed out waiting for IPS disable\n");
6413         } else {
6414                 intel_de_write(dev_priv, IPS_CTL, 0);
6415                 intel_de_posting_read(dev_priv, IPS_CTL);
6416         }
6417
6418         /* We need to wait for a vblank before we can disable the plane. */
6419         intel_wait_for_vblank(dev_priv, crtc->pipe);
6420 }
6421
6422 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
6423 {
6424         if (intel_crtc->overlay)
6425                 (void) intel_overlay_switch_off(intel_crtc->overlay);
6426
6427         /* Let userspace switch the overlay on again. In most cases userspace
6428          * has to recompute where to put it anyway.
6429          */
6430 }
6431
6432 static bool hsw_pre_update_disable_ips(const struct intel_crtc_state *old_crtc_state,
6433                                        const struct intel_crtc_state *new_crtc_state)
6434 {
6435         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6436         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6437
6438         if (!old_crtc_state->ips_enabled)
6439                 return false;
6440
6441         if (needs_modeset(new_crtc_state))
6442                 return true;
6443
6444         /*
6445          * Workaround : Do not read or write the pipe palette/gamma data while
6446          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
6447          *
6448          * Disable IPS before we program the LUT.
6449          */
6450         if (IS_HASWELL(dev_priv) &&
6451             (new_crtc_state->uapi.color_mgmt_changed ||
6452              new_crtc_state->update_pipe) &&
6453             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
6454                 return true;
6455
6456         return !new_crtc_state->ips_enabled;
6457 }
6458
6459 static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_state,
6460                                        const struct intel_crtc_state *new_crtc_state)
6461 {
6462         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6463         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6464
6465         if (!new_crtc_state->ips_enabled)
6466                 return false;
6467
6468         if (needs_modeset(new_crtc_state))
6469                 return true;
6470
6471         /*
6472          * Workaround : Do not read or write the pipe palette/gamma data while
6473          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
6474          *
6475          * Re-enable IPS after the LUT has been programmed.
6476          */
6477         if (IS_HASWELL(dev_priv) &&
6478             (new_crtc_state->uapi.color_mgmt_changed ||
6479              new_crtc_state->update_pipe) &&
6480             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
6481                 return true;
6482
6483         /*
6484          * We can't read out IPS on broadwell, assume the worst and
6485          * forcibly enable IPS on the first fastset.
6486          */
6487         if (new_crtc_state->update_pipe && old_crtc_state->inherited)
6488                 return true;
6489
6490         return !old_crtc_state->ips_enabled;
6491 }
6492
6493 static bool needs_nv12_wa(const struct intel_crtc_state *crtc_state)
6494 {
6495         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
6496
6497         if (!crtc_state->nv12_planes)
6498                 return false;
6499
6500         /* WA Display #0827: Gen9:all */
6501         if (IS_GEN(dev_priv, 9) && !IS_GEMINILAKE(dev_priv))
6502                 return true;
6503
6504         return false;
6505 }
6506
6507 static bool needs_scalerclk_wa(const struct intel_crtc_state *crtc_state)
6508 {
6509         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
6510
6511         /* Wa_2006604312:icl,ehl */
6512         if (crtc_state->scaler_state.scaler_users > 0 && IS_GEN(dev_priv, 11))
6513                 return true;
6514
6515         return false;
6516 }
6517
6518 static bool planes_enabling(const struct intel_crtc_state *old_crtc_state,
6519                             const struct intel_crtc_state *new_crtc_state)
6520 {
6521         return (!old_crtc_state->active_planes || needs_modeset(new_crtc_state)) &&
6522                 new_crtc_state->active_planes;
6523 }
6524
6525 static bool planes_disabling(const struct intel_crtc_state *old_crtc_state,
6526                              const struct intel_crtc_state *new_crtc_state)
6527 {
6528         return old_crtc_state->active_planes &&
6529                 (!new_crtc_state->active_planes || needs_modeset(new_crtc_state));
6530 }
6531
6532 static void intel_post_plane_update(struct intel_atomic_state *state,
6533                                     struct intel_crtc *crtc)
6534 {
6535         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6536         const struct intel_crtc_state *old_crtc_state =
6537                 intel_atomic_get_old_crtc_state(state, crtc);
6538         const struct intel_crtc_state *new_crtc_state =
6539                 intel_atomic_get_new_crtc_state(state, crtc);
6540         enum pipe pipe = crtc->pipe;
6541
6542         intel_frontbuffer_flip(dev_priv, new_crtc_state->fb_bits);
6543
6544         if (new_crtc_state->update_wm_post && new_crtc_state->hw.active)
6545                 intel_update_watermarks(crtc);
6546
6547         if (hsw_post_update_enable_ips(old_crtc_state, new_crtc_state))
6548                 hsw_enable_ips(new_crtc_state);
6549
6550         intel_fbc_post_update(state, crtc);
6551
6552         if (needs_nv12_wa(old_crtc_state) &&
6553             !needs_nv12_wa(new_crtc_state))
6554                 skl_wa_827(dev_priv, pipe, false);
6555
6556         if (needs_scalerclk_wa(old_crtc_state) &&
6557             !needs_scalerclk_wa(new_crtc_state))
6558                 icl_wa_scalerclkgating(dev_priv, pipe, false);
6559 }
6560
6561 static void intel_pre_plane_update(struct intel_atomic_state *state,
6562                                    struct intel_crtc *crtc)
6563 {
6564         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6565         const struct intel_crtc_state *old_crtc_state =
6566                 intel_atomic_get_old_crtc_state(state, crtc);
6567         const struct intel_crtc_state *new_crtc_state =
6568                 intel_atomic_get_new_crtc_state(state, crtc);
6569         enum pipe pipe = crtc->pipe;
6570
6571         if (hsw_pre_update_disable_ips(old_crtc_state, new_crtc_state))
6572                 hsw_disable_ips(old_crtc_state);
6573
6574         if (intel_fbc_pre_update(state, crtc))
6575                 intel_wait_for_vblank(dev_priv, pipe);
6576
6577         /* Display WA 827 */
6578         if (!needs_nv12_wa(old_crtc_state) &&
6579             needs_nv12_wa(new_crtc_state))
6580                 skl_wa_827(dev_priv, pipe, true);
6581
6582         /* Wa_2006604312:icl,ehl */
6583         if (!needs_scalerclk_wa(old_crtc_state) &&
6584             needs_scalerclk_wa(new_crtc_state))
6585                 icl_wa_scalerclkgating(dev_priv, pipe, true);
6586
6587         /*
6588          * Vblank time updates from the shadow to live plane control register
6589          * are blocked if the memory self-refresh mode is active at that
6590          * moment. So to make sure the plane gets truly disabled, disable
6591          * first the self-refresh mode. The self-refresh enable bit in turn
6592          * will be checked/applied by the HW only at the next frame start
6593          * event which is after the vblank start event, so we need to have a
6594          * wait-for-vblank between disabling the plane and the pipe.
6595          */
6596         if (HAS_GMCH(dev_priv) && old_crtc_state->hw.active &&
6597             new_crtc_state->disable_cxsr && intel_set_memory_cxsr(dev_priv, false))
6598                 intel_wait_for_vblank(dev_priv, pipe);
6599
6600         /*
6601          * IVB workaround: must disable low power watermarks for at least
6602          * one frame before enabling scaling.  LP watermarks can be re-enabled
6603          * when scaling is disabled.
6604          *
6605          * WaCxSRDisabledForSpriteScaling:ivb
6606          */
6607         if (old_crtc_state->hw.active &&
6608             new_crtc_state->disable_lp_wm && ilk_disable_lp_wm(dev_priv))
6609                 intel_wait_for_vblank(dev_priv, pipe);
6610
6611         /*
6612          * If we're doing a modeset we don't need to do any
6613          * pre-vblank watermark programming here.
6614          */
6615         if (!needs_modeset(new_crtc_state)) {
6616                 /*
6617                  * For platforms that support atomic watermarks, program the
6618                  * 'intermediate' watermarks immediately.  On pre-gen9 platforms, these
6619                  * will be the intermediate values that are safe for both pre- and
6620                  * post- vblank; when vblank happens, the 'active' values will be set
6621                  * to the final 'target' values and we'll do this again to get the
6622                  * optimal watermarks.  For gen9+ platforms, the values we program here
6623                  * will be the final target values which will get automatically latched
6624                  * at vblank time; no further programming will be necessary.
6625                  *
6626                  * If a platform hasn't been transitioned to atomic watermarks yet,
6627                  * we'll continue to update watermarks the old way, if flags tell
6628                  * us to.
6629                  */
6630                 if (dev_priv->display.initial_watermarks)
6631                         dev_priv->display.initial_watermarks(state, crtc);
6632                 else if (new_crtc_state->update_wm_pre)
6633                         intel_update_watermarks(crtc);
6634         }
6635
6636         /*
6637          * Gen2 reports pipe underruns whenever all planes are disabled.
6638          * So disable underrun reporting before all the planes get disabled.
6639          *
6640          * We do this after .initial_watermarks() so that we have a
6641          * chance of catching underruns with the intermediate watermarks
6642          * vs. the old plane configuration.
6643          */
6644         if (IS_GEN(dev_priv, 2) && planes_disabling(old_crtc_state, new_crtc_state))
6645                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6646 }
6647
6648 static void intel_crtc_disable_planes(struct intel_atomic_state *state,
6649                                       struct intel_crtc *crtc)
6650 {
6651         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6652         const struct intel_crtc_state *new_crtc_state =
6653                 intel_atomic_get_new_crtc_state(state, crtc);
6654         unsigned int update_mask = new_crtc_state->update_planes;
6655         const struct intel_plane_state *old_plane_state;
6656         struct intel_plane *plane;
6657         unsigned fb_bits = 0;
6658         int i;
6659
6660         intel_crtc_dpms_overlay_disable(crtc);
6661
6662         for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
6663                 if (crtc->pipe != plane->pipe ||
6664                     !(update_mask & BIT(plane->id)))
6665                         continue;
6666
6667                 intel_disable_plane(plane, new_crtc_state);
6668
6669                 if (old_plane_state->uapi.visible)
6670                         fb_bits |= plane->frontbuffer_bit;
6671         }
6672
6673         intel_frontbuffer_flip(dev_priv, fb_bits);
6674 }
6675
6676 /*
6677  * intel_connector_primary_encoder - get the primary encoder for a connector
6678  * @connector: connector for which to return the encoder
6679  *
6680  * Returns the primary encoder for a connector. There is a 1:1 mapping from
6681  * all connectors to their encoder, except for DP-MST connectors which have
6682  * both a virtual and a primary encoder. These DP-MST primary encoders can be
6683  * pointed to by as many DP-MST connectors as there are pipes.
6684  */
6685 static struct intel_encoder *
6686 intel_connector_primary_encoder(struct intel_connector *connector)
6687 {
6688         struct intel_encoder *encoder;
6689
6690         if (connector->mst_port)
6691                 return &dp_to_dig_port(connector->mst_port)->base;
6692
6693         encoder = intel_attached_encoder(connector);
6694         drm_WARN_ON(connector->base.dev, !encoder);
6695
6696         return encoder;
6697 }
6698
6699 static void intel_encoders_update_prepare(struct intel_atomic_state *state)
6700 {
6701         struct drm_connector_state *new_conn_state;
6702         struct drm_connector *connector;
6703         int i;
6704
6705         for_each_new_connector_in_state(&state->base, connector, new_conn_state,
6706                                         i) {
6707                 struct intel_connector *intel_connector;
6708                 struct intel_encoder *encoder;
6709                 struct intel_crtc *crtc;
6710
6711                 if (!intel_connector_needs_modeset(state, connector))
6712                         continue;
6713
6714                 intel_connector = to_intel_connector(connector);
6715                 encoder = intel_connector_primary_encoder(intel_connector);
6716                 if (!encoder->update_prepare)
6717                         continue;
6718
6719                 crtc = new_conn_state->crtc ?
6720                         to_intel_crtc(new_conn_state->crtc) : NULL;
6721                 encoder->update_prepare(state, encoder, crtc);
6722         }
6723 }
6724
6725 static void intel_encoders_update_complete(struct intel_atomic_state *state)
6726 {
6727         struct drm_connector_state *new_conn_state;
6728         struct drm_connector *connector;
6729         int i;
6730
6731         for_each_new_connector_in_state(&state->base, connector, new_conn_state,
6732                                         i) {
6733                 struct intel_connector *intel_connector;
6734                 struct intel_encoder *encoder;
6735                 struct intel_crtc *crtc;
6736
6737                 if (!intel_connector_needs_modeset(state, connector))
6738                         continue;
6739
6740                 intel_connector = to_intel_connector(connector);
6741                 encoder = intel_connector_primary_encoder(intel_connector);
6742                 if (!encoder->update_complete)
6743                         continue;
6744
6745                 crtc = new_conn_state->crtc ?
6746                         to_intel_crtc(new_conn_state->crtc) : NULL;
6747                 encoder->update_complete(state, encoder, crtc);
6748         }
6749 }
6750
6751 static void intel_encoders_pre_pll_enable(struct intel_atomic_state *state,
6752                                           struct intel_crtc *crtc)
6753 {
6754         const struct intel_crtc_state *crtc_state =
6755                 intel_atomic_get_new_crtc_state(state, crtc);
6756         const struct drm_connector_state *conn_state;
6757         struct drm_connector *conn;
6758         int i;
6759
6760         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6761                 struct intel_encoder *encoder =
6762                         to_intel_encoder(conn_state->best_encoder);
6763
6764                 if (conn_state->crtc != &crtc->base)
6765                         continue;
6766
6767                 if (encoder->pre_pll_enable)
6768                         encoder->pre_pll_enable(state, encoder,
6769                                                 crtc_state, conn_state);
6770         }
6771 }
6772
6773 static void intel_encoders_pre_enable(struct intel_atomic_state *state,
6774                                       struct intel_crtc *crtc)
6775 {
6776         const struct intel_crtc_state *crtc_state =
6777                 intel_atomic_get_new_crtc_state(state, crtc);
6778         const struct drm_connector_state *conn_state;
6779         struct drm_connector *conn;
6780         int i;
6781
6782         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6783                 struct intel_encoder *encoder =
6784                         to_intel_encoder(conn_state->best_encoder);
6785
6786                 if (conn_state->crtc != &crtc->base)
6787                         continue;
6788
6789                 if (encoder->pre_enable)
6790                         encoder->pre_enable(state, encoder,
6791                                             crtc_state, conn_state);
6792         }
6793 }
6794
6795 static void intel_encoders_enable(struct intel_atomic_state *state,
6796                                   struct intel_crtc *crtc)
6797 {
6798         const struct intel_crtc_state *crtc_state =
6799                 intel_atomic_get_new_crtc_state(state, crtc);
6800         const struct drm_connector_state *conn_state;
6801         struct drm_connector *conn;
6802         int i;
6803
6804         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6805                 struct intel_encoder *encoder =
6806                         to_intel_encoder(conn_state->best_encoder);
6807
6808                 if (conn_state->crtc != &crtc->base)
6809                         continue;
6810
6811                 if (encoder->enable)
6812                         encoder->enable(state, encoder,
6813                                         crtc_state, conn_state);
6814                 intel_opregion_notify_encoder(encoder, true);
6815         }
6816 }
6817
6818 static void intel_encoders_disable(struct intel_atomic_state *state,
6819                                    struct intel_crtc *crtc)
6820 {
6821         const struct intel_crtc_state *old_crtc_state =
6822                 intel_atomic_get_old_crtc_state(state, crtc);
6823         const struct drm_connector_state *old_conn_state;
6824         struct drm_connector *conn;
6825         int i;
6826
6827         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6828                 struct intel_encoder *encoder =
6829                         to_intel_encoder(old_conn_state->best_encoder);
6830
6831                 if (old_conn_state->crtc != &crtc->base)
6832                         continue;
6833
6834                 intel_opregion_notify_encoder(encoder, false);
6835                 if (encoder->disable)
6836                         encoder->disable(state, encoder,
6837                                          old_crtc_state, old_conn_state);
6838         }
6839 }
6840
6841 static void intel_encoders_post_disable(struct intel_atomic_state *state,
6842                                         struct intel_crtc *crtc)
6843 {
6844         const struct intel_crtc_state *old_crtc_state =
6845                 intel_atomic_get_old_crtc_state(state, crtc);
6846         const struct drm_connector_state *old_conn_state;
6847         struct drm_connector *conn;
6848         int i;
6849
6850         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6851                 struct intel_encoder *encoder =
6852                         to_intel_encoder(old_conn_state->best_encoder);
6853
6854                 if (old_conn_state->crtc != &crtc->base)
6855                         continue;
6856
6857                 if (encoder->post_disable)
6858                         encoder->post_disable(state, encoder,
6859                                               old_crtc_state, old_conn_state);
6860         }
6861 }
6862
6863 static void intel_encoders_post_pll_disable(struct intel_atomic_state *state,
6864                                             struct intel_crtc *crtc)
6865 {
6866         const struct intel_crtc_state *old_crtc_state =
6867                 intel_atomic_get_old_crtc_state(state, crtc);
6868         const struct drm_connector_state *old_conn_state;
6869         struct drm_connector *conn;
6870         int i;
6871
6872         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6873                 struct intel_encoder *encoder =
6874                         to_intel_encoder(old_conn_state->best_encoder);
6875
6876                 if (old_conn_state->crtc != &crtc->base)
6877                         continue;
6878
6879                 if (encoder->post_pll_disable)
6880                         encoder->post_pll_disable(state, encoder,
6881                                                   old_crtc_state, old_conn_state);
6882         }
6883 }
6884
6885 static void intel_encoders_update_pipe(struct intel_atomic_state *state,
6886                                        struct intel_crtc *crtc)
6887 {
6888         const struct intel_crtc_state *crtc_state =
6889                 intel_atomic_get_new_crtc_state(state, crtc);
6890         const struct drm_connector_state *conn_state;
6891         struct drm_connector *conn;
6892         int i;
6893
6894         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6895                 struct intel_encoder *encoder =
6896                         to_intel_encoder(conn_state->best_encoder);
6897
6898                 if (conn_state->crtc != &crtc->base)
6899                         continue;
6900
6901                 if (encoder->update_pipe)
6902                         encoder->update_pipe(state, encoder,
6903                                              crtc_state, conn_state);
6904         }
6905 }
6906
6907 static void intel_disable_primary_plane(const struct intel_crtc_state *crtc_state)
6908 {
6909         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6910         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
6911
6912         plane->disable_plane(plane, crtc_state);
6913 }
6914
6915 static void ilk_crtc_enable(struct intel_atomic_state *state,
6916                             struct intel_crtc *crtc)
6917 {
6918         const struct intel_crtc_state *new_crtc_state =
6919                 intel_atomic_get_new_crtc_state(state, crtc);
6920         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6921         enum pipe pipe = crtc->pipe;
6922
6923         if (drm_WARN_ON(&dev_priv->drm, crtc->active))
6924                 return;
6925
6926         /*
6927          * Sometimes spurious CPU pipe underruns happen during FDI
6928          * training, at least with VGA+HDMI cloning. Suppress them.
6929          *
6930          * On ILK we get an occasional spurious CPU pipe underruns
6931          * between eDP port A enable and vdd enable. Also PCH port
6932          * enable seems to result in the occasional CPU pipe underrun.
6933          *
6934          * Spurious PCH underruns also occur during PCH enabling.
6935          */
6936         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6937         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
6938
6939         if (new_crtc_state->has_pch_encoder)
6940                 intel_prepare_shared_dpll(new_crtc_state);
6941
6942         if (intel_crtc_has_dp_encoder(new_crtc_state))
6943                 intel_dp_set_m_n(new_crtc_state, M1_N1);
6944
6945         intel_set_pipe_timings(new_crtc_state);
6946         intel_set_pipe_src_size(new_crtc_state);
6947
6948         if (new_crtc_state->has_pch_encoder)
6949                 intel_cpu_transcoder_set_m_n(new_crtc_state,
6950                                              &new_crtc_state->fdi_m_n, NULL);
6951
6952         ilk_set_pipeconf(new_crtc_state);
6953
6954         crtc->active = true;
6955
6956         intel_encoders_pre_enable(state, crtc);
6957
6958         if (new_crtc_state->has_pch_encoder) {
6959                 /* Note: FDI PLL enabling _must_ be done before we enable the
6960                  * cpu pipes, hence this is separate from all the other fdi/pch
6961                  * enabling. */
6962                 ilk_fdi_pll_enable(new_crtc_state);
6963         } else {
6964                 assert_fdi_tx_disabled(dev_priv, pipe);
6965                 assert_fdi_rx_disabled(dev_priv, pipe);
6966         }
6967
6968         ilk_pfit_enable(new_crtc_state);
6969
6970         /*
6971          * On ILK+ LUT must be loaded before the pipe is running but with
6972          * clocks enabled
6973          */
6974         intel_color_load_luts(new_crtc_state);
6975         intel_color_commit(new_crtc_state);
6976         /* update DSPCNTR to configure gamma for pipe bottom color */
6977         intel_disable_primary_plane(new_crtc_state);
6978
6979         if (dev_priv->display.initial_watermarks)
6980                 dev_priv->display.initial_watermarks(state, crtc);
6981         intel_enable_pipe(new_crtc_state);
6982
6983         if (new_crtc_state->has_pch_encoder)
6984                 ilk_pch_enable(state, new_crtc_state);
6985
6986         intel_crtc_vblank_on(new_crtc_state);
6987
6988         intel_encoders_enable(state, crtc);
6989
6990         if (HAS_PCH_CPT(dev_priv))
6991                 cpt_verify_modeset(dev_priv, pipe);
6992
6993         /*
6994          * Must wait for vblank to avoid spurious PCH FIFO underruns.
6995          * And a second vblank wait is needed at least on ILK with
6996          * some interlaced HDMI modes. Let's do the double wait always
6997          * in case there are more corner cases we don't know about.
6998          */
6999         if (new_crtc_state->has_pch_encoder) {
7000                 intel_wait_for_vblank(dev_priv, pipe);
7001                 intel_wait_for_vblank(dev_priv, pipe);
7002         }
7003         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
7004         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
7005 }
7006
7007 /* IPS only exists on ULT machines and is tied to pipe A. */
7008 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
7009 {
7010         return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A;
7011 }
7012
7013 static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv,
7014                                             enum pipe pipe, bool apply)
7015 {
7016         u32 val = intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe));
7017         u32 mask = DPF_GATING_DIS | DPF_RAM_GATING_DIS | DPFR_GATING_DIS;
7018
7019         if (apply)
7020                 val |= mask;
7021         else
7022                 val &= ~mask;
7023
7024         intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe), val);
7025 }
7026
7027 static void icl_pipe_mbus_enable(struct intel_crtc *crtc)
7028 {
7029         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7030         enum pipe pipe = crtc->pipe;
7031         u32 val;
7032
7033         val = MBUS_DBOX_A_CREDIT(2);
7034
7035         if (INTEL_GEN(dev_priv) >= 12) {
7036                 val |= MBUS_DBOX_BW_CREDIT(2);
7037                 val |= MBUS_DBOX_B_CREDIT(12);
7038         } else {
7039                 val |= MBUS_DBOX_BW_CREDIT(1);
7040                 val |= MBUS_DBOX_B_CREDIT(8);
7041         }
7042
7043         intel_de_write(dev_priv, PIPE_MBUS_DBOX_CTL(pipe), val);
7044 }
7045
7046 static void hsw_set_linetime_wm(const struct intel_crtc_state *crtc_state)
7047 {
7048         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7049         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7050
7051         intel_de_write(dev_priv, WM_LINETIME(crtc->pipe),
7052                        HSW_LINETIME(crtc_state->linetime) |
7053                        HSW_IPS_LINETIME(crtc_state->ips_linetime));
7054 }
7055
7056 static void hsw_set_frame_start_delay(const struct intel_crtc_state *crtc_state)
7057 {
7058         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7059         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7060         i915_reg_t reg = CHICKEN_TRANS(crtc_state->cpu_transcoder);
7061         u32 val;
7062
7063         val = intel_de_read(dev_priv, reg);
7064         val &= ~HSW_FRAME_START_DELAY_MASK;
7065         val |= HSW_FRAME_START_DELAY(0);
7066         intel_de_write(dev_priv, reg, val);
7067 }
7068
7069 static void hsw_crtc_enable(struct intel_atomic_state *state,
7070                             struct intel_crtc *crtc)
7071 {
7072         const struct intel_crtc_state *new_crtc_state =
7073                 intel_atomic_get_new_crtc_state(state, crtc);
7074         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7075         enum pipe pipe = crtc->pipe, hsw_workaround_pipe;
7076         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
7077         bool psl_clkgate_wa;
7078
7079         if (drm_WARN_ON(&dev_priv->drm, crtc->active))
7080                 return;
7081
7082         intel_encoders_pre_pll_enable(state, crtc);
7083
7084         if (new_crtc_state->shared_dpll)
7085                 intel_enable_shared_dpll(new_crtc_state);
7086
7087         intel_encoders_pre_enable(state, crtc);
7088
7089         if (!transcoder_is_dsi(cpu_transcoder))
7090                 intel_set_pipe_timings(new_crtc_state);
7091
7092         intel_set_pipe_src_size(new_crtc_state);
7093
7094         if (cpu_transcoder != TRANSCODER_EDP &&
7095             !transcoder_is_dsi(cpu_transcoder))
7096                 intel_de_write(dev_priv, PIPE_MULT(cpu_transcoder),
7097                                new_crtc_state->pixel_multiplier - 1);
7098
7099         if (new_crtc_state->has_pch_encoder)
7100                 intel_cpu_transcoder_set_m_n(new_crtc_state,
7101                                              &new_crtc_state->fdi_m_n, NULL);
7102
7103         if (!transcoder_is_dsi(cpu_transcoder)) {
7104                 hsw_set_frame_start_delay(new_crtc_state);
7105                 hsw_set_pipeconf(new_crtc_state);
7106         }
7107
7108         if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
7109                 bdw_set_pipemisc(new_crtc_state);
7110
7111         crtc->active = true;
7112
7113         /* Display WA #1180: WaDisableScalarClockGating: glk, cnl */
7114         psl_clkgate_wa = (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) &&
7115                 new_crtc_state->pch_pfit.enabled;
7116         if (psl_clkgate_wa)
7117                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, true);
7118
7119         if (INTEL_GEN(dev_priv) >= 9)
7120                 skl_pfit_enable(new_crtc_state);
7121         else
7122                 ilk_pfit_enable(new_crtc_state);
7123
7124         /*
7125          * On ILK+ LUT must be loaded before the pipe is running but with
7126          * clocks enabled
7127          */
7128         intel_color_load_luts(new_crtc_state);
7129         intel_color_commit(new_crtc_state);
7130         /* update DSPCNTR to configure gamma/csc for pipe bottom color */
7131         if (INTEL_GEN(dev_priv) < 9)
7132                 intel_disable_primary_plane(new_crtc_state);
7133
7134         hsw_set_linetime_wm(new_crtc_state);
7135
7136         if (INTEL_GEN(dev_priv) >= 11)
7137                 icl_set_pipe_chicken(crtc);
7138
7139         if (dev_priv->display.initial_watermarks)
7140                 dev_priv->display.initial_watermarks(state, crtc);
7141
7142         if (INTEL_GEN(dev_priv) >= 11)
7143                 icl_pipe_mbus_enable(crtc);
7144
7145         intel_encoders_enable(state, crtc);
7146
7147         if (psl_clkgate_wa) {
7148                 intel_wait_for_vblank(dev_priv, pipe);
7149                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, false);
7150         }
7151
7152         /* If we change the relative order between pipe/planes enabling, we need
7153          * to change the workaround. */
7154         hsw_workaround_pipe = new_crtc_state->hsw_workaround_pipe;
7155         if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
7156                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
7157                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
7158         }
7159 }
7160
7161 void ilk_pfit_disable(const struct intel_crtc_state *old_crtc_state)
7162 {
7163         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
7164         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7165         enum pipe pipe = crtc->pipe;
7166
7167         /* To avoid upsetting the power well on haswell only disable the pfit if
7168          * it's in use. The hw state code will make sure we get this right. */
7169         if (!old_crtc_state->pch_pfit.enabled)
7170                 return;
7171
7172         intel_de_write(dev_priv, PF_CTL(pipe), 0);
7173         intel_de_write(dev_priv, PF_WIN_POS(pipe), 0);
7174         intel_de_write(dev_priv, PF_WIN_SZ(pipe), 0);
7175 }
7176
7177 static void ilk_crtc_disable(struct intel_atomic_state *state,
7178                              struct intel_crtc *crtc)
7179 {
7180         const struct intel_crtc_state *old_crtc_state =
7181                 intel_atomic_get_old_crtc_state(state, crtc);
7182         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7183         enum pipe pipe = crtc->pipe;
7184
7185         /*
7186          * Sometimes spurious CPU pipe underruns happen when the
7187          * pipe is already disabled, but FDI RX/TX is still enabled.
7188          * Happens at least with VGA+HDMI cloning. Suppress them.
7189          */
7190         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
7191         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
7192
7193         intel_encoders_disable(state, crtc);
7194
7195         intel_crtc_vblank_off(old_crtc_state);
7196
7197         intel_disable_pipe(old_crtc_state);
7198
7199         ilk_pfit_disable(old_crtc_state);
7200
7201         if (old_crtc_state->has_pch_encoder)
7202                 ilk_fdi_disable(crtc);
7203
7204         intel_encoders_post_disable(state, crtc);
7205
7206         if (old_crtc_state->has_pch_encoder) {
7207                 ilk_disable_pch_transcoder(dev_priv, pipe);
7208
7209                 if (HAS_PCH_CPT(dev_priv)) {
7210                         i915_reg_t reg;
7211                         u32 temp;
7212
7213                         /* disable TRANS_DP_CTL */
7214                         reg = TRANS_DP_CTL(pipe);
7215                         temp = intel_de_read(dev_priv, reg);
7216                         temp &= ~(TRANS_DP_OUTPUT_ENABLE |
7217                                   TRANS_DP_PORT_SEL_MASK);
7218                         temp |= TRANS_DP_PORT_SEL_NONE;
7219                         intel_de_write(dev_priv, reg, temp);
7220
7221                         /* disable DPLL_SEL */
7222                         temp = intel_de_read(dev_priv, PCH_DPLL_SEL);
7223                         temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
7224                         intel_de_write(dev_priv, PCH_DPLL_SEL, temp);
7225                 }
7226
7227                 ilk_fdi_pll_disable(crtc);
7228         }
7229
7230         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
7231         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
7232 }
7233
7234 static void hsw_crtc_disable(struct intel_atomic_state *state,
7235                              struct intel_crtc *crtc)
7236 {
7237         /*
7238          * FIXME collapse everything to one hook.
7239          * Need care with mst->ddi interactions.
7240          */
7241         intel_encoders_disable(state, crtc);
7242         intel_encoders_post_disable(state, crtc);
7243 }
7244
7245 static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state)
7246 {
7247         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7248         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7249
7250         if (!crtc_state->gmch_pfit.control)
7251                 return;
7252
7253         /*
7254          * The panel fitter should only be adjusted whilst the pipe is disabled,
7255          * according to register description and PRM.
7256          */
7257         drm_WARN_ON(&dev_priv->drm,
7258                     intel_de_read(dev_priv, PFIT_CONTROL) & PFIT_ENABLE);
7259         assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder);
7260
7261         intel_de_write(dev_priv, PFIT_PGM_RATIOS,
7262                        crtc_state->gmch_pfit.pgm_ratios);
7263         intel_de_write(dev_priv, PFIT_CONTROL, crtc_state->gmch_pfit.control);
7264
7265         /* Border color in case we don't scale up to the full screen. Black by
7266          * default, change to something else for debugging. */
7267         intel_de_write(dev_priv, BCLRPAT(crtc->pipe), 0);
7268 }
7269
7270 bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy)
7271 {
7272         if (phy == PHY_NONE)
7273                 return false;
7274         else if (IS_ROCKETLAKE(dev_priv))
7275                 return phy <= PHY_D;
7276         else if (IS_ELKHARTLAKE(dev_priv))
7277                 return phy <= PHY_C;
7278         else if (INTEL_GEN(dev_priv) >= 11)
7279                 return phy <= PHY_B;
7280         else
7281                 return false;
7282 }
7283
7284 bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy)
7285 {
7286         if (IS_ROCKETLAKE(dev_priv))
7287                 return false;
7288         else if (INTEL_GEN(dev_priv) >= 12)
7289                 return phy >= PHY_D && phy <= PHY_I;
7290         else if (INTEL_GEN(dev_priv) >= 11 && !IS_ELKHARTLAKE(dev_priv))
7291                 return phy >= PHY_C && phy <= PHY_F;
7292         else
7293                 return false;
7294 }
7295
7296 enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port)
7297 {
7298         if (IS_ROCKETLAKE(i915) && port >= PORT_D)
7299                 return (enum phy)port - 1;
7300         else if (IS_ELKHARTLAKE(i915) && port == PORT_D)
7301                 return PHY_A;
7302
7303         return (enum phy)port;
7304 }
7305
7306 enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port)
7307 {
7308         if (!intel_phy_is_tc(dev_priv, intel_port_to_phy(dev_priv, port)))
7309                 return PORT_TC_NONE;
7310
7311         if (INTEL_GEN(dev_priv) >= 12)
7312                 return port - PORT_D;
7313
7314         return port - PORT_C;
7315 }
7316
7317 enum intel_display_power_domain intel_port_to_power_domain(enum port port)
7318 {
7319         switch (port) {
7320         case PORT_A:
7321                 return POWER_DOMAIN_PORT_DDI_A_LANES;
7322         case PORT_B:
7323                 return POWER_DOMAIN_PORT_DDI_B_LANES;
7324         case PORT_C:
7325                 return POWER_DOMAIN_PORT_DDI_C_LANES;
7326         case PORT_D:
7327                 return POWER_DOMAIN_PORT_DDI_D_LANES;
7328         case PORT_E:
7329                 return POWER_DOMAIN_PORT_DDI_E_LANES;
7330         case PORT_F:
7331                 return POWER_DOMAIN_PORT_DDI_F_LANES;
7332         case PORT_G:
7333                 return POWER_DOMAIN_PORT_DDI_G_LANES;
7334         default:
7335                 MISSING_CASE(port);
7336                 return POWER_DOMAIN_PORT_OTHER;
7337         }
7338 }
7339
7340 enum intel_display_power_domain
7341 intel_aux_power_domain(struct intel_digital_port *dig_port)
7342 {
7343         struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
7344         enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port);
7345
7346         if (intel_phy_is_tc(dev_priv, phy) &&
7347             dig_port->tc_mode == TC_PORT_TBT_ALT) {
7348                 switch (dig_port->aux_ch) {
7349                 case AUX_CH_C:
7350                         return POWER_DOMAIN_AUX_C_TBT;
7351                 case AUX_CH_D:
7352                         return POWER_DOMAIN_AUX_D_TBT;
7353                 case AUX_CH_E:
7354                         return POWER_DOMAIN_AUX_E_TBT;
7355                 case AUX_CH_F:
7356                         return POWER_DOMAIN_AUX_F_TBT;
7357                 case AUX_CH_G:
7358                         return POWER_DOMAIN_AUX_G_TBT;
7359                 default:
7360                         MISSING_CASE(dig_port->aux_ch);
7361                         return POWER_DOMAIN_AUX_C_TBT;
7362                 }
7363         }
7364
7365         return intel_legacy_aux_to_power_domain(dig_port->aux_ch);
7366 }
7367
7368 /*
7369  * Converts aux_ch to power_domain without caring about TBT ports for that use
7370  * intel_aux_power_domain()
7371  */
7372 enum intel_display_power_domain
7373 intel_legacy_aux_to_power_domain(enum aux_ch aux_ch)
7374 {
7375         switch (aux_ch) {
7376         case AUX_CH_A:
7377                 return POWER_DOMAIN_AUX_A;
7378         case AUX_CH_B:
7379                 return POWER_DOMAIN_AUX_B;
7380         case AUX_CH_C:
7381                 return POWER_DOMAIN_AUX_C;
7382         case AUX_CH_D:
7383                 return POWER_DOMAIN_AUX_D;
7384         case AUX_CH_E:
7385                 return POWER_DOMAIN_AUX_E;
7386         case AUX_CH_F:
7387                 return POWER_DOMAIN_AUX_F;
7388         case AUX_CH_G:
7389                 return POWER_DOMAIN_AUX_G;
7390         default:
7391                 MISSING_CASE(aux_ch);
7392                 return POWER_DOMAIN_AUX_A;
7393         }
7394 }
7395
7396 static u64 get_crtc_power_domains(struct intel_crtc_state *crtc_state)
7397 {
7398         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7399         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7400         struct drm_encoder *encoder;
7401         enum pipe pipe = crtc->pipe;
7402         u64 mask;
7403         enum transcoder transcoder = crtc_state->cpu_transcoder;
7404
7405         if (!crtc_state->hw.active)
7406                 return 0;
7407
7408         mask = BIT_ULL(POWER_DOMAIN_PIPE(pipe));
7409         mask |= BIT_ULL(POWER_DOMAIN_TRANSCODER(transcoder));
7410         if (crtc_state->pch_pfit.enabled ||
7411             crtc_state->pch_pfit.force_thru)
7412                 mask |= BIT_ULL(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
7413
7414         drm_for_each_encoder_mask(encoder, &dev_priv->drm,
7415                                   crtc_state->uapi.encoder_mask) {
7416                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
7417
7418                 mask |= BIT_ULL(intel_encoder->power_domain);
7419         }
7420
7421         if (HAS_DDI(dev_priv) && crtc_state->has_audio)
7422                 mask |= BIT_ULL(POWER_DOMAIN_AUDIO);
7423
7424         if (crtc_state->shared_dpll)
7425                 mask |= BIT_ULL(POWER_DOMAIN_DISPLAY_CORE);
7426
7427         return mask;
7428 }
7429
7430 static u64
7431 modeset_get_crtc_power_domains(struct intel_crtc_state *crtc_state)
7432 {
7433         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7434         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7435         enum intel_display_power_domain domain;
7436         u64 domains, new_domains, old_domains;
7437
7438         old_domains = crtc->enabled_power_domains;
7439         crtc->enabled_power_domains = new_domains =
7440                 get_crtc_power_domains(crtc_state);
7441
7442         domains = new_domains & ~old_domains;
7443
7444         for_each_power_domain(domain, domains)
7445                 intel_display_power_get(dev_priv, domain);
7446
7447         return old_domains & ~new_domains;
7448 }
7449
7450 static void modeset_put_power_domains(struct drm_i915_private *dev_priv,
7451                                       u64 domains)
7452 {
7453         enum intel_display_power_domain domain;
7454
7455         for_each_power_domain(domain, domains)
7456                 intel_display_power_put_unchecked(dev_priv, domain);
7457 }
7458
7459 static void valleyview_crtc_enable(struct intel_atomic_state *state,
7460                                    struct intel_crtc *crtc)
7461 {
7462         const struct intel_crtc_state *new_crtc_state =
7463                 intel_atomic_get_new_crtc_state(state, crtc);
7464         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7465         enum pipe pipe = crtc->pipe;
7466
7467         if (drm_WARN_ON(&dev_priv->drm, crtc->active))
7468                 return;
7469
7470         if (intel_crtc_has_dp_encoder(new_crtc_state))
7471                 intel_dp_set_m_n(new_crtc_state, M1_N1);
7472
7473         intel_set_pipe_timings(new_crtc_state);
7474         intel_set_pipe_src_size(new_crtc_state);
7475
7476         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
7477                 intel_de_write(dev_priv, CHV_BLEND(pipe), CHV_BLEND_LEGACY);
7478                 intel_de_write(dev_priv, CHV_CANVAS(pipe), 0);
7479         }
7480
7481         i9xx_set_pipeconf(new_crtc_state);
7482
7483         crtc->active = true;
7484
7485         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
7486
7487         intel_encoders_pre_pll_enable(state, crtc);
7488
7489         if (IS_CHERRYVIEW(dev_priv)) {
7490                 chv_prepare_pll(crtc, new_crtc_state);
7491                 chv_enable_pll(crtc, new_crtc_state);
7492         } else {
7493                 vlv_prepare_pll(crtc, new_crtc_state);
7494                 vlv_enable_pll(crtc, new_crtc_state);
7495         }
7496
7497         intel_encoders_pre_enable(state, crtc);
7498
7499         i9xx_pfit_enable(new_crtc_state);
7500
7501         intel_color_load_luts(new_crtc_state);
7502         intel_color_commit(new_crtc_state);
7503         /* update DSPCNTR to configure gamma for pipe bottom color */
7504         intel_disable_primary_plane(new_crtc_state);
7505
7506         dev_priv->display.initial_watermarks(state, crtc);
7507         intel_enable_pipe(new_crtc_state);
7508
7509         intel_crtc_vblank_on(new_crtc_state);
7510
7511         intel_encoders_enable(state, crtc);
7512 }
7513
7514 static void i9xx_set_pll_dividers(const struct intel_crtc_state *crtc_state)
7515 {
7516         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7517         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7518
7519         intel_de_write(dev_priv, FP0(crtc->pipe),
7520                        crtc_state->dpll_hw_state.fp0);
7521         intel_de_write(dev_priv, FP1(crtc->pipe),
7522                        crtc_state->dpll_hw_state.fp1);
7523 }
7524
7525 static void i9xx_crtc_enable(struct intel_atomic_state *state,
7526                              struct intel_crtc *crtc)
7527 {
7528         const struct intel_crtc_state *new_crtc_state =
7529                 intel_atomic_get_new_crtc_state(state, crtc);
7530         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7531         enum pipe pipe = crtc->pipe;
7532
7533         if (drm_WARN_ON(&dev_priv->drm, crtc->active))
7534                 return;
7535
7536         i9xx_set_pll_dividers(new_crtc_state);
7537
7538         if (intel_crtc_has_dp_encoder(new_crtc_state))
7539                 intel_dp_set_m_n(new_crtc_state, M1_N1);
7540
7541         intel_set_pipe_timings(new_crtc_state);
7542         intel_set_pipe_src_size(new_crtc_state);
7543
7544         i9xx_set_pipeconf(new_crtc_state);
7545
7546         crtc->active = true;
7547
7548         if (!IS_GEN(dev_priv, 2))
7549                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
7550
7551         intel_encoders_pre_enable(state, crtc);
7552
7553         i9xx_enable_pll(crtc, new_crtc_state);
7554
7555         i9xx_pfit_enable(new_crtc_state);
7556
7557         intel_color_load_luts(new_crtc_state);
7558         intel_color_commit(new_crtc_state);
7559         /* update DSPCNTR to configure gamma for pipe bottom color */
7560         intel_disable_primary_plane(new_crtc_state);
7561
7562         if (dev_priv->display.initial_watermarks)
7563                 dev_priv->display.initial_watermarks(state, crtc);
7564         else
7565                 intel_update_watermarks(crtc);
7566         intel_enable_pipe(new_crtc_state);
7567
7568         intel_crtc_vblank_on(new_crtc_state);
7569
7570         intel_encoders_enable(state, crtc);
7571
7572         /* prevents spurious underruns */
7573         if (IS_GEN(dev_priv, 2))
7574                 intel_wait_for_vblank(dev_priv, pipe);
7575 }
7576
7577 static void i9xx_pfit_disable(const struct intel_crtc_state *old_crtc_state)
7578 {
7579         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
7580         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7581
7582         if (!old_crtc_state->gmch_pfit.control)
7583                 return;
7584
7585         assert_pipe_disabled(dev_priv, old_crtc_state->cpu_transcoder);
7586
7587         drm_dbg_kms(&dev_priv->drm, "disabling pfit, current: 0x%08x\n",
7588                     intel_de_read(dev_priv, PFIT_CONTROL));
7589         intel_de_write(dev_priv, PFIT_CONTROL, 0);
7590 }
7591
7592 static void i9xx_crtc_disable(struct intel_atomic_state *state,
7593                               struct intel_crtc *crtc)
7594 {
7595         struct intel_crtc_state *old_crtc_state =
7596                 intel_atomic_get_old_crtc_state(state, crtc);
7597         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7598         enum pipe pipe = crtc->pipe;
7599
7600         /*
7601          * On gen2 planes are double buffered but the pipe isn't, so we must
7602          * wait for planes to fully turn off before disabling the pipe.
7603          */
7604         if (IS_GEN(dev_priv, 2))
7605                 intel_wait_for_vblank(dev_priv, pipe);
7606
7607         intel_encoders_disable(state, crtc);
7608
7609         intel_crtc_vblank_off(old_crtc_state);
7610
7611         intel_disable_pipe(old_crtc_state);
7612
7613         i9xx_pfit_disable(old_crtc_state);
7614
7615         intel_encoders_post_disable(state, crtc);
7616
7617         if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DSI)) {
7618                 if (IS_CHERRYVIEW(dev_priv))
7619                         chv_disable_pll(dev_priv, pipe);
7620                 else if (IS_VALLEYVIEW(dev_priv))
7621                         vlv_disable_pll(dev_priv, pipe);
7622                 else
7623                         i9xx_disable_pll(old_crtc_state);
7624         }
7625
7626         intel_encoders_post_pll_disable(state, crtc);
7627
7628         if (!IS_GEN(dev_priv, 2))
7629                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
7630
7631         if (!dev_priv->display.initial_watermarks)
7632                 intel_update_watermarks(crtc);
7633
7634         /* clock the pipe down to 640x480@60 to potentially save power */
7635         if (IS_I830(dev_priv))
7636                 i830_enable_pipe(dev_priv, pipe);
7637 }
7638
7639 static void intel_crtc_disable_noatomic(struct intel_crtc *crtc,
7640                                         struct drm_modeset_acquire_ctx *ctx)
7641 {
7642         struct intel_encoder *encoder;
7643         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7644         struct intel_bw_state *bw_state =
7645                 to_intel_bw_state(dev_priv->bw_obj.state);
7646         struct intel_cdclk_state *cdclk_state =
7647                 to_intel_cdclk_state(dev_priv->cdclk.obj.state);
7648         struct intel_dbuf_state *dbuf_state =
7649                 to_intel_dbuf_state(dev_priv->dbuf.obj.state);
7650         struct intel_crtc_state *crtc_state =
7651                 to_intel_crtc_state(crtc->base.state);
7652         enum intel_display_power_domain domain;
7653         struct intel_plane *plane;
7654         struct drm_atomic_state *state;
7655         struct intel_crtc_state *temp_crtc_state;
7656         enum pipe pipe = crtc->pipe;
7657         u64 domains;
7658         int ret;
7659
7660         if (!crtc_state->hw.active)
7661                 return;
7662
7663         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
7664                 const struct intel_plane_state *plane_state =
7665                         to_intel_plane_state(plane->base.state);
7666
7667                 if (plane_state->uapi.visible)
7668                         intel_plane_disable_noatomic(crtc, plane);
7669         }
7670
7671         state = drm_atomic_state_alloc(&dev_priv->drm);
7672         if (!state) {
7673                 drm_dbg_kms(&dev_priv->drm,
7674                             "failed to disable [CRTC:%d:%s], out of memory",
7675                             crtc->base.base.id, crtc->base.name);
7676                 return;
7677         }
7678
7679         state->acquire_ctx = ctx;
7680
7681         /* Everything's already locked, -EDEADLK can't happen. */
7682         temp_crtc_state = intel_atomic_get_crtc_state(state, crtc);
7683         ret = drm_atomic_add_affected_connectors(state, &crtc->base);
7684
7685         drm_WARN_ON(&dev_priv->drm, IS_ERR(temp_crtc_state) || ret);
7686
7687         dev_priv->display.crtc_disable(to_intel_atomic_state(state), crtc);
7688
7689         drm_atomic_state_put(state);
7690
7691         drm_dbg_kms(&dev_priv->drm,
7692                     "[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
7693                     crtc->base.base.id, crtc->base.name);
7694
7695         crtc->active = false;
7696         crtc->base.enabled = false;
7697
7698         drm_WARN_ON(&dev_priv->drm,
7699                     drm_atomic_set_mode_for_crtc(&crtc_state->uapi, NULL) < 0);
7700         crtc_state->uapi.active = false;
7701         crtc_state->uapi.connector_mask = 0;
7702         crtc_state->uapi.encoder_mask = 0;
7703         intel_crtc_free_hw_state(crtc_state);
7704         memset(&crtc_state->hw, 0, sizeof(crtc_state->hw));
7705
7706         for_each_encoder_on_crtc(&dev_priv->drm, &crtc->base, encoder)
7707                 encoder->base.crtc = NULL;
7708
7709         intel_fbc_disable(crtc);
7710         intel_update_watermarks(crtc);
7711         intel_disable_shared_dpll(crtc_state);
7712
7713         domains = crtc->enabled_power_domains;
7714         for_each_power_domain(domain, domains)
7715                 intel_display_power_put_unchecked(dev_priv, domain);
7716         crtc->enabled_power_domains = 0;
7717
7718         dev_priv->active_pipes &= ~BIT(pipe);
7719         cdclk_state->min_cdclk[pipe] = 0;
7720         cdclk_state->min_voltage_level[pipe] = 0;
7721         cdclk_state->active_pipes &= ~BIT(pipe);
7722
7723         dbuf_state->active_pipes &= ~BIT(pipe);
7724
7725         bw_state->data_rate[pipe] = 0;
7726         bw_state->num_active_planes[pipe] = 0;
7727 }
7728
7729 /*
7730  * turn all crtc's off, but do not adjust state
7731  * This has to be paired with a call to intel_modeset_setup_hw_state.
7732  */
7733 int intel_display_suspend(struct drm_device *dev)
7734 {
7735         struct drm_i915_private *dev_priv = to_i915(dev);
7736         struct drm_atomic_state *state;
7737         int ret;
7738
7739         state = drm_atomic_helper_suspend(dev);
7740         ret = PTR_ERR_OR_ZERO(state);
7741         if (ret)
7742                 drm_err(&dev_priv->drm, "Suspending crtc's failed with %i\n",
7743                         ret);
7744         else
7745                 dev_priv->modeset_restore_state = state;
7746         return ret;
7747 }
7748
7749 void intel_encoder_destroy(struct drm_encoder *encoder)
7750 {
7751         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
7752
7753         drm_encoder_cleanup(encoder);
7754         kfree(intel_encoder);
7755 }
7756
7757 /* Cross check the actual hw state with our own modeset state tracking (and it's
7758  * internal consistency). */
7759 static void intel_connector_verify_state(struct intel_crtc_state *crtc_state,
7760                                          struct drm_connector_state *conn_state)
7761 {
7762         struct intel_connector *connector = to_intel_connector(conn_state->connector);
7763         struct drm_i915_private *i915 = to_i915(connector->base.dev);
7764
7765         drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s]\n",
7766                     connector->base.base.id, connector->base.name);
7767
7768         if (connector->get_hw_state(connector)) {
7769                 struct intel_encoder *encoder = intel_attached_encoder(connector);
7770
7771                 I915_STATE_WARN(!crtc_state,
7772                          "connector enabled without attached crtc\n");
7773
7774                 if (!crtc_state)
7775                         return;
7776
7777                 I915_STATE_WARN(!crtc_state->hw.active,
7778                                 "connector is active, but attached crtc isn't\n");
7779
7780                 if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST)
7781                         return;
7782
7783                 I915_STATE_WARN(conn_state->best_encoder != &encoder->base,
7784                         "atomic encoder doesn't match attached encoder\n");
7785
7786                 I915_STATE_WARN(conn_state->crtc != encoder->base.crtc,
7787                         "attached encoder crtc differs from connector crtc\n");
7788         } else {
7789                 I915_STATE_WARN(crtc_state && crtc_state->hw.active,
7790                                 "attached crtc is active, but connector isn't\n");
7791                 I915_STATE_WARN(!crtc_state && conn_state->best_encoder,
7792                         "best encoder set without crtc!\n");
7793         }
7794 }
7795
7796 static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state)
7797 {
7798         if (crtc_state->hw.enable && crtc_state->has_pch_encoder)
7799                 return crtc_state->fdi_lanes;
7800
7801         return 0;
7802 }
7803
7804 static int ilk_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
7805                                struct intel_crtc_state *pipe_config)
7806 {
7807         struct drm_i915_private *dev_priv = to_i915(dev);
7808         struct drm_atomic_state *state = pipe_config->uapi.state;
7809         struct intel_crtc *other_crtc;
7810         struct intel_crtc_state *other_crtc_state;
7811
7812         drm_dbg_kms(&dev_priv->drm,
7813                     "checking fdi config on pipe %c, lanes %i\n",
7814                     pipe_name(pipe), pipe_config->fdi_lanes);
7815         if (pipe_config->fdi_lanes > 4) {
7816                 drm_dbg_kms(&dev_priv->drm,
7817                             "invalid fdi lane config on pipe %c: %i lanes\n",
7818                             pipe_name(pipe), pipe_config->fdi_lanes);
7819                 return -EINVAL;
7820         }
7821
7822         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
7823                 if (pipe_config->fdi_lanes > 2) {
7824                         drm_dbg_kms(&dev_priv->drm,
7825                                     "only 2 lanes on haswell, required: %i lanes\n",
7826                                     pipe_config->fdi_lanes);
7827                         return -EINVAL;
7828                 } else {
7829                         return 0;
7830                 }
7831         }
7832
7833         if (INTEL_NUM_PIPES(dev_priv) == 2)
7834                 return 0;
7835
7836         /* Ivybridge 3 pipe is really complicated */
7837         switch (pipe) {
7838         case PIPE_A:
7839                 return 0;
7840         case PIPE_B:
7841                 if (pipe_config->fdi_lanes <= 2)
7842                         return 0;
7843
7844                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_C);
7845                 other_crtc_state =
7846                         intel_atomic_get_crtc_state(state, other_crtc);
7847                 if (IS_ERR(other_crtc_state))
7848                         return PTR_ERR(other_crtc_state);
7849
7850                 if (pipe_required_fdi_lanes(other_crtc_state) > 0) {
7851                         drm_dbg_kms(&dev_priv->drm,
7852                                     "invalid shared fdi lane config on pipe %c: %i lanes\n",
7853                                     pipe_name(pipe), pipe_config->fdi_lanes);
7854                         return -EINVAL;
7855                 }
7856                 return 0;
7857         case PIPE_C:
7858                 if (pipe_config->fdi_lanes > 2) {
7859                         drm_dbg_kms(&dev_priv->drm,
7860                                     "only 2 lanes on pipe %c: required %i lanes\n",
7861                                     pipe_name(pipe), pipe_config->fdi_lanes);
7862                         return -EINVAL;
7863                 }
7864
7865                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_B);
7866                 other_crtc_state =
7867                         intel_atomic_get_crtc_state(state, other_crtc);
7868                 if (IS_ERR(other_crtc_state))
7869                         return PTR_ERR(other_crtc_state);
7870
7871                 if (pipe_required_fdi_lanes(other_crtc_state) > 2) {
7872                         drm_dbg_kms(&dev_priv->drm,
7873                                     "fdi link B uses too many lanes to enable link C\n");
7874                         return -EINVAL;
7875                 }
7876                 return 0;
7877         default:
7878                 BUG();
7879         }
7880 }
7881
7882 #define RETRY 1
7883 static int ilk_fdi_compute_config(struct intel_crtc *intel_crtc,
7884                                   struct intel_crtc_state *pipe_config)
7885 {
7886         struct drm_device *dev = intel_crtc->base.dev;
7887         struct drm_i915_private *i915 = to_i915(dev);
7888         const struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
7889         int lane, link_bw, fdi_dotclock, ret;
7890         bool needs_recompute = false;
7891
7892 retry:
7893         /* FDI is a binary signal running at ~2.7GHz, encoding
7894          * each output octet as 10 bits. The actual frequency
7895          * is stored as a divider into a 100MHz clock, and the
7896          * mode pixel clock is stored in units of 1KHz.
7897          * Hence the bw of each lane in terms of the mode signal
7898          * is:
7899          */
7900         link_bw = intel_fdi_link_freq(i915, pipe_config);
7901
7902         fdi_dotclock = adjusted_mode->crtc_clock;
7903
7904         lane = ilk_get_lanes_required(fdi_dotclock, link_bw,
7905                                       pipe_config->pipe_bpp);
7906
7907         pipe_config->fdi_lanes = lane;
7908
7909         intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock,
7910                                link_bw, &pipe_config->fdi_m_n, false, false);
7911
7912         ret = ilk_check_fdi_lanes(dev, intel_crtc->pipe, pipe_config);
7913         if (ret == -EDEADLK)
7914                 return ret;
7915
7916         if (ret == -EINVAL && pipe_config->pipe_bpp > 6*3) {
7917                 pipe_config->pipe_bpp -= 2*3;
7918                 drm_dbg_kms(&i915->drm,
7919                             "fdi link bw constraint, reducing pipe bpp to %i\n",
7920                             pipe_config->pipe_bpp);
7921                 needs_recompute = true;
7922                 pipe_config->bw_constrained = true;
7923
7924                 goto retry;
7925         }
7926
7927         if (needs_recompute)
7928                 return RETRY;
7929
7930         return ret;
7931 }
7932
7933 bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state)
7934 {
7935         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7936         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7937
7938         /* IPS only exists on ULT machines and is tied to pipe A. */
7939         if (!hsw_crtc_supports_ips(crtc))
7940                 return false;
7941
7942         if (!dev_priv->params.enable_ips)
7943                 return false;
7944
7945         if (crtc_state->pipe_bpp > 24)
7946                 return false;
7947
7948         /*
7949          * We compare against max which means we must take
7950          * the increased cdclk requirement into account when
7951          * calculating the new cdclk.
7952          *
7953          * Should measure whether using a lower cdclk w/o IPS
7954          */
7955         if (IS_BROADWELL(dev_priv) &&
7956             crtc_state->pixel_rate > dev_priv->max_cdclk_freq * 95 / 100)
7957                 return false;
7958
7959         return true;
7960 }
7961
7962 static int hsw_compute_ips_config(struct intel_crtc_state *crtc_state)
7963 {
7964         struct drm_i915_private *dev_priv =
7965                 to_i915(crtc_state->uapi.crtc->dev);
7966         struct intel_atomic_state *state =
7967                 to_intel_atomic_state(crtc_state->uapi.state);
7968
7969         crtc_state->ips_enabled = false;
7970
7971         if (!hsw_crtc_state_ips_capable(crtc_state))
7972                 return 0;
7973
7974         /*
7975          * When IPS gets enabled, the pipe CRC changes. Since IPS gets
7976          * enabled and disabled dynamically based on package C states,
7977          * user space can't make reliable use of the CRCs, so let's just
7978          * completely disable it.
7979          */
7980         if (crtc_state->crc_enabled)
7981                 return 0;
7982
7983         /* IPS should be fine as long as at least one plane is enabled. */
7984         if (!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)))
7985                 return 0;
7986
7987         if (IS_BROADWELL(dev_priv)) {
7988                 const struct intel_cdclk_state *cdclk_state;
7989
7990                 cdclk_state = intel_atomic_get_cdclk_state(state);
7991                 if (IS_ERR(cdclk_state))
7992                         return PTR_ERR(cdclk_state);
7993
7994                 /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
7995                 if (crtc_state->pixel_rate > cdclk_state->logical.cdclk * 95 / 100)
7996                         return 0;
7997         }
7998
7999         crtc_state->ips_enabled = true;
8000
8001         return 0;
8002 }
8003
8004 static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc)
8005 {
8006         const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8007
8008         /* GDG double wide on either pipe, otherwise pipe A only */
8009         return INTEL_GEN(dev_priv) < 4 &&
8010                 (crtc->pipe == PIPE_A || IS_I915G(dev_priv));
8011 }
8012
8013 static u32 ilk_pipe_pixel_rate(const struct intel_crtc_state *crtc_state)
8014 {
8015         u32 pixel_rate = crtc_state->hw.adjusted_mode.crtc_clock;
8016         unsigned int pipe_w, pipe_h, pfit_w, pfit_h;
8017
8018         /*
8019          * We only use IF-ID interlacing. If we ever use
8020          * PF-ID we'll need to adjust the pixel_rate here.
8021          */
8022
8023         if (!crtc_state->pch_pfit.enabled)
8024                 return pixel_rate;
8025
8026         pipe_w = crtc_state->pipe_src_w;
8027         pipe_h = crtc_state->pipe_src_h;
8028
8029         pfit_w = drm_rect_width(&crtc_state->pch_pfit.dst);
8030         pfit_h = drm_rect_height(&crtc_state->pch_pfit.dst);
8031
8032         if (pipe_w < pfit_w)
8033                 pipe_w = pfit_w;
8034         if (pipe_h < pfit_h)
8035                 pipe_h = pfit_h;
8036
8037         if (drm_WARN_ON(crtc_state->uapi.crtc->dev,
8038                         !pfit_w || !pfit_h))
8039                 return pixel_rate;
8040
8041         return div_u64(mul_u32_u32(pixel_rate, pipe_w * pipe_h),
8042                        pfit_w * pfit_h);
8043 }
8044
8045 static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state)
8046 {
8047         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
8048
8049         if (HAS_GMCH(dev_priv))
8050                 /* FIXME calculate proper pipe pixel rate for GMCH pfit */
8051                 crtc_state->pixel_rate =
8052                         crtc_state->hw.adjusted_mode.crtc_clock;
8053         else
8054                 crtc_state->pixel_rate =
8055                         ilk_pipe_pixel_rate(crtc_state);
8056 }
8057
8058 static int intel_crtc_compute_config(struct intel_crtc *crtc,
8059                                      struct intel_crtc_state *pipe_config)
8060 {
8061         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8062         const struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
8063         int clock_limit = dev_priv->max_dotclk_freq;
8064
8065         if (INTEL_GEN(dev_priv) < 4) {
8066                 clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
8067
8068                 /*
8069                  * Enable double wide mode when the dot clock
8070                  * is > 90% of the (display) core speed.
8071                  */
8072                 if (intel_crtc_supports_double_wide(crtc) &&
8073                     adjusted_mode->crtc_clock > clock_limit) {
8074                         clock_limit = dev_priv->max_dotclk_freq;
8075                         pipe_config->double_wide = true;
8076                 }
8077         }
8078
8079         if (adjusted_mode->crtc_clock > clock_limit) {
8080                 drm_dbg_kms(&dev_priv->drm,
8081                             "requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
8082                             adjusted_mode->crtc_clock, clock_limit,
8083                             yesno(pipe_config->double_wide));
8084                 return -EINVAL;
8085         }
8086
8087         if ((pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
8088              pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) &&
8089              pipe_config->hw.ctm) {
8090                 /*
8091                  * There is only one pipe CSC unit per pipe, and we need that
8092                  * for output conversion from RGB->YCBCR. So if CTM is already
8093                  * applied we can't support YCBCR420 output.
8094                  */
8095                 drm_dbg_kms(&dev_priv->drm,
8096                             "YCBCR420 and CTM together are not possible\n");
8097                 return -EINVAL;
8098         }
8099
8100         /*
8101          * Pipe horizontal size must be even in:
8102          * - DVO ganged mode
8103          * - LVDS dual channel mode
8104          * - Double wide pipe
8105          */
8106         if (pipe_config->pipe_src_w & 1) {
8107                 if (pipe_config->double_wide) {
8108                         drm_dbg_kms(&dev_priv->drm,
8109                                     "Odd pipe source width not supported with double wide pipe\n");
8110                         return -EINVAL;
8111                 }
8112
8113                 if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_LVDS) &&
8114                     intel_is_dual_link_lvds(dev_priv)) {
8115                         drm_dbg_kms(&dev_priv->drm,
8116                                     "Odd pipe source width not supported with dual link LVDS\n");
8117                         return -EINVAL;
8118                 }
8119         }
8120
8121         /* Cantiga+ cannot handle modes with a hsync front porch of 0.
8122          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
8123          */
8124         if ((INTEL_GEN(dev_priv) > 4 || IS_G4X(dev_priv)) &&
8125                 adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay)
8126                 return -EINVAL;
8127
8128         intel_crtc_compute_pixel_rate(pipe_config);
8129
8130         if (pipe_config->has_pch_encoder)
8131                 return ilk_fdi_compute_config(crtc, pipe_config);
8132
8133         return 0;
8134 }
8135
8136 static void
8137 intel_reduce_m_n_ratio(u32 *num, u32 *den)
8138 {
8139         while (*num > DATA_LINK_M_N_MASK ||
8140                *den > DATA_LINK_M_N_MASK) {
8141                 *num >>= 1;
8142                 *den >>= 1;
8143         }
8144 }
8145
8146 static void compute_m_n(unsigned int m, unsigned int n,
8147                         u32 *ret_m, u32 *ret_n,
8148                         bool constant_n)
8149 {
8150         /*
8151          * Several DP dongles in particular seem to be fussy about
8152          * too large link M/N values. Give N value as 0x8000 that
8153          * should be acceptable by specific devices. 0x8000 is the
8154          * specified fixed N value for asynchronous clock mode,
8155          * which the devices expect also in synchronous clock mode.
8156          */
8157         if (constant_n)
8158                 *ret_n = 0x8000;
8159         else
8160                 *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
8161
8162         *ret_m = div_u64(mul_u32_u32(m, *ret_n), n);
8163         intel_reduce_m_n_ratio(ret_m, ret_n);
8164 }
8165
8166 void
8167 intel_link_compute_m_n(u16 bits_per_pixel, int nlanes,
8168                        int pixel_clock, int link_clock,
8169                        struct intel_link_m_n *m_n,
8170                        bool constant_n, bool fec_enable)
8171 {
8172         u32 data_clock = bits_per_pixel * pixel_clock;
8173
8174         if (fec_enable)
8175                 data_clock = intel_dp_mode_to_fec_clock(data_clock);
8176
8177         m_n->tu = 64;
8178         compute_m_n(data_clock,
8179                     link_clock * nlanes * 8,
8180                     &m_n->gmch_m, &m_n->gmch_n,
8181                     constant_n);
8182
8183         compute_m_n(pixel_clock, link_clock,
8184                     &m_n->link_m, &m_n->link_n,
8185                     constant_n);
8186 }
8187
8188 static void intel_panel_sanitize_ssc(struct drm_i915_private *dev_priv)
8189 {
8190         /*
8191          * There may be no VBT; and if the BIOS enabled SSC we can
8192          * just keep using it to avoid unnecessary flicker.  Whereas if the
8193          * BIOS isn't using it, don't assume it will work even if the VBT
8194          * indicates as much.
8195          */
8196         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
8197                 bool bios_lvds_use_ssc = intel_de_read(dev_priv,
8198                                                        PCH_DREF_CONTROL) &
8199                         DREF_SSC1_ENABLE;
8200
8201                 if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) {
8202                         drm_dbg_kms(&dev_priv->drm,
8203                                     "SSC %s by BIOS, overriding VBT which says %s\n",
8204                                     enableddisabled(bios_lvds_use_ssc),
8205                                     enableddisabled(dev_priv->vbt.lvds_use_ssc));
8206                         dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc;
8207                 }
8208         }
8209 }
8210
8211 static bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
8212 {
8213         if (dev_priv->params.panel_use_ssc >= 0)
8214                 return dev_priv->params.panel_use_ssc != 0;
8215         return dev_priv->vbt.lvds_use_ssc
8216                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
8217 }
8218
8219 static u32 pnv_dpll_compute_fp(struct dpll *dpll)
8220 {
8221         return (1 << dpll->n) << 16 | dpll->m2;
8222 }
8223
8224 static u32 i9xx_dpll_compute_fp(struct dpll *dpll)
8225 {
8226         return dpll->n << 16 | dpll->m1 << 8 | dpll->m2;
8227 }
8228
8229 static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
8230                                      struct intel_crtc_state *crtc_state,
8231                                      struct dpll *reduced_clock)
8232 {
8233         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8234         u32 fp, fp2 = 0;
8235
8236         if (IS_PINEVIEW(dev_priv)) {
8237                 fp = pnv_dpll_compute_fp(&crtc_state->dpll);
8238                 if (reduced_clock)
8239                         fp2 = pnv_dpll_compute_fp(reduced_clock);
8240         } else {
8241                 fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
8242                 if (reduced_clock)
8243                         fp2 = i9xx_dpll_compute_fp(reduced_clock);
8244         }
8245
8246         crtc_state->dpll_hw_state.fp0 = fp;
8247
8248         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8249             reduced_clock) {
8250                 crtc_state->dpll_hw_state.fp1 = fp2;
8251         } else {
8252                 crtc_state->dpll_hw_state.fp1 = fp;
8253         }
8254 }
8255
8256 static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe
8257                 pipe)
8258 {
8259         u32 reg_val;
8260
8261         /*
8262          * PLLB opamp always calibrates to max value of 0x3f, force enable it
8263          * and set it to a reasonable value instead.
8264          */
8265         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
8266         reg_val &= 0xffffff00;
8267         reg_val |= 0x00000030;
8268         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
8269
8270         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
8271         reg_val &= 0x00ffffff;
8272         reg_val |= 0x8c000000;
8273         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
8274
8275         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
8276         reg_val &= 0xffffff00;
8277         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
8278
8279         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
8280         reg_val &= 0x00ffffff;
8281         reg_val |= 0xb0000000;
8282         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
8283 }
8284
8285 static void intel_pch_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
8286                                          const struct intel_link_m_n *m_n)
8287 {
8288         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8289         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8290         enum pipe pipe = crtc->pipe;
8291
8292         intel_de_write(dev_priv, PCH_TRANS_DATA_M1(pipe),
8293                        TU_SIZE(m_n->tu) | m_n->gmch_m);
8294         intel_de_write(dev_priv, PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
8295         intel_de_write(dev_priv, PCH_TRANS_LINK_M1(pipe), m_n->link_m);
8296         intel_de_write(dev_priv, PCH_TRANS_LINK_N1(pipe), m_n->link_n);
8297 }
8298
8299 static bool transcoder_has_m2_n2(struct drm_i915_private *dev_priv,
8300                                  enum transcoder transcoder)
8301 {
8302         if (IS_HASWELL(dev_priv))
8303                 return transcoder == TRANSCODER_EDP;
8304
8305         /*
8306          * Strictly speaking some registers are available before
8307          * gen7, but we only support DRRS on gen7+
8308          */
8309         return IS_GEN(dev_priv, 7) || IS_CHERRYVIEW(dev_priv);
8310 }
8311
8312 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
8313                                          const struct intel_link_m_n *m_n,
8314                                          const struct intel_link_m_n *m2_n2)
8315 {
8316         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8317         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8318         enum pipe pipe = crtc->pipe;
8319         enum transcoder transcoder = crtc_state->cpu_transcoder;
8320
8321         if (INTEL_GEN(dev_priv) >= 5) {
8322                 intel_de_write(dev_priv, PIPE_DATA_M1(transcoder),
8323                                TU_SIZE(m_n->tu) | m_n->gmch_m);
8324                 intel_de_write(dev_priv, PIPE_DATA_N1(transcoder),
8325                                m_n->gmch_n);
8326                 intel_de_write(dev_priv, PIPE_LINK_M1(transcoder),
8327                                m_n->link_m);
8328                 intel_de_write(dev_priv, PIPE_LINK_N1(transcoder),
8329                                m_n->link_n);
8330                 /*
8331                  *  M2_N2 registers are set only if DRRS is supported
8332                  * (to make sure the registers are not unnecessarily accessed).
8333                  */
8334                 if (m2_n2 && crtc_state->has_drrs &&
8335                     transcoder_has_m2_n2(dev_priv, transcoder)) {
8336                         intel_de_write(dev_priv, PIPE_DATA_M2(transcoder),
8337                                        TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
8338                         intel_de_write(dev_priv, PIPE_DATA_N2(transcoder),
8339                                        m2_n2->gmch_n);
8340                         intel_de_write(dev_priv, PIPE_LINK_M2(transcoder),
8341                                        m2_n2->link_m);
8342                         intel_de_write(dev_priv, PIPE_LINK_N2(transcoder),
8343                                        m2_n2->link_n);
8344                 }
8345         } else {
8346                 intel_de_write(dev_priv, PIPE_DATA_M_G4X(pipe),
8347                                TU_SIZE(m_n->tu) | m_n->gmch_m);
8348                 intel_de_write(dev_priv, PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
8349                 intel_de_write(dev_priv, PIPE_LINK_M_G4X(pipe), m_n->link_m);
8350                 intel_de_write(dev_priv, PIPE_LINK_N_G4X(pipe), m_n->link_n);
8351         }
8352 }
8353
8354 void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, enum link_m_n_set m_n)
8355 {
8356         const struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL;
8357         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
8358
8359         if (m_n == M1_N1) {
8360                 dp_m_n = &crtc_state->dp_m_n;
8361                 dp_m2_n2 = &crtc_state->dp_m2_n2;
8362         } else if (m_n == M2_N2) {
8363
8364                 /*
8365                  * M2_N2 registers are not supported. Hence m2_n2 divider value
8366                  * needs to be programmed into M1_N1.
8367                  */
8368                 dp_m_n = &crtc_state->dp_m2_n2;
8369         } else {
8370                 drm_err(&i915->drm, "Unsupported divider value\n");
8371                 return;
8372         }
8373
8374         if (crtc_state->has_pch_encoder)
8375                 intel_pch_transcoder_set_m_n(crtc_state, &crtc_state->dp_m_n);
8376         else
8377                 intel_cpu_transcoder_set_m_n(crtc_state, dp_m_n, dp_m2_n2);
8378 }
8379
8380 static void vlv_compute_dpll(struct intel_crtc *crtc,
8381                              struct intel_crtc_state *pipe_config)
8382 {
8383         pipe_config->dpll_hw_state.dpll = DPLL_INTEGRATED_REF_CLK_VLV |
8384                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
8385         if (crtc->pipe != PIPE_A)
8386                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
8387
8388         /* DPLL not used with DSI, but still need the rest set up */
8389         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
8390                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE |
8391                         DPLL_EXT_BUFFER_ENABLE_VLV;
8392
8393         pipe_config->dpll_hw_state.dpll_md =
8394                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
8395 }
8396
8397 static void chv_compute_dpll(struct intel_crtc *crtc,
8398                              struct intel_crtc_state *pipe_config)
8399 {
8400         pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV |
8401                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
8402         if (crtc->pipe != PIPE_A)
8403                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
8404
8405         /* DPLL not used with DSI, but still need the rest set up */
8406         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
8407                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE;
8408
8409         pipe_config->dpll_hw_state.dpll_md =
8410                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
8411 }
8412
8413 static void vlv_prepare_pll(struct intel_crtc *crtc,
8414                             const struct intel_crtc_state *pipe_config)
8415 {
8416         struct drm_device *dev = crtc->base.dev;
8417         struct drm_i915_private *dev_priv = to_i915(dev);
8418         enum pipe pipe = crtc->pipe;
8419         u32 mdiv;
8420         u32 bestn, bestm1, bestm2, bestp1, bestp2;
8421         u32 coreclk, reg_val;
8422
8423         /* Enable Refclk */
8424         intel_de_write(dev_priv, DPLL(pipe),
8425                        pipe_config->dpll_hw_state.dpll & ~(DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV));
8426
8427         /* No need to actually set up the DPLL with DSI */
8428         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8429                 return;
8430
8431         vlv_dpio_get(dev_priv);
8432
8433         bestn = pipe_config->dpll.n;
8434         bestm1 = pipe_config->dpll.m1;
8435         bestm2 = pipe_config->dpll.m2;
8436         bestp1 = pipe_config->dpll.p1;
8437         bestp2 = pipe_config->dpll.p2;
8438
8439         /* See eDP HDMI DPIO driver vbios notes doc */
8440
8441         /* PLL B needs special handling */
8442         if (pipe == PIPE_B)
8443                 vlv_pllb_recal_opamp(dev_priv, pipe);
8444
8445         /* Set up Tx target for periodic Rcomp update */
8446         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9_BCAST, 0x0100000f);
8447
8448         /* Disable target IRef on PLL */
8449         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW8(pipe));
8450         reg_val &= 0x00ffffff;
8451         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW8(pipe), reg_val);
8452
8453         /* Disable fast lock */
8454         vlv_dpio_write(dev_priv, pipe, VLV_CMN_DW0, 0x610);
8455
8456         /* Set idtafcrecal before PLL is enabled */
8457         mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
8458         mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT));
8459         mdiv |= ((bestn << DPIO_N_SHIFT));
8460         mdiv |= (1 << DPIO_K_SHIFT);
8461
8462         /*
8463          * Post divider depends on pixel clock rate, DAC vs digital (and LVDS,
8464          * but we don't support that).
8465          * Note: don't use the DAC post divider as it seems unstable.
8466          */
8467         mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
8468         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
8469
8470         mdiv |= DPIO_ENABLE_CALIBRATION;
8471         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
8472
8473         /* Set HBR and RBR LPF coefficients */
8474         if (pipe_config->port_clock == 162000 ||
8475             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_ANALOG) ||
8476             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI))
8477                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
8478                                  0x009f0003);
8479         else
8480                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
8481                                  0x00d0000f);
8482
8483         if (intel_crtc_has_dp_encoder(pipe_config)) {
8484                 /* Use SSC source */
8485                 if (pipe == PIPE_A)
8486                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
8487                                          0x0df40000);
8488                 else
8489                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
8490                                          0x0df70000);
8491         } else { /* HDMI or VGA */
8492                 /* Use bend source */
8493                 if (pipe == PIPE_A)
8494                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
8495                                          0x0df70000);
8496                 else
8497                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
8498                                          0x0df40000);
8499         }
8500
8501         coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe));
8502         coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
8503         if (intel_crtc_has_dp_encoder(pipe_config))
8504                 coreclk |= 0x01000000;
8505         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk);
8506
8507         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000);
8508
8509         vlv_dpio_put(dev_priv);
8510 }
8511
8512 static void chv_prepare_pll(struct intel_crtc *crtc,
8513                             const struct intel_crtc_state *pipe_config)
8514 {
8515         struct drm_device *dev = crtc->base.dev;
8516         struct drm_i915_private *dev_priv = to_i915(dev);
8517         enum pipe pipe = crtc->pipe;
8518         enum dpio_channel port = vlv_pipe_to_channel(pipe);
8519         u32 loopfilter, tribuf_calcntr;
8520         u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
8521         u32 dpio_val;
8522         int vco;
8523
8524         /* Enable Refclk and SSC */
8525         intel_de_write(dev_priv, DPLL(pipe),
8526                        pipe_config->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE);
8527
8528         /* No need to actually set up the DPLL with DSI */
8529         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8530                 return;
8531
8532         bestn = pipe_config->dpll.n;
8533         bestm2_frac = pipe_config->dpll.m2 & 0x3fffff;
8534         bestm1 = pipe_config->dpll.m1;
8535         bestm2 = pipe_config->dpll.m2 >> 22;
8536         bestp1 = pipe_config->dpll.p1;
8537         bestp2 = pipe_config->dpll.p2;
8538         vco = pipe_config->dpll.vco;
8539         dpio_val = 0;
8540         loopfilter = 0;
8541
8542         vlv_dpio_get(dev_priv);
8543
8544         /* p1 and p2 divider */
8545         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port),
8546                         5 << DPIO_CHV_S1_DIV_SHIFT |
8547                         bestp1 << DPIO_CHV_P1_DIV_SHIFT |
8548                         bestp2 << DPIO_CHV_P2_DIV_SHIFT |
8549                         1 << DPIO_CHV_K_DIV_SHIFT);
8550
8551         /* Feedback post-divider - m2 */
8552         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2);
8553
8554         /* Feedback refclk divider - n and m1 */
8555         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port),
8556                         DPIO_CHV_M1_DIV_BY_2 |
8557                         1 << DPIO_CHV_N_DIV_SHIFT);
8558
8559         /* M2 fraction division */
8560         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
8561
8562         /* M2 fraction division enable */
8563         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
8564         dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
8565         dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT);
8566         if (bestm2_frac)
8567                 dpio_val |= DPIO_CHV_FRAC_DIV_EN;
8568         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), dpio_val);
8569
8570         /* Program digital lock detect threshold */
8571         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW9(port));
8572         dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
8573                                         DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
8574         dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT);
8575         if (!bestm2_frac)
8576                 dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
8577         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW9(port), dpio_val);
8578
8579         /* Loop filter */
8580         if (vco == 5400000) {
8581                 loopfilter |= (0x3 << DPIO_CHV_PROP_COEFF_SHIFT);
8582                 loopfilter |= (0x8 << DPIO_CHV_INT_COEFF_SHIFT);
8583                 loopfilter |= (0x1 << DPIO_CHV_GAIN_CTRL_SHIFT);
8584                 tribuf_calcntr = 0x9;
8585         } else if (vco <= 6200000) {
8586                 loopfilter |= (0x5 << DPIO_CHV_PROP_COEFF_SHIFT);
8587                 loopfilter |= (0xB << DPIO_CHV_INT_COEFF_SHIFT);
8588                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
8589                 tribuf_calcntr = 0x9;
8590         } else if (vco <= 6480000) {
8591                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
8592                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
8593                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
8594                 tribuf_calcntr = 0x8;
8595         } else {
8596                 /* Not supported. Apply the same limits as in the max case */
8597                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
8598                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
8599                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
8600                 tribuf_calcntr = 0;
8601         }
8602         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter);
8603
8604         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW8(port));
8605         dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
8606         dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT);
8607         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW8(port), dpio_val);
8608
8609         /* AFC Recal */
8610         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port),
8611                         vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) |
8612                         DPIO_AFC_RECAL);
8613
8614         vlv_dpio_put(dev_priv);
8615 }
8616
8617 /**
8618  * vlv_force_pll_on - forcibly enable just the PLL
8619  * @dev_priv: i915 private structure
8620  * @pipe: pipe PLL to enable
8621  * @dpll: PLL configuration
8622  *
8623  * Enable the PLL for @pipe using the supplied @dpll config. To be used
8624  * in cases where we need the PLL enabled even when @pipe is not going to
8625  * be enabled.
8626  */
8627 int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe,
8628                      const struct dpll *dpll)
8629 {
8630         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
8631         struct intel_crtc_state *pipe_config;
8632
8633         pipe_config = intel_crtc_state_alloc(crtc);
8634         if (!pipe_config)
8635                 return -ENOMEM;
8636
8637         pipe_config->cpu_transcoder = (enum transcoder)pipe;
8638         pipe_config->pixel_multiplier = 1;
8639         pipe_config->dpll = *dpll;
8640
8641         if (IS_CHERRYVIEW(dev_priv)) {
8642                 chv_compute_dpll(crtc, pipe_config);
8643                 chv_prepare_pll(crtc, pipe_config);
8644                 chv_enable_pll(crtc, pipe_config);
8645         } else {
8646                 vlv_compute_dpll(crtc, pipe_config);
8647                 vlv_prepare_pll(crtc, pipe_config);
8648                 vlv_enable_pll(crtc, pipe_config);
8649         }
8650
8651         kfree(pipe_config);
8652
8653         return 0;
8654 }
8655
8656 /**
8657  * vlv_force_pll_off - forcibly disable just the PLL
8658  * @dev_priv: i915 private structure
8659  * @pipe: pipe PLL to disable
8660  *
8661  * Disable the PLL for @pipe. To be used in cases where we need
8662  * the PLL enabled even when @pipe is not going to be enabled.
8663  */
8664 void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe)
8665 {
8666         if (IS_CHERRYVIEW(dev_priv))
8667                 chv_disable_pll(dev_priv, pipe);
8668         else
8669                 vlv_disable_pll(dev_priv, pipe);
8670 }
8671
8672 static void i9xx_compute_dpll(struct intel_crtc *crtc,
8673                               struct intel_crtc_state *crtc_state,
8674                               struct dpll *reduced_clock)
8675 {
8676         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8677         u32 dpll;
8678         struct dpll *clock = &crtc_state->dpll;
8679
8680         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
8681
8682         dpll = DPLL_VGA_MODE_DIS;
8683
8684         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
8685                 dpll |= DPLLB_MODE_LVDS;
8686         else
8687                 dpll |= DPLLB_MODE_DAC_SERIAL;
8688
8689         if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
8690             IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
8691                 dpll |= (crtc_state->pixel_multiplier - 1)
8692                         << SDVO_MULTIPLIER_SHIFT_HIRES;
8693         }
8694
8695         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
8696             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
8697                 dpll |= DPLL_SDVO_HIGH_SPEED;
8698
8699         if (intel_crtc_has_dp_encoder(crtc_state))
8700                 dpll |= DPLL_SDVO_HIGH_SPEED;
8701
8702         /* compute bitmask from p1 value */
8703         if (IS_PINEVIEW(dev_priv))
8704                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
8705         else {
8706                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8707                 if (IS_G4X(dev_priv) && reduced_clock)
8708                         dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
8709         }
8710         switch (clock->p2) {
8711         case 5:
8712                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
8713                 break;
8714         case 7:
8715                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
8716                 break;
8717         case 10:
8718                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
8719                 break;
8720         case 14:
8721                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
8722                 break;
8723         }
8724         if (INTEL_GEN(dev_priv) >= 4)
8725                 dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
8726
8727         if (crtc_state->sdvo_tv_clock)
8728                 dpll |= PLL_REF_INPUT_TVCLKINBC;
8729         else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8730                  intel_panel_use_ssc(dev_priv))
8731                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8732         else
8733                 dpll |= PLL_REF_INPUT_DREFCLK;
8734
8735         dpll |= DPLL_VCO_ENABLE;
8736         crtc_state->dpll_hw_state.dpll = dpll;
8737
8738         if (INTEL_GEN(dev_priv) >= 4) {
8739                 u32 dpll_md = (crtc_state->pixel_multiplier - 1)
8740                         << DPLL_MD_UDI_MULTIPLIER_SHIFT;
8741                 crtc_state->dpll_hw_state.dpll_md = dpll_md;
8742         }
8743 }
8744
8745 static void i8xx_compute_dpll(struct intel_crtc *crtc,
8746                               struct intel_crtc_state *crtc_state,
8747                               struct dpll *reduced_clock)
8748 {
8749         struct drm_device *dev = crtc->base.dev;
8750         struct drm_i915_private *dev_priv = to_i915(dev);
8751         u32 dpll;
8752         struct dpll *clock = &crtc_state->dpll;
8753
8754         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
8755
8756         dpll = DPLL_VGA_MODE_DIS;
8757
8758         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8759                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8760         } else {
8761                 if (clock->p1 == 2)
8762                         dpll |= PLL_P1_DIVIDE_BY_TWO;
8763                 else
8764                         dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8765                 if (clock->p2 == 4)
8766                         dpll |= PLL_P2_DIVIDE_BY_4;
8767         }
8768
8769         /*
8770          * Bspec:
8771          * "[Almador Errata}: For the correct operation of the muxed DVO pins
8772          *  (GDEVSELB/I2Cdata, GIRDBY/I2CClk) and (GFRAMEB/DVI_Data,
8773          *  GTRDYB/DVI_Clk): Bit 31 (DPLL VCO Enable) and Bit 30 (2X Clock
8774          *  Enable) must be set to “1” in both the DPLL A Control Register
8775          *  (06014h-06017h) and DPLL B Control Register (06018h-0601Bh)."
8776          *
8777          * For simplicity We simply keep both bits always enabled in
8778          * both DPLLS. The spec says we should disable the DVO 2X clock
8779          * when not needed, but this seems to work fine in practice.
8780          */
8781         if (IS_I830(dev_priv) ||
8782             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO))
8783                 dpll |= DPLL_DVO_2X_MODE;
8784
8785         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8786             intel_panel_use_ssc(dev_priv))
8787                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8788         else
8789                 dpll |= PLL_REF_INPUT_DREFCLK;
8790
8791         dpll |= DPLL_VCO_ENABLE;
8792         crtc_state->dpll_hw_state.dpll = dpll;
8793 }
8794
8795 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state)
8796 {
8797         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8798         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8799         enum pipe pipe = crtc->pipe;
8800         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
8801         const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
8802         u32 crtc_vtotal, crtc_vblank_end;
8803         int vsyncshift = 0;
8804
8805         /* We need to be careful not to changed the adjusted mode, for otherwise
8806          * the hw state checker will get angry at the mismatch. */
8807         crtc_vtotal = adjusted_mode->crtc_vtotal;
8808         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
8809
8810         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
8811                 /* the chip adds 2 halflines automatically */
8812                 crtc_vtotal -= 1;
8813                 crtc_vblank_end -= 1;
8814
8815                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
8816                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
8817                 else
8818                         vsyncshift = adjusted_mode->crtc_hsync_start -
8819                                 adjusted_mode->crtc_htotal / 2;
8820                 if (vsyncshift < 0)
8821                         vsyncshift += adjusted_mode->crtc_htotal;
8822         }
8823
8824         if (INTEL_GEN(dev_priv) > 3)
8825                 intel_de_write(dev_priv, VSYNCSHIFT(cpu_transcoder),
8826                                vsyncshift);
8827
8828         intel_de_write(dev_priv, HTOTAL(cpu_transcoder),
8829                        (adjusted_mode->crtc_hdisplay - 1) | ((adjusted_mode->crtc_htotal - 1) << 16));
8830         intel_de_write(dev_priv, HBLANK(cpu_transcoder),
8831                        (adjusted_mode->crtc_hblank_start - 1) | ((adjusted_mode->crtc_hblank_end - 1) << 16));
8832         intel_de_write(dev_priv, HSYNC(cpu_transcoder),
8833                        (adjusted_mode->crtc_hsync_start - 1) | ((adjusted_mode->crtc_hsync_end - 1) << 16));
8834
8835         intel_de_write(dev_priv, VTOTAL(cpu_transcoder),
8836                        (adjusted_mode->crtc_vdisplay - 1) | ((crtc_vtotal - 1) << 16));
8837         intel_de_write(dev_priv, VBLANK(cpu_transcoder),
8838                        (adjusted_mode->crtc_vblank_start - 1) | ((crtc_vblank_end - 1) << 16));
8839         intel_de_write(dev_priv, VSYNC(cpu_transcoder),
8840                        (adjusted_mode->crtc_vsync_start - 1) | ((adjusted_mode->crtc_vsync_end - 1) << 16));
8841
8842         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
8843          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
8844          * documented on the DDI_FUNC_CTL register description, EDP Input Select
8845          * bits. */
8846         if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP &&
8847             (pipe == PIPE_B || pipe == PIPE_C))
8848                 intel_de_write(dev_priv, VTOTAL(pipe),
8849                                intel_de_read(dev_priv, VTOTAL(cpu_transcoder)));
8850
8851 }
8852
8853 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state)
8854 {
8855         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8856         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8857         enum pipe pipe = crtc->pipe;
8858
8859         /* pipesrc controls the size that is scaled from, which should
8860          * always be the user's requested size.
8861          */
8862         intel_de_write(dev_priv, PIPESRC(pipe),
8863                        ((crtc_state->pipe_src_w - 1) << 16) | (crtc_state->pipe_src_h - 1));
8864 }
8865
8866 static bool intel_pipe_is_interlaced(const struct intel_crtc_state *crtc_state)
8867 {
8868         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
8869         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
8870
8871         if (IS_GEN(dev_priv, 2))
8872                 return false;
8873
8874         if (INTEL_GEN(dev_priv) >= 9 ||
8875             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
8876                 return intel_de_read(dev_priv, PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK_HSW;
8877         else
8878                 return intel_de_read(dev_priv, PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK;
8879 }
8880
8881 static void intel_get_pipe_timings(struct intel_crtc *crtc,
8882                                    struct intel_crtc_state *pipe_config)
8883 {
8884         struct drm_device *dev = crtc->base.dev;
8885         struct drm_i915_private *dev_priv = to_i915(dev);
8886         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
8887         u32 tmp;
8888
8889         tmp = intel_de_read(dev_priv, HTOTAL(cpu_transcoder));
8890         pipe_config->hw.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
8891         pipe_config->hw.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
8892
8893         if (!transcoder_is_dsi(cpu_transcoder)) {
8894                 tmp = intel_de_read(dev_priv, HBLANK(cpu_transcoder));
8895                 pipe_config->hw.adjusted_mode.crtc_hblank_start =
8896                                                         (tmp & 0xffff) + 1;
8897                 pipe_config->hw.adjusted_mode.crtc_hblank_end =
8898                                                 ((tmp >> 16) & 0xffff) + 1;
8899         }
8900         tmp = intel_de_read(dev_priv, HSYNC(cpu_transcoder));
8901         pipe_config->hw.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
8902         pipe_config->hw.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
8903
8904         tmp = intel_de_read(dev_priv, VTOTAL(cpu_transcoder));
8905         pipe_config->hw.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
8906         pipe_config->hw.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
8907
8908         if (!transcoder_is_dsi(cpu_transcoder)) {
8909                 tmp = intel_de_read(dev_priv, VBLANK(cpu_transcoder));
8910                 pipe_config->hw.adjusted_mode.crtc_vblank_start =
8911                                                         (tmp & 0xffff) + 1;
8912                 pipe_config->hw.adjusted_mode.crtc_vblank_end =
8913                                                 ((tmp >> 16) & 0xffff) + 1;
8914         }
8915         tmp = intel_de_read(dev_priv, VSYNC(cpu_transcoder));
8916         pipe_config->hw.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
8917         pipe_config->hw.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
8918
8919         if (intel_pipe_is_interlaced(pipe_config)) {
8920                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
8921                 pipe_config->hw.adjusted_mode.crtc_vtotal += 1;
8922                 pipe_config->hw.adjusted_mode.crtc_vblank_end += 1;
8923         }
8924 }
8925
8926 static void intel_get_pipe_src_size(struct intel_crtc *crtc,
8927                                     struct intel_crtc_state *pipe_config)
8928 {
8929         struct drm_device *dev = crtc->base.dev;
8930         struct drm_i915_private *dev_priv = to_i915(dev);
8931         u32 tmp;
8932
8933         tmp = intel_de_read(dev_priv, PIPESRC(crtc->pipe));
8934         pipe_config->pipe_src_h = (tmp & 0xffff) + 1;
8935         pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1;
8936
8937         pipe_config->hw.mode.vdisplay = pipe_config->pipe_src_h;
8938         pipe_config->hw.mode.hdisplay = pipe_config->pipe_src_w;
8939 }
8940
8941 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
8942                                  struct intel_crtc_state *pipe_config)
8943 {
8944         mode->hdisplay = pipe_config->hw.adjusted_mode.crtc_hdisplay;
8945         mode->htotal = pipe_config->hw.adjusted_mode.crtc_htotal;
8946         mode->hsync_start = pipe_config->hw.adjusted_mode.crtc_hsync_start;
8947         mode->hsync_end = pipe_config->hw.adjusted_mode.crtc_hsync_end;
8948
8949         mode->vdisplay = pipe_config->hw.adjusted_mode.crtc_vdisplay;
8950         mode->vtotal = pipe_config->hw.adjusted_mode.crtc_vtotal;
8951         mode->vsync_start = pipe_config->hw.adjusted_mode.crtc_vsync_start;
8952         mode->vsync_end = pipe_config->hw.adjusted_mode.crtc_vsync_end;
8953
8954         mode->flags = pipe_config->hw.adjusted_mode.flags;
8955         mode->type = DRM_MODE_TYPE_DRIVER;
8956
8957         mode->clock = pipe_config->hw.adjusted_mode.crtc_clock;
8958
8959         drm_mode_set_name(mode);
8960 }
8961
8962 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
8963 {
8964         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8965         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8966         u32 pipeconf;
8967
8968         pipeconf = 0;
8969
8970         /* we keep both pipes enabled on 830 */
8971         if (IS_I830(dev_priv))
8972                 pipeconf |= intel_de_read(dev_priv, PIPECONF(crtc->pipe)) & PIPECONF_ENABLE;
8973
8974         if (crtc_state->double_wide)
8975                 pipeconf |= PIPECONF_DOUBLE_WIDE;
8976
8977         /* only g4x and later have fancy bpc/dither controls */
8978         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
8979             IS_CHERRYVIEW(dev_priv)) {
8980                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
8981                 if (crtc_state->dither && crtc_state->pipe_bpp != 30)
8982                         pipeconf |= PIPECONF_DITHER_EN |
8983                                     PIPECONF_DITHER_TYPE_SP;
8984
8985                 switch (crtc_state->pipe_bpp) {
8986                 case 18:
8987                         pipeconf |= PIPECONF_6BPC;
8988                         break;
8989                 case 24:
8990                         pipeconf |= PIPECONF_8BPC;
8991                         break;
8992                 case 30:
8993                         pipeconf |= PIPECONF_10BPC;
8994                         break;
8995                 default:
8996                         /* Case prevented by intel_choose_pipe_bpp_dither. */
8997                         BUG();
8998                 }
8999         }
9000
9001         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
9002                 if (INTEL_GEN(dev_priv) < 4 ||
9003                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
9004                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
9005                 else
9006                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
9007         } else {
9008                 pipeconf |= PIPECONF_PROGRESSIVE;
9009         }
9010
9011         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
9012              crtc_state->limited_color_range)
9013                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
9014
9015         pipeconf |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
9016
9017         pipeconf |= PIPECONF_FRAME_START_DELAY(0);
9018
9019         intel_de_write(dev_priv, PIPECONF(crtc->pipe), pipeconf);
9020         intel_de_posting_read(dev_priv, PIPECONF(crtc->pipe));
9021 }
9022
9023 static int i8xx_crtc_compute_clock(struct intel_crtc *crtc,
9024                                    struct intel_crtc_state *crtc_state)
9025 {
9026         struct drm_device *dev = crtc->base.dev;
9027         struct drm_i915_private *dev_priv = to_i915(dev);
9028         const struct intel_limit *limit;
9029         int refclk = 48000;
9030
9031         memset(&crtc_state->dpll_hw_state, 0,
9032                sizeof(crtc_state->dpll_hw_state));
9033
9034         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
9035                 if (intel_panel_use_ssc(dev_priv)) {
9036                         refclk = dev_priv->vbt.lvds_ssc_freq;
9037                         drm_dbg_kms(&dev_priv->drm,
9038                                     "using SSC reference clock of %d kHz\n",
9039                                     refclk);
9040                 }
9041
9042                 limit = &intel_limits_i8xx_lvds;
9043         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO)) {
9044                 limit = &intel_limits_i8xx_dvo;
9045         } else {
9046                 limit = &intel_limits_i8xx_dac;
9047         }
9048
9049         if (!crtc_state->clock_set &&
9050             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
9051                                  refclk, NULL, &crtc_state->dpll)) {
9052                 drm_err(&dev_priv->drm,
9053                         "Couldn't find PLL settings for mode!\n");
9054                 return -EINVAL;
9055         }
9056
9057         i8xx_compute_dpll(crtc, crtc_state, NULL);
9058
9059         return 0;
9060 }
9061
9062 static int g4x_crtc_compute_clock(struct intel_crtc *crtc,
9063                                   struct intel_crtc_state *crtc_state)
9064 {
9065         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9066         const struct intel_limit *limit;
9067         int refclk = 96000;
9068
9069         memset(&crtc_state->dpll_hw_state, 0,
9070                sizeof(crtc_state->dpll_hw_state));
9071
9072         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
9073                 if (intel_panel_use_ssc(dev_priv)) {
9074                         refclk = dev_priv->vbt.lvds_ssc_freq;
9075                         drm_dbg_kms(&dev_priv->drm,
9076                                     "using SSC reference clock of %d kHz\n",
9077                                     refclk);
9078                 }
9079
9080                 if (intel_is_dual_link_lvds(dev_priv))
9081                         limit = &intel_limits_g4x_dual_channel_lvds;
9082                 else
9083                         limit = &intel_limits_g4x_single_channel_lvds;
9084         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) ||
9085                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
9086                 limit = &intel_limits_g4x_hdmi;
9087         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) {
9088                 limit = &intel_limits_g4x_sdvo;
9089         } else {
9090                 /* The option is for other outputs */
9091                 limit = &intel_limits_i9xx_sdvo;
9092         }
9093
9094         if (!crtc_state->clock_set &&
9095             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
9096                                 refclk, NULL, &crtc_state->dpll)) {
9097                 drm_err(&dev_priv->drm,
9098                         "Couldn't find PLL settings for mode!\n");
9099                 return -EINVAL;
9100         }
9101
9102         i9xx_compute_dpll(crtc, crtc_state, NULL);
9103
9104         return 0;
9105 }
9106
9107 static int pnv_crtc_compute_clock(struct intel_crtc *crtc,
9108                                   struct intel_crtc_state *crtc_state)
9109 {
9110         struct drm_device *dev = crtc->base.dev;
9111         struct drm_i915_private *dev_priv = to_i915(dev);
9112         const struct intel_limit *limit;
9113         int refclk = 96000;
9114
9115         memset(&crtc_state->dpll_hw_state, 0,
9116                sizeof(crtc_state->dpll_hw_state));
9117
9118         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
9119                 if (intel_panel_use_ssc(dev_priv)) {
9120                         refclk = dev_priv->vbt.lvds_ssc_freq;
9121                         drm_dbg_kms(&dev_priv->drm,
9122                                     "using SSC reference clock of %d kHz\n",
9123                                     refclk);
9124                 }
9125
9126                 limit = &pnv_limits_lvds;
9127         } else {
9128                 limit = &pnv_limits_sdvo;
9129         }
9130
9131         if (!crtc_state->clock_set &&
9132             !pnv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
9133                                 refclk, NULL, &crtc_state->dpll)) {
9134                 drm_err(&dev_priv->drm,
9135                         "Couldn't find PLL settings for mode!\n");
9136                 return -EINVAL;
9137         }
9138
9139         i9xx_compute_dpll(crtc, crtc_state, NULL);
9140
9141         return 0;
9142 }
9143
9144 static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
9145                                    struct intel_crtc_state *crtc_state)
9146 {
9147         struct drm_device *dev = crtc->base.dev;
9148         struct drm_i915_private *dev_priv = to_i915(dev);
9149         const struct intel_limit *limit;
9150         int refclk = 96000;
9151
9152         memset(&crtc_state->dpll_hw_state, 0,
9153                sizeof(crtc_state->dpll_hw_state));
9154
9155         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
9156                 if (intel_panel_use_ssc(dev_priv)) {
9157                         refclk = dev_priv->vbt.lvds_ssc_freq;
9158                         drm_dbg_kms(&dev_priv->drm,
9159                                     "using SSC reference clock of %d kHz\n",
9160                                     refclk);
9161                 }
9162
9163                 limit = &intel_limits_i9xx_lvds;
9164         } else {
9165                 limit = &intel_limits_i9xx_sdvo;
9166         }
9167
9168         if (!crtc_state->clock_set &&
9169             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
9170                                  refclk, NULL, &crtc_state->dpll)) {
9171                 drm_err(&dev_priv->drm,
9172                         "Couldn't find PLL settings for mode!\n");
9173                 return -EINVAL;
9174         }
9175
9176         i9xx_compute_dpll(crtc, crtc_state, NULL);
9177
9178         return 0;
9179 }
9180
9181 static int chv_crtc_compute_clock(struct intel_crtc *crtc,
9182                                   struct intel_crtc_state *crtc_state)
9183 {
9184         int refclk = 100000;
9185         const struct intel_limit *limit = &intel_limits_chv;
9186         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
9187
9188         memset(&crtc_state->dpll_hw_state, 0,
9189                sizeof(crtc_state->dpll_hw_state));
9190
9191         if (!crtc_state->clock_set &&
9192             !chv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
9193                                 refclk, NULL, &crtc_state->dpll)) {
9194                 drm_err(&i915->drm, "Couldn't find PLL settings for mode!\n");
9195                 return -EINVAL;
9196         }
9197
9198         chv_compute_dpll(crtc, crtc_state);
9199
9200         return 0;
9201 }
9202
9203 static int vlv_crtc_compute_clock(struct intel_crtc *crtc,
9204                                   struct intel_crtc_state *crtc_state)
9205 {
9206         int refclk = 100000;
9207         const struct intel_limit *limit = &intel_limits_vlv;
9208         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
9209
9210         memset(&crtc_state->dpll_hw_state, 0,
9211                sizeof(crtc_state->dpll_hw_state));
9212
9213         if (!crtc_state->clock_set &&
9214             !vlv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
9215                                 refclk, NULL, &crtc_state->dpll)) {
9216                 drm_err(&i915->drm,  "Couldn't find PLL settings for mode!\n");
9217                 return -EINVAL;
9218         }
9219
9220         vlv_compute_dpll(crtc, crtc_state);
9221
9222         return 0;
9223 }
9224
9225 static bool i9xx_has_pfit(struct drm_i915_private *dev_priv)
9226 {
9227         if (IS_I830(dev_priv))
9228                 return false;
9229
9230         return INTEL_GEN(dev_priv) >= 4 ||
9231                 IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
9232 }
9233
9234 static void i9xx_get_pfit_config(struct intel_crtc_state *crtc_state)
9235 {
9236         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9237         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9238         u32 tmp;
9239
9240         if (!i9xx_has_pfit(dev_priv))
9241                 return;
9242
9243         tmp = intel_de_read(dev_priv, PFIT_CONTROL);
9244         if (!(tmp & PFIT_ENABLE))
9245                 return;
9246
9247         /* Check whether the pfit is attached to our pipe. */
9248         if (INTEL_GEN(dev_priv) < 4) {
9249                 if (crtc->pipe != PIPE_B)
9250                         return;
9251         } else {
9252                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
9253                         return;
9254         }
9255
9256         crtc_state->gmch_pfit.control = tmp;
9257         crtc_state->gmch_pfit.pgm_ratios =
9258                 intel_de_read(dev_priv, PFIT_PGM_RATIOS);
9259 }
9260
9261 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
9262                                struct intel_crtc_state *pipe_config)
9263 {
9264         struct drm_device *dev = crtc->base.dev;
9265         struct drm_i915_private *dev_priv = to_i915(dev);
9266         enum pipe pipe = crtc->pipe;
9267         struct dpll clock;
9268         u32 mdiv;
9269         int refclk = 100000;
9270
9271         /* In case of DSI, DPLL will not be used */
9272         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
9273                 return;
9274
9275         vlv_dpio_get(dev_priv);
9276         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
9277         vlv_dpio_put(dev_priv);
9278
9279         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
9280         clock.m2 = mdiv & DPIO_M2DIV_MASK;
9281         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
9282         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
9283         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
9284
9285         pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
9286 }
9287
9288 static void
9289 i9xx_get_initial_plane_config(struct intel_crtc *crtc,
9290                               struct intel_initial_plane_config *plane_config)
9291 {
9292         struct drm_device *dev = crtc->base.dev;
9293         struct drm_i915_private *dev_priv = to_i915(dev);
9294         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
9295         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
9296         enum pipe pipe;
9297         u32 val, base, offset;
9298         int fourcc, pixel_format;
9299         unsigned int aligned_height;
9300         struct drm_framebuffer *fb;
9301         struct intel_framebuffer *intel_fb;
9302
9303         if (!plane->get_hw_state(plane, &pipe))
9304                 return;
9305
9306         drm_WARN_ON(dev, pipe != crtc->pipe);
9307
9308         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
9309         if (!intel_fb) {
9310                 drm_dbg_kms(&dev_priv->drm, "failed to alloc fb\n");
9311                 return;
9312         }
9313
9314         fb = &intel_fb->base;
9315
9316         fb->dev = dev;
9317
9318         val = intel_de_read(dev_priv, DSPCNTR(i9xx_plane));
9319
9320         if (INTEL_GEN(dev_priv) >= 4) {
9321                 if (val & DISPPLANE_TILED) {
9322                         plane_config->tiling = I915_TILING_X;
9323                         fb->modifier = I915_FORMAT_MOD_X_TILED;
9324                 }
9325
9326                 if (val & DISPPLANE_ROTATE_180)
9327                         plane_config->rotation = DRM_MODE_ROTATE_180;
9328         }
9329
9330         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B &&
9331             val & DISPPLANE_MIRROR)
9332                 plane_config->rotation |= DRM_MODE_REFLECT_X;
9333
9334         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
9335         fourcc = i9xx_format_to_fourcc(pixel_format);
9336         fb->format = drm_format_info(fourcc);
9337
9338         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
9339                 offset = intel_de_read(dev_priv, DSPOFFSET(i9xx_plane));
9340                 base = intel_de_read(dev_priv, DSPSURF(i9xx_plane)) & 0xfffff000;
9341         } else if (INTEL_GEN(dev_priv) >= 4) {
9342                 if (plane_config->tiling)
9343                         offset = intel_de_read(dev_priv,
9344                                                DSPTILEOFF(i9xx_plane));
9345                 else
9346                         offset = intel_de_read(dev_priv,
9347                                                DSPLINOFF(i9xx_plane));
9348                 base = intel_de_read(dev_priv, DSPSURF(i9xx_plane)) & 0xfffff000;
9349         } else {
9350                 base = intel_de_read(dev_priv, DSPADDR(i9xx_plane));
9351         }
9352         plane_config->base = base;
9353
9354         val = intel_de_read(dev_priv, PIPESRC(pipe));
9355         fb->width = ((val >> 16) & 0xfff) + 1;
9356         fb->height = ((val >> 0) & 0xfff) + 1;
9357
9358         val = intel_de_read(dev_priv, DSPSTRIDE(i9xx_plane));
9359         fb->pitches[0] = val & 0xffffffc0;
9360
9361         aligned_height = intel_fb_align_height(fb, 0, fb->height);
9362
9363         plane_config->size = fb->pitches[0] * aligned_height;
9364
9365         drm_dbg_kms(&dev_priv->drm,
9366                     "%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
9367                     crtc->base.name, plane->base.name, fb->width, fb->height,
9368                     fb->format->cpp[0] * 8, base, fb->pitches[0],
9369                     plane_config->size);
9370
9371         plane_config->fb = intel_fb;
9372 }
9373
9374 static void chv_crtc_clock_get(struct intel_crtc *crtc,
9375                                struct intel_crtc_state *pipe_config)
9376 {
9377         struct drm_device *dev = crtc->base.dev;
9378         struct drm_i915_private *dev_priv = to_i915(dev);
9379         enum pipe pipe = crtc->pipe;
9380         enum dpio_channel port = vlv_pipe_to_channel(pipe);
9381         struct dpll clock;
9382         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
9383         int refclk = 100000;
9384
9385         /* In case of DSI, DPLL will not be used */
9386         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
9387                 return;
9388
9389         vlv_dpio_get(dev_priv);
9390         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
9391         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
9392         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
9393         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
9394         pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
9395         vlv_dpio_put(dev_priv);
9396
9397         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
9398         clock.m2 = (pll_dw0 & 0xff) << 22;
9399         if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
9400                 clock.m2 |= pll_dw2 & 0x3fffff;
9401         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
9402         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
9403         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
9404
9405         pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
9406 }
9407
9408 static enum intel_output_format
9409 bdw_get_pipemisc_output_format(struct intel_crtc *crtc)
9410 {
9411         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9412         u32 tmp;
9413
9414         tmp = intel_de_read(dev_priv, PIPEMISC(crtc->pipe));
9415
9416         if (tmp & PIPEMISC_YUV420_ENABLE) {
9417                 /* We support 4:2:0 in full blend mode only */
9418                 drm_WARN_ON(&dev_priv->drm,
9419                             (tmp & PIPEMISC_YUV420_MODE_FULL_BLEND) == 0);
9420
9421                 return INTEL_OUTPUT_FORMAT_YCBCR420;
9422         } else if (tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV) {
9423                 return INTEL_OUTPUT_FORMAT_YCBCR444;
9424         } else {
9425                 return INTEL_OUTPUT_FORMAT_RGB;
9426         }
9427 }
9428
9429 static void i9xx_get_pipe_color_config(struct intel_crtc_state *crtc_state)
9430 {
9431         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9432         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
9433         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9434         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
9435         u32 tmp;
9436
9437         tmp = intel_de_read(dev_priv, DSPCNTR(i9xx_plane));
9438
9439         if (tmp & DISPPLANE_GAMMA_ENABLE)
9440                 crtc_state->gamma_enable = true;
9441
9442         if (!HAS_GMCH(dev_priv) &&
9443             tmp & DISPPLANE_PIPE_CSC_ENABLE)
9444                 crtc_state->csc_enable = true;
9445 }
9446
9447 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
9448                                  struct intel_crtc_state *pipe_config)
9449 {
9450         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9451         enum intel_display_power_domain power_domain;
9452         intel_wakeref_t wakeref;
9453         u32 tmp;
9454         bool ret;
9455
9456         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
9457         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
9458         if (!wakeref)
9459                 return false;
9460
9461         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
9462         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9463         pipe_config->shared_dpll = NULL;
9464
9465         ret = false;
9466
9467         tmp = intel_de_read(dev_priv, PIPECONF(crtc->pipe));
9468         if (!(tmp & PIPECONF_ENABLE))
9469                 goto out;
9470
9471         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
9472             IS_CHERRYVIEW(dev_priv)) {
9473                 switch (tmp & PIPECONF_BPC_MASK) {
9474                 case PIPECONF_6BPC:
9475                         pipe_config->pipe_bpp = 18;
9476                         break;
9477                 case PIPECONF_8BPC:
9478                         pipe_config->pipe_bpp = 24;
9479                         break;
9480                 case PIPECONF_10BPC:
9481                         pipe_config->pipe_bpp = 30;
9482                         break;
9483                 default:
9484                         break;
9485                 }
9486         }
9487
9488         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
9489             (tmp & PIPECONF_COLOR_RANGE_SELECT))
9490                 pipe_config->limited_color_range = true;
9491
9492         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_I9XX) >>
9493                 PIPECONF_GAMMA_MODE_SHIFT;
9494
9495         if (IS_CHERRYVIEW(dev_priv))
9496                 pipe_config->cgm_mode = intel_de_read(dev_priv,
9497                                                       CGM_PIPE_MODE(crtc->pipe));
9498
9499         i9xx_get_pipe_color_config(pipe_config);
9500         intel_color_get_config(pipe_config);
9501
9502         if (INTEL_GEN(dev_priv) < 4)
9503                 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
9504
9505         intel_get_pipe_timings(crtc, pipe_config);
9506         intel_get_pipe_src_size(crtc, pipe_config);
9507
9508         i9xx_get_pfit_config(pipe_config);
9509
9510         if (INTEL_GEN(dev_priv) >= 4) {
9511                 /* No way to read it out on pipes B and C */
9512                 if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
9513                         tmp = dev_priv->chv_dpll_md[crtc->pipe];
9514                 else
9515                         tmp = intel_de_read(dev_priv, DPLL_MD(crtc->pipe));
9516                 pipe_config->pixel_multiplier =
9517                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
9518                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
9519                 pipe_config->dpll_hw_state.dpll_md = tmp;
9520         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
9521                    IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
9522                 tmp = intel_de_read(dev_priv, DPLL(crtc->pipe));
9523                 pipe_config->pixel_multiplier =
9524                         ((tmp & SDVO_MULTIPLIER_MASK)
9525                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
9526         } else {
9527                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
9528                  * port and will be fixed up in the encoder->get_config
9529                  * function. */
9530                 pipe_config->pixel_multiplier = 1;
9531         }
9532         pipe_config->dpll_hw_state.dpll = intel_de_read(dev_priv,
9533                                                         DPLL(crtc->pipe));
9534         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
9535                 pipe_config->dpll_hw_state.fp0 = intel_de_read(dev_priv,
9536                                                                FP0(crtc->pipe));
9537                 pipe_config->dpll_hw_state.fp1 = intel_de_read(dev_priv,
9538                                                                FP1(crtc->pipe));
9539         } else {
9540                 /* Mask out read-only status bits. */
9541                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
9542                                                      DPLL_PORTC_READY_MASK |
9543                                                      DPLL_PORTB_READY_MASK);
9544         }
9545
9546         if (IS_CHERRYVIEW(dev_priv))
9547                 chv_crtc_clock_get(crtc, pipe_config);
9548         else if (IS_VALLEYVIEW(dev_priv))
9549                 vlv_crtc_clock_get(crtc, pipe_config);
9550         else
9551                 i9xx_crtc_clock_get(crtc, pipe_config);
9552
9553         /*
9554          * Normally the dotclock is filled in by the encoder .get_config()
9555          * but in case the pipe is enabled w/o any ports we need a sane
9556          * default.
9557          */
9558         pipe_config->hw.adjusted_mode.crtc_clock =
9559                 pipe_config->port_clock / pipe_config->pixel_multiplier;
9560
9561         ret = true;
9562
9563 out:
9564         intel_display_power_put(dev_priv, power_domain, wakeref);
9565
9566         return ret;
9567 }
9568
9569 static void ilk_init_pch_refclk(struct drm_i915_private *dev_priv)
9570 {
9571         struct intel_encoder *encoder;
9572         int i;
9573         u32 val, final;
9574         bool has_lvds = false;
9575         bool has_cpu_edp = false;
9576         bool has_panel = false;
9577         bool has_ck505 = false;
9578         bool can_ssc = false;
9579         bool using_ssc_source = false;
9580
9581         /* We need to take the global config into account */
9582         for_each_intel_encoder(&dev_priv->drm, encoder) {
9583                 switch (encoder->type) {
9584                 case INTEL_OUTPUT_LVDS:
9585                         has_panel = true;
9586                         has_lvds = true;
9587                         break;
9588                 case INTEL_OUTPUT_EDP:
9589                         has_panel = true;
9590                         if (encoder->port == PORT_A)
9591                                 has_cpu_edp = true;
9592                         break;
9593                 default:
9594                         break;
9595                 }
9596         }
9597
9598         if (HAS_PCH_IBX(dev_priv)) {
9599                 has_ck505 = dev_priv->vbt.display_clock_mode;
9600                 can_ssc = has_ck505;
9601         } else {
9602                 has_ck505 = false;
9603                 can_ssc = true;
9604         }
9605
9606         /* Check if any DPLLs are using the SSC source */
9607         for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++) {
9608                 u32 temp = intel_de_read(dev_priv, PCH_DPLL(i));
9609
9610                 if (!(temp & DPLL_VCO_ENABLE))
9611                         continue;
9612
9613                 if ((temp & PLL_REF_INPUT_MASK) ==
9614                     PLLB_REF_INPUT_SPREADSPECTRUMIN) {
9615                         using_ssc_source = true;
9616                         break;
9617                 }
9618         }
9619
9620         drm_dbg_kms(&dev_priv->drm,
9621                     "has_panel %d has_lvds %d has_ck505 %d using_ssc_source %d\n",
9622                     has_panel, has_lvds, has_ck505, using_ssc_source);
9623
9624         /* Ironlake: try to setup display ref clock before DPLL
9625          * enabling. This is only under driver's control after
9626          * PCH B stepping, previous chipset stepping should be
9627          * ignoring this setting.
9628          */
9629         val = intel_de_read(dev_priv, PCH_DREF_CONTROL);
9630
9631         /* As we must carefully and slowly disable/enable each source in turn,
9632          * compute the final state we want first and check if we need to
9633          * make any changes at all.
9634          */
9635         final = val;
9636         final &= ~DREF_NONSPREAD_SOURCE_MASK;
9637         if (has_ck505)
9638                 final |= DREF_NONSPREAD_CK505_ENABLE;
9639         else
9640                 final |= DREF_NONSPREAD_SOURCE_ENABLE;
9641
9642         final &= ~DREF_SSC_SOURCE_MASK;
9643         final &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9644         final &= ~DREF_SSC1_ENABLE;
9645
9646         if (has_panel) {
9647                 final |= DREF_SSC_SOURCE_ENABLE;
9648
9649                 if (intel_panel_use_ssc(dev_priv) && can_ssc)
9650                         final |= DREF_SSC1_ENABLE;
9651
9652                 if (has_cpu_edp) {
9653                         if (intel_panel_use_ssc(dev_priv) && can_ssc)
9654                                 final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
9655                         else
9656                                 final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
9657                 } else
9658                         final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9659         } else if (using_ssc_source) {
9660                 final |= DREF_SSC_SOURCE_ENABLE;
9661                 final |= DREF_SSC1_ENABLE;
9662         }
9663
9664         if (final == val)
9665                 return;
9666
9667         /* Always enable nonspread source */
9668         val &= ~DREF_NONSPREAD_SOURCE_MASK;
9669
9670         if (has_ck505)
9671                 val |= DREF_NONSPREAD_CK505_ENABLE;
9672         else
9673                 val |= DREF_NONSPREAD_SOURCE_ENABLE;
9674
9675         if (has_panel) {
9676                 val &= ~DREF_SSC_SOURCE_MASK;
9677                 val |= DREF_SSC_SOURCE_ENABLE;
9678
9679                 /* SSC must be turned on before enabling the CPU output  */
9680                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
9681                         drm_dbg_kms(&dev_priv->drm, "Using SSC on panel\n");
9682                         val |= DREF_SSC1_ENABLE;
9683                 } else
9684                         val &= ~DREF_SSC1_ENABLE;
9685
9686                 /* Get SSC going before enabling the outputs */
9687                 intel_de_write(dev_priv, PCH_DREF_CONTROL, val);
9688                 intel_de_posting_read(dev_priv, PCH_DREF_CONTROL);
9689                 udelay(200);
9690
9691                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9692
9693                 /* Enable CPU source on CPU attached eDP */
9694                 if (has_cpu_edp) {
9695                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
9696                                 drm_dbg_kms(&dev_priv->drm,
9697                                             "Using SSC on eDP\n");
9698                                 val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
9699                         } else
9700                                 val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
9701                 } else
9702                         val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9703
9704                 intel_de_write(dev_priv, PCH_DREF_CONTROL, val);
9705                 intel_de_posting_read(dev_priv, PCH_DREF_CONTROL);
9706                 udelay(200);
9707         } else {
9708                 drm_dbg_kms(&dev_priv->drm, "Disabling CPU source output\n");
9709
9710                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9711
9712                 /* Turn off CPU output */
9713                 val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9714
9715                 intel_de_write(dev_priv, PCH_DREF_CONTROL, val);
9716                 intel_de_posting_read(dev_priv, PCH_DREF_CONTROL);
9717                 udelay(200);
9718
9719                 if (!using_ssc_source) {
9720                         drm_dbg_kms(&dev_priv->drm, "Disabling SSC source\n");
9721
9722                         /* Turn off the SSC source */
9723                         val &= ~DREF_SSC_SOURCE_MASK;
9724                         val |= DREF_SSC_SOURCE_DISABLE;
9725
9726                         /* Turn off SSC1 */
9727                         val &= ~DREF_SSC1_ENABLE;
9728
9729                         intel_de_write(dev_priv, PCH_DREF_CONTROL, val);
9730                         intel_de_posting_read(dev_priv, PCH_DREF_CONTROL);
9731                         udelay(200);
9732                 }
9733         }
9734
9735         BUG_ON(val != final);
9736 }
9737
9738 static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv)
9739 {
9740         u32 tmp;
9741
9742         tmp = intel_de_read(dev_priv, SOUTH_CHICKEN2);
9743         tmp |= FDI_MPHY_IOSFSB_RESET_CTL;
9744         intel_de_write(dev_priv, SOUTH_CHICKEN2, tmp);
9745
9746         if (wait_for_us(intel_de_read(dev_priv, SOUTH_CHICKEN2) &
9747                         FDI_MPHY_IOSFSB_RESET_STATUS, 100))
9748                 drm_err(&dev_priv->drm, "FDI mPHY reset assert timeout\n");
9749
9750         tmp = intel_de_read(dev_priv, SOUTH_CHICKEN2);
9751         tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL;
9752         intel_de_write(dev_priv, SOUTH_CHICKEN2, tmp);
9753
9754         if (wait_for_us((intel_de_read(dev_priv, SOUTH_CHICKEN2) &
9755                          FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100))
9756                 drm_err(&dev_priv->drm, "FDI mPHY reset de-assert timeout\n");
9757 }
9758
9759 /* WaMPhyProgramming:hsw */
9760 static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv)
9761 {
9762         u32 tmp;
9763
9764         tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY);
9765         tmp &= ~(0xFF << 24);
9766         tmp |= (0x12 << 24);
9767         intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY);
9768
9769         tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY);
9770         tmp |= (1 << 11);
9771         intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY);
9772
9773         tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY);
9774         tmp |= (1 << 11);
9775         intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY);
9776
9777         tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY);
9778         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
9779         intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY);
9780
9781         tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY);
9782         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
9783         intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY);
9784
9785         tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY);
9786         tmp &= ~(7 << 13);
9787         tmp |= (5 << 13);
9788         intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY);
9789
9790         tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY);
9791         tmp &= ~(7 << 13);
9792         tmp |= (5 << 13);
9793         intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY);
9794
9795         tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY);
9796         tmp &= ~0xFF;
9797         tmp |= 0x1C;
9798         intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY);
9799
9800         tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY);
9801         tmp &= ~0xFF;
9802         tmp |= 0x1C;
9803         intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY);
9804
9805         tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY);
9806         tmp &= ~(0xFF << 16);
9807         tmp |= (0x1C << 16);
9808         intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY);
9809
9810         tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY);
9811         tmp &= ~(0xFF << 16);
9812         tmp |= (0x1C << 16);
9813         intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY);
9814
9815         tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY);
9816         tmp |= (1 << 27);
9817         intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY);
9818
9819         tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY);
9820         tmp |= (1 << 27);
9821         intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY);
9822
9823         tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY);
9824         tmp &= ~(0xF << 28);
9825         tmp |= (4 << 28);
9826         intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY);
9827
9828         tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY);
9829         tmp &= ~(0xF << 28);
9830         tmp |= (4 << 28);
9831         intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY);
9832 }
9833
9834 /* Implements 3 different sequences from BSpec chapter "Display iCLK
9835  * Programming" based on the parameters passed:
9836  * - Sequence to enable CLKOUT_DP
9837  * - Sequence to enable CLKOUT_DP without spread
9838  * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O
9839  */
9840 static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv,
9841                                  bool with_spread, bool with_fdi)
9842 {
9843         u32 reg, tmp;
9844
9845         if (drm_WARN(&dev_priv->drm, with_fdi && !with_spread,
9846                      "FDI requires downspread\n"))
9847                 with_spread = true;
9848         if (drm_WARN(&dev_priv->drm, HAS_PCH_LPT_LP(dev_priv) &&
9849                      with_fdi, "LP PCH doesn't have FDI\n"))
9850                 with_fdi = false;
9851
9852         mutex_lock(&dev_priv->sb_lock);
9853
9854         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9855         tmp &= ~SBI_SSCCTL_DISABLE;
9856         tmp |= SBI_SSCCTL_PATHALT;
9857         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9858
9859         udelay(24);
9860
9861         if (with_spread) {
9862                 tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9863                 tmp &= ~SBI_SSCCTL_PATHALT;
9864                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9865
9866                 if (with_fdi) {
9867                         lpt_reset_fdi_mphy(dev_priv);
9868                         lpt_program_fdi_mphy(dev_priv);
9869                 }
9870         }
9871
9872         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
9873         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
9874         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
9875         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
9876
9877         mutex_unlock(&dev_priv->sb_lock);
9878 }
9879
9880 /* Sequence to disable CLKOUT_DP */
9881 void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv)
9882 {
9883         u32 reg, tmp;
9884
9885         mutex_lock(&dev_priv->sb_lock);
9886
9887         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
9888         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
9889         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
9890         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
9891
9892         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9893         if (!(tmp & SBI_SSCCTL_DISABLE)) {
9894                 if (!(tmp & SBI_SSCCTL_PATHALT)) {
9895                         tmp |= SBI_SSCCTL_PATHALT;
9896                         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9897                         udelay(32);
9898                 }
9899                 tmp |= SBI_SSCCTL_DISABLE;
9900                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9901         }
9902
9903         mutex_unlock(&dev_priv->sb_lock);
9904 }
9905
9906 #define BEND_IDX(steps) ((50 + (steps)) / 5)
9907
9908 static const u16 sscdivintphase[] = {
9909         [BEND_IDX( 50)] = 0x3B23,
9910         [BEND_IDX( 45)] = 0x3B23,
9911         [BEND_IDX( 40)] = 0x3C23,
9912         [BEND_IDX( 35)] = 0x3C23,
9913         [BEND_IDX( 30)] = 0x3D23,
9914         [BEND_IDX( 25)] = 0x3D23,
9915         [BEND_IDX( 20)] = 0x3E23,
9916         [BEND_IDX( 15)] = 0x3E23,
9917         [BEND_IDX( 10)] = 0x3F23,
9918         [BEND_IDX(  5)] = 0x3F23,
9919         [BEND_IDX(  0)] = 0x0025,
9920         [BEND_IDX( -5)] = 0x0025,
9921         [BEND_IDX(-10)] = 0x0125,
9922         [BEND_IDX(-15)] = 0x0125,
9923         [BEND_IDX(-20)] = 0x0225,
9924         [BEND_IDX(-25)] = 0x0225,
9925         [BEND_IDX(-30)] = 0x0325,
9926         [BEND_IDX(-35)] = 0x0325,
9927         [BEND_IDX(-40)] = 0x0425,
9928         [BEND_IDX(-45)] = 0x0425,
9929         [BEND_IDX(-50)] = 0x0525,
9930 };
9931
9932 /*
9933  * Bend CLKOUT_DP
9934  * steps -50 to 50 inclusive, in steps of 5
9935  * < 0 slow down the clock, > 0 speed up the clock, 0 == no bend (135MHz)
9936  * change in clock period = -(steps / 10) * 5.787 ps
9937  */
9938 static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps)
9939 {
9940         u32 tmp;
9941         int idx = BEND_IDX(steps);
9942
9943         if (drm_WARN_ON(&dev_priv->drm, steps % 5 != 0))
9944                 return;
9945
9946         if (drm_WARN_ON(&dev_priv->drm, idx >= ARRAY_SIZE(sscdivintphase)))
9947                 return;
9948
9949         mutex_lock(&dev_priv->sb_lock);
9950
9951         if (steps % 10 != 0)
9952                 tmp = 0xAAAAAAAB;
9953         else
9954                 tmp = 0x00000000;
9955         intel_sbi_write(dev_priv, SBI_SSCDITHPHASE, tmp, SBI_ICLK);
9956
9957         tmp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE, SBI_ICLK);
9958         tmp &= 0xffff0000;
9959         tmp |= sscdivintphase[idx];
9960         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE, tmp, SBI_ICLK);
9961
9962         mutex_unlock(&dev_priv->sb_lock);
9963 }
9964
9965 #undef BEND_IDX
9966
9967 static bool spll_uses_pch_ssc(struct drm_i915_private *dev_priv)
9968 {
9969         u32 fuse_strap = intel_de_read(dev_priv, FUSE_STRAP);
9970         u32 ctl = intel_de_read(dev_priv, SPLL_CTL);
9971
9972         if ((ctl & SPLL_PLL_ENABLE) == 0)
9973                 return false;
9974
9975         if ((ctl & SPLL_REF_MASK) == SPLL_REF_MUXED_SSC &&
9976             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
9977                 return true;
9978
9979         if (IS_BROADWELL(dev_priv) &&
9980             (ctl & SPLL_REF_MASK) == SPLL_REF_PCH_SSC_BDW)
9981                 return true;
9982
9983         return false;
9984 }
9985
9986 static bool wrpll_uses_pch_ssc(struct drm_i915_private *dev_priv,
9987                                enum intel_dpll_id id)
9988 {
9989         u32 fuse_strap = intel_de_read(dev_priv, FUSE_STRAP);
9990         u32 ctl = intel_de_read(dev_priv, WRPLL_CTL(id));
9991
9992         if ((ctl & WRPLL_PLL_ENABLE) == 0)
9993                 return false;
9994
9995         if ((ctl & WRPLL_REF_MASK) == WRPLL_REF_PCH_SSC)
9996                 return true;
9997
9998         if ((IS_BROADWELL(dev_priv) || IS_HSW_ULT(dev_priv)) &&
9999             (ctl & WRPLL_REF_MASK) == WRPLL_REF_MUXED_SSC_BDW &&
10000             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
10001                 return true;
10002
10003         return false;
10004 }
10005
10006 static void lpt_init_pch_refclk(struct drm_i915_private *dev_priv)
10007 {
10008         struct intel_encoder *encoder;
10009         bool has_fdi = false;
10010
10011         for_each_intel_encoder(&dev_priv->drm, encoder) {
10012                 switch (encoder->type) {
10013                 case INTEL_OUTPUT_ANALOG:
10014                         has_fdi = true;
10015                         break;
10016                 default:
10017                         break;
10018                 }
10019         }
10020
10021         /*
10022          * The BIOS may have decided to use the PCH SSC
10023          * reference so we must not disable it until the
10024          * relevant PLLs have stopped relying on it. We'll
10025          * just leave the PCH SSC reference enabled in case
10026          * any active PLL is using it. It will get disabled
10027          * after runtime suspend if we don't have FDI.
10028          *
10029          * TODO: Move the whole reference clock handling
10030          * to the modeset sequence proper so that we can
10031          * actually enable/disable/reconfigure these things
10032          * safely. To do that we need to introduce a real
10033          * clock hierarchy. That would also allow us to do
10034          * clock bending finally.
10035          */
10036         dev_priv->pch_ssc_use = 0;
10037
10038         if (spll_uses_pch_ssc(dev_priv)) {
10039                 drm_dbg_kms(&dev_priv->drm, "SPLL using PCH SSC\n");
10040                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_SPLL);
10041         }
10042
10043         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL1)) {
10044                 drm_dbg_kms(&dev_priv->drm, "WRPLL1 using PCH SSC\n");
10045                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_WRPLL1);
10046         }
10047
10048         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL2)) {
10049                 drm_dbg_kms(&dev_priv->drm, "WRPLL2 using PCH SSC\n");
10050                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_WRPLL2);
10051         }
10052
10053         if (dev_priv->pch_ssc_use)
10054                 return;
10055
10056         if (has_fdi) {
10057                 lpt_bend_clkout_dp(dev_priv, 0);
10058                 lpt_enable_clkout_dp(dev_priv, true, true);
10059         } else {
10060                 lpt_disable_clkout_dp(dev_priv);
10061         }
10062 }
10063
10064 /*
10065  * Initialize reference clocks when the driver loads
10066  */
10067 void intel_init_pch_refclk(struct drm_i915_private *dev_priv)
10068 {
10069         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
10070                 ilk_init_pch_refclk(dev_priv);
10071         else if (HAS_PCH_LPT(dev_priv))
10072                 lpt_init_pch_refclk(dev_priv);
10073 }
10074
10075 static void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state)
10076 {
10077         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
10078         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10079         enum pipe pipe = crtc->pipe;
10080         u32 val;
10081
10082         val = 0;
10083
10084         switch (crtc_state->pipe_bpp) {
10085         case 18:
10086                 val |= PIPECONF_6BPC;
10087                 break;
10088         case 24:
10089                 val |= PIPECONF_8BPC;
10090                 break;
10091         case 30:
10092                 val |= PIPECONF_10BPC;
10093                 break;
10094         case 36:
10095                 val |= PIPECONF_12BPC;
10096                 break;
10097         default:
10098                 /* Case prevented by intel_choose_pipe_bpp_dither. */
10099                 BUG();
10100         }
10101
10102         if (crtc_state->dither)
10103                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
10104
10105         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
10106                 val |= PIPECONF_INTERLACED_ILK;
10107         else
10108                 val |= PIPECONF_PROGRESSIVE;
10109
10110         /*
10111          * This would end up with an odd purple hue over
10112          * the entire display. Make sure we don't do it.
10113          */
10114         drm_WARN_ON(&dev_priv->drm, crtc_state->limited_color_range &&
10115                     crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB);
10116
10117         if (crtc_state->limited_color_range &&
10118             !intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
10119                 val |= PIPECONF_COLOR_RANGE_SELECT;
10120
10121         if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
10122                 val |= PIPECONF_OUTPUT_COLORSPACE_YUV709;
10123
10124         val |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
10125
10126         val |= PIPECONF_FRAME_START_DELAY(0);
10127
10128         intel_de_write(dev_priv, PIPECONF(pipe), val);
10129         intel_de_posting_read(dev_priv, PIPECONF(pipe));
10130 }
10131
10132 static void hsw_set_pipeconf(const struct intel_crtc_state *crtc_state)
10133 {
10134         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
10135         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10136         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
10137         u32 val = 0;
10138
10139         if (IS_HASWELL(dev_priv) && crtc_state->dither)
10140                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
10141
10142         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
10143                 val |= PIPECONF_INTERLACED_ILK;
10144         else
10145                 val |= PIPECONF_PROGRESSIVE;
10146
10147         if (IS_HASWELL(dev_priv) &&
10148             crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
10149                 val |= PIPECONF_OUTPUT_COLORSPACE_YUV_HSW;
10150
10151         intel_de_write(dev_priv, PIPECONF(cpu_transcoder), val);
10152         intel_de_posting_read(dev_priv, PIPECONF(cpu_transcoder));
10153 }
10154
10155 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state)
10156 {
10157         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
10158         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10159         u32 val = 0;
10160
10161         switch (crtc_state->pipe_bpp) {
10162         case 18:
10163                 val |= PIPEMISC_DITHER_6_BPC;
10164                 break;
10165         case 24:
10166                 val |= PIPEMISC_DITHER_8_BPC;
10167                 break;
10168         case 30:
10169                 val |= PIPEMISC_DITHER_10_BPC;
10170                 break;
10171         case 36:
10172                 val |= PIPEMISC_DITHER_12_BPC;
10173                 break;
10174         default:
10175                 MISSING_CASE(crtc_state->pipe_bpp);
10176                 break;
10177         }
10178
10179         if (crtc_state->dither)
10180                 val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
10181
10182         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
10183             crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444)
10184                 val |= PIPEMISC_OUTPUT_COLORSPACE_YUV;
10185
10186         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
10187                 val |= PIPEMISC_YUV420_ENABLE |
10188                         PIPEMISC_YUV420_MODE_FULL_BLEND;
10189
10190         if (INTEL_GEN(dev_priv) >= 11 &&
10191             (crtc_state->active_planes & ~(icl_hdr_plane_mask() |
10192                                            BIT(PLANE_CURSOR))) == 0)
10193                 val |= PIPEMISC_HDR_MODE_PRECISION;
10194
10195         if (INTEL_GEN(dev_priv) >= 12)
10196                 val |= PIPEMISC_PIXEL_ROUNDING_TRUNC;
10197
10198         intel_de_write(dev_priv, PIPEMISC(crtc->pipe), val);
10199 }
10200
10201 int bdw_get_pipemisc_bpp(struct intel_crtc *crtc)
10202 {
10203         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10204         u32 tmp;
10205
10206         tmp = intel_de_read(dev_priv, PIPEMISC(crtc->pipe));
10207
10208         switch (tmp & PIPEMISC_DITHER_BPC_MASK) {
10209         case PIPEMISC_DITHER_6_BPC:
10210                 return 18;
10211         case PIPEMISC_DITHER_8_BPC:
10212                 return 24;
10213         case PIPEMISC_DITHER_10_BPC:
10214                 return 30;
10215         case PIPEMISC_DITHER_12_BPC:
10216                 return 36;
10217         default:
10218                 MISSING_CASE(tmp);
10219                 return 0;
10220         }
10221 }
10222
10223 int ilk_get_lanes_required(int target_clock, int link_bw, int bpp)
10224 {
10225         /*
10226          * Account for spread spectrum to avoid
10227          * oversubscribing the link. Max center spread
10228          * is 2.5%; use 5% for safety's sake.
10229          */
10230         u32 bps = target_clock * bpp * 21 / 20;
10231         return DIV_ROUND_UP(bps, link_bw * 8);
10232 }
10233
10234 static bool ilk_needs_fb_cb_tune(struct dpll *dpll, int factor)
10235 {
10236         return i9xx_dpll_compute_m(dpll) < factor * dpll->n;
10237 }
10238
10239 static void ilk_compute_dpll(struct intel_crtc *crtc,
10240                              struct intel_crtc_state *crtc_state,
10241                              struct dpll *reduced_clock)
10242 {
10243         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10244         u32 dpll, fp, fp2;
10245         int factor;
10246
10247         /* Enable autotuning of the PLL clock (if permissible) */
10248         factor = 21;
10249         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
10250                 if ((intel_panel_use_ssc(dev_priv) &&
10251                      dev_priv->vbt.lvds_ssc_freq == 100000) ||
10252                     (HAS_PCH_IBX(dev_priv) &&
10253                      intel_is_dual_link_lvds(dev_priv)))
10254                         factor = 25;
10255         } else if (crtc_state->sdvo_tv_clock) {
10256                 factor = 20;
10257         }
10258
10259         fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
10260
10261         if (ilk_needs_fb_cb_tune(&crtc_state->dpll, factor))
10262                 fp |= FP_CB_TUNE;
10263
10264         if (reduced_clock) {
10265                 fp2 = i9xx_dpll_compute_fp(reduced_clock);
10266
10267                 if (reduced_clock->m < factor * reduced_clock->n)
10268                         fp2 |= FP_CB_TUNE;
10269         } else {
10270                 fp2 = fp;
10271         }
10272
10273         dpll = 0;
10274
10275         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
10276                 dpll |= DPLLB_MODE_LVDS;
10277         else
10278                 dpll |= DPLLB_MODE_DAC_SERIAL;
10279
10280         dpll |= (crtc_state->pixel_multiplier - 1)
10281                 << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
10282
10283         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
10284             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
10285                 dpll |= DPLL_SDVO_HIGH_SPEED;
10286
10287         if (intel_crtc_has_dp_encoder(crtc_state))
10288                 dpll |= DPLL_SDVO_HIGH_SPEED;
10289
10290         /*
10291          * The high speed IO clock is only really required for
10292          * SDVO/HDMI/DP, but we also enable it for CRT to make it
10293          * possible to share the DPLL between CRT and HDMI. Enabling
10294          * the clock needlessly does no real harm, except use up a
10295          * bit of power potentially.
10296          *
10297          * We'll limit this to IVB with 3 pipes, since it has only two
10298          * DPLLs and so DPLL sharing is the only way to get three pipes
10299          * driving PCH ports at the same time. On SNB we could do this,
10300          * and potentially avoid enabling the second DPLL, but it's not
10301          * clear if it''s a win or loss power wise. No point in doing
10302          * this on ILK at all since it has a fixed DPLL<->pipe mapping.
10303          */
10304         if (INTEL_NUM_PIPES(dev_priv) == 3 &&
10305             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG))
10306                 dpll |= DPLL_SDVO_HIGH_SPEED;
10307
10308         /* compute bitmask from p1 value */
10309         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
10310         /* also FPA1 */
10311         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
10312
10313         switch (crtc_state->dpll.p2) {
10314         case 5:
10315                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
10316                 break;
10317         case 7:
10318                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
10319                 break;
10320         case 10:
10321                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
10322                 break;
10323         case 14:
10324                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
10325                 break;
10326         }
10327
10328         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
10329             intel_panel_use_ssc(dev_priv))
10330                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
10331         else
10332                 dpll |= PLL_REF_INPUT_DREFCLK;
10333
10334         dpll |= DPLL_VCO_ENABLE;
10335
10336         crtc_state->dpll_hw_state.dpll = dpll;
10337         crtc_state->dpll_hw_state.fp0 = fp;
10338         crtc_state->dpll_hw_state.fp1 = fp2;
10339 }
10340
10341 static int ilk_crtc_compute_clock(struct intel_crtc *crtc,
10342                                   struct intel_crtc_state *crtc_state)
10343 {
10344         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10345         struct intel_atomic_state *state =
10346                 to_intel_atomic_state(crtc_state->uapi.state);
10347         const struct intel_limit *limit;
10348         int refclk = 120000;
10349
10350         memset(&crtc_state->dpll_hw_state, 0,
10351                sizeof(crtc_state->dpll_hw_state));
10352
10353         /* CPU eDP is the only output that doesn't need a PCH PLL of its own. */
10354         if (!crtc_state->has_pch_encoder)
10355                 return 0;
10356
10357         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
10358                 if (intel_panel_use_ssc(dev_priv)) {
10359                         drm_dbg_kms(&dev_priv->drm,
10360                                     "using SSC reference clock of %d kHz\n",
10361                                     dev_priv->vbt.lvds_ssc_freq);
10362                         refclk = dev_priv->vbt.lvds_ssc_freq;
10363                 }
10364
10365                 if (intel_is_dual_link_lvds(dev_priv)) {
10366                         if (refclk == 100000)
10367                                 limit = &ilk_limits_dual_lvds_100m;
10368                         else
10369                                 limit = &ilk_limits_dual_lvds;
10370                 } else {
10371                         if (refclk == 100000)
10372                                 limit = &ilk_limits_single_lvds_100m;
10373                         else
10374                                 limit = &ilk_limits_single_lvds;
10375                 }
10376         } else {
10377                 limit = &ilk_limits_dac;
10378         }
10379
10380         if (!crtc_state->clock_set &&
10381             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
10382                                 refclk, NULL, &crtc_state->dpll)) {
10383                 drm_err(&dev_priv->drm,
10384                         "Couldn't find PLL settings for mode!\n");
10385                 return -EINVAL;
10386         }
10387
10388         ilk_compute_dpll(crtc, crtc_state, NULL);
10389
10390         if (!intel_reserve_shared_dplls(state, crtc, NULL)) {
10391                 drm_dbg_kms(&dev_priv->drm,
10392                             "failed to find PLL for pipe %c\n",
10393                             pipe_name(crtc->pipe));
10394                 return -EINVAL;
10395         }
10396
10397         return 0;
10398 }
10399
10400 static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
10401                                          struct intel_link_m_n *m_n)
10402 {
10403         struct drm_device *dev = crtc->base.dev;
10404         struct drm_i915_private *dev_priv = to_i915(dev);
10405         enum pipe pipe = crtc->pipe;
10406
10407         m_n->link_m = intel_de_read(dev_priv, PCH_TRANS_LINK_M1(pipe));
10408         m_n->link_n = intel_de_read(dev_priv, PCH_TRANS_LINK_N1(pipe));
10409         m_n->gmch_m = intel_de_read(dev_priv, PCH_TRANS_DATA_M1(pipe))
10410                 & ~TU_SIZE_MASK;
10411         m_n->gmch_n = intel_de_read(dev_priv, PCH_TRANS_DATA_N1(pipe));
10412         m_n->tu = ((intel_de_read(dev_priv, PCH_TRANS_DATA_M1(pipe))
10413                     & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
10414 }
10415
10416 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
10417                                          enum transcoder transcoder,
10418                                          struct intel_link_m_n *m_n,
10419                                          struct intel_link_m_n *m2_n2)
10420 {
10421         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10422         enum pipe pipe = crtc->pipe;
10423
10424         if (INTEL_GEN(dev_priv) >= 5) {
10425                 m_n->link_m = intel_de_read(dev_priv,
10426                                             PIPE_LINK_M1(transcoder));
10427                 m_n->link_n = intel_de_read(dev_priv,
10428                                             PIPE_LINK_N1(transcoder));
10429                 m_n->gmch_m = intel_de_read(dev_priv,
10430                                             PIPE_DATA_M1(transcoder))
10431                         & ~TU_SIZE_MASK;
10432                 m_n->gmch_n = intel_de_read(dev_priv,
10433                                             PIPE_DATA_N1(transcoder));
10434                 m_n->tu = ((intel_de_read(dev_priv, PIPE_DATA_M1(transcoder))
10435                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
10436
10437                 if (m2_n2 && transcoder_has_m2_n2(dev_priv, transcoder)) {
10438                         m2_n2->link_m = intel_de_read(dev_priv,
10439                                                       PIPE_LINK_M2(transcoder));
10440                         m2_n2->link_n = intel_de_read(dev_priv,
10441                                                              PIPE_LINK_N2(transcoder));
10442                         m2_n2->gmch_m = intel_de_read(dev_priv,
10443                                                              PIPE_DATA_M2(transcoder))
10444                                         & ~TU_SIZE_MASK;
10445                         m2_n2->gmch_n = intel_de_read(dev_priv,
10446                                                              PIPE_DATA_N2(transcoder));
10447                         m2_n2->tu = ((intel_de_read(dev_priv, PIPE_DATA_M2(transcoder))
10448                                         & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
10449                 }
10450         } else {
10451                 m_n->link_m = intel_de_read(dev_priv, PIPE_LINK_M_G4X(pipe));
10452                 m_n->link_n = intel_de_read(dev_priv, PIPE_LINK_N_G4X(pipe));
10453                 m_n->gmch_m = intel_de_read(dev_priv, PIPE_DATA_M_G4X(pipe))
10454                         & ~TU_SIZE_MASK;
10455                 m_n->gmch_n = intel_de_read(dev_priv, PIPE_DATA_N_G4X(pipe));
10456                 m_n->tu = ((intel_de_read(dev_priv, PIPE_DATA_M_G4X(pipe))
10457                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
10458         }
10459 }
10460
10461 void intel_dp_get_m_n(struct intel_crtc *crtc,
10462                       struct intel_crtc_state *pipe_config)
10463 {
10464         if (pipe_config->has_pch_encoder)
10465                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
10466         else
10467                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
10468                                              &pipe_config->dp_m_n,
10469                                              &pipe_config->dp_m2_n2);
10470 }
10471
10472 static void ilk_get_fdi_m_n_config(struct intel_crtc *crtc,
10473                                    struct intel_crtc_state *pipe_config)
10474 {
10475         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
10476                                      &pipe_config->fdi_m_n, NULL);
10477 }
10478
10479 static void ilk_get_pfit_pos_size(struct intel_crtc_state *crtc_state,
10480                                   u32 pos, u32 size)
10481 {
10482         drm_rect_init(&crtc_state->pch_pfit.dst,
10483                       pos >> 16, pos & 0xffff,
10484                       size >> 16, size & 0xffff);
10485 }
10486
10487 static void skl_get_pfit_config(struct intel_crtc_state *crtc_state)
10488 {
10489         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
10490         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10491         struct intel_crtc_scaler_state *scaler_state = &crtc_state->scaler_state;
10492         int id = -1;
10493         int i;
10494
10495         /* find scaler attached to this pipe */
10496         for (i = 0; i < crtc->num_scalers; i++) {
10497                 u32 ctl, pos, size;
10498
10499                 ctl = intel_de_read(dev_priv, SKL_PS_CTRL(crtc->pipe, i));
10500                 if ((ctl & (PS_SCALER_EN | PS_PLANE_SEL_MASK)) != PS_SCALER_EN)
10501                         continue;
10502
10503                 id = i;
10504                 crtc_state->pch_pfit.enabled = true;
10505
10506                 pos = intel_de_read(dev_priv, SKL_PS_WIN_POS(crtc->pipe, i));
10507                 size = intel_de_read(dev_priv, SKL_PS_WIN_SZ(crtc->pipe, i));
10508
10509                 ilk_get_pfit_pos_size(crtc_state, pos, size);
10510
10511                 scaler_state->scalers[i].in_use = true;
10512                 break;
10513         }
10514
10515         scaler_state->scaler_id = id;
10516         if (id >= 0)
10517                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
10518         else
10519                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
10520 }
10521
10522 static void
10523 skl_get_initial_plane_config(struct intel_crtc *crtc,
10524                              struct intel_initial_plane_config *plane_config)
10525 {
10526         struct drm_device *dev = crtc->base.dev;
10527         struct drm_i915_private *dev_priv = to_i915(dev);
10528         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
10529         enum plane_id plane_id = plane->id;
10530         enum pipe pipe;
10531         u32 val, base, offset, stride_mult, tiling, alpha;
10532         int fourcc, pixel_format;
10533         unsigned int aligned_height;
10534         struct drm_framebuffer *fb;
10535         struct intel_framebuffer *intel_fb;
10536
10537         if (!plane->get_hw_state(plane, &pipe))
10538                 return;
10539
10540         drm_WARN_ON(dev, pipe != crtc->pipe);
10541
10542         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
10543         if (!intel_fb) {
10544                 drm_dbg_kms(&dev_priv->drm, "failed to alloc fb\n");
10545                 return;
10546         }
10547
10548         fb = &intel_fb->base;
10549
10550         fb->dev = dev;
10551
10552         val = intel_de_read(dev_priv, PLANE_CTL(pipe, plane_id));
10553
10554         if (INTEL_GEN(dev_priv) >= 11)
10555                 pixel_format = val & ICL_PLANE_CTL_FORMAT_MASK;
10556         else
10557                 pixel_format = val & PLANE_CTL_FORMAT_MASK;
10558
10559         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
10560                 alpha = intel_de_read(dev_priv,
10561                                       PLANE_COLOR_CTL(pipe, plane_id));
10562                 alpha &= PLANE_COLOR_ALPHA_MASK;
10563         } else {
10564                 alpha = val & PLANE_CTL_ALPHA_MASK;
10565         }
10566
10567         fourcc = skl_format_to_fourcc(pixel_format,
10568                                       val & PLANE_CTL_ORDER_RGBX, alpha);
10569         fb->format = drm_format_info(fourcc);
10570
10571         tiling = val & PLANE_CTL_TILED_MASK;
10572         switch (tiling) {
10573         case PLANE_CTL_TILED_LINEAR:
10574                 fb->modifier = DRM_FORMAT_MOD_LINEAR;
10575                 break;
10576         case PLANE_CTL_TILED_X:
10577                 plane_config->tiling = I915_TILING_X;
10578                 fb->modifier = I915_FORMAT_MOD_X_TILED;
10579                 break;
10580         case PLANE_CTL_TILED_Y:
10581                 plane_config->tiling = I915_TILING_Y;
10582                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
10583                         fb->modifier = INTEL_GEN(dev_priv) >= 12 ?
10584                                 I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS :
10585                                 I915_FORMAT_MOD_Y_TILED_CCS;
10586                 else if (val & PLANE_CTL_MEDIA_DECOMPRESSION_ENABLE)
10587                         fb->modifier = I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS;
10588                 else
10589                         fb->modifier = I915_FORMAT_MOD_Y_TILED;
10590                 break;
10591         case PLANE_CTL_TILED_YF:
10592                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
10593                         fb->modifier = I915_FORMAT_MOD_Yf_TILED_CCS;
10594                 else
10595                         fb->modifier = I915_FORMAT_MOD_Yf_TILED;
10596                 break;
10597         default:
10598                 MISSING_CASE(tiling);
10599                 goto error;
10600         }
10601
10602         /*
10603          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
10604          * while i915 HW rotation is clockwise, thats why this swapping.
10605          */
10606         switch (val & PLANE_CTL_ROTATE_MASK) {
10607         case PLANE_CTL_ROTATE_0:
10608                 plane_config->rotation = DRM_MODE_ROTATE_0;
10609                 break;
10610         case PLANE_CTL_ROTATE_90:
10611                 plane_config->rotation = DRM_MODE_ROTATE_270;
10612                 break;
10613         case PLANE_CTL_ROTATE_180:
10614                 plane_config->rotation = DRM_MODE_ROTATE_180;
10615                 break;
10616         case PLANE_CTL_ROTATE_270:
10617                 plane_config->rotation = DRM_MODE_ROTATE_90;
10618                 break;
10619         }
10620
10621         if (INTEL_GEN(dev_priv) >= 10 &&
10622             val & PLANE_CTL_FLIP_HORIZONTAL)
10623                 plane_config->rotation |= DRM_MODE_REFLECT_X;
10624
10625         base = intel_de_read(dev_priv, PLANE_SURF(pipe, plane_id)) & 0xfffff000;
10626         plane_config->base = base;
10627
10628         offset = intel_de_read(dev_priv, PLANE_OFFSET(pipe, plane_id));
10629
10630         val = intel_de_read(dev_priv, PLANE_SIZE(pipe, plane_id));
10631         fb->height = ((val >> 16) & 0xffff) + 1;
10632         fb->width = ((val >> 0) & 0xffff) + 1;
10633
10634         val = intel_de_read(dev_priv, PLANE_STRIDE(pipe, plane_id));
10635         stride_mult = skl_plane_stride_mult(fb, 0, DRM_MODE_ROTATE_0);
10636         fb->pitches[0] = (val & 0x3ff) * stride_mult;
10637
10638         aligned_height = intel_fb_align_height(fb, 0, fb->height);
10639
10640         plane_config->size = fb->pitches[0] * aligned_height;
10641
10642         drm_dbg_kms(&dev_priv->drm,
10643                     "%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
10644                     crtc->base.name, plane->base.name, fb->width, fb->height,
10645                     fb->format->cpp[0] * 8, base, fb->pitches[0],
10646                     plane_config->size);
10647
10648         plane_config->fb = intel_fb;
10649         return;
10650
10651 error:
10652         kfree(intel_fb);
10653 }
10654
10655 static void ilk_get_pfit_config(struct intel_crtc_state *crtc_state)
10656 {
10657         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
10658         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10659         u32 ctl, pos, size;
10660
10661         ctl = intel_de_read(dev_priv, PF_CTL(crtc->pipe));
10662         if ((ctl & PF_ENABLE) == 0)
10663                 return;
10664
10665         crtc_state->pch_pfit.enabled = true;
10666
10667         pos = intel_de_read(dev_priv, PF_WIN_POS(crtc->pipe));
10668         size = intel_de_read(dev_priv, PF_WIN_SZ(crtc->pipe));
10669
10670         ilk_get_pfit_pos_size(crtc_state, pos, size);
10671
10672         /*
10673          * We currently do not free assignements of panel fitters on
10674          * ivb/hsw (since we don't use the higher upscaling modes which
10675          * differentiates them) so just WARN about this case for now.
10676          */
10677         drm_WARN_ON(&dev_priv->drm, IS_GEN(dev_priv, 7) &&
10678                     (ctl & PF_PIPE_SEL_MASK_IVB) != PF_PIPE_SEL_IVB(crtc->pipe));
10679 }
10680
10681 static bool ilk_get_pipe_config(struct intel_crtc *crtc,
10682                                 struct intel_crtc_state *pipe_config)
10683 {
10684         struct drm_device *dev = crtc->base.dev;
10685         struct drm_i915_private *dev_priv = to_i915(dev);
10686         enum intel_display_power_domain power_domain;
10687         intel_wakeref_t wakeref;
10688         u32 tmp;
10689         bool ret;
10690
10691         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
10692         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
10693         if (!wakeref)
10694                 return false;
10695
10696         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
10697         pipe_config->shared_dpll = NULL;
10698
10699         ret = false;
10700         tmp = intel_de_read(dev_priv, PIPECONF(crtc->pipe));
10701         if (!(tmp & PIPECONF_ENABLE))
10702                 goto out;
10703
10704         switch (tmp & PIPECONF_BPC_MASK) {
10705         case PIPECONF_6BPC:
10706                 pipe_config->pipe_bpp = 18;
10707                 break;
10708         case PIPECONF_8BPC:
10709                 pipe_config->pipe_bpp = 24;
10710                 break;
10711         case PIPECONF_10BPC:
10712                 pipe_config->pipe_bpp = 30;
10713                 break;
10714         case PIPECONF_12BPC:
10715                 pipe_config->pipe_bpp = 36;
10716                 break;
10717         default:
10718                 break;
10719         }
10720
10721         if (tmp & PIPECONF_COLOR_RANGE_SELECT)
10722                 pipe_config->limited_color_range = true;
10723
10724         switch (tmp & PIPECONF_OUTPUT_COLORSPACE_MASK) {
10725         case PIPECONF_OUTPUT_COLORSPACE_YUV601:
10726         case PIPECONF_OUTPUT_COLORSPACE_YUV709:
10727                 pipe_config->output_format = INTEL_OUTPUT_FORMAT_YCBCR444;
10728                 break;
10729         default:
10730                 pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
10731                 break;
10732         }
10733
10734         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_ILK) >>
10735                 PIPECONF_GAMMA_MODE_SHIFT;
10736
10737         pipe_config->csc_mode = intel_de_read(dev_priv,
10738                                               PIPE_CSC_MODE(crtc->pipe));
10739
10740         i9xx_get_pipe_color_config(pipe_config);
10741         intel_color_get_config(pipe_config);
10742
10743         if (intel_de_read(dev_priv, PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
10744                 struct intel_shared_dpll *pll;
10745                 enum intel_dpll_id pll_id;
10746
10747                 pipe_config->has_pch_encoder = true;
10748
10749                 tmp = intel_de_read(dev_priv, FDI_RX_CTL(crtc->pipe));
10750                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
10751                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
10752
10753                 ilk_get_fdi_m_n_config(crtc, pipe_config);
10754
10755                 if (HAS_PCH_IBX(dev_priv)) {
10756                         /*
10757                          * The pipe->pch transcoder and pch transcoder->pll
10758                          * mapping is fixed.
10759                          */
10760                         pll_id = (enum intel_dpll_id) crtc->pipe;
10761                 } else {
10762                         tmp = intel_de_read(dev_priv, PCH_DPLL_SEL);
10763                         if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
10764                                 pll_id = DPLL_ID_PCH_PLL_B;
10765                         else
10766                                 pll_id= DPLL_ID_PCH_PLL_A;
10767                 }
10768
10769                 pipe_config->shared_dpll =
10770                         intel_get_shared_dpll_by_id(dev_priv, pll_id);
10771                 pll = pipe_config->shared_dpll;
10772
10773                 drm_WARN_ON(dev, !pll->info->funcs->get_hw_state(dev_priv, pll,
10774                                                  &pipe_config->dpll_hw_state));
10775
10776                 tmp = pipe_config->dpll_hw_state.dpll;
10777                 pipe_config->pixel_multiplier =
10778                         ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
10779                          >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
10780
10781                 ilk_pch_clock_get(crtc, pipe_config);
10782         } else {
10783                 pipe_config->pixel_multiplier = 1;
10784         }
10785
10786         intel_get_pipe_timings(crtc, pipe_config);
10787         intel_get_pipe_src_size(crtc, pipe_config);
10788
10789         ilk_get_pfit_config(pipe_config);
10790
10791         ret = true;
10792
10793 out:
10794         intel_display_power_put(dev_priv, power_domain, wakeref);
10795
10796         return ret;
10797 }
10798
10799 static int hsw_crtc_compute_clock(struct intel_crtc *crtc,
10800                                   struct intel_crtc_state *crtc_state)
10801 {
10802         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10803         struct intel_atomic_state *state =
10804                 to_intel_atomic_state(crtc_state->uapi.state);
10805
10806         if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) ||
10807             INTEL_GEN(dev_priv) >= 11) {
10808                 struct intel_encoder *encoder =
10809                         intel_get_crtc_new_encoder(state, crtc_state);
10810
10811                 if (!intel_reserve_shared_dplls(state, crtc, encoder)) {
10812                         drm_dbg_kms(&dev_priv->drm,
10813                                     "failed to find PLL for pipe %c\n",
10814                                     pipe_name(crtc->pipe));
10815                         return -EINVAL;
10816                 }
10817         }
10818
10819         return 0;
10820 }
10821
10822 static void cnl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port,
10823                             struct intel_crtc_state *pipe_config)
10824 {
10825         enum intel_dpll_id id;
10826         u32 temp;
10827
10828         temp = intel_de_read(dev_priv, DPCLKA_CFGCR0) & DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
10829         id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port);
10830
10831         if (drm_WARN_ON(&dev_priv->drm, id < SKL_DPLL0 || id > SKL_DPLL2))
10832                 return;
10833
10834         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10835 }
10836
10837 static void icl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port,
10838                             struct intel_crtc_state *pipe_config)
10839 {
10840         enum phy phy = intel_port_to_phy(dev_priv, port);
10841         enum icl_port_dpll_id port_dpll_id;
10842         enum intel_dpll_id id;
10843         u32 temp;
10844
10845         if (intel_phy_is_combo(dev_priv, phy)) {
10846                 u32 mask, shift;
10847
10848                 if (IS_ROCKETLAKE(dev_priv)) {
10849                         mask = RKL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy);
10850                         shift = RKL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy);
10851                 } else {
10852                         mask = ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy);
10853                         shift = ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy);
10854                 }
10855
10856                 temp = intel_de_read(dev_priv, ICL_DPCLKA_CFGCR0) & mask;
10857                 id = temp >> shift;
10858                 port_dpll_id = ICL_PORT_DPLL_DEFAULT;
10859         } else if (intel_phy_is_tc(dev_priv, phy)) {
10860                 u32 clk_sel = intel_de_read(dev_priv, DDI_CLK_SEL(port)) & DDI_CLK_SEL_MASK;
10861
10862                 if (clk_sel == DDI_CLK_SEL_MG) {
10863                         id = icl_tc_port_to_pll_id(intel_port_to_tc(dev_priv,
10864                                                                     port));
10865                         port_dpll_id = ICL_PORT_DPLL_MG_PHY;
10866                 } else {
10867                         drm_WARN_ON(&dev_priv->drm,
10868                                     clk_sel < DDI_CLK_SEL_TBT_162);
10869                         id = DPLL_ID_ICL_TBTPLL;
10870                         port_dpll_id = ICL_PORT_DPLL_DEFAULT;
10871                 }
10872         } else {
10873                 drm_WARN(&dev_priv->drm, 1, "Invalid port %x\n", port);
10874                 return;
10875         }
10876
10877         pipe_config->icl_port_dplls[port_dpll_id].pll =
10878                 intel_get_shared_dpll_by_id(dev_priv, id);
10879
10880         icl_set_active_port_dpll(pipe_config, port_dpll_id);
10881 }
10882
10883 static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv,
10884                                 enum port port,
10885                                 struct intel_crtc_state *pipe_config)
10886 {
10887         enum intel_dpll_id id;
10888
10889         switch (port) {
10890         case PORT_A:
10891                 id = DPLL_ID_SKL_DPLL0;
10892                 break;
10893         case PORT_B:
10894                 id = DPLL_ID_SKL_DPLL1;
10895                 break;
10896         case PORT_C:
10897                 id = DPLL_ID_SKL_DPLL2;
10898                 break;
10899         default:
10900                 drm_err(&dev_priv->drm, "Incorrect port type\n");
10901                 return;
10902         }
10903
10904         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10905 }
10906
10907 static void skl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port,
10908                             struct intel_crtc_state *pipe_config)
10909 {
10910         enum intel_dpll_id id;
10911         u32 temp;
10912
10913         temp = intel_de_read(dev_priv, DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port);
10914         id = temp >> (port * 3 + 1);
10915
10916         if (drm_WARN_ON(&dev_priv->drm, id < SKL_DPLL0 || id > SKL_DPLL3))
10917                 return;
10918
10919         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10920 }
10921
10922 static void hsw_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port,
10923                             struct intel_crtc_state *pipe_config)
10924 {
10925         enum intel_dpll_id id;
10926         u32 ddi_pll_sel = intel_de_read(dev_priv, PORT_CLK_SEL(port));
10927
10928         switch (ddi_pll_sel) {
10929         case PORT_CLK_SEL_WRPLL1:
10930                 id = DPLL_ID_WRPLL1;
10931                 break;
10932         case PORT_CLK_SEL_WRPLL2:
10933                 id = DPLL_ID_WRPLL2;
10934                 break;
10935         case PORT_CLK_SEL_SPLL:
10936                 id = DPLL_ID_SPLL;
10937                 break;
10938         case PORT_CLK_SEL_LCPLL_810:
10939                 id = DPLL_ID_LCPLL_810;
10940                 break;
10941         case PORT_CLK_SEL_LCPLL_1350:
10942                 id = DPLL_ID_LCPLL_1350;
10943                 break;
10944         case PORT_CLK_SEL_LCPLL_2700:
10945                 id = DPLL_ID_LCPLL_2700;
10946                 break;
10947         default:
10948                 MISSING_CASE(ddi_pll_sel);
10949                 fallthrough;
10950         case PORT_CLK_SEL_NONE:
10951                 return;
10952         }
10953
10954         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10955 }
10956
10957 static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
10958                                      struct intel_crtc_state *pipe_config,
10959                                      u64 *power_domain_mask,
10960                                      intel_wakeref_t *wakerefs)
10961 {
10962         struct drm_device *dev = crtc->base.dev;
10963         struct drm_i915_private *dev_priv = to_i915(dev);
10964         enum intel_display_power_domain power_domain;
10965         unsigned long panel_transcoder_mask = BIT(TRANSCODER_EDP);
10966         unsigned long enabled_panel_transcoders = 0;
10967         enum transcoder panel_transcoder;
10968         intel_wakeref_t wf;
10969         u32 tmp;
10970
10971         if (INTEL_GEN(dev_priv) >= 11)
10972                 panel_transcoder_mask |=
10973                         BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1);
10974
10975         /*
10976          * The pipe->transcoder mapping is fixed with the exception of the eDP
10977          * and DSI transcoders handled below.
10978          */
10979         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
10980
10981         /*
10982          * XXX: Do intel_display_power_get_if_enabled before reading this (for
10983          * consistency and less surprising code; it's in always on power).
10984          */
10985         for_each_cpu_transcoder_masked(dev_priv, panel_transcoder,
10986                                        panel_transcoder_mask) {
10987                 bool force_thru = false;
10988                 enum pipe trans_pipe;
10989
10990                 tmp = intel_de_read(dev_priv,
10991                                     TRANS_DDI_FUNC_CTL(panel_transcoder));
10992                 if (!(tmp & TRANS_DDI_FUNC_ENABLE))
10993                         continue;
10994
10995                 /*
10996                  * Log all enabled ones, only use the first one.
10997                  *
10998                  * FIXME: This won't work for two separate DSI displays.
10999                  */
11000                 enabled_panel_transcoders |= BIT(panel_transcoder);
11001                 if (enabled_panel_transcoders != BIT(panel_transcoder))
11002                         continue;
11003
11004                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
11005                 default:
11006                         drm_WARN(dev, 1,
11007                                  "unknown pipe linked to transcoder %s\n",
11008                                  transcoder_name(panel_transcoder));
11009                         fallthrough;
11010                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
11011                         force_thru = true;
11012                         fallthrough;
11013                 case TRANS_DDI_EDP_INPUT_A_ON:
11014                         trans_pipe = PIPE_A;
11015                         break;
11016                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
11017                         trans_pipe = PIPE_B;
11018                         break;
11019                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
11020                         trans_pipe = PIPE_C;
11021                         break;
11022                 case TRANS_DDI_EDP_INPUT_D_ONOFF:
11023                         trans_pipe = PIPE_D;
11024                         break;
11025                 }
11026
11027                 if (trans_pipe == crtc->pipe) {
11028                         pipe_config->cpu_transcoder = panel_transcoder;
11029                         pipe_config->pch_pfit.force_thru = force_thru;
11030                 }
11031         }
11032
11033         /*
11034          * Valid combos: none, eDP, DSI0, DSI1, DSI0+DSI1
11035          */
11036         drm_WARN_ON(dev, (enabled_panel_transcoders & BIT(TRANSCODER_EDP)) &&
11037                     enabled_panel_transcoders != BIT(TRANSCODER_EDP));
11038
11039         power_domain = POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder);
11040         drm_WARN_ON(dev, *power_domain_mask & BIT_ULL(power_domain));
11041
11042         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
11043         if (!wf)
11044                 return false;
11045
11046         wakerefs[power_domain] = wf;
11047         *power_domain_mask |= BIT_ULL(power_domain);
11048
11049         tmp = intel_de_read(dev_priv, PIPECONF(pipe_config->cpu_transcoder));
11050
11051         return tmp & PIPECONF_ENABLE;
11052 }
11053
11054 static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
11055                                          struct intel_crtc_state *pipe_config,
11056                                          u64 *power_domain_mask,
11057                                          intel_wakeref_t *wakerefs)
11058 {
11059         struct drm_device *dev = crtc->base.dev;
11060         struct drm_i915_private *dev_priv = to_i915(dev);
11061         enum intel_display_power_domain power_domain;
11062         enum transcoder cpu_transcoder;
11063         intel_wakeref_t wf;
11064         enum port port;
11065         u32 tmp;
11066
11067         for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) {
11068                 if (port == PORT_A)
11069                         cpu_transcoder = TRANSCODER_DSI_A;
11070                 else
11071                         cpu_transcoder = TRANSCODER_DSI_C;
11072
11073                 power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
11074                 drm_WARN_ON(dev, *power_domain_mask & BIT_ULL(power_domain));
11075
11076                 wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
11077                 if (!wf)
11078                         continue;
11079
11080                 wakerefs[power_domain] = wf;
11081                 *power_domain_mask |= BIT_ULL(power_domain);
11082
11083                 /*
11084                  * The PLL needs to be enabled with a valid divider
11085                  * configuration, otherwise accessing DSI registers will hang
11086                  * the machine. See BSpec North Display Engine
11087                  * registers/MIPI[BXT]. We can break out here early, since we
11088                  * need the same DSI PLL to be enabled for both DSI ports.
11089                  */
11090                 if (!bxt_dsi_pll_is_enabled(dev_priv))
11091                         break;
11092
11093                 /* XXX: this works for video mode only */
11094                 tmp = intel_de_read(dev_priv, BXT_MIPI_PORT_CTRL(port));
11095                 if (!(tmp & DPI_ENABLE))
11096                         continue;
11097
11098                 tmp = intel_de_read(dev_priv, MIPI_CTRL(port));
11099                 if ((tmp & BXT_PIPE_SELECT_MASK) != BXT_PIPE_SELECT(crtc->pipe))
11100                         continue;
11101
11102                 pipe_config->cpu_transcoder = cpu_transcoder;
11103                 break;
11104         }
11105
11106         return transcoder_is_dsi(pipe_config->cpu_transcoder);
11107 }
11108
11109 static void hsw_get_ddi_port_state(struct intel_crtc *crtc,
11110                                    struct intel_crtc_state *pipe_config)
11111 {
11112         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11113         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
11114         struct intel_shared_dpll *pll;
11115         enum port port;
11116         u32 tmp;
11117
11118         if (transcoder_is_dsi(cpu_transcoder)) {
11119                 port = (cpu_transcoder == TRANSCODER_DSI_A) ?
11120                                                 PORT_A : PORT_B;
11121         } else {
11122                 tmp = intel_de_read(dev_priv,
11123                                     TRANS_DDI_FUNC_CTL(cpu_transcoder));
11124                 if (INTEL_GEN(dev_priv) >= 12)
11125                         port = TGL_TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp);
11126                 else
11127                         port = TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp);
11128         }
11129
11130         if (INTEL_GEN(dev_priv) >= 11)
11131                 icl_get_ddi_pll(dev_priv, port, pipe_config);
11132         else if (IS_CANNONLAKE(dev_priv))
11133                 cnl_get_ddi_pll(dev_priv, port, pipe_config);
11134         else if (IS_GEN9_BC(dev_priv))
11135                 skl_get_ddi_pll(dev_priv, port, pipe_config);
11136         else if (IS_GEN9_LP(dev_priv))
11137                 bxt_get_ddi_pll(dev_priv, port, pipe_config);
11138         else
11139                 hsw_get_ddi_pll(dev_priv, port, pipe_config);
11140
11141         pll = pipe_config->shared_dpll;
11142         if (pll) {
11143                 drm_WARN_ON(&dev_priv->drm,
11144                             !pll->info->funcs->get_hw_state(dev_priv, pll,
11145                                                 &pipe_config->dpll_hw_state));
11146         }
11147
11148         /*
11149          * Haswell has only FDI/PCH transcoder A. It is which is connected to
11150          * DDI E. So just check whether this pipe is wired to DDI E and whether
11151          * the PCH transcoder is on.
11152          */
11153         if (INTEL_GEN(dev_priv) < 9 &&
11154             (port == PORT_E) && intel_de_read(dev_priv, LPT_TRANSCONF) & TRANS_ENABLE) {
11155                 pipe_config->has_pch_encoder = true;
11156
11157                 tmp = intel_de_read(dev_priv, FDI_RX_CTL(PIPE_A));
11158                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
11159                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
11160
11161                 ilk_get_fdi_m_n_config(crtc, pipe_config);
11162         }
11163 }
11164
11165 static bool hsw_get_pipe_config(struct intel_crtc *crtc,
11166                                 struct intel_crtc_state *pipe_config)
11167 {
11168         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11169         intel_wakeref_t wakerefs[POWER_DOMAIN_NUM], wf;
11170         enum intel_display_power_domain power_domain;
11171         u64 power_domain_mask;
11172         bool active;
11173         u32 tmp;
11174
11175         pipe_config->master_transcoder = INVALID_TRANSCODER;
11176
11177         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
11178         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
11179         if (!wf)
11180                 return false;
11181
11182         wakerefs[power_domain] = wf;
11183         power_domain_mask = BIT_ULL(power_domain);
11184
11185         pipe_config->shared_dpll = NULL;
11186
11187         active = hsw_get_transcoder_state(crtc, pipe_config,
11188                                           &power_domain_mask, wakerefs);
11189
11190         if (IS_GEN9_LP(dev_priv) &&
11191             bxt_get_dsi_transcoder_state(crtc, pipe_config,
11192                                          &power_domain_mask, wakerefs)) {
11193                 drm_WARN_ON(&dev_priv->drm, active);
11194                 active = true;
11195         }
11196
11197         if (!active)
11198                 goto out;
11199
11200         if (!transcoder_is_dsi(pipe_config->cpu_transcoder) ||
11201             INTEL_GEN(dev_priv) >= 11) {
11202                 hsw_get_ddi_port_state(crtc, pipe_config);
11203                 intel_get_pipe_timings(crtc, pipe_config);
11204         }
11205
11206         intel_get_pipe_src_size(crtc, pipe_config);
11207
11208         if (IS_HASWELL(dev_priv)) {
11209                 u32 tmp = intel_de_read(dev_priv,
11210                                         PIPECONF(pipe_config->cpu_transcoder));
11211
11212                 if (tmp & PIPECONF_OUTPUT_COLORSPACE_YUV_HSW)
11213                         pipe_config->output_format = INTEL_OUTPUT_FORMAT_YCBCR444;
11214                 else
11215                         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
11216         } else {
11217                 pipe_config->output_format =
11218                         bdw_get_pipemisc_output_format(crtc);
11219
11220                 /*
11221                  * Currently there is no interface defined to
11222                  * check user preference between RGB/YCBCR444
11223                  * or YCBCR420. So the only possible case for
11224                  * YCBCR444 usage is driving YCBCR420 output
11225                  * with LSPCON, when pipe is configured for
11226                  * YCBCR444 output and LSPCON takes care of
11227                  * downsampling it.
11228                  */
11229                 pipe_config->lspcon_downsampling =
11230                         pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR444;
11231         }
11232
11233         pipe_config->gamma_mode = intel_de_read(dev_priv,
11234                                                 GAMMA_MODE(crtc->pipe));
11235
11236         pipe_config->csc_mode = intel_de_read(dev_priv,
11237                                               PIPE_CSC_MODE(crtc->pipe));
11238
11239         if (INTEL_GEN(dev_priv) >= 9) {
11240                 tmp = intel_de_read(dev_priv, SKL_BOTTOM_COLOR(crtc->pipe));
11241
11242                 if (tmp & SKL_BOTTOM_COLOR_GAMMA_ENABLE)
11243                         pipe_config->gamma_enable = true;
11244
11245                 if (tmp & SKL_BOTTOM_COLOR_CSC_ENABLE)
11246                         pipe_config->csc_enable = true;
11247         } else {
11248                 i9xx_get_pipe_color_config(pipe_config);
11249         }
11250
11251         intel_color_get_config(pipe_config);
11252
11253         tmp = intel_de_read(dev_priv, WM_LINETIME(crtc->pipe));
11254         pipe_config->linetime = REG_FIELD_GET(HSW_LINETIME_MASK, tmp);
11255         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
11256                 pipe_config->ips_linetime =
11257                         REG_FIELD_GET(HSW_IPS_LINETIME_MASK, tmp);
11258
11259         power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
11260         drm_WARN_ON(&dev_priv->drm, power_domain_mask & BIT_ULL(power_domain));
11261
11262         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
11263         if (wf) {
11264                 wakerefs[power_domain] = wf;
11265                 power_domain_mask |= BIT_ULL(power_domain);
11266
11267                 if (INTEL_GEN(dev_priv) >= 9)
11268                         skl_get_pfit_config(pipe_config);
11269                 else
11270                         ilk_get_pfit_config(pipe_config);
11271         }
11272
11273         if (hsw_crtc_supports_ips(crtc)) {
11274                 if (IS_HASWELL(dev_priv))
11275                         pipe_config->ips_enabled = intel_de_read(dev_priv,
11276                                                                  IPS_CTL) & IPS_ENABLE;
11277                 else {
11278                         /*
11279                          * We cannot readout IPS state on broadwell, set to
11280                          * true so we can set it to a defined state on first
11281                          * commit.
11282                          */
11283                         pipe_config->ips_enabled = true;
11284                 }
11285         }
11286
11287         if (pipe_config->cpu_transcoder != TRANSCODER_EDP &&
11288             !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
11289                 pipe_config->pixel_multiplier =
11290                         intel_de_read(dev_priv,
11291                                       PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
11292         } else {
11293                 pipe_config->pixel_multiplier = 1;
11294         }
11295
11296 out:
11297         for_each_power_domain(power_domain, power_domain_mask)
11298                 intel_display_power_put(dev_priv,
11299                                         power_domain, wakerefs[power_domain]);
11300
11301         return active;
11302 }
11303
11304 static u32 intel_cursor_base(const struct intel_plane_state *plane_state)
11305 {
11306         struct drm_i915_private *dev_priv =
11307                 to_i915(plane_state->uapi.plane->dev);
11308         const struct drm_framebuffer *fb = plane_state->hw.fb;
11309         const struct drm_i915_gem_object *obj = intel_fb_obj(fb);
11310         u32 base;
11311
11312         if (INTEL_INFO(dev_priv)->display.cursor_needs_physical)
11313                 base = sg_dma_address(obj->mm.pages->sgl);
11314         else
11315                 base = intel_plane_ggtt_offset(plane_state);
11316
11317         return base + plane_state->color_plane[0].offset;
11318 }
11319
11320 static u32 intel_cursor_position(const struct intel_plane_state *plane_state)
11321 {
11322         int x = plane_state->uapi.dst.x1;
11323         int y = plane_state->uapi.dst.y1;
11324         u32 pos = 0;
11325
11326         if (x < 0) {
11327                 pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
11328                 x = -x;
11329         }
11330         pos |= x << CURSOR_X_SHIFT;
11331
11332         if (y < 0) {
11333                 pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
11334                 y = -y;
11335         }
11336         pos |= y << CURSOR_Y_SHIFT;
11337
11338         return pos;
11339 }
11340
11341 static bool intel_cursor_size_ok(const struct intel_plane_state *plane_state)
11342 {
11343         const struct drm_mode_config *config =
11344                 &plane_state->uapi.plane->dev->mode_config;
11345         int width = drm_rect_width(&plane_state->uapi.dst);
11346         int height = drm_rect_height(&plane_state->uapi.dst);
11347
11348         return width > 0 && width <= config->cursor_width &&
11349                 height > 0 && height <= config->cursor_height;
11350 }
11351
11352 static int intel_cursor_check_surface(struct intel_plane_state *plane_state)
11353 {
11354         struct drm_i915_private *dev_priv =
11355                 to_i915(plane_state->uapi.plane->dev);
11356         unsigned int rotation = plane_state->hw.rotation;
11357         int src_x, src_y;
11358         u32 offset;
11359         int ret;
11360
11361         ret = intel_plane_compute_gtt(plane_state);
11362         if (ret)
11363                 return ret;
11364
11365         if (!plane_state->uapi.visible)
11366                 return 0;
11367
11368         src_x = plane_state->uapi.src.x1 >> 16;
11369         src_y = plane_state->uapi.src.y1 >> 16;
11370
11371         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
11372         offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
11373                                                     plane_state, 0);
11374
11375         if (src_x != 0 || src_y != 0) {
11376                 drm_dbg_kms(&dev_priv->drm,
11377                             "Arbitrary cursor panning not supported\n");
11378                 return -EINVAL;
11379         }
11380
11381         /*
11382          * Put the final coordinates back so that the src
11383          * coordinate checks will see the right values.
11384          */
11385         drm_rect_translate_to(&plane_state->uapi.src,
11386                               src_x << 16, src_y << 16);
11387
11388         /* ILK+ do this automagically in hardware */
11389         if (HAS_GMCH(dev_priv) && rotation & DRM_MODE_ROTATE_180) {
11390                 const struct drm_framebuffer *fb = plane_state->hw.fb;
11391                 int src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
11392                 int src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
11393
11394                 offset += (src_h * src_w - 1) * fb->format->cpp[0];
11395         }
11396
11397         plane_state->color_plane[0].offset = offset;
11398         plane_state->color_plane[0].x = src_x;
11399         plane_state->color_plane[0].y = src_y;
11400
11401         return 0;
11402 }
11403
11404 static int intel_check_cursor(struct intel_crtc_state *crtc_state,
11405                               struct intel_plane_state *plane_state)
11406 {
11407         const struct drm_framebuffer *fb = plane_state->hw.fb;
11408         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
11409         int ret;
11410
11411         if (fb && fb->modifier != DRM_FORMAT_MOD_LINEAR) {
11412                 drm_dbg_kms(&i915->drm, "cursor cannot be tiled\n");
11413                 return -EINVAL;
11414         }
11415
11416         ret = drm_atomic_helper_check_plane_state(&plane_state->uapi,
11417                                                   &crtc_state->uapi,
11418                                                   DRM_PLANE_HELPER_NO_SCALING,
11419                                                   DRM_PLANE_HELPER_NO_SCALING,
11420                                                   true, true);
11421         if (ret)
11422                 return ret;
11423
11424         /* Use the unclipped src/dst rectangles, which we program to hw */
11425         plane_state->uapi.src = drm_plane_state_src(&plane_state->uapi);
11426         plane_state->uapi.dst = drm_plane_state_dest(&plane_state->uapi);
11427
11428         ret = intel_cursor_check_surface(plane_state);
11429         if (ret)
11430                 return ret;
11431
11432         if (!plane_state->uapi.visible)
11433                 return 0;
11434
11435         ret = intel_plane_check_src_coordinates(plane_state);
11436         if (ret)
11437                 return ret;
11438
11439         return 0;
11440 }
11441
11442 static unsigned int
11443 i845_cursor_max_stride(struct intel_plane *plane,
11444                        u32 pixel_format, u64 modifier,
11445                        unsigned int rotation)
11446 {
11447         return 2048;
11448 }
11449
11450 static u32 i845_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state)
11451 {
11452         u32 cntl = 0;
11453
11454         if (crtc_state->gamma_enable)
11455                 cntl |= CURSOR_GAMMA_ENABLE;
11456
11457         return cntl;
11458 }
11459
11460 static u32 i845_cursor_ctl(const struct intel_crtc_state *crtc_state,
11461                            const struct intel_plane_state *plane_state)
11462 {
11463         return CURSOR_ENABLE |
11464                 CURSOR_FORMAT_ARGB |
11465                 CURSOR_STRIDE(plane_state->color_plane[0].stride);
11466 }
11467
11468 static bool i845_cursor_size_ok(const struct intel_plane_state *plane_state)
11469 {
11470         int width = drm_rect_width(&plane_state->uapi.dst);
11471
11472         /*
11473          * 845g/865g are only limited by the width of their cursors,
11474          * the height is arbitrary up to the precision of the register.
11475          */
11476         return intel_cursor_size_ok(plane_state) && IS_ALIGNED(width, 64);
11477 }
11478
11479 static int i845_check_cursor(struct intel_crtc_state *crtc_state,
11480                              struct intel_plane_state *plane_state)
11481 {
11482         const struct drm_framebuffer *fb = plane_state->hw.fb;
11483         struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
11484         int ret;
11485
11486         ret = intel_check_cursor(crtc_state, plane_state);
11487         if (ret)
11488                 return ret;
11489
11490         /* if we want to turn off the cursor ignore width and height */
11491         if (!fb)
11492                 return 0;
11493
11494         /* Check for which cursor types we support */
11495         if (!i845_cursor_size_ok(plane_state)) {
11496                 drm_dbg_kms(&i915->drm,
11497                             "Cursor dimension %dx%d not supported\n",
11498                             drm_rect_width(&plane_state->uapi.dst),
11499                             drm_rect_height(&plane_state->uapi.dst));
11500                 return -EINVAL;
11501         }
11502
11503         drm_WARN_ON(&i915->drm, plane_state->uapi.visible &&
11504                     plane_state->color_plane[0].stride != fb->pitches[0]);
11505
11506         switch (fb->pitches[0]) {
11507         case 256:
11508         case 512:
11509         case 1024:
11510         case 2048:
11511                 break;
11512         default:
11513                  drm_dbg_kms(&i915->drm, "Invalid cursor stride (%u)\n",
11514                              fb->pitches[0]);
11515                 return -EINVAL;
11516         }
11517
11518         plane_state->ctl = i845_cursor_ctl(crtc_state, plane_state);
11519
11520         return 0;
11521 }
11522
11523 static void i845_update_cursor(struct intel_plane *plane,
11524                                const struct intel_crtc_state *crtc_state,
11525                                const struct intel_plane_state *plane_state)
11526 {
11527         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11528         u32 cntl = 0, base = 0, pos = 0, size = 0;
11529         unsigned long irqflags;
11530
11531         if (plane_state && plane_state->uapi.visible) {
11532                 unsigned int width = drm_rect_width(&plane_state->uapi.dst);
11533                 unsigned int height = drm_rect_height(&plane_state->uapi.dst);
11534
11535                 cntl = plane_state->ctl |
11536                         i845_cursor_ctl_crtc(crtc_state);
11537
11538                 size = (height << 12) | width;
11539
11540                 base = intel_cursor_base(plane_state);
11541                 pos = intel_cursor_position(plane_state);
11542         }
11543
11544         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
11545
11546         /* On these chipsets we can only modify the base/size/stride
11547          * whilst the cursor is disabled.
11548          */
11549         if (plane->cursor.base != base ||
11550             plane->cursor.size != size ||
11551             plane->cursor.cntl != cntl) {
11552                 intel_de_write_fw(dev_priv, CURCNTR(PIPE_A), 0);
11553                 intel_de_write_fw(dev_priv, CURBASE(PIPE_A), base);
11554                 intel_de_write_fw(dev_priv, CURSIZE, size);
11555                 intel_de_write_fw(dev_priv, CURPOS(PIPE_A), pos);
11556                 intel_de_write_fw(dev_priv, CURCNTR(PIPE_A), cntl);
11557
11558                 plane->cursor.base = base;
11559                 plane->cursor.size = size;
11560                 plane->cursor.cntl = cntl;
11561         } else {
11562                 intel_de_write_fw(dev_priv, CURPOS(PIPE_A), pos);
11563         }
11564
11565         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
11566 }
11567
11568 static void i845_disable_cursor(struct intel_plane *plane,
11569                                 const struct intel_crtc_state *crtc_state)
11570 {
11571         i845_update_cursor(plane, crtc_state, NULL);
11572 }
11573
11574 static bool i845_cursor_get_hw_state(struct intel_plane *plane,
11575                                      enum pipe *pipe)
11576 {
11577         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11578         enum intel_display_power_domain power_domain;
11579         intel_wakeref_t wakeref;
11580         bool ret;
11581
11582         power_domain = POWER_DOMAIN_PIPE(PIPE_A);
11583         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
11584         if (!wakeref)
11585                 return false;
11586
11587         ret = intel_de_read(dev_priv, CURCNTR(PIPE_A)) & CURSOR_ENABLE;
11588
11589         *pipe = PIPE_A;
11590
11591         intel_display_power_put(dev_priv, power_domain, wakeref);
11592
11593         return ret;
11594 }
11595
11596 static unsigned int
11597 i9xx_cursor_max_stride(struct intel_plane *plane,
11598                        u32 pixel_format, u64 modifier,
11599                        unsigned int rotation)
11600 {
11601         return plane->base.dev->mode_config.cursor_width * 4;
11602 }
11603
11604 static u32 i9xx_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state)
11605 {
11606         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
11607         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11608         u32 cntl = 0;
11609
11610         if (INTEL_GEN(dev_priv) >= 11)
11611                 return cntl;
11612
11613         if (crtc_state->gamma_enable)
11614                 cntl = MCURSOR_GAMMA_ENABLE;
11615
11616         if (crtc_state->csc_enable)
11617                 cntl |= MCURSOR_PIPE_CSC_ENABLE;
11618
11619         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
11620                 cntl |= MCURSOR_PIPE_SELECT(crtc->pipe);
11621
11622         return cntl;
11623 }
11624
11625 static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state,
11626                            const struct intel_plane_state *plane_state)
11627 {
11628         struct drm_i915_private *dev_priv =
11629                 to_i915(plane_state->uapi.plane->dev);
11630         u32 cntl = 0;
11631
11632         if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
11633                 cntl |= MCURSOR_TRICKLE_FEED_DISABLE;
11634
11635         switch (drm_rect_width(&plane_state->uapi.dst)) {
11636         case 64:
11637                 cntl |= MCURSOR_MODE_64_ARGB_AX;
11638                 break;
11639         case 128:
11640                 cntl |= MCURSOR_MODE_128_ARGB_AX;
11641                 break;
11642         case 256:
11643                 cntl |= MCURSOR_MODE_256_ARGB_AX;
11644                 break;
11645         default:
11646                 MISSING_CASE(drm_rect_width(&plane_state->uapi.dst));
11647                 return 0;
11648         }
11649
11650         if (plane_state->hw.rotation & DRM_MODE_ROTATE_180)
11651                 cntl |= MCURSOR_ROTATE_180;
11652
11653         return cntl;
11654 }
11655
11656 static bool i9xx_cursor_size_ok(const struct intel_plane_state *plane_state)
11657 {
11658         struct drm_i915_private *dev_priv =
11659                 to_i915(plane_state->uapi.plane->dev);
11660         int width = drm_rect_width(&plane_state->uapi.dst);
11661         int height = drm_rect_height(&plane_state->uapi.dst);
11662
11663         if (!intel_cursor_size_ok(plane_state))
11664                 return false;
11665
11666         /* Cursor width is limited to a few power-of-two sizes */
11667         switch (width) {
11668         case 256:
11669         case 128:
11670         case 64:
11671                 break;
11672         default:
11673                 return false;
11674         }
11675
11676         /*
11677          * IVB+ have CUR_FBC_CTL which allows an arbitrary cursor
11678          * height from 8 lines up to the cursor width, when the
11679          * cursor is not rotated. Everything else requires square
11680          * cursors.
11681          */
11682         if (HAS_CUR_FBC(dev_priv) &&
11683             plane_state->hw.rotation & DRM_MODE_ROTATE_0) {
11684                 if (height < 8 || height > width)
11685                         return false;
11686         } else {
11687                 if (height != width)
11688                         return false;
11689         }
11690
11691         return true;
11692 }
11693
11694 static int i9xx_check_cursor(struct intel_crtc_state *crtc_state,
11695                              struct intel_plane_state *plane_state)
11696 {
11697         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
11698         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11699         const struct drm_framebuffer *fb = plane_state->hw.fb;
11700         enum pipe pipe = plane->pipe;
11701         int ret;
11702
11703         ret = intel_check_cursor(crtc_state, plane_state);
11704         if (ret)
11705                 return ret;
11706
11707         /* if we want to turn off the cursor ignore width and height */
11708         if (!fb)
11709                 return 0;
11710
11711         /* Check for which cursor types we support */
11712         if (!i9xx_cursor_size_ok(plane_state)) {
11713                 drm_dbg(&dev_priv->drm,
11714                         "Cursor dimension %dx%d not supported\n",
11715                         drm_rect_width(&plane_state->uapi.dst),
11716                         drm_rect_height(&plane_state->uapi.dst));
11717                 return -EINVAL;
11718         }
11719
11720         drm_WARN_ON(&dev_priv->drm, plane_state->uapi.visible &&
11721                     plane_state->color_plane[0].stride != fb->pitches[0]);
11722
11723         if (fb->pitches[0] !=
11724             drm_rect_width(&plane_state->uapi.dst) * fb->format->cpp[0]) {
11725                 drm_dbg_kms(&dev_priv->drm,
11726                             "Invalid cursor stride (%u) (cursor width %d)\n",
11727                             fb->pitches[0],
11728                             drm_rect_width(&plane_state->uapi.dst));
11729                 return -EINVAL;
11730         }
11731
11732         /*
11733          * There's something wrong with the cursor on CHV pipe C.
11734          * If it straddles the left edge of the screen then
11735          * moving it away from the edge or disabling it often
11736          * results in a pipe underrun, and often that can lead to
11737          * dead pipe (constant underrun reported, and it scans
11738          * out just a solid color). To recover from that, the
11739          * display power well must be turned off and on again.
11740          * Refuse the put the cursor into that compromised position.
11741          */
11742         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_C &&
11743             plane_state->uapi.visible && plane_state->uapi.dst.x1 < 0) {
11744                 drm_dbg_kms(&dev_priv->drm,
11745                             "CHV cursor C not allowed to straddle the left screen edge\n");
11746                 return -EINVAL;
11747         }
11748
11749         plane_state->ctl = i9xx_cursor_ctl(crtc_state, plane_state);
11750
11751         return 0;
11752 }
11753
11754 static void i9xx_update_cursor(struct intel_plane *plane,
11755                                const struct intel_crtc_state *crtc_state,
11756                                const struct intel_plane_state *plane_state)
11757 {
11758         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11759         enum pipe pipe = plane->pipe;
11760         u32 cntl = 0, base = 0, pos = 0, fbc_ctl = 0;
11761         unsigned long irqflags;
11762
11763         if (plane_state && plane_state->uapi.visible) {
11764                 unsigned width = drm_rect_width(&plane_state->uapi.dst);
11765                 unsigned height = drm_rect_height(&plane_state->uapi.dst);
11766
11767                 cntl = plane_state->ctl |
11768                         i9xx_cursor_ctl_crtc(crtc_state);
11769
11770                 if (width != height)
11771                         fbc_ctl = CUR_FBC_CTL_EN | (height - 1);
11772
11773                 base = intel_cursor_base(plane_state);
11774                 pos = intel_cursor_position(plane_state);
11775         }
11776
11777         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
11778
11779         /*
11780          * On some platforms writing CURCNTR first will also
11781          * cause CURPOS to be armed by the CURBASE write.
11782          * Without the CURCNTR write the CURPOS write would
11783          * arm itself. Thus we always update CURCNTR before
11784          * CURPOS.
11785          *
11786          * On other platforms CURPOS always requires the
11787          * CURBASE write to arm the update. Additonally
11788          * a write to any of the cursor register will cancel
11789          * an already armed cursor update. Thus leaving out
11790          * the CURBASE write after CURPOS could lead to a
11791          * cursor that doesn't appear to move, or even change
11792          * shape. Thus we always write CURBASE.
11793          *
11794          * The other registers are armed by by the CURBASE write
11795          * except when the plane is getting enabled at which time
11796          * the CURCNTR write arms the update.
11797          */
11798
11799         if (INTEL_GEN(dev_priv) >= 9)
11800                 skl_write_cursor_wm(plane, crtc_state);
11801
11802         if (plane->cursor.base != base ||
11803             plane->cursor.size != fbc_ctl ||
11804             plane->cursor.cntl != cntl) {
11805                 if (HAS_CUR_FBC(dev_priv))
11806                         intel_de_write_fw(dev_priv, CUR_FBC_CTL(pipe),
11807                                           fbc_ctl);
11808                 intel_de_write_fw(dev_priv, CURCNTR(pipe), cntl);
11809                 intel_de_write_fw(dev_priv, CURPOS(pipe), pos);
11810                 intel_de_write_fw(dev_priv, CURBASE(pipe), base);
11811
11812                 plane->cursor.base = base;
11813                 plane->cursor.size = fbc_ctl;
11814                 plane->cursor.cntl = cntl;
11815         } else {
11816                 intel_de_write_fw(dev_priv, CURPOS(pipe), pos);
11817                 intel_de_write_fw(dev_priv, CURBASE(pipe), base);
11818         }
11819
11820         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
11821 }
11822
11823 static void i9xx_disable_cursor(struct intel_plane *plane,
11824                                 const struct intel_crtc_state *crtc_state)
11825 {
11826         i9xx_update_cursor(plane, crtc_state, NULL);
11827 }
11828
11829 static bool i9xx_cursor_get_hw_state(struct intel_plane *plane,
11830                                      enum pipe *pipe)
11831 {
11832         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11833         enum intel_display_power_domain power_domain;
11834         intel_wakeref_t wakeref;
11835         bool ret;
11836         u32 val;
11837
11838         /*
11839          * Not 100% correct for planes that can move between pipes,
11840          * but that's only the case for gen2-3 which don't have any
11841          * display power wells.
11842          */
11843         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
11844         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
11845         if (!wakeref)
11846                 return false;
11847
11848         val = intel_de_read(dev_priv, CURCNTR(plane->pipe));
11849
11850         ret = val & MCURSOR_MODE;
11851
11852         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
11853                 *pipe = plane->pipe;
11854         else
11855                 *pipe = (val & MCURSOR_PIPE_SELECT_MASK) >>
11856                         MCURSOR_PIPE_SELECT_SHIFT;
11857
11858         intel_display_power_put(dev_priv, power_domain, wakeref);
11859
11860         return ret;
11861 }
11862
11863 /* VESA 640x480x72Hz mode to set on the pipe */
11864 static const struct drm_display_mode load_detect_mode = {
11865         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
11866                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
11867 };
11868
11869 struct drm_framebuffer *
11870 intel_framebuffer_create(struct drm_i915_gem_object *obj,
11871                          struct drm_mode_fb_cmd2 *mode_cmd)
11872 {
11873         struct intel_framebuffer *intel_fb;
11874         int ret;
11875
11876         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
11877         if (!intel_fb)
11878                 return ERR_PTR(-ENOMEM);
11879
11880         ret = intel_framebuffer_init(intel_fb, obj, mode_cmd);
11881         if (ret)
11882                 goto err;
11883
11884         return &intel_fb->base;
11885
11886 err:
11887         kfree(intel_fb);
11888         return ERR_PTR(ret);
11889 }
11890
11891 static int intel_modeset_disable_planes(struct drm_atomic_state *state,
11892                                         struct drm_crtc *crtc)
11893 {
11894         struct drm_plane *plane;
11895         struct drm_plane_state *plane_state;
11896         int ret, i;
11897
11898         ret = drm_atomic_add_affected_planes(state, crtc);
11899         if (ret)
11900                 return ret;
11901
11902         for_each_new_plane_in_state(state, plane, plane_state, i) {
11903                 if (plane_state->crtc != crtc)
11904                         continue;
11905
11906                 ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
11907                 if (ret)
11908                         return ret;
11909
11910                 drm_atomic_set_fb_for_plane(plane_state, NULL);
11911         }
11912
11913         return 0;
11914 }
11915
11916 int intel_get_load_detect_pipe(struct drm_connector *connector,
11917                                struct intel_load_detect_pipe *old,
11918                                struct drm_modeset_acquire_ctx *ctx)
11919 {
11920         struct intel_crtc *intel_crtc;
11921         struct intel_encoder *intel_encoder =
11922                 intel_attached_encoder(to_intel_connector(connector));
11923         struct drm_crtc *possible_crtc;
11924         struct drm_encoder *encoder = &intel_encoder->base;
11925         struct drm_crtc *crtc = NULL;
11926         struct drm_device *dev = encoder->dev;
11927         struct drm_i915_private *dev_priv = to_i915(dev);
11928         struct drm_mode_config *config = &dev->mode_config;
11929         struct drm_atomic_state *state = NULL, *restore_state = NULL;
11930         struct drm_connector_state *connector_state;
11931         struct intel_crtc_state *crtc_state;
11932         int ret, i = -1;
11933
11934         drm_dbg_kms(&dev_priv->drm, "[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
11935                     connector->base.id, connector->name,
11936                     encoder->base.id, encoder->name);
11937
11938         old->restore_state = NULL;
11939
11940         drm_WARN_ON(dev, !drm_modeset_is_locked(&config->connection_mutex));
11941
11942         /*
11943          * Algorithm gets a little messy:
11944          *
11945          *   - if the connector already has an assigned crtc, use it (but make
11946          *     sure it's on first)
11947          *
11948          *   - try to find the first unused crtc that can drive this connector,
11949          *     and use that if we find one
11950          */
11951
11952         /* See if we already have a CRTC for this connector */
11953         if (connector->state->crtc) {
11954                 crtc = connector->state->crtc;
11955
11956                 ret = drm_modeset_lock(&crtc->mutex, ctx);
11957                 if (ret)
11958                         goto fail;
11959
11960                 /* Make sure the crtc and connector are running */
11961                 goto found;
11962         }
11963
11964         /* Find an unused one (if possible) */
11965         for_each_crtc(dev, possible_crtc) {
11966                 i++;
11967                 if (!(encoder->possible_crtcs & (1 << i)))
11968                         continue;
11969
11970                 ret = drm_modeset_lock(&possible_crtc->mutex, ctx);
11971                 if (ret)
11972                         goto fail;
11973
11974                 if (possible_crtc->state->enable) {
11975                         drm_modeset_unlock(&possible_crtc->mutex);
11976                         continue;
11977                 }
11978
11979                 crtc = possible_crtc;
11980                 break;
11981         }
11982
11983         /*
11984          * If we didn't find an unused CRTC, don't use any.
11985          */
11986         if (!crtc) {
11987                 drm_dbg_kms(&dev_priv->drm,
11988                             "no pipe available for load-detect\n");
11989                 ret = -ENODEV;
11990                 goto fail;
11991         }
11992
11993 found:
11994         intel_crtc = to_intel_crtc(crtc);
11995
11996         state = drm_atomic_state_alloc(dev);
11997         restore_state = drm_atomic_state_alloc(dev);
11998         if (!state || !restore_state) {
11999                 ret = -ENOMEM;
12000                 goto fail;
12001         }
12002
12003         state->acquire_ctx = ctx;
12004         restore_state->acquire_ctx = ctx;
12005
12006         connector_state = drm_atomic_get_connector_state(state, connector);
12007         if (IS_ERR(connector_state)) {
12008                 ret = PTR_ERR(connector_state);
12009                 goto fail;
12010         }
12011
12012         ret = drm_atomic_set_crtc_for_connector(connector_state, crtc);
12013         if (ret)
12014                 goto fail;
12015
12016         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
12017         if (IS_ERR(crtc_state)) {
12018                 ret = PTR_ERR(crtc_state);
12019                 goto fail;
12020         }
12021
12022         crtc_state->uapi.active = true;
12023
12024         ret = drm_atomic_set_mode_for_crtc(&crtc_state->uapi,
12025                                            &load_detect_mode);
12026         if (ret)
12027                 goto fail;
12028
12029         ret = intel_modeset_disable_planes(state, crtc);
12030         if (ret)
12031                 goto fail;
12032
12033         ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector));
12034         if (!ret)
12035                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, crtc));
12036         if (!ret)
12037                 ret = drm_atomic_add_affected_planes(restore_state, crtc);
12038         if (ret) {
12039                 drm_dbg_kms(&dev_priv->drm,
12040                             "Failed to create a copy of old state to restore: %i\n",
12041                             ret);
12042                 goto fail;
12043         }
12044
12045         ret = drm_atomic_commit(state);
12046         if (ret) {
12047                 drm_dbg_kms(&dev_priv->drm,
12048                             "failed to set mode on load-detect pipe\n");
12049                 goto fail;
12050         }
12051
12052         old->restore_state = restore_state;
12053         drm_atomic_state_put(state);
12054
12055         /* let the connector get through one full cycle before testing */
12056         intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
12057         return true;
12058
12059 fail:
12060         if (state) {
12061                 drm_atomic_state_put(state);
12062                 state = NULL;
12063         }
12064         if (restore_state) {
12065                 drm_atomic_state_put(restore_state);
12066                 restore_state = NULL;
12067         }
12068
12069         if (ret == -EDEADLK)
12070                 return ret;
12071
12072         return false;
12073 }
12074
12075 void intel_release_load_detect_pipe(struct drm_connector *connector,
12076                                     struct intel_load_detect_pipe *old,
12077                                     struct drm_modeset_acquire_ctx *ctx)
12078 {
12079         struct intel_encoder *intel_encoder =
12080                 intel_attached_encoder(to_intel_connector(connector));
12081         struct drm_i915_private *i915 = to_i915(intel_encoder->base.dev);
12082         struct drm_encoder *encoder = &intel_encoder->base;
12083         struct drm_atomic_state *state = old->restore_state;
12084         int ret;
12085
12086         drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
12087                     connector->base.id, connector->name,
12088                     encoder->base.id, encoder->name);
12089
12090         if (!state)
12091                 return;
12092
12093         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
12094         if (ret)
12095                 drm_dbg_kms(&i915->drm,
12096                             "Couldn't release load detect pipe: %i\n", ret);
12097         drm_atomic_state_put(state);
12098 }
12099
12100 static int i9xx_pll_refclk(struct drm_device *dev,
12101                            const struct intel_crtc_state *pipe_config)
12102 {
12103         struct drm_i915_private *dev_priv = to_i915(dev);
12104         u32 dpll = pipe_config->dpll_hw_state.dpll;
12105
12106         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
12107                 return dev_priv->vbt.lvds_ssc_freq;
12108         else if (HAS_PCH_SPLIT(dev_priv))
12109                 return 120000;
12110         else if (!IS_GEN(dev_priv, 2))
12111                 return 96000;
12112         else
12113                 return 48000;
12114 }
12115
12116 /* Returns the clock of the currently programmed mode of the given pipe. */
12117 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
12118                                 struct intel_crtc_state *pipe_config)
12119 {
12120         struct drm_device *dev = crtc->base.dev;
12121         struct drm_i915_private *dev_priv = to_i915(dev);
12122         enum pipe pipe = crtc->pipe;
12123         u32 dpll = pipe_config->dpll_hw_state.dpll;
12124         u32 fp;
12125         struct dpll clock;
12126         int port_clock;
12127         int refclk = i9xx_pll_refclk(dev, pipe_config);
12128
12129         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
12130                 fp = pipe_config->dpll_hw_state.fp0;
12131         else
12132                 fp = pipe_config->dpll_hw_state.fp1;
12133
12134         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
12135         if (IS_PINEVIEW(dev_priv)) {
12136                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
12137                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
12138         } else {
12139                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
12140                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
12141         }
12142
12143         if (!IS_GEN(dev_priv, 2)) {
12144                 if (IS_PINEVIEW(dev_priv))
12145                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
12146                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
12147                 else
12148                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
12149                                DPLL_FPA01_P1_POST_DIV_SHIFT);
12150
12151                 switch (dpll & DPLL_MODE_MASK) {
12152                 case DPLLB_MODE_DAC_SERIAL:
12153                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
12154                                 5 : 10;
12155                         break;
12156                 case DPLLB_MODE_LVDS:
12157                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
12158                                 7 : 14;
12159                         break;
12160                 default:
12161                         drm_dbg_kms(&dev_priv->drm,
12162                                     "Unknown DPLL mode %08x in programmed "
12163                                     "mode\n", (int)(dpll & DPLL_MODE_MASK));
12164                         return;
12165                 }
12166
12167                 if (IS_PINEVIEW(dev_priv))
12168                         port_clock = pnv_calc_dpll_params(refclk, &clock);
12169                 else
12170                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
12171         } else {
12172                 u32 lvds = IS_I830(dev_priv) ? 0 : intel_de_read(dev_priv,
12173                                                                  LVDS);
12174                 bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN);
12175
12176                 if (is_lvds) {
12177                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
12178                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
12179
12180                         if (lvds & LVDS_CLKB_POWER_UP)
12181                                 clock.p2 = 7;
12182                         else
12183                                 clock.p2 = 14;
12184                 } else {
12185                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
12186                                 clock.p1 = 2;
12187                         else {
12188                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
12189                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
12190                         }
12191                         if (dpll & PLL_P2_DIVIDE_BY_4)
12192                                 clock.p2 = 4;
12193                         else
12194                                 clock.p2 = 2;
12195                 }
12196
12197                 port_clock = i9xx_calc_dpll_params(refclk, &clock);
12198         }
12199
12200         /*
12201          * This value includes pixel_multiplier. We will use
12202          * port_clock to compute adjusted_mode.crtc_clock in the
12203          * encoder's get_config() function.
12204          */
12205         pipe_config->port_clock = port_clock;
12206 }
12207
12208 int intel_dotclock_calculate(int link_freq,
12209                              const struct intel_link_m_n *m_n)
12210 {
12211         /*
12212          * The calculation for the data clock is:
12213          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
12214          * But we want to avoid losing precison if possible, so:
12215          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
12216          *
12217          * and the link clock is simpler:
12218          * link_clock = (m * link_clock) / n
12219          */
12220
12221         if (!m_n->link_n)
12222                 return 0;
12223
12224         return div_u64(mul_u32_u32(m_n->link_m, link_freq), m_n->link_n);
12225 }
12226
12227 static void ilk_pch_clock_get(struct intel_crtc *crtc,
12228                               struct intel_crtc_state *pipe_config)
12229 {
12230         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12231
12232         /* read out port_clock from the DPLL */
12233         i9xx_crtc_clock_get(crtc, pipe_config);
12234
12235         /*
12236          * In case there is an active pipe without active ports,
12237          * we may need some idea for the dotclock anyway.
12238          * Calculate one based on the FDI configuration.
12239          */
12240         pipe_config->hw.adjusted_mode.crtc_clock =
12241                 intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
12242                                          &pipe_config->fdi_m_n);
12243 }
12244
12245 static void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
12246                                    struct intel_crtc *crtc)
12247 {
12248         memset(crtc_state, 0, sizeof(*crtc_state));
12249
12250         __drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
12251
12252         crtc_state->cpu_transcoder = INVALID_TRANSCODER;
12253         crtc_state->master_transcoder = INVALID_TRANSCODER;
12254         crtc_state->hsw_workaround_pipe = INVALID_PIPE;
12255         crtc_state->output_format = INTEL_OUTPUT_FORMAT_INVALID;
12256         crtc_state->scaler_state.scaler_id = -1;
12257         crtc_state->mst_master_transcoder = INVALID_TRANSCODER;
12258 }
12259
12260 static struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
12261 {
12262         struct intel_crtc_state *crtc_state;
12263
12264         crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
12265
12266         if (crtc_state)
12267                 intel_crtc_state_reset(crtc_state, crtc);
12268
12269         return crtc_state;
12270 }
12271
12272 /* Returns the currently programmed mode of the given encoder. */
12273 struct drm_display_mode *
12274 intel_encoder_current_mode(struct intel_encoder *encoder)
12275 {
12276         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
12277         struct intel_crtc_state *crtc_state;
12278         struct drm_display_mode *mode;
12279         struct intel_crtc *crtc;
12280         enum pipe pipe;
12281
12282         if (!encoder->get_hw_state(encoder, &pipe))
12283                 return NULL;
12284
12285         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
12286
12287         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
12288         if (!mode)
12289                 return NULL;
12290
12291         crtc_state = intel_crtc_state_alloc(crtc);
12292         if (!crtc_state) {
12293                 kfree(mode);
12294                 return NULL;
12295         }
12296
12297         if (!dev_priv->display.get_pipe_config(crtc, crtc_state)) {
12298                 kfree(crtc_state);
12299                 kfree(mode);
12300                 return NULL;
12301         }
12302
12303         encoder->get_config(encoder, crtc_state);
12304
12305         intel_mode_from_pipe_config(mode, crtc_state);
12306
12307         kfree(crtc_state);
12308
12309         return mode;
12310 }
12311
12312 static void intel_crtc_destroy(struct drm_crtc *crtc)
12313 {
12314         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
12315
12316         drm_crtc_cleanup(crtc);
12317         kfree(intel_crtc);
12318 }
12319
12320 /**
12321  * intel_wm_need_update - Check whether watermarks need updating
12322  * @cur: current plane state
12323  * @new: new plane state
12324  *
12325  * Check current plane state versus the new one to determine whether
12326  * watermarks need to be recalculated.
12327  *
12328  * Returns true or false.
12329  */
12330 static bool intel_wm_need_update(const struct intel_plane_state *cur,
12331                                  struct intel_plane_state *new)
12332 {
12333         /* Update watermarks on tiling or size changes. */
12334         if (new->uapi.visible != cur->uapi.visible)
12335                 return true;
12336
12337         if (!cur->hw.fb || !new->hw.fb)
12338                 return false;
12339
12340         if (cur->hw.fb->modifier != new->hw.fb->modifier ||
12341             cur->hw.rotation != new->hw.rotation ||
12342             drm_rect_width(&new->uapi.src) != drm_rect_width(&cur->uapi.src) ||
12343             drm_rect_height(&new->uapi.src) != drm_rect_height(&cur->uapi.src) ||
12344             drm_rect_width(&new->uapi.dst) != drm_rect_width(&cur->uapi.dst) ||
12345             drm_rect_height(&new->uapi.dst) != drm_rect_height(&cur->uapi.dst))
12346                 return true;
12347
12348         return false;
12349 }
12350
12351 static bool needs_scaling(const struct intel_plane_state *state)
12352 {
12353         int src_w = drm_rect_width(&state->uapi.src) >> 16;
12354         int src_h = drm_rect_height(&state->uapi.src) >> 16;
12355         int dst_w = drm_rect_width(&state->uapi.dst);
12356         int dst_h = drm_rect_height(&state->uapi.dst);
12357
12358         return (src_w != dst_w || src_h != dst_h);
12359 }
12360
12361 int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state,
12362                                     struct intel_crtc_state *crtc_state,
12363                                     const struct intel_plane_state *old_plane_state,
12364                                     struct intel_plane_state *plane_state)
12365 {
12366         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
12367         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
12368         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12369         bool mode_changed = needs_modeset(crtc_state);
12370         bool was_crtc_enabled = old_crtc_state->hw.active;
12371         bool is_crtc_enabled = crtc_state->hw.active;
12372         bool turn_off, turn_on, visible, was_visible;
12373         int ret;
12374
12375         if (INTEL_GEN(dev_priv) >= 9 && plane->id != PLANE_CURSOR) {
12376                 ret = skl_update_scaler_plane(crtc_state, plane_state);
12377                 if (ret)
12378                         return ret;
12379         }
12380
12381         was_visible = old_plane_state->uapi.visible;
12382         visible = plane_state->uapi.visible;
12383
12384         if (!was_crtc_enabled && drm_WARN_ON(&dev_priv->drm, was_visible))
12385                 was_visible = false;
12386
12387         /*
12388          * Visibility is calculated as if the crtc was on, but
12389          * after scaler setup everything depends on it being off
12390          * when the crtc isn't active.
12391          *
12392          * FIXME this is wrong for watermarks. Watermarks should also
12393          * be computed as if the pipe would be active. Perhaps move
12394          * per-plane wm computation to the .check_plane() hook, and
12395          * only combine the results from all planes in the current place?
12396          */
12397         if (!is_crtc_enabled) {
12398                 intel_plane_set_invisible(crtc_state, plane_state);
12399                 visible = false;
12400         }
12401
12402         if (!was_visible && !visible)
12403                 return 0;
12404
12405         turn_off = was_visible && (!visible || mode_changed);
12406         turn_on = visible && (!was_visible || mode_changed);
12407
12408         drm_dbg_atomic(&dev_priv->drm,
12409                        "[CRTC:%d:%s] with [PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n",
12410                        crtc->base.base.id, crtc->base.name,
12411                        plane->base.base.id, plane->base.name,
12412                        was_visible, visible,
12413                        turn_off, turn_on, mode_changed);
12414
12415         if (turn_on) {
12416                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
12417                         crtc_state->update_wm_pre = true;
12418
12419                 /* must disable cxsr around plane enable/disable */
12420                 if (plane->id != PLANE_CURSOR)
12421                         crtc_state->disable_cxsr = true;
12422         } else if (turn_off) {
12423                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
12424                         crtc_state->update_wm_post = true;
12425
12426                 /* must disable cxsr around plane enable/disable */
12427                 if (plane->id != PLANE_CURSOR)
12428                         crtc_state->disable_cxsr = true;
12429         } else if (intel_wm_need_update(old_plane_state, plane_state)) {
12430                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) {
12431                         /* FIXME bollocks */
12432                         crtc_state->update_wm_pre = true;
12433                         crtc_state->update_wm_post = true;
12434                 }
12435         }
12436
12437         if (visible || was_visible)
12438                 crtc_state->fb_bits |= plane->frontbuffer_bit;
12439
12440         /*
12441          * ILK/SNB DVSACNTR/Sprite Enable
12442          * IVB SPR_CTL/Sprite Enable
12443          * "When in Self Refresh Big FIFO mode, a write to enable the
12444          *  plane will be internally buffered and delayed while Big FIFO
12445          *  mode is exiting."
12446          *
12447          * Which means that enabling the sprite can take an extra frame
12448          * when we start in big FIFO mode (LP1+). Thus we need to drop
12449          * down to LP0 and wait for vblank in order to make sure the
12450          * sprite gets enabled on the next vblank after the register write.
12451          * Doing otherwise would risk enabling the sprite one frame after
12452          * we've already signalled flip completion. We can resume LP1+
12453          * once the sprite has been enabled.
12454          *
12455          *
12456          * WaCxSRDisabledForSpriteScaling:ivb
12457          * IVB SPR_SCALE/Scaling Enable
12458          * "Low Power watermarks must be disabled for at least one
12459          *  frame before enabling sprite scaling, and kept disabled
12460          *  until sprite scaling is disabled."
12461          *
12462          * ILK/SNB DVSASCALE/Scaling Enable
12463          * "When in Self Refresh Big FIFO mode, scaling enable will be
12464          *  masked off while Big FIFO mode is exiting."
12465          *
12466          * Despite the w/a only being listed for IVB we assume that
12467          * the ILK/SNB note has similar ramifications, hence we apply
12468          * the w/a on all three platforms.
12469          *
12470          * With experimental results seems this is needed also for primary
12471          * plane, not only sprite plane.
12472          */
12473         if (plane->id != PLANE_CURSOR &&
12474             (IS_GEN_RANGE(dev_priv, 5, 6) ||
12475              IS_IVYBRIDGE(dev_priv)) &&
12476             (turn_on || (!needs_scaling(old_plane_state) &&
12477                          needs_scaling(plane_state))))
12478                 crtc_state->disable_lp_wm = true;
12479
12480         return 0;
12481 }
12482
12483 static bool encoders_cloneable(const struct intel_encoder *a,
12484                                const struct intel_encoder *b)
12485 {
12486         /* masks could be asymmetric, so check both ways */
12487         return a == b || (a->cloneable & (1 << b->type) &&
12488                           b->cloneable & (1 << a->type));
12489 }
12490
12491 static bool check_single_encoder_cloning(struct drm_atomic_state *state,
12492                                          struct intel_crtc *crtc,
12493                                          struct intel_encoder *encoder)
12494 {
12495         struct intel_encoder *source_encoder;
12496         struct drm_connector *connector;
12497         struct drm_connector_state *connector_state;
12498         int i;
12499
12500         for_each_new_connector_in_state(state, connector, connector_state, i) {
12501                 if (connector_state->crtc != &crtc->base)
12502                         continue;
12503
12504                 source_encoder =
12505                         to_intel_encoder(connector_state->best_encoder);
12506                 if (!encoders_cloneable(encoder, source_encoder))
12507                         return false;
12508         }
12509
12510         return true;
12511 }
12512
12513 static int icl_add_linked_planes(struct intel_atomic_state *state)
12514 {
12515         struct intel_plane *plane, *linked;
12516         struct intel_plane_state *plane_state, *linked_plane_state;
12517         int i;
12518
12519         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12520                 linked = plane_state->planar_linked_plane;
12521
12522                 if (!linked)
12523                         continue;
12524
12525                 linked_plane_state = intel_atomic_get_plane_state(state, linked);
12526                 if (IS_ERR(linked_plane_state))
12527                         return PTR_ERR(linked_plane_state);
12528
12529                 drm_WARN_ON(state->base.dev,
12530                             linked_plane_state->planar_linked_plane != plane);
12531                 drm_WARN_ON(state->base.dev,
12532                             linked_plane_state->planar_slave == plane_state->planar_slave);
12533         }
12534
12535         return 0;
12536 }
12537
12538 static int icl_check_nv12_planes(struct intel_crtc_state *crtc_state)
12539 {
12540         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
12541         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12542         struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->uapi.state);
12543         struct intel_plane *plane, *linked;
12544         struct intel_plane_state *plane_state;
12545         int i;
12546
12547         if (INTEL_GEN(dev_priv) < 11)
12548                 return 0;
12549
12550         /*
12551          * Destroy all old plane links and make the slave plane invisible
12552          * in the crtc_state->active_planes mask.
12553          */
12554         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12555                 if (plane->pipe != crtc->pipe || !plane_state->planar_linked_plane)
12556                         continue;
12557
12558                 plane_state->planar_linked_plane = NULL;
12559                 if (plane_state->planar_slave && !plane_state->uapi.visible) {
12560                         crtc_state->active_planes &= ~BIT(plane->id);
12561                         crtc_state->update_planes |= BIT(plane->id);
12562                 }
12563
12564                 plane_state->planar_slave = false;
12565         }
12566
12567         if (!crtc_state->nv12_planes)
12568                 return 0;
12569
12570         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12571                 struct intel_plane_state *linked_state = NULL;
12572
12573                 if (plane->pipe != crtc->pipe ||
12574                     !(crtc_state->nv12_planes & BIT(plane->id)))
12575                         continue;
12576
12577                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, linked) {
12578                         if (!icl_is_nv12_y_plane(dev_priv, linked->id))
12579                                 continue;
12580
12581                         if (crtc_state->active_planes & BIT(linked->id))
12582                                 continue;
12583
12584                         linked_state = intel_atomic_get_plane_state(state, linked);
12585                         if (IS_ERR(linked_state))
12586                                 return PTR_ERR(linked_state);
12587
12588                         break;
12589                 }
12590
12591                 if (!linked_state) {
12592                         drm_dbg_kms(&dev_priv->drm,
12593                                     "Need %d free Y planes for planar YUV\n",
12594                                     hweight8(crtc_state->nv12_planes));
12595
12596                         return -EINVAL;
12597                 }
12598
12599                 plane_state->planar_linked_plane = linked;
12600
12601                 linked_state->planar_slave = true;
12602                 linked_state->planar_linked_plane = plane;
12603                 crtc_state->active_planes |= BIT(linked->id);
12604                 crtc_state->update_planes |= BIT(linked->id);
12605                 drm_dbg_kms(&dev_priv->drm, "Using %s as Y plane for %s\n",
12606                             linked->base.name, plane->base.name);
12607
12608                 /* Copy parameters to slave plane */
12609                 linked_state->ctl = plane_state->ctl | PLANE_CTL_YUV420_Y_PLANE;
12610                 linked_state->color_ctl = plane_state->color_ctl;
12611                 linked_state->view = plane_state->view;
12612                 memcpy(linked_state->color_plane, plane_state->color_plane,
12613                        sizeof(linked_state->color_plane));
12614
12615                 intel_plane_copy_uapi_to_hw_state(linked_state, plane_state);
12616                 linked_state->uapi.src = plane_state->uapi.src;
12617                 linked_state->uapi.dst = plane_state->uapi.dst;
12618
12619                 if (icl_is_hdr_plane(dev_priv, plane->id)) {
12620                         if (linked->id == PLANE_SPRITE5)
12621                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_7;
12622                         else if (linked->id == PLANE_SPRITE4)
12623                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_6;
12624                         else if (linked->id == PLANE_SPRITE3)
12625                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_5_RKL;
12626                         else if (linked->id == PLANE_SPRITE2)
12627                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_4_RKL;
12628                         else
12629                                 MISSING_CASE(linked->id);
12630                 }
12631         }
12632
12633         return 0;
12634 }
12635
12636 static bool c8_planes_changed(const struct intel_crtc_state *new_crtc_state)
12637 {
12638         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
12639         struct intel_atomic_state *state =
12640                 to_intel_atomic_state(new_crtc_state->uapi.state);
12641         const struct intel_crtc_state *old_crtc_state =
12642                 intel_atomic_get_old_crtc_state(state, crtc);
12643
12644         return !old_crtc_state->c8_planes != !new_crtc_state->c8_planes;
12645 }
12646
12647 static u16 hsw_linetime_wm(const struct intel_crtc_state *crtc_state)
12648 {
12649         const struct drm_display_mode *adjusted_mode =
12650                 &crtc_state->hw.adjusted_mode;
12651         int linetime_wm;
12652
12653         if (!crtc_state->hw.enable)
12654                 return 0;
12655
12656         linetime_wm = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
12657                                         adjusted_mode->crtc_clock);
12658
12659         return min(linetime_wm, 0x1ff);
12660 }
12661
12662 static u16 hsw_ips_linetime_wm(const struct intel_crtc_state *crtc_state,
12663                                const struct intel_cdclk_state *cdclk_state)
12664 {
12665         const struct drm_display_mode *adjusted_mode =
12666                 &crtc_state->hw.adjusted_mode;
12667         int linetime_wm;
12668
12669         if (!crtc_state->hw.enable)
12670                 return 0;
12671
12672         linetime_wm = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
12673                                         cdclk_state->logical.cdclk);
12674
12675         return min(linetime_wm, 0x1ff);
12676 }
12677
12678 static u16 skl_linetime_wm(const struct intel_crtc_state *crtc_state)
12679 {
12680         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
12681         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12682         const struct drm_display_mode *adjusted_mode =
12683                 &crtc_state->hw.adjusted_mode;
12684         int linetime_wm;
12685
12686         if (!crtc_state->hw.enable)
12687                 return 0;
12688
12689         linetime_wm = DIV_ROUND_UP(adjusted_mode->crtc_htotal * 1000 * 8,
12690                                    crtc_state->pixel_rate);
12691
12692         /* Display WA #1135: BXT:ALL GLK:ALL */
12693         if (IS_GEN9_LP(dev_priv) && dev_priv->ipc_enabled)
12694                 linetime_wm /= 2;
12695
12696         return min(linetime_wm, 0x1ff);
12697 }
12698
12699 static int hsw_compute_linetime_wm(struct intel_atomic_state *state,
12700                                    struct intel_crtc *crtc)
12701 {
12702         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12703         struct intel_crtc_state *crtc_state =
12704                 intel_atomic_get_new_crtc_state(state, crtc);
12705         const struct intel_cdclk_state *cdclk_state;
12706
12707         if (INTEL_GEN(dev_priv) >= 9)
12708                 crtc_state->linetime = skl_linetime_wm(crtc_state);
12709         else
12710                 crtc_state->linetime = hsw_linetime_wm(crtc_state);
12711
12712         if (!hsw_crtc_supports_ips(crtc))
12713                 return 0;
12714
12715         cdclk_state = intel_atomic_get_cdclk_state(state);
12716         if (IS_ERR(cdclk_state))
12717                 return PTR_ERR(cdclk_state);
12718
12719         crtc_state->ips_linetime = hsw_ips_linetime_wm(crtc_state,
12720                                                        cdclk_state);
12721
12722         return 0;
12723 }
12724
12725 static int intel_crtc_atomic_check(struct intel_atomic_state *state,
12726                                    struct intel_crtc *crtc)
12727 {
12728         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12729         struct intel_crtc_state *crtc_state =
12730                 intel_atomic_get_new_crtc_state(state, crtc);
12731         bool mode_changed = needs_modeset(crtc_state);
12732         int ret;
12733
12734         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv) &&
12735             mode_changed && !crtc_state->hw.active)
12736                 crtc_state->update_wm_post = true;
12737
12738         if (mode_changed && crtc_state->hw.enable &&
12739             dev_priv->display.crtc_compute_clock &&
12740             !drm_WARN_ON(&dev_priv->drm, crtc_state->shared_dpll)) {
12741                 ret = dev_priv->display.crtc_compute_clock(crtc, crtc_state);
12742                 if (ret)
12743                         return ret;
12744         }
12745
12746         /*
12747          * May need to update pipe gamma enable bits
12748          * when C8 planes are getting enabled/disabled.
12749          */
12750         if (c8_planes_changed(crtc_state))
12751                 crtc_state->uapi.color_mgmt_changed = true;
12752
12753         if (mode_changed || crtc_state->update_pipe ||
12754             crtc_state->uapi.color_mgmt_changed) {
12755                 ret = intel_color_check(crtc_state);
12756                 if (ret)
12757                         return ret;
12758         }
12759
12760         if (dev_priv->display.compute_pipe_wm) {
12761                 ret = dev_priv->display.compute_pipe_wm(crtc_state);
12762                 if (ret) {
12763                         drm_dbg_kms(&dev_priv->drm,
12764                                     "Target pipe watermarks are invalid\n");
12765                         return ret;
12766                 }
12767         }
12768
12769         if (dev_priv->display.compute_intermediate_wm) {
12770                 if (drm_WARN_ON(&dev_priv->drm,
12771                                 !dev_priv->display.compute_pipe_wm))
12772                         return 0;
12773
12774                 /*
12775                  * Calculate 'intermediate' watermarks that satisfy both the
12776                  * old state and the new state.  We can program these
12777                  * immediately.
12778                  */
12779                 ret = dev_priv->display.compute_intermediate_wm(crtc_state);
12780                 if (ret) {
12781                         drm_dbg_kms(&dev_priv->drm,
12782                                     "No valid intermediate pipe watermarks are possible\n");
12783                         return ret;
12784                 }
12785         }
12786
12787         if (INTEL_GEN(dev_priv) >= 9) {
12788                 if (mode_changed || crtc_state->update_pipe) {
12789                         ret = skl_update_scaler_crtc(crtc_state);
12790                         if (ret)
12791                                 return ret;
12792                 }
12793
12794                 ret = intel_atomic_setup_scalers(dev_priv, crtc, crtc_state);
12795                 if (ret)
12796                         return ret;
12797         }
12798
12799         if (HAS_IPS(dev_priv)) {
12800                 ret = hsw_compute_ips_config(crtc_state);
12801                 if (ret)
12802                         return ret;
12803         }
12804
12805         if (INTEL_GEN(dev_priv) >= 9 ||
12806             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
12807                 ret = hsw_compute_linetime_wm(state, crtc);
12808                 if (ret)
12809                         return ret;
12810
12811         }
12812
12813         if (!mode_changed)
12814                 intel_psr2_sel_fetch_update(state, crtc);
12815
12816         return 0;
12817 }
12818
12819 static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
12820 {
12821         struct intel_connector *connector;
12822         struct drm_connector_list_iter conn_iter;
12823
12824         drm_connector_list_iter_begin(dev, &conn_iter);
12825         for_each_intel_connector_iter(connector, &conn_iter) {
12826                 if (connector->base.state->crtc)
12827                         drm_connector_put(&connector->base);
12828
12829                 if (connector->base.encoder) {
12830                         connector->base.state->best_encoder =
12831                                 connector->base.encoder;
12832                         connector->base.state->crtc =
12833                                 connector->base.encoder->crtc;
12834
12835                         drm_connector_get(&connector->base);
12836                 } else {
12837                         connector->base.state->best_encoder = NULL;
12838                         connector->base.state->crtc = NULL;
12839                 }
12840         }
12841         drm_connector_list_iter_end(&conn_iter);
12842 }
12843
12844 static int
12845 compute_sink_pipe_bpp(const struct drm_connector_state *conn_state,
12846                       struct intel_crtc_state *pipe_config)
12847 {
12848         struct drm_connector *connector = conn_state->connector;
12849         struct drm_i915_private *i915 = to_i915(pipe_config->uapi.crtc->dev);
12850         const struct drm_display_info *info = &connector->display_info;
12851         int bpp;
12852
12853         switch (conn_state->max_bpc) {
12854         case 6 ... 7:
12855                 bpp = 6 * 3;
12856                 break;
12857         case 8 ... 9:
12858                 bpp = 8 * 3;
12859                 break;
12860         case 10 ... 11:
12861                 bpp = 10 * 3;
12862                 break;
12863         case 12:
12864                 bpp = 12 * 3;
12865                 break;
12866         default:
12867                 return -EINVAL;
12868         }
12869
12870         if (bpp < pipe_config->pipe_bpp) {
12871                 drm_dbg_kms(&i915->drm,
12872                             "[CONNECTOR:%d:%s] Limiting display bpp to %d instead of "
12873                             "EDID bpp %d, requested bpp %d, max platform bpp %d\n",
12874                             connector->base.id, connector->name,
12875                             bpp, 3 * info->bpc,
12876                             3 * conn_state->max_requested_bpc,
12877                             pipe_config->pipe_bpp);
12878
12879                 pipe_config->pipe_bpp = bpp;
12880         }
12881
12882         return 0;
12883 }
12884
12885 static int
12886 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
12887                           struct intel_crtc_state *pipe_config)
12888 {
12889         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12890         struct drm_atomic_state *state = pipe_config->uapi.state;
12891         struct drm_connector *connector;
12892         struct drm_connector_state *connector_state;
12893         int bpp, i;
12894
12895         if ((IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
12896             IS_CHERRYVIEW(dev_priv)))
12897                 bpp = 10*3;
12898         else if (INTEL_GEN(dev_priv) >= 5)
12899                 bpp = 12*3;
12900         else
12901                 bpp = 8*3;
12902
12903         pipe_config->pipe_bpp = bpp;
12904
12905         /* Clamp display bpp to connector max bpp */
12906         for_each_new_connector_in_state(state, connector, connector_state, i) {
12907                 int ret;
12908
12909                 if (connector_state->crtc != &crtc->base)
12910                         continue;
12911
12912                 ret = compute_sink_pipe_bpp(connector_state, pipe_config);
12913                 if (ret)
12914                         return ret;
12915         }
12916
12917         return 0;
12918 }
12919
12920 static void intel_dump_crtc_timings(struct drm_i915_private *i915,
12921                                     const struct drm_display_mode *mode)
12922 {
12923         drm_dbg_kms(&i915->drm, "crtc timings: %d %d %d %d %d %d %d %d %d, "
12924                     "type: 0x%x flags: 0x%x\n",
12925                     mode->crtc_clock,
12926                     mode->crtc_hdisplay, mode->crtc_hsync_start,
12927                     mode->crtc_hsync_end, mode->crtc_htotal,
12928                     mode->crtc_vdisplay, mode->crtc_vsync_start,
12929                     mode->crtc_vsync_end, mode->crtc_vtotal,
12930                     mode->type, mode->flags);
12931 }
12932
12933 static void
12934 intel_dump_m_n_config(const struct intel_crtc_state *pipe_config,
12935                       const char *id, unsigned int lane_count,
12936                       const struct intel_link_m_n *m_n)
12937 {
12938         struct drm_i915_private *i915 = to_i915(pipe_config->uapi.crtc->dev);
12939
12940         drm_dbg_kms(&i915->drm,
12941                     "%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
12942                     id, lane_count,
12943                     m_n->gmch_m, m_n->gmch_n,
12944                     m_n->link_m, m_n->link_n, m_n->tu);
12945 }
12946
12947 static void
12948 intel_dump_infoframe(struct drm_i915_private *dev_priv,
12949                      const union hdmi_infoframe *frame)
12950 {
12951         if (!drm_debug_enabled(DRM_UT_KMS))
12952                 return;
12953
12954         hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, frame);
12955 }
12956
12957 static void
12958 intel_dump_dp_vsc_sdp(struct drm_i915_private *dev_priv,
12959                       const struct drm_dp_vsc_sdp *vsc)
12960 {
12961         if (!drm_debug_enabled(DRM_UT_KMS))
12962                 return;
12963
12964         drm_dp_vsc_sdp_log(KERN_DEBUG, dev_priv->drm.dev, vsc);
12965 }
12966
12967 #define OUTPUT_TYPE(x) [INTEL_OUTPUT_ ## x] = #x
12968
12969 static const char * const output_type_str[] = {
12970         OUTPUT_TYPE(UNUSED),
12971         OUTPUT_TYPE(ANALOG),
12972         OUTPUT_TYPE(DVO),
12973         OUTPUT_TYPE(SDVO),
12974         OUTPUT_TYPE(LVDS),
12975         OUTPUT_TYPE(TVOUT),
12976         OUTPUT_TYPE(HDMI),
12977         OUTPUT_TYPE(DP),
12978         OUTPUT_TYPE(EDP),
12979         OUTPUT_TYPE(DSI),
12980         OUTPUT_TYPE(DDI),
12981         OUTPUT_TYPE(DP_MST),
12982 };
12983
12984 #undef OUTPUT_TYPE
12985
12986 static void snprintf_output_types(char *buf, size_t len,
12987                                   unsigned int output_types)
12988 {
12989         char *str = buf;
12990         int i;
12991
12992         str[0] = '\0';
12993
12994         for (i = 0; i < ARRAY_SIZE(output_type_str); i++) {
12995                 int r;
12996
12997                 if ((output_types & BIT(i)) == 0)
12998                         continue;
12999
13000                 r = snprintf(str, len, "%s%s",
13001                              str != buf ? "," : "", output_type_str[i]);
13002                 if (r >= len)
13003                         break;
13004                 str += r;
13005                 len -= r;
13006
13007                 output_types &= ~BIT(i);
13008         }
13009
13010         WARN_ON_ONCE(output_types != 0);
13011 }
13012
13013 static const char * const output_format_str[] = {
13014         [INTEL_OUTPUT_FORMAT_INVALID] = "Invalid",
13015         [INTEL_OUTPUT_FORMAT_RGB] = "RGB",
13016         [INTEL_OUTPUT_FORMAT_YCBCR420] = "YCBCR4:2:0",
13017         [INTEL_OUTPUT_FORMAT_YCBCR444] = "YCBCR4:4:4",
13018 };
13019
13020 static const char *output_formats(enum intel_output_format format)
13021 {
13022         if (format >= ARRAY_SIZE(output_format_str))
13023                 format = INTEL_OUTPUT_FORMAT_INVALID;
13024         return output_format_str[format];
13025 }
13026
13027 static void intel_dump_plane_state(const struct intel_plane_state *plane_state)
13028 {
13029         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
13030         struct drm_i915_private *i915 = to_i915(plane->base.dev);
13031         const struct drm_framebuffer *fb = plane_state->hw.fb;
13032         struct drm_format_name_buf format_name;
13033
13034         if (!fb) {
13035                 drm_dbg_kms(&i915->drm,
13036                             "[PLANE:%d:%s] fb: [NOFB], visible: %s\n",
13037                             plane->base.base.id, plane->base.name,
13038                             yesno(plane_state->uapi.visible));
13039                 return;
13040         }
13041
13042         drm_dbg_kms(&i915->drm,
13043                     "[PLANE:%d:%s] fb: [FB:%d] %ux%u format = %s, visible: %s\n",
13044                     plane->base.base.id, plane->base.name,
13045                     fb->base.id, fb->width, fb->height,
13046                     drm_get_format_name(fb->format->format, &format_name),
13047                     yesno(plane_state->uapi.visible));
13048         drm_dbg_kms(&i915->drm, "\trotation: 0x%x, scaler: %d\n",
13049                     plane_state->hw.rotation, plane_state->scaler_id);
13050         if (plane_state->uapi.visible)
13051                 drm_dbg_kms(&i915->drm,
13052                             "\tsrc: " DRM_RECT_FP_FMT " dst: " DRM_RECT_FMT "\n",
13053                             DRM_RECT_FP_ARG(&plane_state->uapi.src),
13054                             DRM_RECT_ARG(&plane_state->uapi.dst));
13055 }
13056
13057 static void intel_dump_pipe_config(const struct intel_crtc_state *pipe_config,
13058                                    struct intel_atomic_state *state,
13059                                    const char *context)
13060 {
13061         struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
13062         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13063         const struct intel_plane_state *plane_state;
13064         struct intel_plane *plane;
13065         char buf[64];
13066         int i;
13067
13068         drm_dbg_kms(&dev_priv->drm, "[CRTC:%d:%s] enable: %s %s\n",
13069                     crtc->base.base.id, crtc->base.name,
13070                     yesno(pipe_config->hw.enable), context);
13071
13072         if (!pipe_config->hw.enable)
13073                 goto dump_planes;
13074
13075         snprintf_output_types(buf, sizeof(buf), pipe_config->output_types);
13076         drm_dbg_kms(&dev_priv->drm,
13077                     "active: %s, output_types: %s (0x%x), output format: %s\n",
13078                     yesno(pipe_config->hw.active),
13079                     buf, pipe_config->output_types,
13080                     output_formats(pipe_config->output_format));
13081
13082         drm_dbg_kms(&dev_priv->drm,
13083                     "cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
13084                     transcoder_name(pipe_config->cpu_transcoder),
13085                     pipe_config->pipe_bpp, pipe_config->dither);
13086
13087         drm_dbg_kms(&dev_priv->drm,
13088                     "port sync: master transcoder: %s, slave transcoder bitmask = 0x%x\n",
13089                     transcoder_name(pipe_config->master_transcoder),
13090                     pipe_config->sync_mode_slaves_mask);
13091
13092         if (pipe_config->has_pch_encoder)
13093                 intel_dump_m_n_config(pipe_config, "fdi",
13094                                       pipe_config->fdi_lanes,
13095                                       &pipe_config->fdi_m_n);
13096
13097         if (intel_crtc_has_dp_encoder(pipe_config)) {
13098                 intel_dump_m_n_config(pipe_config, "dp m_n",
13099                                 pipe_config->lane_count, &pipe_config->dp_m_n);
13100                 if (pipe_config->has_drrs)
13101                         intel_dump_m_n_config(pipe_config, "dp m2_n2",
13102                                               pipe_config->lane_count,
13103                                               &pipe_config->dp_m2_n2);
13104         }
13105
13106         drm_dbg_kms(&dev_priv->drm,
13107                     "audio: %i, infoframes: %i, infoframes enabled: 0x%x\n",
13108                     pipe_config->has_audio, pipe_config->has_infoframe,
13109                     pipe_config->infoframes.enable);
13110
13111         if (pipe_config->infoframes.enable &
13112             intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GENERAL_CONTROL))
13113                 drm_dbg_kms(&dev_priv->drm, "GCP: 0x%x\n",
13114                             pipe_config->infoframes.gcp);
13115         if (pipe_config->infoframes.enable &
13116             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_AVI))
13117                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.avi);
13118         if (pipe_config->infoframes.enable &
13119             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_SPD))
13120                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.spd);
13121         if (pipe_config->infoframes.enable &
13122             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_VENDOR))
13123                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.hdmi);
13124         if (pipe_config->infoframes.enable &
13125             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_DRM))
13126                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.drm);
13127         if (pipe_config->infoframes.enable &
13128             intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GAMUT_METADATA))
13129                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.drm);
13130         if (pipe_config->infoframes.enable &
13131             intel_hdmi_infoframe_enable(DP_SDP_VSC))
13132                 intel_dump_dp_vsc_sdp(dev_priv, &pipe_config->infoframes.vsc);
13133
13134         drm_dbg_kms(&dev_priv->drm, "requested mode:\n");
13135         drm_mode_debug_printmodeline(&pipe_config->hw.mode);
13136         drm_dbg_kms(&dev_priv->drm, "adjusted mode:\n");
13137         drm_mode_debug_printmodeline(&pipe_config->hw.adjusted_mode);
13138         intel_dump_crtc_timings(dev_priv, &pipe_config->hw.adjusted_mode);
13139         drm_dbg_kms(&dev_priv->drm,
13140                     "port clock: %d, pipe src size: %dx%d, pixel rate %d\n",
13141                     pipe_config->port_clock,
13142                     pipe_config->pipe_src_w, pipe_config->pipe_src_h,
13143                     pipe_config->pixel_rate);
13144
13145         drm_dbg_kms(&dev_priv->drm, "linetime: %d, ips linetime: %d\n",
13146                     pipe_config->linetime, pipe_config->ips_linetime);
13147
13148         if (INTEL_GEN(dev_priv) >= 9)
13149                 drm_dbg_kms(&dev_priv->drm,
13150                             "num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
13151                             crtc->num_scalers,
13152                             pipe_config->scaler_state.scaler_users,
13153                             pipe_config->scaler_state.scaler_id);
13154
13155         if (HAS_GMCH(dev_priv))
13156                 drm_dbg_kms(&dev_priv->drm,
13157                             "gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
13158                             pipe_config->gmch_pfit.control,
13159                             pipe_config->gmch_pfit.pgm_ratios,
13160                             pipe_config->gmch_pfit.lvds_border_bits);
13161         else
13162                 drm_dbg_kms(&dev_priv->drm,
13163                             "pch pfit: " DRM_RECT_FMT ", %s, force thru: %s\n",
13164                             DRM_RECT_ARG(&pipe_config->pch_pfit.dst),
13165                             enableddisabled(pipe_config->pch_pfit.enabled),
13166                             yesno(pipe_config->pch_pfit.force_thru));
13167
13168         drm_dbg_kms(&dev_priv->drm, "ips: %i, double wide: %i\n",
13169                     pipe_config->ips_enabled, pipe_config->double_wide);
13170
13171         intel_dpll_dump_hw_state(dev_priv, &pipe_config->dpll_hw_state);
13172
13173         if (IS_CHERRYVIEW(dev_priv))
13174                 drm_dbg_kms(&dev_priv->drm,
13175                             "cgm_mode: 0x%x gamma_mode: 0x%x gamma_enable: %d csc_enable: %d\n",
13176                             pipe_config->cgm_mode, pipe_config->gamma_mode,
13177                             pipe_config->gamma_enable, pipe_config->csc_enable);
13178         else
13179                 drm_dbg_kms(&dev_priv->drm,
13180                             "csc_mode: 0x%x gamma_mode: 0x%x gamma_enable: %d csc_enable: %d\n",
13181                             pipe_config->csc_mode, pipe_config->gamma_mode,
13182                             pipe_config->gamma_enable, pipe_config->csc_enable);
13183
13184         drm_dbg_kms(&dev_priv->drm, "MST master transcoder: %s\n",
13185                     transcoder_name(pipe_config->mst_master_transcoder));
13186
13187 dump_planes:
13188         if (!state)
13189                 return;
13190
13191         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
13192                 if (plane->pipe == crtc->pipe)
13193                         intel_dump_plane_state(plane_state);
13194         }
13195 }
13196
13197 static bool check_digital_port_conflicts(struct intel_atomic_state *state)
13198 {
13199         struct drm_device *dev = state->base.dev;
13200         struct drm_connector *connector;
13201         struct drm_connector_list_iter conn_iter;
13202         unsigned int used_ports = 0;
13203         unsigned int used_mst_ports = 0;
13204         bool ret = true;
13205
13206         /*
13207          * We're going to peek into connector->state,
13208          * hence connection_mutex must be held.
13209          */
13210         drm_modeset_lock_assert_held(&dev->mode_config.connection_mutex);
13211
13212         /*
13213          * Walk the connector list instead of the encoder
13214          * list to detect the problem on ddi platforms
13215          * where there's just one encoder per digital port.
13216          */
13217         drm_connector_list_iter_begin(dev, &conn_iter);
13218         drm_for_each_connector_iter(connector, &conn_iter) {
13219                 struct drm_connector_state *connector_state;
13220                 struct intel_encoder *encoder;
13221
13222                 connector_state =
13223                         drm_atomic_get_new_connector_state(&state->base,
13224                                                            connector);
13225                 if (!connector_state)
13226                         connector_state = connector->state;
13227
13228                 if (!connector_state->best_encoder)
13229                         continue;
13230
13231                 encoder = to_intel_encoder(connector_state->best_encoder);
13232
13233                 drm_WARN_ON(dev, !connector_state->crtc);
13234
13235                 switch (encoder->type) {
13236                 case INTEL_OUTPUT_DDI:
13237                         if (drm_WARN_ON(dev, !HAS_DDI(to_i915(dev))))
13238                                 break;
13239                         fallthrough;
13240                 case INTEL_OUTPUT_DP:
13241                 case INTEL_OUTPUT_HDMI:
13242                 case INTEL_OUTPUT_EDP:
13243                         /* the same port mustn't appear more than once */
13244                         if (used_ports & BIT(encoder->port))
13245                                 ret = false;
13246
13247                         used_ports |= BIT(encoder->port);
13248                         break;
13249                 case INTEL_OUTPUT_DP_MST:
13250                         used_mst_ports |=
13251                                 1 << encoder->port;
13252                         break;
13253                 default:
13254                         break;
13255                 }
13256         }
13257         drm_connector_list_iter_end(&conn_iter);
13258
13259         /* can't mix MST and SST/HDMI on the same port */
13260         if (used_ports & used_mst_ports)
13261                 return false;
13262
13263         return ret;
13264 }
13265
13266 static void
13267 intel_crtc_copy_uapi_to_hw_state_nomodeset(struct intel_crtc_state *crtc_state)
13268 {
13269         intel_crtc_copy_color_blobs(crtc_state);
13270 }
13271
13272 static void
13273 intel_crtc_copy_uapi_to_hw_state(struct intel_crtc_state *crtc_state)
13274 {
13275         crtc_state->hw.enable = crtc_state->uapi.enable;
13276         crtc_state->hw.active = crtc_state->uapi.active;
13277         crtc_state->hw.mode = crtc_state->uapi.mode;
13278         crtc_state->hw.adjusted_mode = crtc_state->uapi.adjusted_mode;
13279         intel_crtc_copy_uapi_to_hw_state_nomodeset(crtc_state);
13280 }
13281
13282 static void intel_crtc_copy_hw_to_uapi_state(struct intel_crtc_state *crtc_state)
13283 {
13284         crtc_state->uapi.enable = crtc_state->hw.enable;
13285         crtc_state->uapi.active = crtc_state->hw.active;
13286         drm_WARN_ON(crtc_state->uapi.crtc->dev,
13287                     drm_atomic_set_mode_for_crtc(&crtc_state->uapi, &crtc_state->hw.mode) < 0);
13288
13289         crtc_state->uapi.adjusted_mode = crtc_state->hw.adjusted_mode;
13290
13291         /* copy color blobs to uapi */
13292         drm_property_replace_blob(&crtc_state->uapi.degamma_lut,
13293                                   crtc_state->hw.degamma_lut);
13294         drm_property_replace_blob(&crtc_state->uapi.gamma_lut,
13295                                   crtc_state->hw.gamma_lut);
13296         drm_property_replace_blob(&crtc_state->uapi.ctm,
13297                                   crtc_state->hw.ctm);
13298 }
13299
13300 static int
13301 intel_crtc_prepare_cleared_state(struct intel_crtc_state *crtc_state)
13302 {
13303         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
13304         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13305         struct intel_crtc_state *saved_state;
13306
13307         saved_state = intel_crtc_state_alloc(crtc);
13308         if (!saved_state)
13309                 return -ENOMEM;
13310
13311         /* free the old crtc_state->hw members */
13312         intel_crtc_free_hw_state(crtc_state);
13313
13314         /* FIXME: before the switch to atomic started, a new pipe_config was
13315          * kzalloc'd. Code that depends on any field being zero should be
13316          * fixed, so that the crtc_state can be safely duplicated. For now,
13317          * only fields that are know to not cause problems are preserved. */
13318
13319         saved_state->uapi = crtc_state->uapi;
13320         saved_state->scaler_state = crtc_state->scaler_state;
13321         saved_state->shared_dpll = crtc_state->shared_dpll;
13322         saved_state->dpll_hw_state = crtc_state->dpll_hw_state;
13323         memcpy(saved_state->icl_port_dplls, crtc_state->icl_port_dplls,
13324                sizeof(saved_state->icl_port_dplls));
13325         saved_state->crc_enabled = crtc_state->crc_enabled;
13326         if (IS_G4X(dev_priv) ||
13327             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
13328                 saved_state->wm = crtc_state->wm;
13329
13330         memcpy(crtc_state, saved_state, sizeof(*crtc_state));
13331         kfree(saved_state);
13332
13333         intel_crtc_copy_uapi_to_hw_state(crtc_state);
13334
13335         return 0;
13336 }
13337
13338 static int
13339 intel_modeset_pipe_config(struct intel_crtc_state *pipe_config)
13340 {
13341         struct drm_crtc *crtc = pipe_config->uapi.crtc;
13342         struct drm_atomic_state *state = pipe_config->uapi.state;
13343         struct drm_i915_private *i915 = to_i915(pipe_config->uapi.crtc->dev);
13344         struct drm_connector *connector;
13345         struct drm_connector_state *connector_state;
13346         int base_bpp, ret, i;
13347         bool retry = true;
13348
13349         pipe_config->cpu_transcoder =
13350                 (enum transcoder) to_intel_crtc(crtc)->pipe;
13351
13352         /*
13353          * Sanitize sync polarity flags based on requested ones. If neither
13354          * positive or negative polarity is requested, treat this as meaning
13355          * negative polarity.
13356          */
13357         if (!(pipe_config->hw.adjusted_mode.flags &
13358               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
13359                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
13360
13361         if (!(pipe_config->hw.adjusted_mode.flags &
13362               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
13363                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
13364
13365         ret = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
13366                                         pipe_config);
13367         if (ret)
13368                 return ret;
13369
13370         base_bpp = pipe_config->pipe_bpp;
13371
13372         /*
13373          * Determine the real pipe dimensions. Note that stereo modes can
13374          * increase the actual pipe size due to the frame doubling and
13375          * insertion of additional space for blanks between the frame. This
13376          * is stored in the crtc timings. We use the requested mode to do this
13377          * computation to clearly distinguish it from the adjusted mode, which
13378          * can be changed by the connectors in the below retry loop.
13379          */
13380         drm_mode_get_hv_timing(&pipe_config->hw.mode,
13381                                &pipe_config->pipe_src_w,
13382                                &pipe_config->pipe_src_h);
13383
13384         for_each_new_connector_in_state(state, connector, connector_state, i) {
13385                 struct intel_encoder *encoder =
13386                         to_intel_encoder(connector_state->best_encoder);
13387
13388                 if (connector_state->crtc != crtc)
13389                         continue;
13390
13391                 if (!check_single_encoder_cloning(state, to_intel_crtc(crtc), encoder)) {
13392                         drm_dbg_kms(&i915->drm,
13393                                     "rejecting invalid cloning configuration\n");
13394                         return -EINVAL;
13395                 }
13396
13397                 /*
13398                  * Determine output_types before calling the .compute_config()
13399                  * hooks so that the hooks can use this information safely.
13400                  */
13401                 if (encoder->compute_output_type)
13402                         pipe_config->output_types |=
13403                                 BIT(encoder->compute_output_type(encoder, pipe_config,
13404                                                                  connector_state));
13405                 else
13406                         pipe_config->output_types |= BIT(encoder->type);
13407         }
13408
13409 encoder_retry:
13410         /* Ensure the port clock defaults are reset when retrying. */
13411         pipe_config->port_clock = 0;
13412         pipe_config->pixel_multiplier = 1;
13413
13414         /* Fill in default crtc timings, allow encoders to overwrite them. */
13415         drm_mode_set_crtcinfo(&pipe_config->hw.adjusted_mode,
13416                               CRTC_STEREO_DOUBLE);
13417
13418         /* Pass our mode to the connectors and the CRTC to give them a chance to
13419          * adjust it according to limitations or connector properties, and also
13420          * a chance to reject the mode entirely.
13421          */
13422         for_each_new_connector_in_state(state, connector, connector_state, i) {
13423                 struct intel_encoder *encoder =
13424                         to_intel_encoder(connector_state->best_encoder);
13425
13426                 if (connector_state->crtc != crtc)
13427                         continue;
13428
13429                 ret = encoder->compute_config(encoder, pipe_config,
13430                                               connector_state);
13431                 if (ret < 0) {
13432                         if (ret != -EDEADLK)
13433                                 drm_dbg_kms(&i915->drm,
13434                                             "Encoder config failure: %d\n",
13435                                             ret);
13436                         return ret;
13437                 }
13438         }
13439
13440         /* Set default port clock if not overwritten by the encoder. Needs to be
13441          * done afterwards in case the encoder adjusts the mode. */
13442         if (!pipe_config->port_clock)
13443                 pipe_config->port_clock = pipe_config->hw.adjusted_mode.crtc_clock
13444                         * pipe_config->pixel_multiplier;
13445
13446         ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
13447         if (ret == -EDEADLK)
13448                 return ret;
13449         if (ret < 0) {
13450                 drm_dbg_kms(&i915->drm, "CRTC fixup failed\n");
13451                 return ret;
13452         }
13453
13454         if (ret == RETRY) {
13455                 if (drm_WARN(&i915->drm, !retry,
13456                              "loop in pipe configuration computation\n"))
13457                         return -EINVAL;
13458
13459                 drm_dbg_kms(&i915->drm, "CRTC bw constrained, retrying\n");
13460                 retry = false;
13461                 goto encoder_retry;
13462         }
13463
13464         /* Dithering seems to not pass-through bits correctly when it should, so
13465          * only enable it on 6bpc panels and when its not a compliance
13466          * test requesting 6bpc video pattern.
13467          */
13468         pipe_config->dither = (pipe_config->pipe_bpp == 6*3) &&
13469                 !pipe_config->dither_force_disable;
13470         drm_dbg_kms(&i915->drm,
13471                     "hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
13472                     base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
13473
13474         return 0;
13475 }
13476
13477 static int
13478 intel_modeset_pipe_config_late(struct intel_crtc_state *crtc_state)
13479 {
13480         struct intel_atomic_state *state =
13481                 to_intel_atomic_state(crtc_state->uapi.state);
13482         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
13483         struct drm_connector_state *conn_state;
13484         struct drm_connector *connector;
13485         int i;
13486
13487         for_each_new_connector_in_state(&state->base, connector,
13488                                         conn_state, i) {
13489                 struct intel_encoder *encoder =
13490                         to_intel_encoder(conn_state->best_encoder);
13491                 int ret;
13492
13493                 if (conn_state->crtc != &crtc->base ||
13494                     !encoder->compute_config_late)
13495                         continue;
13496
13497                 ret = encoder->compute_config_late(encoder, crtc_state,
13498                                                    conn_state);
13499                 if (ret)
13500                         return ret;
13501         }
13502
13503         return 0;
13504 }
13505
13506 bool intel_fuzzy_clock_check(int clock1, int clock2)
13507 {
13508         int diff;
13509
13510         if (clock1 == clock2)
13511                 return true;
13512
13513         if (!clock1 || !clock2)
13514                 return false;
13515
13516         diff = abs(clock1 - clock2);
13517
13518         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
13519                 return true;
13520
13521         return false;
13522 }
13523
13524 static bool
13525 intel_compare_m_n(unsigned int m, unsigned int n,
13526                   unsigned int m2, unsigned int n2,
13527                   bool exact)
13528 {
13529         if (m == m2 && n == n2)
13530                 return true;
13531
13532         if (exact || !m || !n || !m2 || !n2)
13533                 return false;
13534
13535         BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
13536
13537         if (n > n2) {
13538                 while (n > n2) {
13539                         m2 <<= 1;
13540                         n2 <<= 1;
13541                 }
13542         } else if (n < n2) {
13543                 while (n < n2) {
13544                         m <<= 1;
13545                         n <<= 1;
13546                 }
13547         }
13548
13549         if (n != n2)
13550                 return false;
13551
13552         return intel_fuzzy_clock_check(m, m2);
13553 }
13554
13555 static bool
13556 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
13557                        const struct intel_link_m_n *m2_n2,
13558                        bool exact)
13559 {
13560         return m_n->tu == m2_n2->tu &&
13561                 intel_compare_m_n(m_n->gmch_m, m_n->gmch_n,
13562                                   m2_n2->gmch_m, m2_n2->gmch_n, exact) &&
13563                 intel_compare_m_n(m_n->link_m, m_n->link_n,
13564                                   m2_n2->link_m, m2_n2->link_n, exact);
13565 }
13566
13567 static bool
13568 intel_compare_infoframe(const union hdmi_infoframe *a,
13569                         const union hdmi_infoframe *b)
13570 {
13571         return memcmp(a, b, sizeof(*a)) == 0;
13572 }
13573
13574 static bool
13575 intel_compare_dp_vsc_sdp(const struct drm_dp_vsc_sdp *a,
13576                          const struct drm_dp_vsc_sdp *b)
13577 {
13578         return memcmp(a, b, sizeof(*a)) == 0;
13579 }
13580
13581 static void
13582 pipe_config_infoframe_mismatch(struct drm_i915_private *dev_priv,
13583                                bool fastset, const char *name,
13584                                const union hdmi_infoframe *a,
13585                                const union hdmi_infoframe *b)
13586 {
13587         if (fastset) {
13588                 if (!drm_debug_enabled(DRM_UT_KMS))
13589                         return;
13590
13591                 drm_dbg_kms(&dev_priv->drm,
13592                             "fastset mismatch in %s infoframe\n", name);
13593                 drm_dbg_kms(&dev_priv->drm, "expected:\n");
13594                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, a);
13595                 drm_dbg_kms(&dev_priv->drm, "found:\n");
13596                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, b);
13597         } else {
13598                 drm_err(&dev_priv->drm, "mismatch in %s infoframe\n", name);
13599                 drm_err(&dev_priv->drm, "expected:\n");
13600                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, a);
13601                 drm_err(&dev_priv->drm, "found:\n");
13602                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, b);
13603         }
13604 }
13605
13606 static void
13607 pipe_config_dp_vsc_sdp_mismatch(struct drm_i915_private *dev_priv,
13608                                 bool fastset, const char *name,
13609                                 const struct drm_dp_vsc_sdp *a,
13610                                 const struct drm_dp_vsc_sdp *b)
13611 {
13612         if (fastset) {
13613                 if (!drm_debug_enabled(DRM_UT_KMS))
13614                         return;
13615
13616                 drm_dbg_kms(&dev_priv->drm,
13617                             "fastset mismatch in %s dp sdp\n", name);
13618                 drm_dbg_kms(&dev_priv->drm, "expected:\n");
13619                 drm_dp_vsc_sdp_log(KERN_DEBUG, dev_priv->drm.dev, a);
13620                 drm_dbg_kms(&dev_priv->drm, "found:\n");
13621                 drm_dp_vsc_sdp_log(KERN_DEBUG, dev_priv->drm.dev, b);
13622         } else {
13623                 drm_err(&dev_priv->drm, "mismatch in %s dp sdp\n", name);
13624                 drm_err(&dev_priv->drm, "expected:\n");
13625                 drm_dp_vsc_sdp_log(KERN_ERR, dev_priv->drm.dev, a);
13626                 drm_err(&dev_priv->drm, "found:\n");
13627                 drm_dp_vsc_sdp_log(KERN_ERR, dev_priv->drm.dev, b);
13628         }
13629 }
13630
13631 static void __printf(4, 5)
13632 pipe_config_mismatch(bool fastset, const struct intel_crtc *crtc,
13633                      const char *name, const char *format, ...)
13634 {
13635         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
13636         struct va_format vaf;
13637         va_list args;
13638
13639         va_start(args, format);
13640         vaf.fmt = format;
13641         vaf.va = &args;
13642
13643         if (fastset)
13644                 drm_dbg_kms(&i915->drm,
13645                             "[CRTC:%d:%s] fastset mismatch in %s %pV\n",
13646                             crtc->base.base.id, crtc->base.name, name, &vaf);
13647         else
13648                 drm_err(&i915->drm, "[CRTC:%d:%s] mismatch in %s %pV\n",
13649                         crtc->base.base.id, crtc->base.name, name, &vaf);
13650
13651         va_end(args);
13652 }
13653
13654 static bool fastboot_enabled(struct drm_i915_private *dev_priv)
13655 {
13656         if (dev_priv->params.fastboot != -1)
13657                 return dev_priv->params.fastboot;
13658
13659         /* Enable fastboot by default on Skylake and newer */
13660         if (INTEL_GEN(dev_priv) >= 9)
13661                 return true;
13662
13663         /* Enable fastboot by default on VLV and CHV */
13664         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
13665                 return true;
13666
13667         /* Disabled by default on all others */
13668         return false;
13669 }
13670
13671 static bool
13672 intel_pipe_config_compare(const struct intel_crtc_state *current_config,
13673                           const struct intel_crtc_state *pipe_config,
13674                           bool fastset)
13675 {
13676         struct drm_i915_private *dev_priv = to_i915(current_config->uapi.crtc->dev);
13677         struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
13678         bool ret = true;
13679         u32 bp_gamma = 0;
13680         bool fixup_inherited = fastset &&
13681                 current_config->inherited && !pipe_config->inherited;
13682
13683         if (fixup_inherited && !fastboot_enabled(dev_priv)) {
13684                 drm_dbg_kms(&dev_priv->drm,
13685                             "initial modeset and fastboot not set\n");
13686                 ret = false;
13687         }
13688
13689 #define PIPE_CONF_CHECK_X(name) do { \
13690         if (current_config->name != pipe_config->name) { \
13691                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13692                                      "(expected 0x%08x, found 0x%08x)", \
13693                                      current_config->name, \
13694                                      pipe_config->name); \
13695                 ret = false; \
13696         } \
13697 } while (0)
13698
13699 #define PIPE_CONF_CHECK_I(name) do { \
13700         if (current_config->name != pipe_config->name) { \
13701                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13702                                      "(expected %i, found %i)", \
13703                                      current_config->name, \
13704                                      pipe_config->name); \
13705                 ret = false; \
13706         } \
13707 } while (0)
13708
13709 #define PIPE_CONF_CHECK_BOOL(name) do { \
13710         if (current_config->name != pipe_config->name) { \
13711                 pipe_config_mismatch(fastset, crtc,  __stringify(name), \
13712                                      "(expected %s, found %s)", \
13713                                      yesno(current_config->name), \
13714                                      yesno(pipe_config->name)); \
13715                 ret = false; \
13716         } \
13717 } while (0)
13718
13719 /*
13720  * Checks state where we only read out the enabling, but not the entire
13721  * state itself (like full infoframes or ELD for audio). These states
13722  * require a full modeset on bootup to fix up.
13723  */
13724 #define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) do { \
13725         if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \
13726                 PIPE_CONF_CHECK_BOOL(name); \
13727         } else { \
13728                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13729                                      "unable to verify whether state matches exactly, forcing modeset (expected %s, found %s)", \
13730                                      yesno(current_config->name), \
13731                                      yesno(pipe_config->name)); \
13732                 ret = false; \
13733         } \
13734 } while (0)
13735
13736 #define PIPE_CONF_CHECK_P(name) do { \
13737         if (current_config->name != pipe_config->name) { \
13738                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13739                                      "(expected %p, found %p)", \
13740                                      current_config->name, \
13741                                      pipe_config->name); \
13742                 ret = false; \
13743         } \
13744 } while (0)
13745
13746 #define PIPE_CONF_CHECK_M_N(name) do { \
13747         if (!intel_compare_link_m_n(&current_config->name, \
13748                                     &pipe_config->name,\
13749                                     !fastset)) { \
13750                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13751                                      "(expected tu %i gmch %i/%i link %i/%i, " \
13752                                      "found tu %i, gmch %i/%i link %i/%i)", \
13753                                      current_config->name.tu, \
13754                                      current_config->name.gmch_m, \
13755                                      current_config->name.gmch_n, \
13756                                      current_config->name.link_m, \
13757                                      current_config->name.link_n, \
13758                                      pipe_config->name.tu, \
13759                                      pipe_config->name.gmch_m, \
13760                                      pipe_config->name.gmch_n, \
13761                                      pipe_config->name.link_m, \
13762                                      pipe_config->name.link_n); \
13763                 ret = false; \
13764         } \
13765 } while (0)
13766
13767 /* This is required for BDW+ where there is only one set of registers for
13768  * switching between high and low RR.
13769  * This macro can be used whenever a comparison has to be made between one
13770  * hw state and multiple sw state variables.
13771  */
13772 #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) do { \
13773         if (!intel_compare_link_m_n(&current_config->name, \
13774                                     &pipe_config->name, !fastset) && \
13775             !intel_compare_link_m_n(&current_config->alt_name, \
13776                                     &pipe_config->name, !fastset)) { \
13777                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13778                                      "(expected tu %i gmch %i/%i link %i/%i, " \
13779                                      "or tu %i gmch %i/%i link %i/%i, " \
13780                                      "found tu %i, gmch %i/%i link %i/%i)", \
13781                                      current_config->name.tu, \
13782                                      current_config->name.gmch_m, \
13783                                      current_config->name.gmch_n, \
13784                                      current_config->name.link_m, \
13785                                      current_config->name.link_n, \
13786                                      current_config->alt_name.tu, \
13787                                      current_config->alt_name.gmch_m, \
13788                                      current_config->alt_name.gmch_n, \
13789                                      current_config->alt_name.link_m, \
13790                                      current_config->alt_name.link_n, \
13791                                      pipe_config->name.tu, \
13792                                      pipe_config->name.gmch_m, \
13793                                      pipe_config->name.gmch_n, \
13794                                      pipe_config->name.link_m, \
13795                                      pipe_config->name.link_n); \
13796                 ret = false; \
13797         } \
13798 } while (0)
13799
13800 #define PIPE_CONF_CHECK_FLAGS(name, mask) do { \
13801         if ((current_config->name ^ pipe_config->name) & (mask)) { \
13802                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13803                                      "(%x) (expected %i, found %i)", \
13804                                      (mask), \
13805                                      current_config->name & (mask), \
13806                                      pipe_config->name & (mask)); \
13807                 ret = false; \
13808         } \
13809 } while (0)
13810
13811 #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) do { \
13812         if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
13813                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13814                                      "(expected %i, found %i)", \
13815                                      current_config->name, \
13816                                      pipe_config->name); \
13817                 ret = false; \
13818         } \
13819 } while (0)
13820
13821 #define PIPE_CONF_CHECK_INFOFRAME(name) do { \
13822         if (!intel_compare_infoframe(&current_config->infoframes.name, \
13823                                      &pipe_config->infoframes.name)) { \
13824                 pipe_config_infoframe_mismatch(dev_priv, fastset, __stringify(name), \
13825                                                &current_config->infoframes.name, \
13826                                                &pipe_config->infoframes.name); \
13827                 ret = false; \
13828         } \
13829 } while (0)
13830
13831 #define PIPE_CONF_CHECK_DP_VSC_SDP(name) do { \
13832         if (!current_config->has_psr && !pipe_config->has_psr && \
13833             !intel_compare_dp_vsc_sdp(&current_config->infoframes.name, \
13834                                       &pipe_config->infoframes.name)) { \
13835                 pipe_config_dp_vsc_sdp_mismatch(dev_priv, fastset, __stringify(name), \
13836                                                 &current_config->infoframes.name, \
13837                                                 &pipe_config->infoframes.name); \
13838                 ret = false; \
13839         } \
13840 } while (0)
13841
13842 #define PIPE_CONF_CHECK_COLOR_LUT(name1, name2, bit_precision) do { \
13843         if (current_config->name1 != pipe_config->name1) { \
13844                 pipe_config_mismatch(fastset, crtc, __stringify(name1), \
13845                                 "(expected %i, found %i, won't compare lut values)", \
13846                                 current_config->name1, \
13847                                 pipe_config->name1); \
13848                 ret = false;\
13849         } else { \
13850                 if (!intel_color_lut_equal(current_config->name2, \
13851                                         pipe_config->name2, pipe_config->name1, \
13852                                         bit_precision)) { \
13853                         pipe_config_mismatch(fastset, crtc, __stringify(name2), \
13854                                         "hw_state doesn't match sw_state"); \
13855                         ret = false; \
13856                 } \
13857         } \
13858 } while (0)
13859
13860 #define PIPE_CONF_QUIRK(quirk) \
13861         ((current_config->quirks | pipe_config->quirks) & (quirk))
13862
13863         PIPE_CONF_CHECK_I(cpu_transcoder);
13864
13865         PIPE_CONF_CHECK_BOOL(has_pch_encoder);
13866         PIPE_CONF_CHECK_I(fdi_lanes);
13867         PIPE_CONF_CHECK_M_N(fdi_m_n);
13868
13869         PIPE_CONF_CHECK_I(lane_count);
13870         PIPE_CONF_CHECK_X(lane_lat_optim_mask);
13871
13872         if (INTEL_GEN(dev_priv) < 8) {
13873                 PIPE_CONF_CHECK_M_N(dp_m_n);
13874
13875                 if (current_config->has_drrs)
13876                         PIPE_CONF_CHECK_M_N(dp_m2_n2);
13877         } else
13878                 PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
13879
13880         PIPE_CONF_CHECK_X(output_types);
13881
13882         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hdisplay);
13883         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_htotal);
13884         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hblank_start);
13885         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hblank_end);
13886         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hsync_start);
13887         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hsync_end);
13888
13889         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vdisplay);
13890         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vtotal);
13891         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vblank_start);
13892         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vblank_end);
13893         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vsync_start);
13894         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vsync_end);
13895
13896         PIPE_CONF_CHECK_I(pixel_multiplier);
13897         PIPE_CONF_CHECK_I(output_format);
13898         PIPE_CONF_CHECK_BOOL(has_hdmi_sink);
13899         if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
13900             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
13901                 PIPE_CONF_CHECK_BOOL(limited_color_range);
13902
13903         PIPE_CONF_CHECK_BOOL(hdmi_scrambling);
13904         PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio);
13905         PIPE_CONF_CHECK_BOOL(has_infoframe);
13906         PIPE_CONF_CHECK_BOOL(fec_enable);
13907
13908         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio);
13909
13910         PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13911                               DRM_MODE_FLAG_INTERLACE);
13912
13913         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
13914                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13915                                       DRM_MODE_FLAG_PHSYNC);
13916                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13917                                       DRM_MODE_FLAG_NHSYNC);
13918                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13919                                       DRM_MODE_FLAG_PVSYNC);
13920                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13921                                       DRM_MODE_FLAG_NVSYNC);
13922         }
13923
13924         PIPE_CONF_CHECK_X(gmch_pfit.control);
13925         /* pfit ratios are autocomputed by the hw on gen4+ */
13926         if (INTEL_GEN(dev_priv) < 4)
13927                 PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios);
13928         PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
13929
13930         /*
13931          * Changing the EDP transcoder input mux
13932          * (A_ONOFF vs. A_ON) requires a full modeset.
13933          */
13934         PIPE_CONF_CHECK_BOOL(pch_pfit.force_thru);
13935
13936         if (!fastset) {
13937                 PIPE_CONF_CHECK_I(pipe_src_w);
13938                 PIPE_CONF_CHECK_I(pipe_src_h);
13939
13940                 PIPE_CONF_CHECK_BOOL(pch_pfit.enabled);
13941                 if (current_config->pch_pfit.enabled) {
13942                         PIPE_CONF_CHECK_I(pch_pfit.dst.x1);
13943                         PIPE_CONF_CHECK_I(pch_pfit.dst.y1);
13944                         PIPE_CONF_CHECK_I(pch_pfit.dst.x2);
13945                         PIPE_CONF_CHECK_I(pch_pfit.dst.y2);
13946                 }
13947
13948                 PIPE_CONF_CHECK_I(scaler_state.scaler_id);
13949                 PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate);
13950
13951                 PIPE_CONF_CHECK_X(gamma_mode);
13952                 if (IS_CHERRYVIEW(dev_priv))
13953                         PIPE_CONF_CHECK_X(cgm_mode);
13954                 else
13955                         PIPE_CONF_CHECK_X(csc_mode);
13956                 PIPE_CONF_CHECK_BOOL(gamma_enable);
13957                 PIPE_CONF_CHECK_BOOL(csc_enable);
13958
13959                 PIPE_CONF_CHECK_I(linetime);
13960                 PIPE_CONF_CHECK_I(ips_linetime);
13961
13962                 bp_gamma = intel_color_get_gamma_bit_precision(pipe_config);
13963                 if (bp_gamma)
13964                         PIPE_CONF_CHECK_COLOR_LUT(gamma_mode, hw.gamma_lut, bp_gamma);
13965         }
13966
13967         PIPE_CONF_CHECK_BOOL(double_wide);
13968
13969         PIPE_CONF_CHECK_P(shared_dpll);
13970         PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
13971         PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
13972         PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
13973         PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
13974         PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
13975         PIPE_CONF_CHECK_X(dpll_hw_state.spll);
13976         PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
13977         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
13978         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
13979         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0);
13980         PIPE_CONF_CHECK_X(dpll_hw_state.ebb0);
13981         PIPE_CONF_CHECK_X(dpll_hw_state.ebb4);
13982         PIPE_CONF_CHECK_X(dpll_hw_state.pll0);
13983         PIPE_CONF_CHECK_X(dpll_hw_state.pll1);
13984         PIPE_CONF_CHECK_X(dpll_hw_state.pll2);
13985         PIPE_CONF_CHECK_X(dpll_hw_state.pll3);
13986         PIPE_CONF_CHECK_X(dpll_hw_state.pll6);
13987         PIPE_CONF_CHECK_X(dpll_hw_state.pll8);
13988         PIPE_CONF_CHECK_X(dpll_hw_state.pll9);
13989         PIPE_CONF_CHECK_X(dpll_hw_state.pll10);
13990         PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12);
13991         PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl);
13992         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1);
13993         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl);
13994         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0);
13995         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1);
13996         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf);
13997         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock);
13998         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc);
13999         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias);
14000         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias);
14001
14002         PIPE_CONF_CHECK_X(dsi_pll.ctrl);
14003         PIPE_CONF_CHECK_X(dsi_pll.div);
14004
14005         if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5)
14006                 PIPE_CONF_CHECK_I(pipe_bpp);
14007
14008         PIPE_CONF_CHECK_CLOCK_FUZZY(hw.adjusted_mode.crtc_clock);
14009         PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
14010
14011         PIPE_CONF_CHECK_I(min_voltage_level);
14012
14013         PIPE_CONF_CHECK_X(infoframes.enable);
14014         PIPE_CONF_CHECK_X(infoframes.gcp);
14015         PIPE_CONF_CHECK_INFOFRAME(avi);
14016         PIPE_CONF_CHECK_INFOFRAME(spd);
14017         PIPE_CONF_CHECK_INFOFRAME(hdmi);
14018         PIPE_CONF_CHECK_INFOFRAME(drm);
14019         PIPE_CONF_CHECK_DP_VSC_SDP(vsc);
14020
14021         PIPE_CONF_CHECK_X(sync_mode_slaves_mask);
14022         PIPE_CONF_CHECK_I(master_transcoder);
14023
14024         PIPE_CONF_CHECK_I(dsc.compression_enable);
14025         PIPE_CONF_CHECK_I(dsc.dsc_split);
14026         PIPE_CONF_CHECK_I(dsc.compressed_bpp);
14027
14028         PIPE_CONF_CHECK_I(mst_master_transcoder);
14029
14030 #undef PIPE_CONF_CHECK_X
14031 #undef PIPE_CONF_CHECK_I
14032 #undef PIPE_CONF_CHECK_BOOL
14033 #undef PIPE_CONF_CHECK_BOOL_INCOMPLETE
14034 #undef PIPE_CONF_CHECK_P
14035 #undef PIPE_CONF_CHECK_FLAGS
14036 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
14037 #undef PIPE_CONF_CHECK_COLOR_LUT
14038 #undef PIPE_CONF_QUIRK
14039
14040         return ret;
14041 }
14042
14043 static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv,
14044                                            const struct intel_crtc_state *pipe_config)
14045 {
14046         if (pipe_config->has_pch_encoder) {
14047                 int fdi_dotclock = intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
14048                                                             &pipe_config->fdi_m_n);
14049                 int dotclock = pipe_config->hw.adjusted_mode.crtc_clock;
14050
14051                 /*
14052                  * FDI already provided one idea for the dotclock.
14053                  * Yell if the encoder disagrees.
14054                  */
14055                 drm_WARN(&dev_priv->drm,
14056                          !intel_fuzzy_clock_check(fdi_dotclock, dotclock),
14057                          "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
14058                          fdi_dotclock, dotclock);
14059         }
14060 }
14061
14062 static void verify_wm_state(struct intel_crtc *crtc,
14063                             struct intel_crtc_state *new_crtc_state)
14064 {
14065         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14066         struct skl_hw_state {
14067                 struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
14068                 struct skl_ddb_entry ddb_uv[I915_MAX_PLANES];
14069                 struct skl_pipe_wm wm;
14070         } *hw;
14071         struct skl_pipe_wm *sw_wm;
14072         struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
14073         u8 hw_enabled_slices;
14074         const enum pipe pipe = crtc->pipe;
14075         int plane, level, max_level = ilk_wm_max_level(dev_priv);
14076
14077         if (INTEL_GEN(dev_priv) < 9 || !new_crtc_state->hw.active)
14078                 return;
14079
14080         hw = kzalloc(sizeof(*hw), GFP_KERNEL);
14081         if (!hw)
14082                 return;
14083
14084         skl_pipe_wm_get_hw_state(crtc, &hw->wm);
14085         sw_wm = &new_crtc_state->wm.skl.optimal;
14086
14087         skl_pipe_ddb_get_hw_state(crtc, hw->ddb_y, hw->ddb_uv);
14088
14089         hw_enabled_slices = intel_enabled_dbuf_slices_mask(dev_priv);
14090
14091         if (INTEL_GEN(dev_priv) >= 11 &&
14092             hw_enabled_slices != dev_priv->dbuf.enabled_slices)
14093                 drm_err(&dev_priv->drm,
14094                         "mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n",
14095                         dev_priv->dbuf.enabled_slices,
14096                         hw_enabled_slices);
14097
14098         /* planes */
14099         for_each_universal_plane(dev_priv, pipe, plane) {
14100                 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
14101
14102                 hw_plane_wm = &hw->wm.planes[plane];
14103                 sw_plane_wm = &sw_wm->planes[plane];
14104
14105                 /* Watermarks */
14106                 for (level = 0; level <= max_level; level++) {
14107                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
14108                                                 &sw_plane_wm->wm[level]) ||
14109                             (level == 0 && skl_wm_level_equals(&hw_plane_wm->wm[level],
14110                                                                &sw_plane_wm->sagv_wm0)))
14111                                 continue;
14112
14113                         drm_err(&dev_priv->drm,
14114                                 "mismatch in WM pipe %c plane %d level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
14115                                 pipe_name(pipe), plane + 1, level,
14116                                 sw_plane_wm->wm[level].plane_en,
14117                                 sw_plane_wm->wm[level].plane_res_b,
14118                                 sw_plane_wm->wm[level].plane_res_l,
14119                                 hw_plane_wm->wm[level].plane_en,
14120                                 hw_plane_wm->wm[level].plane_res_b,
14121                                 hw_plane_wm->wm[level].plane_res_l);
14122                 }
14123
14124                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
14125                                          &sw_plane_wm->trans_wm)) {
14126                         drm_err(&dev_priv->drm,
14127                                 "mismatch in trans WM pipe %c plane %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
14128                                 pipe_name(pipe), plane + 1,
14129                                 sw_plane_wm->trans_wm.plane_en,
14130                                 sw_plane_wm->trans_wm.plane_res_b,
14131                                 sw_plane_wm->trans_wm.plane_res_l,
14132                                 hw_plane_wm->trans_wm.plane_en,
14133                                 hw_plane_wm->trans_wm.plane_res_b,
14134                                 hw_plane_wm->trans_wm.plane_res_l);
14135                 }
14136
14137                 /* DDB */
14138                 hw_ddb_entry = &hw->ddb_y[plane];
14139                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[plane];
14140
14141                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
14142                         drm_err(&dev_priv->drm,
14143                                 "mismatch in DDB state pipe %c plane %d (expected (%u,%u), found (%u,%u))\n",
14144                                 pipe_name(pipe), plane + 1,
14145                                 sw_ddb_entry->start, sw_ddb_entry->end,
14146                                 hw_ddb_entry->start, hw_ddb_entry->end);
14147                 }
14148         }
14149
14150         /*
14151          * cursor
14152          * If the cursor plane isn't active, we may not have updated it's ddb
14153          * allocation. In that case since the ddb allocation will be updated
14154          * once the plane becomes visible, we can skip this check
14155          */
14156         if (1) {
14157                 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
14158
14159                 hw_plane_wm = &hw->wm.planes[PLANE_CURSOR];
14160                 sw_plane_wm = &sw_wm->planes[PLANE_CURSOR];
14161
14162                 /* Watermarks */
14163                 for (level = 0; level <= max_level; level++) {
14164                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
14165                                                 &sw_plane_wm->wm[level]) ||
14166                             (level == 0 && skl_wm_level_equals(&hw_plane_wm->wm[level],
14167                                                                &sw_plane_wm->sagv_wm0)))
14168                                 continue;
14169
14170                         drm_err(&dev_priv->drm,
14171                                 "mismatch in WM pipe %c cursor level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
14172                                 pipe_name(pipe), level,
14173                                 sw_plane_wm->wm[level].plane_en,
14174                                 sw_plane_wm->wm[level].plane_res_b,
14175                                 sw_plane_wm->wm[level].plane_res_l,
14176                                 hw_plane_wm->wm[level].plane_en,
14177                                 hw_plane_wm->wm[level].plane_res_b,
14178                                 hw_plane_wm->wm[level].plane_res_l);
14179                 }
14180
14181                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
14182                                          &sw_plane_wm->trans_wm)) {
14183                         drm_err(&dev_priv->drm,
14184                                 "mismatch in trans WM pipe %c cursor (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
14185                                 pipe_name(pipe),
14186                                 sw_plane_wm->trans_wm.plane_en,
14187                                 sw_plane_wm->trans_wm.plane_res_b,
14188                                 sw_plane_wm->trans_wm.plane_res_l,
14189                                 hw_plane_wm->trans_wm.plane_en,
14190                                 hw_plane_wm->trans_wm.plane_res_b,
14191                                 hw_plane_wm->trans_wm.plane_res_l);
14192                 }
14193
14194                 /* DDB */
14195                 hw_ddb_entry = &hw->ddb_y[PLANE_CURSOR];
14196                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR];
14197
14198                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
14199                         drm_err(&dev_priv->drm,
14200                                 "mismatch in DDB state pipe %c cursor (expected (%u,%u), found (%u,%u))\n",
14201                                 pipe_name(pipe),
14202                                 sw_ddb_entry->start, sw_ddb_entry->end,
14203                                 hw_ddb_entry->start, hw_ddb_entry->end);
14204                 }
14205         }
14206
14207         kfree(hw);
14208 }
14209
14210 static void
14211 verify_connector_state(struct intel_atomic_state *state,
14212                        struct intel_crtc *crtc)
14213 {
14214         struct drm_connector *connector;
14215         struct drm_connector_state *new_conn_state;
14216         int i;
14217
14218         for_each_new_connector_in_state(&state->base, connector, new_conn_state, i) {
14219                 struct drm_encoder *encoder = connector->encoder;
14220                 struct intel_crtc_state *crtc_state = NULL;
14221
14222                 if (new_conn_state->crtc != &crtc->base)
14223                         continue;
14224
14225                 if (crtc)
14226                         crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
14227
14228                 intel_connector_verify_state(crtc_state, new_conn_state);
14229
14230                 I915_STATE_WARN(new_conn_state->best_encoder != encoder,
14231                      "connector's atomic encoder doesn't match legacy encoder\n");
14232         }
14233 }
14234
14235 static void
14236 verify_encoder_state(struct drm_i915_private *dev_priv, struct intel_atomic_state *state)
14237 {
14238         struct intel_encoder *encoder;
14239         struct drm_connector *connector;
14240         struct drm_connector_state *old_conn_state, *new_conn_state;
14241         int i;
14242
14243         for_each_intel_encoder(&dev_priv->drm, encoder) {
14244                 bool enabled = false, found = false;
14245                 enum pipe pipe;
14246
14247                 drm_dbg_kms(&dev_priv->drm, "[ENCODER:%d:%s]\n",
14248                             encoder->base.base.id,
14249                             encoder->base.name);
14250
14251                 for_each_oldnew_connector_in_state(&state->base, connector, old_conn_state,
14252                                                    new_conn_state, i) {
14253                         if (old_conn_state->best_encoder == &encoder->base)
14254                                 found = true;
14255
14256                         if (new_conn_state->best_encoder != &encoder->base)
14257                                 continue;
14258                         found = enabled = true;
14259
14260                         I915_STATE_WARN(new_conn_state->crtc !=
14261                                         encoder->base.crtc,
14262                              "connector's crtc doesn't match encoder crtc\n");
14263                 }
14264
14265                 if (!found)
14266                         continue;
14267
14268                 I915_STATE_WARN(!!encoder->base.crtc != enabled,
14269                      "encoder's enabled state mismatch "
14270                      "(expected %i, found %i)\n",
14271                      !!encoder->base.crtc, enabled);
14272
14273                 if (!encoder->base.crtc) {
14274                         bool active;
14275
14276                         active = encoder->get_hw_state(encoder, &pipe);
14277                         I915_STATE_WARN(active,
14278                              "encoder detached but still enabled on pipe %c.\n",
14279                              pipe_name(pipe));
14280                 }
14281         }
14282 }
14283
14284 static void
14285 verify_crtc_state(struct intel_crtc *crtc,
14286                   struct intel_crtc_state *old_crtc_state,
14287                   struct intel_crtc_state *new_crtc_state)
14288 {
14289         struct drm_device *dev = crtc->base.dev;
14290         struct drm_i915_private *dev_priv = to_i915(dev);
14291         struct intel_encoder *encoder;
14292         struct intel_crtc_state *pipe_config = old_crtc_state;
14293         struct drm_atomic_state *state = old_crtc_state->uapi.state;
14294         bool active;
14295
14296         __drm_atomic_helper_crtc_destroy_state(&old_crtc_state->uapi);
14297         intel_crtc_free_hw_state(old_crtc_state);
14298         intel_crtc_state_reset(old_crtc_state, crtc);
14299         old_crtc_state->uapi.state = state;
14300
14301         drm_dbg_kms(&dev_priv->drm, "[CRTC:%d:%s]\n", crtc->base.base.id,
14302                     crtc->base.name);
14303
14304         active = dev_priv->display.get_pipe_config(crtc, pipe_config);
14305
14306         /* we keep both pipes enabled on 830 */
14307         if (IS_I830(dev_priv))
14308                 active = new_crtc_state->hw.active;
14309
14310         I915_STATE_WARN(new_crtc_state->hw.active != active,
14311                         "crtc active state doesn't match with hw state "
14312                         "(expected %i, found %i)\n",
14313                         new_crtc_state->hw.active, active);
14314
14315         I915_STATE_WARN(crtc->active != new_crtc_state->hw.active,
14316                         "transitional active state does not match atomic hw state "
14317                         "(expected %i, found %i)\n",
14318                         new_crtc_state->hw.active, crtc->active);
14319
14320         for_each_encoder_on_crtc(dev, &crtc->base, encoder) {
14321                 enum pipe pipe;
14322
14323                 active = encoder->get_hw_state(encoder, &pipe);
14324                 I915_STATE_WARN(active != new_crtc_state->hw.active,
14325                                 "[ENCODER:%i] active %i with crtc active %i\n",
14326                                 encoder->base.base.id, active,
14327                                 new_crtc_state->hw.active);
14328
14329                 I915_STATE_WARN(active && crtc->pipe != pipe,
14330                                 "Encoder connected to wrong pipe %c\n",
14331                                 pipe_name(pipe));
14332
14333                 if (active)
14334                         encoder->get_config(encoder, pipe_config);
14335         }
14336
14337         intel_crtc_compute_pixel_rate(pipe_config);
14338
14339         if (!new_crtc_state->hw.active)
14340                 return;
14341
14342         intel_pipe_config_sanity_check(dev_priv, pipe_config);
14343
14344         if (!intel_pipe_config_compare(new_crtc_state,
14345                                        pipe_config, false)) {
14346                 I915_STATE_WARN(1, "pipe state doesn't match!\n");
14347                 intel_dump_pipe_config(pipe_config, NULL, "[hw state]");
14348                 intel_dump_pipe_config(new_crtc_state, NULL, "[sw state]");
14349         }
14350 }
14351
14352 static void
14353 intel_verify_planes(struct intel_atomic_state *state)
14354 {
14355         struct intel_plane *plane;
14356         const struct intel_plane_state *plane_state;
14357         int i;
14358
14359         for_each_new_intel_plane_in_state(state, plane,
14360                                           plane_state, i)
14361                 assert_plane(plane, plane_state->planar_slave ||
14362                              plane_state->uapi.visible);
14363 }
14364
14365 static void
14366 verify_single_dpll_state(struct drm_i915_private *dev_priv,
14367                          struct intel_shared_dpll *pll,
14368                          struct intel_crtc *crtc,
14369                          struct intel_crtc_state *new_crtc_state)
14370 {
14371         struct intel_dpll_hw_state dpll_hw_state;
14372         unsigned int crtc_mask;
14373         bool active;
14374
14375         memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
14376
14377         drm_dbg_kms(&dev_priv->drm, "%s\n", pll->info->name);
14378
14379         active = pll->info->funcs->get_hw_state(dev_priv, pll, &dpll_hw_state);
14380
14381         if (!(pll->info->flags & INTEL_DPLL_ALWAYS_ON)) {
14382                 I915_STATE_WARN(!pll->on && pll->active_mask,
14383                      "pll in active use but not on in sw tracking\n");
14384                 I915_STATE_WARN(pll->on && !pll->active_mask,
14385                      "pll is on but not used by any active crtc\n");
14386                 I915_STATE_WARN(pll->on != active,
14387                      "pll on state mismatch (expected %i, found %i)\n",
14388                      pll->on, active);
14389         }
14390
14391         if (!crtc) {
14392                 I915_STATE_WARN(pll->active_mask & ~pll->state.crtc_mask,
14393                                 "more active pll users than references: %x vs %x\n",
14394                                 pll->active_mask, pll->state.crtc_mask);
14395
14396                 return;
14397         }
14398
14399         crtc_mask = drm_crtc_mask(&crtc->base);
14400
14401         if (new_crtc_state->hw.active)
14402                 I915_STATE_WARN(!(pll->active_mask & crtc_mask),
14403                                 "pll active mismatch (expected pipe %c in active mask 0x%02x)\n",
14404                                 pipe_name(crtc->pipe), pll->active_mask);
14405         else
14406                 I915_STATE_WARN(pll->active_mask & crtc_mask,
14407                                 "pll active mismatch (didn't expect pipe %c in active mask 0x%02x)\n",
14408                                 pipe_name(crtc->pipe), pll->active_mask);
14409
14410         I915_STATE_WARN(!(pll->state.crtc_mask & crtc_mask),
14411                         "pll enabled crtcs mismatch (expected 0x%x in 0x%02x)\n",
14412                         crtc_mask, pll->state.crtc_mask);
14413
14414         I915_STATE_WARN(pll->on && memcmp(&pll->state.hw_state,
14415                                           &dpll_hw_state,
14416                                           sizeof(dpll_hw_state)),
14417                         "pll hw state mismatch\n");
14418 }
14419
14420 static void
14421 verify_shared_dpll_state(struct intel_crtc *crtc,
14422                          struct intel_crtc_state *old_crtc_state,
14423                          struct intel_crtc_state *new_crtc_state)
14424 {
14425         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14426
14427         if (new_crtc_state->shared_dpll)
14428                 verify_single_dpll_state(dev_priv, new_crtc_state->shared_dpll, crtc, new_crtc_state);
14429
14430         if (old_crtc_state->shared_dpll &&
14431             old_crtc_state->shared_dpll != new_crtc_state->shared_dpll) {
14432                 unsigned int crtc_mask = drm_crtc_mask(&crtc->base);
14433                 struct intel_shared_dpll *pll = old_crtc_state->shared_dpll;
14434
14435                 I915_STATE_WARN(pll->active_mask & crtc_mask,
14436                                 "pll active mismatch (didn't expect pipe %c in active mask)\n",
14437                                 pipe_name(crtc->pipe));
14438                 I915_STATE_WARN(pll->state.crtc_mask & crtc_mask,
14439                                 "pll enabled crtcs mismatch (found %x in enabled mask)\n",
14440                                 pipe_name(crtc->pipe));
14441         }
14442 }
14443
14444 static void
14445 intel_modeset_verify_crtc(struct intel_crtc *crtc,
14446                           struct intel_atomic_state *state,
14447                           struct intel_crtc_state *old_crtc_state,
14448                           struct intel_crtc_state *new_crtc_state)
14449 {
14450         if (!needs_modeset(new_crtc_state) && !new_crtc_state->update_pipe)
14451                 return;
14452
14453         verify_wm_state(crtc, new_crtc_state);
14454         verify_connector_state(state, crtc);
14455         verify_crtc_state(crtc, old_crtc_state, new_crtc_state);
14456         verify_shared_dpll_state(crtc, old_crtc_state, new_crtc_state);
14457 }
14458
14459 static void
14460 verify_disabled_dpll_state(struct drm_i915_private *dev_priv)
14461 {
14462         int i;
14463
14464         for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++)
14465                 verify_single_dpll_state(dev_priv,
14466                                          &dev_priv->dpll.shared_dplls[i],
14467                                          NULL, NULL);
14468 }
14469
14470 static void
14471 intel_modeset_verify_disabled(struct drm_i915_private *dev_priv,
14472                               struct intel_atomic_state *state)
14473 {
14474         verify_encoder_state(dev_priv, state);
14475         verify_connector_state(state, NULL);
14476         verify_disabled_dpll_state(dev_priv);
14477 }
14478
14479 static void
14480 intel_crtc_update_active_timings(const struct intel_crtc_state *crtc_state)
14481 {
14482         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
14483         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14484         const struct drm_display_mode *adjusted_mode =
14485                 &crtc_state->hw.adjusted_mode;
14486
14487         drm_calc_timestamping_constants(&crtc->base, adjusted_mode);
14488
14489         crtc->mode_flags = crtc_state->mode_flags;
14490
14491         /*
14492          * The scanline counter increments at the leading edge of hsync.
14493          *
14494          * On most platforms it starts counting from vtotal-1 on the
14495          * first active line. That means the scanline counter value is
14496          * always one less than what we would expect. Ie. just after
14497          * start of vblank, which also occurs at start of hsync (on the
14498          * last active line), the scanline counter will read vblank_start-1.
14499          *
14500          * On gen2 the scanline counter starts counting from 1 instead
14501          * of vtotal-1, so we have to subtract one (or rather add vtotal-1
14502          * to keep the value positive), instead of adding one.
14503          *
14504          * On HSW+ the behaviour of the scanline counter depends on the output
14505          * type. For DP ports it behaves like most other platforms, but on HDMI
14506          * there's an extra 1 line difference. So we need to add two instead of
14507          * one to the value.
14508          *
14509          * On VLV/CHV DSI the scanline counter would appear to increment
14510          * approx. 1/3 of a scanline before start of vblank. Unfortunately
14511          * that means we can't tell whether we're in vblank or not while
14512          * we're on that particular line. We must still set scanline_offset
14513          * to 1 so that the vblank timestamps come out correct when we query
14514          * the scanline counter from within the vblank interrupt handler.
14515          * However if queried just before the start of vblank we'll get an
14516          * answer that's slightly in the future.
14517          */
14518         if (IS_GEN(dev_priv, 2)) {
14519                 int vtotal;
14520
14521                 vtotal = adjusted_mode->crtc_vtotal;
14522                 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
14523                         vtotal /= 2;
14524
14525                 crtc->scanline_offset = vtotal - 1;
14526         } else if (HAS_DDI(dev_priv) &&
14527                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
14528                 crtc->scanline_offset = 2;
14529         } else {
14530                 crtc->scanline_offset = 1;
14531         }
14532 }
14533
14534 static void intel_modeset_clear_plls(struct intel_atomic_state *state)
14535 {
14536         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14537         struct intel_crtc_state *new_crtc_state;
14538         struct intel_crtc *crtc;
14539         int i;
14540
14541         if (!dev_priv->display.crtc_compute_clock)
14542                 return;
14543
14544         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
14545                 if (!needs_modeset(new_crtc_state))
14546                         continue;
14547
14548                 intel_release_shared_dplls(state, crtc);
14549         }
14550 }
14551
14552 /*
14553  * This implements the workaround described in the "notes" section of the mode
14554  * set sequence documentation. When going from no pipes or single pipe to
14555  * multiple pipes, and planes are enabled after the pipe, we need to wait at
14556  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
14557  */
14558 static int hsw_mode_set_planes_workaround(struct intel_atomic_state *state)
14559 {
14560         struct intel_crtc_state *crtc_state;
14561         struct intel_crtc *crtc;
14562         struct intel_crtc_state *first_crtc_state = NULL;
14563         struct intel_crtc_state *other_crtc_state = NULL;
14564         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
14565         int i;
14566
14567         /* look at all crtc's that are going to be enabled in during modeset */
14568         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
14569                 if (!crtc_state->hw.active ||
14570                     !needs_modeset(crtc_state))
14571                         continue;
14572
14573                 if (first_crtc_state) {
14574                         other_crtc_state = crtc_state;
14575                         break;
14576                 } else {
14577                         first_crtc_state = crtc_state;
14578                         first_pipe = crtc->pipe;
14579                 }
14580         }
14581
14582         /* No workaround needed? */
14583         if (!first_crtc_state)
14584                 return 0;
14585
14586         /* w/a possibly needed, check how many crtc's are already enabled. */
14587         for_each_intel_crtc(state->base.dev, crtc) {
14588                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
14589                 if (IS_ERR(crtc_state))
14590                         return PTR_ERR(crtc_state);
14591
14592                 crtc_state->hsw_workaround_pipe = INVALID_PIPE;
14593
14594                 if (!crtc_state->hw.active ||
14595                     needs_modeset(crtc_state))
14596                         continue;
14597
14598                 /* 2 or more enabled crtcs means no need for w/a */
14599                 if (enabled_pipe != INVALID_PIPE)
14600                         return 0;
14601
14602                 enabled_pipe = crtc->pipe;
14603         }
14604
14605         if (enabled_pipe != INVALID_PIPE)
14606                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
14607         else if (other_crtc_state)
14608                 other_crtc_state->hsw_workaround_pipe = first_pipe;
14609
14610         return 0;
14611 }
14612
14613 u8 intel_calc_active_pipes(struct intel_atomic_state *state,
14614                            u8 active_pipes)
14615 {
14616         const struct intel_crtc_state *crtc_state;
14617         struct intel_crtc *crtc;
14618         int i;
14619
14620         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
14621                 if (crtc_state->hw.active)
14622                         active_pipes |= BIT(crtc->pipe);
14623                 else
14624                         active_pipes &= ~BIT(crtc->pipe);
14625         }
14626
14627         return active_pipes;
14628 }
14629
14630 static int intel_modeset_checks(struct intel_atomic_state *state)
14631 {
14632         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14633         int ret;
14634
14635         state->modeset = true;
14636         state->active_pipes = intel_calc_active_pipes(state, dev_priv->active_pipes);
14637
14638         if (state->active_pipes != dev_priv->active_pipes) {
14639                 ret = _intel_atomic_lock_global_state(state);
14640                 if (ret)
14641                         return ret;
14642         }
14643
14644         if (IS_HASWELL(dev_priv))
14645                 return hsw_mode_set_planes_workaround(state);
14646
14647         return 0;
14648 }
14649
14650 /*
14651  * Handle calculation of various watermark data at the end of the atomic check
14652  * phase.  The code here should be run after the per-crtc and per-plane 'check'
14653  * handlers to ensure that all derived state has been updated.
14654  */
14655 static int calc_watermark_data(struct intel_atomic_state *state)
14656 {
14657         struct drm_device *dev = state->base.dev;
14658         struct drm_i915_private *dev_priv = to_i915(dev);
14659
14660         /* Is there platform-specific watermark information to calculate? */
14661         if (dev_priv->display.compute_global_watermarks)
14662                 return dev_priv->display.compute_global_watermarks(state);
14663
14664         return 0;
14665 }
14666
14667 static void intel_crtc_check_fastset(const struct intel_crtc_state *old_crtc_state,
14668                                      struct intel_crtc_state *new_crtc_state)
14669 {
14670         if (!intel_pipe_config_compare(old_crtc_state, new_crtc_state, true))
14671                 return;
14672
14673         new_crtc_state->uapi.mode_changed = false;
14674         new_crtc_state->update_pipe = true;
14675 }
14676
14677 static void intel_crtc_copy_fastset(const struct intel_crtc_state *old_crtc_state,
14678                                     struct intel_crtc_state *new_crtc_state)
14679 {
14680         /*
14681          * If we're not doing the full modeset we want to
14682          * keep the current M/N values as they may be
14683          * sufficiently different to the computed values
14684          * to cause problems.
14685          *
14686          * FIXME: should really copy more fuzzy state here
14687          */
14688         new_crtc_state->fdi_m_n = old_crtc_state->fdi_m_n;
14689         new_crtc_state->dp_m_n = old_crtc_state->dp_m_n;
14690         new_crtc_state->dp_m2_n2 = old_crtc_state->dp_m2_n2;
14691         new_crtc_state->has_drrs = old_crtc_state->has_drrs;
14692 }
14693
14694 static int intel_crtc_add_planes_to_state(struct intel_atomic_state *state,
14695                                           struct intel_crtc *crtc,
14696                                           u8 plane_ids_mask)
14697 {
14698         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14699         struct intel_plane *plane;
14700
14701         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
14702                 struct intel_plane_state *plane_state;
14703
14704                 if ((plane_ids_mask & BIT(plane->id)) == 0)
14705                         continue;
14706
14707                 plane_state = intel_atomic_get_plane_state(state, plane);
14708                 if (IS_ERR(plane_state))
14709                         return PTR_ERR(plane_state);
14710         }
14711
14712         return 0;
14713 }
14714
14715 static bool active_planes_affects_min_cdclk(struct drm_i915_private *dev_priv)
14716 {
14717         /* See {hsw,vlv,ivb}_plane_ratio() */
14718         return IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv) ||
14719                 IS_CHERRYVIEW(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
14720                 IS_IVYBRIDGE(dev_priv) || (INTEL_GEN(dev_priv) >= 11);
14721 }
14722
14723 static int intel_atomic_check_planes(struct intel_atomic_state *state)
14724 {
14725         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14726         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14727         struct intel_plane_state *plane_state;
14728         struct intel_plane *plane;
14729         struct intel_crtc *crtc;
14730         int i, ret;
14731
14732         ret = icl_add_linked_planes(state);
14733         if (ret)
14734                 return ret;
14735
14736         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
14737                 ret = intel_plane_atomic_check(state, plane);
14738                 if (ret) {
14739                         drm_dbg_atomic(&dev_priv->drm,
14740                                        "[PLANE:%d:%s] atomic driver check failed\n",
14741                                        plane->base.base.id, plane->base.name);
14742                         return ret;
14743                 }
14744         }
14745
14746         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14747                                             new_crtc_state, i) {
14748                 u8 old_active_planes, new_active_planes;
14749
14750                 ret = icl_check_nv12_planes(new_crtc_state);
14751                 if (ret)
14752                         return ret;
14753
14754                 /*
14755                  * On some platforms the number of active planes affects
14756                  * the planes' minimum cdclk calculation. Add such planes
14757                  * to the state before we compute the minimum cdclk.
14758                  */
14759                 if (!active_planes_affects_min_cdclk(dev_priv))
14760                         continue;
14761
14762                 old_active_planes = old_crtc_state->active_planes & ~BIT(PLANE_CURSOR);
14763                 new_active_planes = new_crtc_state->active_planes & ~BIT(PLANE_CURSOR);
14764
14765                 /*
14766                  * Not only the number of planes, but if the plane configuration had
14767                  * changed might already mean we need to recompute min CDCLK,
14768                  * because different planes might consume different amount of Dbuf bandwidth
14769                  * according to formula: Bw per plane = Pixel rate * bpp * pipe/plane scale factor
14770                  */
14771                 if (old_active_planes == new_active_planes)
14772                         continue;
14773
14774                 ret = intel_crtc_add_planes_to_state(state, crtc, new_active_planes);
14775                 if (ret)
14776                         return ret;
14777         }
14778
14779         return 0;
14780 }
14781
14782 static int intel_atomic_check_cdclk(struct intel_atomic_state *state,
14783                                     bool *need_cdclk_calc)
14784 {
14785         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14786         struct intel_cdclk_state *new_cdclk_state;
14787         struct intel_plane_state *plane_state;
14788         struct intel_bw_state *new_bw_state;
14789         struct intel_plane *plane;
14790         int min_cdclk = 0;
14791         enum pipe pipe;
14792         int ret;
14793         int i;
14794         /*
14795          * active_planes bitmask has been updated, and potentially
14796          * affected planes are part of the state. We can now
14797          * compute the minimum cdclk for each plane.
14798          */
14799         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
14800                 ret = intel_plane_calc_min_cdclk(state, plane, need_cdclk_calc);
14801                 if (ret)
14802                         return ret;
14803         }
14804
14805         new_cdclk_state = intel_atomic_get_new_cdclk_state(state);
14806
14807         if (new_cdclk_state && new_cdclk_state->force_min_cdclk_changed)
14808                 *need_cdclk_calc = true;
14809
14810         ret = dev_priv->display.bw_calc_min_cdclk(state);
14811         if (ret)
14812                 return ret;
14813
14814         new_bw_state = intel_atomic_get_new_bw_state(state);
14815
14816         if (!new_cdclk_state || !new_bw_state)
14817                 return 0;
14818
14819         for_each_pipe(dev_priv, pipe) {
14820                 min_cdclk = max(new_cdclk_state->min_cdclk[pipe], min_cdclk);
14821
14822                 /*
14823                  * Currently do this change only if we need to increase
14824                  */
14825                 if (new_bw_state->min_cdclk > min_cdclk)
14826                         *need_cdclk_calc = true;
14827         }
14828
14829         return 0;
14830 }
14831
14832 static int intel_atomic_check_crtcs(struct intel_atomic_state *state)
14833 {
14834         struct intel_crtc_state *crtc_state;
14835         struct intel_crtc *crtc;
14836         int i;
14837
14838         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
14839                 int ret = intel_crtc_atomic_check(state, crtc);
14840                 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
14841                 if (ret) {
14842                         drm_dbg_atomic(&i915->drm,
14843                                        "[CRTC:%d:%s] atomic driver check failed\n",
14844                                        crtc->base.base.id, crtc->base.name);
14845                         return ret;
14846                 }
14847         }
14848
14849         return 0;
14850 }
14851
14852 static bool intel_cpu_transcoders_need_modeset(struct intel_atomic_state *state,
14853                                                u8 transcoders)
14854 {
14855         const struct intel_crtc_state *new_crtc_state;
14856         struct intel_crtc *crtc;
14857         int i;
14858
14859         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
14860                 if (new_crtc_state->hw.enable &&
14861                     transcoders & BIT(new_crtc_state->cpu_transcoder) &&
14862                     needs_modeset(new_crtc_state))
14863                         return true;
14864         }
14865
14866         return false;
14867 }
14868
14869 /**
14870  * intel_atomic_check - validate state object
14871  * @dev: drm device
14872  * @_state: state to validate
14873  */
14874 static int intel_atomic_check(struct drm_device *dev,
14875                               struct drm_atomic_state *_state)
14876 {
14877         struct drm_i915_private *dev_priv = to_i915(dev);
14878         struct intel_atomic_state *state = to_intel_atomic_state(_state);
14879         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14880         struct intel_crtc *crtc;
14881         int ret, i;
14882         bool any_ms = false;
14883
14884         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14885                                             new_crtc_state, i) {
14886                 if (new_crtc_state->inherited != old_crtc_state->inherited)
14887                         new_crtc_state->uapi.mode_changed = true;
14888         }
14889
14890         ret = drm_atomic_helper_check_modeset(dev, &state->base);
14891         if (ret)
14892                 goto fail;
14893
14894         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14895                                             new_crtc_state, i) {
14896                 if (!needs_modeset(new_crtc_state)) {
14897                         /* Light copy */
14898                         intel_crtc_copy_uapi_to_hw_state_nomodeset(new_crtc_state);
14899
14900                         continue;
14901                 }
14902
14903                 ret = intel_crtc_prepare_cleared_state(new_crtc_state);
14904                 if (ret)
14905                         goto fail;
14906
14907                 if (!new_crtc_state->hw.enable)
14908                         continue;
14909
14910                 ret = intel_modeset_pipe_config(new_crtc_state);
14911                 if (ret)
14912                         goto fail;
14913         }
14914
14915         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14916                                             new_crtc_state, i) {
14917                 if (!needs_modeset(new_crtc_state))
14918                         continue;
14919
14920                 ret = intel_modeset_pipe_config_late(new_crtc_state);
14921                 if (ret)
14922                         goto fail;
14923
14924                 intel_crtc_check_fastset(old_crtc_state, new_crtc_state);
14925         }
14926
14927         /**
14928          * Check if fastset is allowed by external dependencies like other
14929          * pipes and transcoders.
14930          *
14931          * Right now it only forces a fullmodeset when the MST master
14932          * transcoder did not changed but the pipe of the master transcoder
14933          * needs a fullmodeset so all slaves also needs to do a fullmodeset or
14934          * in case of port synced crtcs, if one of the synced crtcs
14935          * needs a full modeset, all other synced crtcs should be
14936          * forced a full modeset.
14937          */
14938         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
14939                 if (!new_crtc_state->hw.enable || needs_modeset(new_crtc_state))
14940                         continue;
14941
14942                 if (intel_dp_mst_is_slave_trans(new_crtc_state)) {
14943                         enum transcoder master = new_crtc_state->mst_master_transcoder;
14944
14945                         if (intel_cpu_transcoders_need_modeset(state, BIT(master))) {
14946                                 new_crtc_state->uapi.mode_changed = true;
14947                                 new_crtc_state->update_pipe = false;
14948                         }
14949                 }
14950
14951                 if (is_trans_port_sync_mode(new_crtc_state)) {
14952                         u8 trans = new_crtc_state->sync_mode_slaves_mask;
14953
14954                         if (new_crtc_state->master_transcoder != INVALID_TRANSCODER)
14955                                 trans |= BIT(new_crtc_state->master_transcoder);
14956
14957                         if (intel_cpu_transcoders_need_modeset(state, trans)) {
14958                                 new_crtc_state->uapi.mode_changed = true;
14959                                 new_crtc_state->update_pipe = false;
14960                         }
14961                 }
14962         }
14963
14964         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14965                                             new_crtc_state, i) {
14966                 if (needs_modeset(new_crtc_state)) {
14967                         any_ms = true;
14968                         continue;
14969                 }
14970
14971                 if (!new_crtc_state->update_pipe)
14972                         continue;
14973
14974                 intel_crtc_copy_fastset(old_crtc_state, new_crtc_state);
14975         }
14976
14977         if (any_ms && !check_digital_port_conflicts(state)) {
14978                 drm_dbg_kms(&dev_priv->drm,
14979                             "rejecting conflicting digital port configuration\n");
14980                 ret = -EINVAL;
14981                 goto fail;
14982         }
14983
14984         ret = drm_dp_mst_atomic_check(&state->base);
14985         if (ret)
14986                 goto fail;
14987
14988         ret = intel_atomic_check_planes(state);
14989         if (ret)
14990                 goto fail;
14991
14992         /*
14993          * distrust_bios_wm will force a full dbuf recomputation
14994          * but the hardware state will only get updated accordingly
14995          * if state->modeset==true. Hence distrust_bios_wm==true &&
14996          * state->modeset==false is an invalid combination which
14997          * would cause the hardware and software dbuf state to get
14998          * out of sync. We must prevent that.
14999          *
15000          * FIXME clean up this mess and introduce better
15001          * state tracking for dbuf.
15002          */
15003         if (dev_priv->wm.distrust_bios_wm)
15004                 any_ms = true;
15005
15006         intel_fbc_choose_crtc(dev_priv, state);
15007         ret = calc_watermark_data(state);
15008         if (ret)
15009                 goto fail;
15010
15011         ret = intel_bw_atomic_check(state);
15012         if (ret)
15013                 goto fail;
15014
15015         ret = intel_atomic_check_cdclk(state, &any_ms);
15016         if (ret)
15017                 goto fail;
15018
15019         if (any_ms) {
15020                 ret = intel_modeset_checks(state);
15021                 if (ret)
15022                         goto fail;
15023
15024                 ret = intel_modeset_calc_cdclk(state);
15025                 if (ret)
15026                         return ret;
15027
15028                 intel_modeset_clear_plls(state);
15029         }
15030
15031         ret = intel_atomic_check_crtcs(state);
15032         if (ret)
15033                 goto fail;
15034
15035         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
15036                                             new_crtc_state, i) {
15037                 if (!needs_modeset(new_crtc_state) &&
15038                     !new_crtc_state->update_pipe)
15039                         continue;
15040
15041                 intel_dump_pipe_config(new_crtc_state, state,
15042                                        needs_modeset(new_crtc_state) ?
15043                                        "[modeset]" : "[fastset]");
15044         }
15045
15046         return 0;
15047
15048  fail:
15049         if (ret == -EDEADLK)
15050                 return ret;
15051
15052         /*
15053          * FIXME would probably be nice to know which crtc specifically
15054          * caused the failure, in cases where we can pinpoint it.
15055          */
15056         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
15057                                             new_crtc_state, i)
15058                 intel_dump_pipe_config(new_crtc_state, state, "[failed]");
15059
15060         return ret;
15061 }
15062
15063 static int intel_atomic_prepare_commit(struct intel_atomic_state *state)
15064 {
15065         struct intel_crtc_state *crtc_state;
15066         struct intel_crtc *crtc;
15067         int i, ret;
15068
15069         ret = drm_atomic_helper_prepare_planes(state->base.dev, &state->base);
15070         if (ret < 0)
15071                 return ret;
15072
15073         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
15074                 bool mode_changed = needs_modeset(crtc_state);
15075
15076                 if (mode_changed || crtc_state->update_pipe ||
15077                     crtc_state->uapi.color_mgmt_changed) {
15078                         intel_dsb_prepare(crtc_state);
15079                 }
15080         }
15081
15082         return 0;
15083 }
15084
15085 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
15086 {
15087         struct drm_device *dev = crtc->base.dev;
15088         struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)];
15089
15090         if (!vblank->max_vblank_count)
15091                 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
15092
15093         return crtc->base.funcs->get_vblank_counter(&crtc->base);
15094 }
15095
15096 void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
15097                                   struct intel_crtc_state *crtc_state)
15098 {
15099         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
15100
15101         if (!IS_GEN(dev_priv, 2) || crtc_state->active_planes)
15102                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
15103
15104         if (crtc_state->has_pch_encoder) {
15105                 enum pipe pch_transcoder =
15106                         intel_crtc_pch_transcoder(crtc);
15107
15108                 intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
15109         }
15110 }
15111
15112 static void intel_pipe_fastset(const struct intel_crtc_state *old_crtc_state,
15113                                const struct intel_crtc_state *new_crtc_state)
15114 {
15115         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
15116         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
15117
15118         /*
15119          * Update pipe size and adjust fitter if needed: the reason for this is
15120          * that in compute_mode_changes we check the native mode (not the pfit
15121          * mode) to see if we can flip rather than do a full mode set. In the
15122          * fastboot case, we'll flip, but if we don't update the pipesrc and
15123          * pfit state, we'll end up with a big fb scanned out into the wrong
15124          * sized surface.
15125          */
15126         intel_set_pipe_src_size(new_crtc_state);
15127
15128         /* on skylake this is done by detaching scalers */
15129         if (INTEL_GEN(dev_priv) >= 9) {
15130                 skl_detach_scalers(new_crtc_state);
15131
15132                 if (new_crtc_state->pch_pfit.enabled)
15133                         skl_pfit_enable(new_crtc_state);
15134         } else if (HAS_PCH_SPLIT(dev_priv)) {
15135                 if (new_crtc_state->pch_pfit.enabled)
15136                         ilk_pfit_enable(new_crtc_state);
15137                 else if (old_crtc_state->pch_pfit.enabled)
15138                         ilk_pfit_disable(old_crtc_state);
15139         }
15140
15141         /*
15142          * The register is supposedly single buffered so perhaps
15143          * not 100% correct to do this here. But SKL+ calculate
15144          * this based on the adjust pixel rate so pfit changes do
15145          * affect it and so it must be updated for fastsets.
15146          * HSW/BDW only really need this here for fastboot, after
15147          * that the value should not change without a full modeset.
15148          */
15149         if (INTEL_GEN(dev_priv) >= 9 ||
15150             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
15151                 hsw_set_linetime_wm(new_crtc_state);
15152
15153         if (INTEL_GEN(dev_priv) >= 11)
15154                 icl_set_pipe_chicken(crtc);
15155 }
15156
15157 static void commit_pipe_config(struct intel_atomic_state *state,
15158                                struct intel_crtc *crtc)
15159 {
15160         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
15161         const struct intel_crtc_state *old_crtc_state =
15162                 intel_atomic_get_old_crtc_state(state, crtc);
15163         const struct intel_crtc_state *new_crtc_state =
15164                 intel_atomic_get_new_crtc_state(state, crtc);
15165         bool modeset = needs_modeset(new_crtc_state);
15166
15167         /*
15168          * During modesets pipe configuration was programmed as the
15169          * CRTC was enabled.
15170          */
15171         if (!modeset) {
15172                 if (new_crtc_state->uapi.color_mgmt_changed ||
15173                     new_crtc_state->update_pipe)
15174                         intel_color_commit(new_crtc_state);
15175
15176                 if (INTEL_GEN(dev_priv) >= 9)
15177                         skl_detach_scalers(new_crtc_state);
15178
15179                 if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
15180                         bdw_set_pipemisc(new_crtc_state);
15181
15182                 if (new_crtc_state->update_pipe)
15183                         intel_pipe_fastset(old_crtc_state, new_crtc_state);
15184
15185                 intel_psr2_program_trans_man_trk_ctl(new_crtc_state);
15186         }
15187
15188         if (dev_priv->display.atomic_update_watermarks)
15189                 dev_priv->display.atomic_update_watermarks(state, crtc);
15190 }
15191
15192 static void intel_enable_crtc(struct intel_atomic_state *state,
15193                               struct intel_crtc *crtc)
15194 {
15195         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
15196         const struct intel_crtc_state *new_crtc_state =
15197                 intel_atomic_get_new_crtc_state(state, crtc);
15198
15199         if (!needs_modeset(new_crtc_state))
15200                 return;
15201
15202         intel_crtc_update_active_timings(new_crtc_state);
15203
15204         dev_priv->display.crtc_enable(state, crtc);
15205
15206         /* vblanks work again, re-enable pipe CRC. */
15207         intel_crtc_enable_pipe_crc(crtc);
15208 }
15209
15210 static void intel_update_crtc(struct intel_atomic_state *state,
15211                               struct intel_crtc *crtc)
15212 {
15213         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
15214         const struct intel_crtc_state *old_crtc_state =
15215                 intel_atomic_get_old_crtc_state(state, crtc);
15216         struct intel_crtc_state *new_crtc_state =
15217                 intel_atomic_get_new_crtc_state(state, crtc);
15218         bool modeset = needs_modeset(new_crtc_state);
15219
15220         if (!modeset) {
15221                 if (new_crtc_state->preload_luts &&
15222                     (new_crtc_state->uapi.color_mgmt_changed ||
15223                      new_crtc_state->update_pipe))
15224                         intel_color_load_luts(new_crtc_state);
15225
15226                 intel_pre_plane_update(state, crtc);
15227
15228                 if (new_crtc_state->update_pipe)
15229                         intel_encoders_update_pipe(state, crtc);
15230         }
15231
15232         if (new_crtc_state->update_pipe && !new_crtc_state->enable_fbc)
15233                 intel_fbc_disable(crtc);
15234         else
15235                 intel_fbc_enable(state, crtc);
15236
15237         /* Perform vblank evasion around commit operation */
15238         intel_pipe_update_start(new_crtc_state);
15239
15240         commit_pipe_config(state, crtc);
15241
15242         if (INTEL_GEN(dev_priv) >= 9)
15243                 skl_update_planes_on_crtc(state, crtc);
15244         else
15245                 i9xx_update_planes_on_crtc(state, crtc);
15246
15247         intel_pipe_update_end(new_crtc_state);
15248
15249         /*
15250          * We usually enable FIFO underrun interrupts as part of the
15251          * CRTC enable sequence during modesets.  But when we inherit a
15252          * valid pipe configuration from the BIOS we need to take care
15253          * of enabling them on the CRTC's first fastset.
15254          */
15255         if (new_crtc_state->update_pipe && !modeset &&
15256             old_crtc_state->inherited)
15257                 intel_crtc_arm_fifo_underrun(crtc, new_crtc_state);
15258 }
15259
15260
15261 static void intel_old_crtc_state_disables(struct intel_atomic_state *state,
15262                                           struct intel_crtc_state *old_crtc_state,
15263                                           struct intel_crtc_state *new_crtc_state,
15264                                           struct intel_crtc *crtc)
15265 {
15266         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
15267
15268         intel_crtc_disable_planes(state, crtc);
15269
15270         /*
15271          * We need to disable pipe CRC before disabling the pipe,
15272          * or we race against vblank off.
15273          */
15274         intel_crtc_disable_pipe_crc(crtc);
15275
15276         dev_priv->display.crtc_disable(state, crtc);
15277         crtc->active = false;
15278         intel_fbc_disable(crtc);
15279         intel_disable_shared_dpll(old_crtc_state);
15280
15281         /* FIXME unify this for all platforms */
15282         if (!new_crtc_state->hw.active &&
15283             !HAS_GMCH(dev_priv) &&
15284             dev_priv->display.initial_watermarks)
15285                 dev_priv->display.initial_watermarks(state, crtc);
15286 }
15287
15288 static void intel_commit_modeset_disables(struct intel_atomic_state *state)
15289 {
15290         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
15291         struct intel_crtc *crtc;
15292         u32 handled = 0;
15293         int i;
15294
15295         /* Only disable port sync and MST slaves */
15296         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
15297                                             new_crtc_state, i) {
15298                 if (!needs_modeset(new_crtc_state))
15299                         continue;
15300
15301                 if (!old_crtc_state->hw.active)
15302                         continue;
15303
15304                 /* In case of Transcoder port Sync master slave CRTCs can be
15305                  * assigned in any order and we need to make sure that
15306                  * slave CRTCs are disabled first and then master CRTC since
15307                  * Slave vblanks are masked till Master Vblanks.
15308                  */
15309                 if (!is_trans_port_sync_slave(old_crtc_state) &&
15310                     !intel_dp_mst_is_slave_trans(old_crtc_state))
15311                         continue;
15312
15313                 intel_pre_plane_update(state, crtc);
15314                 intel_old_crtc_state_disables(state, old_crtc_state,
15315                                               new_crtc_state, crtc);
15316                 handled |= BIT(crtc->pipe);
15317         }
15318
15319         /* Disable everything else left on */
15320         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
15321                                             new_crtc_state, i) {
15322                 if (!needs_modeset(new_crtc_state) ||
15323                     (handled & BIT(crtc->pipe)))
15324                         continue;
15325
15326                 intel_pre_plane_update(state, crtc);
15327                 if (old_crtc_state->hw.active)
15328                         intel_old_crtc_state_disables(state, old_crtc_state,
15329                                                       new_crtc_state, crtc);
15330         }
15331 }
15332
15333 static void intel_commit_modeset_enables(struct intel_atomic_state *state)
15334 {
15335         struct intel_crtc_state *new_crtc_state;
15336         struct intel_crtc *crtc;
15337         int i;
15338
15339         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
15340                 if (!new_crtc_state->hw.active)
15341                         continue;
15342
15343                 intel_enable_crtc(state, crtc);
15344                 intel_update_crtc(state, crtc);
15345         }
15346 }
15347
15348 static void skl_commit_modeset_enables(struct intel_atomic_state *state)
15349 {
15350         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
15351         struct intel_crtc *crtc;
15352         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
15353         struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
15354         u8 update_pipes = 0, modeset_pipes = 0;
15355         int i;
15356
15357         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
15358                 enum pipe pipe = crtc->pipe;
15359
15360                 if (!new_crtc_state->hw.active)
15361                         continue;
15362
15363                 /* ignore allocations for crtc's that have been turned off. */
15364                 if (!needs_modeset(new_crtc_state)) {
15365                         entries[pipe] = old_crtc_state->wm.skl.ddb;
15366                         update_pipes |= BIT(pipe);
15367                 } else {
15368                         modeset_pipes |= BIT(pipe);
15369                 }
15370         }
15371
15372         /*
15373          * Whenever the number of active pipes changes, we need to make sure we
15374          * update the pipes in the right order so that their ddb allocations
15375          * never overlap with each other between CRTC updates. Otherwise we'll
15376          * cause pipe underruns and other bad stuff.
15377          *
15378          * So first lets enable all pipes that do not need a fullmodeset as
15379          * those don't have any external dependency.
15380          */
15381         while (update_pipes) {
15382                 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
15383                                                     new_crtc_state, i) {
15384                         enum pipe pipe = crtc->pipe;
15385
15386                         if ((update_pipes & BIT(pipe)) == 0)
15387                                 continue;
15388
15389                         if (skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
15390                                                         entries, I915_MAX_PIPES, pipe))
15391                                 continue;
15392
15393                         entries[pipe] = new_crtc_state->wm.skl.ddb;
15394                         update_pipes &= ~BIT(pipe);
15395
15396                         intel_update_crtc(state, crtc);
15397
15398                         /*
15399                          * If this is an already active pipe, it's DDB changed,
15400                          * and this isn't the last pipe that needs updating
15401                          * then we need to wait for a vblank to pass for the
15402                          * new ddb allocation to take effect.
15403                          */
15404                         if (!skl_ddb_entry_equal(&new_crtc_state->wm.skl.ddb,
15405                                                  &old_crtc_state->wm.skl.ddb) &&
15406                             (update_pipes | modeset_pipes))
15407                                 intel_wait_for_vblank(dev_priv, pipe);
15408                 }
15409         }
15410
15411         update_pipes = modeset_pipes;
15412
15413         /*
15414          * Enable all pipes that needs a modeset and do not depends on other
15415          * pipes
15416          */
15417         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
15418                 enum pipe pipe = crtc->pipe;
15419
15420                 if ((modeset_pipes & BIT(pipe)) == 0)
15421                         continue;
15422
15423                 if (intel_dp_mst_is_slave_trans(new_crtc_state) ||
15424                     is_trans_port_sync_master(new_crtc_state))
15425                         continue;
15426
15427                 modeset_pipes &= ~BIT(pipe);
15428
15429                 intel_enable_crtc(state, crtc);
15430         }
15431
15432         /*
15433          * Then we enable all remaining pipes that depend on other
15434          * pipes: MST slaves and port sync masters.
15435          */
15436         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
15437                 enum pipe pipe = crtc->pipe;
15438
15439                 if ((modeset_pipes & BIT(pipe)) == 0)
15440                         continue;
15441
15442                 modeset_pipes &= ~BIT(pipe);
15443
15444                 intel_enable_crtc(state, crtc);
15445         }
15446
15447         /*
15448          * Finally we do the plane updates/etc. for all pipes that got enabled.
15449          */
15450         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
15451                 enum pipe pipe = crtc->pipe;
15452
15453                 if ((update_pipes & BIT(pipe)) == 0)
15454                         continue;
15455
15456                 drm_WARN_ON(&dev_priv->drm, skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
15457                                                                         entries, I915_MAX_PIPES, pipe));
15458
15459                 entries[pipe] = new_crtc_state->wm.skl.ddb;
15460                 update_pipes &= ~BIT(pipe);
15461
15462                 intel_update_crtc(state, crtc);
15463         }
15464
15465         drm_WARN_ON(&dev_priv->drm, modeset_pipes);
15466         drm_WARN_ON(&dev_priv->drm, update_pipes);
15467 }
15468
15469 static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv)
15470 {
15471         struct intel_atomic_state *state, *next;
15472         struct llist_node *freed;
15473
15474         freed = llist_del_all(&dev_priv->atomic_helper.free_list);
15475         llist_for_each_entry_safe(state, next, freed, freed)
15476                 drm_atomic_state_put(&state->base);
15477 }
15478
15479 static void intel_atomic_helper_free_state_worker(struct work_struct *work)
15480 {
15481         struct drm_i915_private *dev_priv =
15482                 container_of(work, typeof(*dev_priv), atomic_helper.free_work);
15483
15484         intel_atomic_helper_free_state(dev_priv);
15485 }
15486
15487 static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state)
15488 {
15489         struct wait_queue_entry wait_fence, wait_reset;
15490         struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev);
15491
15492         init_wait_entry(&wait_fence, 0);
15493         init_wait_entry(&wait_reset, 0);
15494         for (;;) {
15495                 prepare_to_wait(&intel_state->commit_ready.wait,
15496                                 &wait_fence, TASK_UNINTERRUPTIBLE);
15497                 prepare_to_wait(bit_waitqueue(&dev_priv->gt.reset.flags,
15498                                               I915_RESET_MODESET),
15499                                 &wait_reset, TASK_UNINTERRUPTIBLE);
15500
15501
15502                 if (i915_sw_fence_done(&intel_state->commit_ready) ||
15503                     test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags))
15504                         break;
15505
15506                 schedule();
15507         }
15508         finish_wait(&intel_state->commit_ready.wait, &wait_fence);
15509         finish_wait(bit_waitqueue(&dev_priv->gt.reset.flags,
15510                                   I915_RESET_MODESET),
15511                     &wait_reset);
15512 }
15513
15514 static void intel_cleanup_dsbs(struct intel_atomic_state *state)
15515 {
15516         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
15517         struct intel_crtc *crtc;
15518         int i;
15519
15520         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
15521                                             new_crtc_state, i)
15522                 intel_dsb_cleanup(old_crtc_state);
15523 }
15524
15525 static void intel_atomic_cleanup_work(struct work_struct *work)
15526 {
15527         struct intel_atomic_state *state =
15528                 container_of(work, struct intel_atomic_state, base.commit_work);
15529         struct drm_i915_private *i915 = to_i915(state->base.dev);
15530
15531         intel_cleanup_dsbs(state);
15532         drm_atomic_helper_cleanup_planes(&i915->drm, &state->base);
15533         drm_atomic_helper_commit_cleanup_done(&state->base);
15534         drm_atomic_state_put(&state->base);
15535
15536         intel_atomic_helper_free_state(i915);
15537 }
15538
15539 static void intel_atomic_commit_tail(struct intel_atomic_state *state)
15540 {
15541         struct drm_device *dev = state->base.dev;
15542         struct drm_i915_private *dev_priv = to_i915(dev);
15543         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
15544         struct intel_crtc *crtc;
15545         u64 put_domains[I915_MAX_PIPES] = {};
15546         intel_wakeref_t wakeref = 0;
15547         int i;
15548
15549         intel_atomic_commit_fence_wait(state);
15550
15551         drm_atomic_helper_wait_for_dependencies(&state->base);
15552
15553         if (state->modeset)
15554                 wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
15555
15556         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
15557                                             new_crtc_state, i) {
15558                 if (needs_modeset(new_crtc_state) ||
15559                     new_crtc_state->update_pipe) {
15560
15561                         put_domains[crtc->pipe] =
15562                                 modeset_get_crtc_power_domains(new_crtc_state);
15563                 }
15564         }
15565
15566         intel_commit_modeset_disables(state);
15567
15568         /* FIXME: Eventually get rid of our crtc->config pointer */
15569         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
15570                 crtc->config = new_crtc_state;
15571
15572         if (state->modeset) {
15573                 drm_atomic_helper_update_legacy_modeset_state(dev, &state->base);
15574
15575                 intel_set_cdclk_pre_plane_update(state);
15576
15577                 intel_modeset_verify_disabled(dev_priv, state);
15578         }
15579
15580         intel_sagv_pre_plane_update(state);
15581
15582         /* Complete the events for pipes that have now been disabled */
15583         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
15584                 bool modeset = needs_modeset(new_crtc_state);
15585
15586                 /* Complete events for now disable pipes here. */
15587                 if (modeset && !new_crtc_state->hw.active && new_crtc_state->uapi.event) {
15588                         spin_lock_irq(&dev->event_lock);
15589                         drm_crtc_send_vblank_event(&crtc->base,
15590                                                    new_crtc_state->uapi.event);
15591                         spin_unlock_irq(&dev->event_lock);
15592
15593                         new_crtc_state->uapi.event = NULL;
15594                 }
15595         }
15596
15597         if (state->modeset)
15598                 intel_encoders_update_prepare(state);
15599
15600         intel_dbuf_pre_plane_update(state);
15601
15602         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
15603         dev_priv->display.commit_modeset_enables(state);
15604
15605         if (state->modeset) {
15606                 intel_encoders_update_complete(state);
15607
15608                 intel_set_cdclk_post_plane_update(state);
15609         }
15610
15611         /* FIXME: We should call drm_atomic_helper_commit_hw_done() here
15612          * already, but still need the state for the delayed optimization. To
15613          * fix this:
15614          * - wrap the optimization/post_plane_update stuff into a per-crtc work.
15615          * - schedule that vblank worker _before_ calling hw_done
15616          * - at the start of commit_tail, cancel it _synchrously
15617          * - switch over to the vblank wait helper in the core after that since
15618          *   we don't need out special handling any more.
15619          */
15620         drm_atomic_helper_wait_for_flip_done(dev, &state->base);
15621
15622         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
15623                 if (new_crtc_state->hw.active &&
15624                     !needs_modeset(new_crtc_state) &&
15625                     !new_crtc_state->preload_luts &&
15626                     (new_crtc_state->uapi.color_mgmt_changed ||
15627                      new_crtc_state->update_pipe))
15628                         intel_color_load_luts(new_crtc_state);
15629         }
15630
15631         /*
15632          * Now that the vblank has passed, we can go ahead and program the
15633          * optimal watermarks on platforms that need two-step watermark
15634          * programming.
15635          *
15636          * TODO: Move this (and other cleanup) to an async worker eventually.
15637          */
15638         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
15639                                             new_crtc_state, i) {
15640                 /*
15641                  * Gen2 reports pipe underruns whenever all planes are disabled.
15642                  * So re-enable underrun reporting after some planes get enabled.
15643                  *
15644                  * We do this before .optimize_watermarks() so that we have a
15645                  * chance of catching underruns with the intermediate watermarks
15646                  * vs. the new plane configuration.
15647                  */
15648                 if (IS_GEN(dev_priv, 2) && planes_enabling(old_crtc_state, new_crtc_state))
15649                         intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
15650
15651                 if (dev_priv->display.optimize_watermarks)
15652                         dev_priv->display.optimize_watermarks(state, crtc);
15653         }
15654
15655         intel_dbuf_post_plane_update(state);
15656
15657         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
15658                 intel_post_plane_update(state, crtc);
15659
15660                 if (put_domains[i])
15661                         modeset_put_power_domains(dev_priv, put_domains[i]);
15662
15663                 intel_modeset_verify_crtc(crtc, state, old_crtc_state, new_crtc_state);
15664
15665                 /*
15666                  * DSB cleanup is done in cleanup_work aligning with framebuffer
15667                  * cleanup. So copy and reset the dsb structure to sync with
15668                  * commit_done and later do dsb cleanup in cleanup_work.
15669                  */
15670                 old_crtc_state->dsb = fetch_and_zero(&new_crtc_state->dsb);
15671         }
15672
15673         /* Underruns don't always raise interrupts, so check manually */
15674         intel_check_cpu_fifo_underruns(dev_priv);
15675         intel_check_pch_fifo_underruns(dev_priv);
15676
15677         if (state->modeset)
15678                 intel_verify_planes(state);
15679
15680         intel_sagv_post_plane_update(state);
15681
15682         drm_atomic_helper_commit_hw_done(&state->base);
15683
15684         if (state->modeset) {
15685                 /* As one of the primary mmio accessors, KMS has a high
15686                  * likelihood of triggering bugs in unclaimed access. After we
15687                  * finish modesetting, see if an error has been flagged, and if
15688                  * so enable debugging for the next modeset - and hope we catch
15689                  * the culprit.
15690                  */
15691                 intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore);
15692                 intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET, wakeref);
15693         }
15694         intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
15695
15696         /*
15697          * Defer the cleanup of the old state to a separate worker to not
15698          * impede the current task (userspace for blocking modesets) that
15699          * are executed inline. For out-of-line asynchronous modesets/flips,
15700          * deferring to a new worker seems overkill, but we would place a
15701          * schedule point (cond_resched()) here anyway to keep latencies
15702          * down.
15703          */
15704         INIT_WORK(&state->base.commit_work, intel_atomic_cleanup_work);
15705         queue_work(system_highpri_wq, &state->base.commit_work);
15706 }
15707
15708 static void intel_atomic_commit_work(struct work_struct *work)
15709 {
15710         struct intel_atomic_state *state =
15711                 container_of(work, struct intel_atomic_state, base.commit_work);
15712
15713         intel_atomic_commit_tail(state);
15714 }
15715
15716 static int __i915_sw_fence_call
15717 intel_atomic_commit_ready(struct i915_sw_fence *fence,
15718                           enum i915_sw_fence_notify notify)
15719 {
15720         struct intel_atomic_state *state =
15721                 container_of(fence, struct intel_atomic_state, commit_ready);
15722
15723         switch (notify) {
15724         case FENCE_COMPLETE:
15725                 /* we do blocking waits in the worker, nothing to do here */
15726                 break;
15727         case FENCE_FREE:
15728                 {
15729                         struct intel_atomic_helper *helper =
15730                                 &to_i915(state->base.dev)->atomic_helper;
15731
15732                         if (llist_add(&state->freed, &helper->free_list))
15733                                 schedule_work(&helper->free_work);
15734                         break;
15735                 }
15736         }
15737
15738         return NOTIFY_DONE;
15739 }
15740
15741 static void intel_atomic_track_fbs(struct intel_atomic_state *state)
15742 {
15743         struct intel_plane_state *old_plane_state, *new_plane_state;
15744         struct intel_plane *plane;
15745         int i;
15746
15747         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
15748                                              new_plane_state, i)
15749                 intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->hw.fb),
15750                                         to_intel_frontbuffer(new_plane_state->hw.fb),
15751                                         plane->frontbuffer_bit);
15752 }
15753
15754 static void assert_global_state_locked(struct drm_i915_private *dev_priv)
15755 {
15756         struct intel_crtc *crtc;
15757
15758         for_each_intel_crtc(&dev_priv->drm, crtc)
15759                 drm_modeset_lock_assert_held(&crtc->base.mutex);
15760 }
15761
15762 static int intel_atomic_commit(struct drm_device *dev,
15763                                struct drm_atomic_state *_state,
15764                                bool nonblock)
15765 {
15766         struct intel_atomic_state *state = to_intel_atomic_state(_state);
15767         struct drm_i915_private *dev_priv = to_i915(dev);
15768         int ret = 0;
15769
15770         state->wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
15771
15772         drm_atomic_state_get(&state->base);
15773         i915_sw_fence_init(&state->commit_ready,
15774                            intel_atomic_commit_ready);
15775
15776         /*
15777          * The intel_legacy_cursor_update() fast path takes care
15778          * of avoiding the vblank waits for simple cursor
15779          * movement and flips. For cursor on/off and size changes,
15780          * we want to perform the vblank waits so that watermark
15781          * updates happen during the correct frames. Gen9+ have
15782          * double buffered watermarks and so shouldn't need this.
15783          *
15784          * Unset state->legacy_cursor_update before the call to
15785          * drm_atomic_helper_setup_commit() because otherwise
15786          * drm_atomic_helper_wait_for_flip_done() is a noop and
15787          * we get FIFO underruns because we didn't wait
15788          * for vblank.
15789          *
15790          * FIXME doing watermarks and fb cleanup from a vblank worker
15791          * (assuming we had any) would solve these problems.
15792          */
15793         if (INTEL_GEN(dev_priv) < 9 && state->base.legacy_cursor_update) {
15794                 struct intel_crtc_state *new_crtc_state;
15795                 struct intel_crtc *crtc;
15796                 int i;
15797
15798                 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
15799                         if (new_crtc_state->wm.need_postvbl_update ||
15800                             new_crtc_state->update_wm_post)
15801                                 state->base.legacy_cursor_update = false;
15802         }
15803
15804         ret = intel_atomic_prepare_commit(state);
15805         if (ret) {
15806                 drm_dbg_atomic(&dev_priv->drm,
15807                                "Preparing state failed with %i\n", ret);
15808                 i915_sw_fence_commit(&state->commit_ready);
15809                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
15810                 return ret;
15811         }
15812
15813         ret = drm_atomic_helper_setup_commit(&state->base, nonblock);
15814         if (!ret)
15815                 ret = drm_atomic_helper_swap_state(&state->base, true);
15816         if (!ret)
15817                 intel_atomic_swap_global_state(state);
15818
15819         if (ret) {
15820                 struct intel_crtc_state *new_crtc_state;
15821                 struct intel_crtc *crtc;
15822                 int i;
15823
15824                 i915_sw_fence_commit(&state->commit_ready);
15825
15826                 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
15827                         intel_dsb_cleanup(new_crtc_state);
15828
15829                 drm_atomic_helper_cleanup_planes(dev, &state->base);
15830                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
15831                 return ret;
15832         }
15833         dev_priv->wm.distrust_bios_wm = false;
15834         intel_shared_dpll_swap_state(state);
15835         intel_atomic_track_fbs(state);
15836
15837         if (state->global_state_changed) {
15838                 assert_global_state_locked(dev_priv);
15839
15840                 dev_priv->active_pipes = state->active_pipes;
15841         }
15842
15843         drm_atomic_state_get(&state->base);
15844         INIT_WORK(&state->base.commit_work, intel_atomic_commit_work);
15845
15846         i915_sw_fence_commit(&state->commit_ready);
15847         if (nonblock && state->modeset) {
15848                 queue_work(dev_priv->modeset_wq, &state->base.commit_work);
15849         } else if (nonblock) {
15850                 queue_work(dev_priv->flip_wq, &state->base.commit_work);
15851         } else {
15852                 if (state->modeset)
15853                         flush_workqueue(dev_priv->modeset_wq);
15854                 intel_atomic_commit_tail(state);
15855         }
15856
15857         return 0;
15858 }
15859
15860 struct wait_rps_boost {
15861         struct wait_queue_entry wait;
15862
15863         struct drm_crtc *crtc;
15864         struct i915_request *request;
15865 };
15866
15867 static int do_rps_boost(struct wait_queue_entry *_wait,
15868                         unsigned mode, int sync, void *key)
15869 {
15870         struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait);
15871         struct i915_request *rq = wait->request;
15872
15873         /*
15874          * If we missed the vblank, but the request is already running it
15875          * is reasonable to assume that it will complete before the next
15876          * vblank without our intervention, so leave RPS alone.
15877          */
15878         if (!i915_request_started(rq))
15879                 intel_rps_boost(rq);
15880         i915_request_put(rq);
15881
15882         drm_crtc_vblank_put(wait->crtc);
15883
15884         list_del(&wait->wait.entry);
15885         kfree(wait);
15886         return 1;
15887 }
15888
15889 static void add_rps_boost_after_vblank(struct drm_crtc *crtc,
15890                                        struct dma_fence *fence)
15891 {
15892         struct wait_rps_boost *wait;
15893
15894         if (!dma_fence_is_i915(fence))
15895                 return;
15896
15897         if (INTEL_GEN(to_i915(crtc->dev)) < 6)
15898                 return;
15899
15900         if (drm_crtc_vblank_get(crtc))
15901                 return;
15902
15903         wait = kmalloc(sizeof(*wait), GFP_KERNEL);
15904         if (!wait) {
15905                 drm_crtc_vblank_put(crtc);
15906                 return;
15907         }
15908
15909         wait->request = to_request(dma_fence_get(fence));
15910         wait->crtc = crtc;
15911
15912         wait->wait.func = do_rps_boost;
15913         wait->wait.flags = 0;
15914
15915         add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait);
15916 }
15917
15918 static int intel_plane_pin_fb(struct intel_plane_state *plane_state)
15919 {
15920         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
15921         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
15922         struct drm_framebuffer *fb = plane_state->hw.fb;
15923         struct i915_vma *vma;
15924
15925         if (plane->id == PLANE_CURSOR &&
15926             INTEL_INFO(dev_priv)->display.cursor_needs_physical) {
15927                 struct drm_i915_gem_object *obj = intel_fb_obj(fb);
15928                 const int align = intel_cursor_alignment(dev_priv);
15929                 int err;
15930
15931                 err = i915_gem_object_attach_phys(obj, align);
15932                 if (err)
15933                         return err;
15934         }
15935
15936         vma = intel_pin_and_fence_fb_obj(fb,
15937                                          &plane_state->view,
15938                                          intel_plane_uses_fence(plane_state),
15939                                          &plane_state->flags);
15940         if (IS_ERR(vma))
15941                 return PTR_ERR(vma);
15942
15943         plane_state->vma = vma;
15944
15945         return 0;
15946 }
15947
15948 static void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
15949 {
15950         struct i915_vma *vma;
15951
15952         vma = fetch_and_zero(&old_plane_state->vma);
15953         if (vma)
15954                 intel_unpin_fb_vma(vma, old_plane_state->flags);
15955 }
15956
15957 static void fb_obj_bump_render_priority(struct drm_i915_gem_object *obj)
15958 {
15959         struct i915_sched_attr attr = {
15960                 .priority = I915_USER_PRIORITY(I915_PRIORITY_DISPLAY),
15961         };
15962
15963         i915_gem_object_wait_priority(obj, 0, &attr);
15964 }
15965
15966 /**
15967  * intel_prepare_plane_fb - Prepare fb for usage on plane
15968  * @_plane: drm plane to prepare for
15969  * @_new_plane_state: the plane state being prepared
15970  *
15971  * Prepares a framebuffer for usage on a display plane.  Generally this
15972  * involves pinning the underlying object and updating the frontbuffer tracking
15973  * bits.  Some older platforms need special physical address handling for
15974  * cursor planes.
15975  *
15976  * Returns 0 on success, negative error code on failure.
15977  */
15978 int
15979 intel_prepare_plane_fb(struct drm_plane *_plane,
15980                        struct drm_plane_state *_new_plane_state)
15981 {
15982         struct intel_plane *plane = to_intel_plane(_plane);
15983         struct intel_plane_state *new_plane_state =
15984                 to_intel_plane_state(_new_plane_state);
15985         struct intel_atomic_state *state =
15986                 to_intel_atomic_state(new_plane_state->uapi.state);
15987         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
15988         const struct intel_plane_state *old_plane_state =
15989                 intel_atomic_get_old_plane_state(state, plane);
15990         struct drm_i915_gem_object *obj = intel_fb_obj(new_plane_state->hw.fb);
15991         struct drm_i915_gem_object *old_obj = intel_fb_obj(old_plane_state->hw.fb);
15992         int ret;
15993
15994         if (old_obj) {
15995                 const struct intel_crtc_state *crtc_state =
15996                         intel_atomic_get_new_crtc_state(state,
15997                                                         to_intel_crtc(old_plane_state->hw.crtc));
15998
15999                 /* Big Hammer, we also need to ensure that any pending
16000                  * MI_WAIT_FOR_EVENT inside a user batch buffer on the
16001                  * current scanout is retired before unpinning the old
16002                  * framebuffer. Note that we rely on userspace rendering
16003                  * into the buffer attached to the pipe they are waiting
16004                  * on. If not, userspace generates a GPU hang with IPEHR
16005                  * point to the MI_WAIT_FOR_EVENT.
16006                  *
16007                  * This should only fail upon a hung GPU, in which case we
16008                  * can safely continue.
16009                  */
16010                 if (needs_modeset(crtc_state)) {
16011                         ret = i915_sw_fence_await_reservation(&state->commit_ready,
16012                                                               old_obj->base.resv, NULL,
16013                                                               false, 0,
16014                                                               GFP_KERNEL);
16015                         if (ret < 0)
16016                                 return ret;
16017                 }
16018         }
16019
16020         if (new_plane_state->uapi.fence) { /* explicit fencing */
16021                 ret = i915_sw_fence_await_dma_fence(&state->commit_ready,
16022                                                     new_plane_state->uapi.fence,
16023                                                     i915_fence_timeout(dev_priv),
16024                                                     GFP_KERNEL);
16025                 if (ret < 0)
16026                         return ret;
16027         }
16028
16029         if (!obj)
16030                 return 0;
16031
16032         ret = i915_gem_object_pin_pages(obj);
16033         if (ret)
16034                 return ret;
16035
16036         ret = intel_plane_pin_fb(new_plane_state);
16037
16038         i915_gem_object_unpin_pages(obj);
16039         if (ret)
16040                 return ret;
16041
16042         fb_obj_bump_render_priority(obj);
16043         i915_gem_object_flush_frontbuffer(obj, ORIGIN_DIRTYFB);
16044
16045         if (!new_plane_state->uapi.fence) { /* implicit fencing */
16046                 struct dma_fence *fence;
16047
16048                 ret = i915_sw_fence_await_reservation(&state->commit_ready,
16049                                                       obj->base.resv, NULL,
16050                                                       false,
16051                                                       i915_fence_timeout(dev_priv),
16052                                                       GFP_KERNEL);
16053                 if (ret < 0)
16054                         goto unpin_fb;
16055
16056                 fence = dma_resv_get_excl_rcu(obj->base.resv);
16057                 if (fence) {
16058                         add_rps_boost_after_vblank(new_plane_state->hw.crtc,
16059                                                    fence);
16060                         dma_fence_put(fence);
16061                 }
16062         } else {
16063                 add_rps_boost_after_vblank(new_plane_state->hw.crtc,
16064                                            new_plane_state->uapi.fence);
16065         }
16066
16067         /*
16068          * We declare pageflips to be interactive and so merit a small bias
16069          * towards upclocking to deliver the frame on time. By only changing
16070          * the RPS thresholds to sample more regularly and aim for higher
16071          * clocks we can hopefully deliver low power workloads (like kodi)
16072          * that are not quite steady state without resorting to forcing
16073          * maximum clocks following a vblank miss (see do_rps_boost()).
16074          */
16075         if (!state->rps_interactive) {
16076                 intel_rps_mark_interactive(&dev_priv->gt.rps, true);
16077                 state->rps_interactive = true;
16078         }
16079
16080         return 0;
16081
16082 unpin_fb:
16083         intel_plane_unpin_fb(new_plane_state);
16084
16085         return ret;
16086 }
16087
16088 /**
16089  * intel_cleanup_plane_fb - Cleans up an fb after plane use
16090  * @plane: drm plane to clean up for
16091  * @_old_plane_state: the state from the previous modeset
16092  *
16093  * Cleans up a framebuffer that has just been removed from a plane.
16094  */
16095 void
16096 intel_cleanup_plane_fb(struct drm_plane *plane,
16097                        struct drm_plane_state *_old_plane_state)
16098 {
16099         struct intel_plane_state *old_plane_state =
16100                 to_intel_plane_state(_old_plane_state);
16101         struct intel_atomic_state *state =
16102                 to_intel_atomic_state(old_plane_state->uapi.state);
16103         struct drm_i915_private *dev_priv = to_i915(plane->dev);
16104         struct drm_i915_gem_object *obj = intel_fb_obj(old_plane_state->hw.fb);
16105
16106         if (!obj)
16107                 return;
16108
16109         if (state->rps_interactive) {
16110                 intel_rps_mark_interactive(&dev_priv->gt.rps, false);
16111                 state->rps_interactive = false;
16112         }
16113
16114         /* Should only be called after a successful intel_prepare_plane_fb()! */
16115         intel_plane_unpin_fb(old_plane_state);
16116 }
16117
16118 /**
16119  * intel_plane_destroy - destroy a plane
16120  * @plane: plane to destroy
16121  *
16122  * Common destruction function for all types of planes (primary, cursor,
16123  * sprite).
16124  */
16125 void intel_plane_destroy(struct drm_plane *plane)
16126 {
16127         drm_plane_cleanup(plane);
16128         kfree(to_intel_plane(plane));
16129 }
16130
16131 static bool i8xx_plane_format_mod_supported(struct drm_plane *_plane,
16132                                             u32 format, u64 modifier)
16133 {
16134         switch (modifier) {
16135         case DRM_FORMAT_MOD_LINEAR:
16136         case I915_FORMAT_MOD_X_TILED:
16137                 break;
16138         default:
16139                 return false;
16140         }
16141
16142         switch (format) {
16143         case DRM_FORMAT_C8:
16144         case DRM_FORMAT_RGB565:
16145         case DRM_FORMAT_XRGB1555:
16146         case DRM_FORMAT_XRGB8888:
16147                 return modifier == DRM_FORMAT_MOD_LINEAR ||
16148                         modifier == I915_FORMAT_MOD_X_TILED;
16149         default:
16150                 return false;
16151         }
16152 }
16153
16154 static bool i965_plane_format_mod_supported(struct drm_plane *_plane,
16155                                             u32 format, u64 modifier)
16156 {
16157         switch (modifier) {
16158         case DRM_FORMAT_MOD_LINEAR:
16159         case I915_FORMAT_MOD_X_TILED:
16160                 break;
16161         default:
16162                 return false;
16163         }
16164
16165         switch (format) {
16166         case DRM_FORMAT_C8:
16167         case DRM_FORMAT_RGB565:
16168         case DRM_FORMAT_XRGB8888:
16169         case DRM_FORMAT_XBGR8888:
16170         case DRM_FORMAT_ARGB8888:
16171         case DRM_FORMAT_ABGR8888:
16172         case DRM_FORMAT_XRGB2101010:
16173         case DRM_FORMAT_XBGR2101010:
16174         case DRM_FORMAT_ARGB2101010:
16175         case DRM_FORMAT_ABGR2101010:
16176         case DRM_FORMAT_XBGR16161616F:
16177                 return modifier == DRM_FORMAT_MOD_LINEAR ||
16178                         modifier == I915_FORMAT_MOD_X_TILED;
16179         default:
16180                 return false;
16181         }
16182 }
16183
16184 static bool intel_cursor_format_mod_supported(struct drm_plane *_plane,
16185                                               u32 format, u64 modifier)
16186 {
16187         return modifier == DRM_FORMAT_MOD_LINEAR &&
16188                 format == DRM_FORMAT_ARGB8888;
16189 }
16190
16191 static const struct drm_plane_funcs i965_plane_funcs = {
16192         .update_plane = drm_atomic_helper_update_plane,
16193         .disable_plane = drm_atomic_helper_disable_plane,
16194         .destroy = intel_plane_destroy,
16195         .atomic_duplicate_state = intel_plane_duplicate_state,
16196         .atomic_destroy_state = intel_plane_destroy_state,
16197         .format_mod_supported = i965_plane_format_mod_supported,
16198 };
16199
16200 static const struct drm_plane_funcs i8xx_plane_funcs = {
16201         .update_plane = drm_atomic_helper_update_plane,
16202         .disable_plane = drm_atomic_helper_disable_plane,
16203         .destroy = intel_plane_destroy,
16204         .atomic_duplicate_state = intel_plane_duplicate_state,
16205         .atomic_destroy_state = intel_plane_destroy_state,
16206         .format_mod_supported = i8xx_plane_format_mod_supported,
16207 };
16208
16209 static int
16210 intel_legacy_cursor_update(struct drm_plane *_plane,
16211                            struct drm_crtc *_crtc,
16212                            struct drm_framebuffer *fb,
16213                            int crtc_x, int crtc_y,
16214                            unsigned int crtc_w, unsigned int crtc_h,
16215                            u32 src_x, u32 src_y,
16216                            u32 src_w, u32 src_h,
16217                            struct drm_modeset_acquire_ctx *ctx)
16218 {
16219         struct intel_plane *plane = to_intel_plane(_plane);
16220         struct intel_crtc *crtc = to_intel_crtc(_crtc);
16221         struct intel_plane_state *old_plane_state =
16222                 to_intel_plane_state(plane->base.state);
16223         struct intel_plane_state *new_plane_state;
16224         struct intel_crtc_state *crtc_state =
16225                 to_intel_crtc_state(crtc->base.state);
16226         struct intel_crtc_state *new_crtc_state;
16227         int ret;
16228
16229         /*
16230          * When crtc is inactive or there is a modeset pending,
16231          * wait for it to complete in the slowpath
16232          */
16233         if (!crtc_state->hw.active || needs_modeset(crtc_state) ||
16234             crtc_state->update_pipe)
16235                 goto slow;
16236
16237         /*
16238          * Don't do an async update if there is an outstanding commit modifying
16239          * the plane.  This prevents our async update's changes from getting
16240          * overridden by a previous synchronous update's state.
16241          */
16242         if (old_plane_state->uapi.commit &&
16243             !try_wait_for_completion(&old_plane_state->uapi.commit->hw_done))
16244                 goto slow;
16245
16246         /*
16247          * If any parameters change that may affect watermarks,
16248          * take the slowpath. Only changing fb or position should be
16249          * in the fastpath.
16250          */
16251         if (old_plane_state->uapi.crtc != &crtc->base ||
16252             old_plane_state->uapi.src_w != src_w ||
16253             old_plane_state->uapi.src_h != src_h ||
16254             old_plane_state->uapi.crtc_w != crtc_w ||
16255             old_plane_state->uapi.crtc_h != crtc_h ||
16256             !old_plane_state->uapi.fb != !fb)
16257                 goto slow;
16258
16259         new_plane_state = to_intel_plane_state(intel_plane_duplicate_state(&plane->base));
16260         if (!new_plane_state)
16261                 return -ENOMEM;
16262
16263         new_crtc_state = to_intel_crtc_state(intel_crtc_duplicate_state(&crtc->base));
16264         if (!new_crtc_state) {
16265                 ret = -ENOMEM;
16266                 goto out_free;
16267         }
16268
16269         drm_atomic_set_fb_for_plane(&new_plane_state->uapi, fb);
16270
16271         new_plane_state->uapi.src_x = src_x;
16272         new_plane_state->uapi.src_y = src_y;
16273         new_plane_state->uapi.src_w = src_w;
16274         new_plane_state->uapi.src_h = src_h;
16275         new_plane_state->uapi.crtc_x = crtc_x;
16276         new_plane_state->uapi.crtc_y = crtc_y;
16277         new_plane_state->uapi.crtc_w = crtc_w;
16278         new_plane_state->uapi.crtc_h = crtc_h;
16279
16280         intel_plane_copy_uapi_to_hw_state(new_plane_state, new_plane_state);
16281
16282         ret = intel_plane_atomic_check_with_state(crtc_state, new_crtc_state,
16283                                                   old_plane_state, new_plane_state);
16284         if (ret)
16285                 goto out_free;
16286
16287         ret = intel_plane_pin_fb(new_plane_state);
16288         if (ret)
16289                 goto out_free;
16290
16291         intel_frontbuffer_flush(to_intel_frontbuffer(new_plane_state->hw.fb),
16292                                 ORIGIN_FLIP);
16293         intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->hw.fb),
16294                                 to_intel_frontbuffer(new_plane_state->hw.fb),
16295                                 plane->frontbuffer_bit);
16296
16297         /* Swap plane state */
16298         plane->base.state = &new_plane_state->uapi;
16299
16300         /*
16301          * We cannot swap crtc_state as it may be in use by an atomic commit or
16302          * page flip that's running simultaneously. If we swap crtc_state and
16303          * destroy the old state, we will cause a use-after-free there.
16304          *
16305          * Only update active_planes, which is needed for our internal
16306          * bookkeeping. Either value will do the right thing when updating
16307          * planes atomically. If the cursor was part of the atomic update then
16308          * we would have taken the slowpath.
16309          */
16310         crtc_state->active_planes = new_crtc_state->active_planes;
16311
16312         if (new_plane_state->uapi.visible)
16313                 intel_update_plane(plane, crtc_state, new_plane_state);
16314         else
16315                 intel_disable_plane(plane, crtc_state);
16316
16317         intel_plane_unpin_fb(old_plane_state);
16318
16319 out_free:
16320         if (new_crtc_state)
16321                 intel_crtc_destroy_state(&crtc->base, &new_crtc_state->uapi);
16322         if (ret)
16323                 intel_plane_destroy_state(&plane->base, &new_plane_state->uapi);
16324         else
16325                 intel_plane_destroy_state(&plane->base, &old_plane_state->uapi);
16326         return ret;
16327
16328 slow:
16329         return drm_atomic_helper_update_plane(&plane->base, &crtc->base, fb,
16330                                               crtc_x, crtc_y, crtc_w, crtc_h,
16331                                               src_x, src_y, src_w, src_h, ctx);
16332 }
16333
16334 static const struct drm_plane_funcs intel_cursor_plane_funcs = {
16335         .update_plane = intel_legacy_cursor_update,
16336         .disable_plane = drm_atomic_helper_disable_plane,
16337         .destroy = intel_plane_destroy,
16338         .atomic_duplicate_state = intel_plane_duplicate_state,
16339         .atomic_destroy_state = intel_plane_destroy_state,
16340         .format_mod_supported = intel_cursor_format_mod_supported,
16341 };
16342
16343 static bool i9xx_plane_has_fbc(struct drm_i915_private *dev_priv,
16344                                enum i9xx_plane_id i9xx_plane)
16345 {
16346         if (!HAS_FBC(dev_priv))
16347                 return false;
16348
16349         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
16350                 return i9xx_plane == PLANE_A; /* tied to pipe A */
16351         else if (IS_IVYBRIDGE(dev_priv))
16352                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B ||
16353                         i9xx_plane == PLANE_C;
16354         else if (INTEL_GEN(dev_priv) >= 4)
16355                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B;
16356         else
16357                 return i9xx_plane == PLANE_A;
16358 }
16359
16360 static struct intel_plane *
16361 intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
16362 {
16363         struct intel_plane *plane;
16364         const struct drm_plane_funcs *plane_funcs;
16365         unsigned int supported_rotations;
16366         const u32 *formats;
16367         int num_formats;
16368         int ret, zpos;
16369
16370         if (INTEL_GEN(dev_priv) >= 9)
16371                 return skl_universal_plane_create(dev_priv, pipe,
16372                                                   PLANE_PRIMARY);
16373
16374         plane = intel_plane_alloc();
16375         if (IS_ERR(plane))
16376                 return plane;
16377
16378         plane->pipe = pipe;
16379         /*
16380          * On gen2/3 only plane A can do FBC, but the panel fitter and LVDS
16381          * port is hooked to pipe B. Hence we want plane A feeding pipe B.
16382          */
16383         if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4 &&
16384             INTEL_NUM_PIPES(dev_priv) == 2)
16385                 plane->i9xx_plane = (enum i9xx_plane_id) !pipe;
16386         else
16387                 plane->i9xx_plane = (enum i9xx_plane_id) pipe;
16388         plane->id = PLANE_PRIMARY;
16389         plane->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, plane->id);
16390
16391         plane->has_fbc = i9xx_plane_has_fbc(dev_priv, plane->i9xx_plane);
16392         if (plane->has_fbc) {
16393                 struct intel_fbc *fbc = &dev_priv->fbc;
16394
16395                 fbc->possible_framebuffer_bits |= plane->frontbuffer_bit;
16396         }
16397
16398         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
16399                 formats = vlv_primary_formats;
16400                 num_formats = ARRAY_SIZE(vlv_primary_formats);
16401         } else if (INTEL_GEN(dev_priv) >= 4) {
16402                 /*
16403                  * WaFP16GammaEnabling:ivb
16404                  * "Workaround : When using the 64-bit format, the plane
16405                  *  output on each color channel has one quarter amplitude.
16406                  *  It can be brought up to full amplitude by using pipe
16407                  *  gamma correction or pipe color space conversion to
16408                  *  multiply the plane output by four."
16409                  *
16410                  * There is no dedicated plane gamma for the primary plane,
16411                  * and using the pipe gamma/csc could conflict with other
16412                  * planes, so we choose not to expose fp16 on IVB primary
16413                  * planes. HSW primary planes no longer have this problem.
16414                  */
16415                 if (IS_IVYBRIDGE(dev_priv)) {
16416                         formats = ivb_primary_formats;
16417                         num_formats = ARRAY_SIZE(ivb_primary_formats);
16418                 } else {
16419                         formats = i965_primary_formats;
16420                         num_formats = ARRAY_SIZE(i965_primary_formats);
16421                 }
16422         } else {
16423                 formats = i8xx_primary_formats;
16424                 num_formats = ARRAY_SIZE(i8xx_primary_formats);
16425         }
16426
16427         if (INTEL_GEN(dev_priv) >= 4)
16428                 plane_funcs = &i965_plane_funcs;
16429         else
16430                 plane_funcs = &i8xx_plane_funcs;
16431
16432         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
16433                 plane->min_cdclk = vlv_plane_min_cdclk;
16434         else if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
16435                 plane->min_cdclk = hsw_plane_min_cdclk;
16436         else if (IS_IVYBRIDGE(dev_priv))
16437                 plane->min_cdclk = ivb_plane_min_cdclk;
16438         else
16439                 plane->min_cdclk = i9xx_plane_min_cdclk;
16440
16441         plane->max_stride = i9xx_plane_max_stride;
16442         plane->update_plane = i9xx_update_plane;
16443         plane->disable_plane = i9xx_disable_plane;
16444         plane->get_hw_state = i9xx_plane_get_hw_state;
16445         plane->check_plane = i9xx_plane_check;
16446
16447         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
16448                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
16449                                                0, plane_funcs,
16450                                                formats, num_formats,
16451                                                i9xx_format_modifiers,
16452                                                DRM_PLANE_TYPE_PRIMARY,
16453                                                "primary %c", pipe_name(pipe));
16454         else
16455                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
16456                                                0, plane_funcs,
16457                                                formats, num_formats,
16458                                                i9xx_format_modifiers,
16459                                                DRM_PLANE_TYPE_PRIMARY,
16460                                                "plane %c",
16461                                                plane_name(plane->i9xx_plane));
16462         if (ret)
16463                 goto fail;
16464
16465         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
16466                 supported_rotations =
16467                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180 |
16468                         DRM_MODE_REFLECT_X;
16469         } else if (INTEL_GEN(dev_priv) >= 4) {
16470                 supported_rotations =
16471                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180;
16472         } else {
16473                 supported_rotations = DRM_MODE_ROTATE_0;
16474         }
16475
16476         if (INTEL_GEN(dev_priv) >= 4)
16477                 drm_plane_create_rotation_property(&plane->base,
16478                                                    DRM_MODE_ROTATE_0,
16479                                                    supported_rotations);
16480
16481         zpos = 0;
16482         drm_plane_create_zpos_immutable_property(&plane->base, zpos);
16483
16484         drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs);
16485
16486         return plane;
16487
16488 fail:
16489         intel_plane_free(plane);
16490
16491         return ERR_PTR(ret);
16492 }
16493
16494 static struct intel_plane *
16495 intel_cursor_plane_create(struct drm_i915_private *dev_priv,
16496                           enum pipe pipe)
16497 {
16498         struct intel_plane *cursor;
16499         int ret, zpos;
16500
16501         cursor = intel_plane_alloc();
16502         if (IS_ERR(cursor))
16503                 return cursor;
16504
16505         cursor->pipe = pipe;
16506         cursor->i9xx_plane = (enum i9xx_plane_id) pipe;
16507         cursor->id = PLANE_CURSOR;
16508         cursor->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, cursor->id);
16509
16510         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
16511                 cursor->max_stride = i845_cursor_max_stride;
16512                 cursor->update_plane = i845_update_cursor;
16513                 cursor->disable_plane = i845_disable_cursor;
16514                 cursor->get_hw_state = i845_cursor_get_hw_state;
16515                 cursor->check_plane = i845_check_cursor;
16516         } else {
16517                 cursor->max_stride = i9xx_cursor_max_stride;
16518                 cursor->update_plane = i9xx_update_cursor;
16519                 cursor->disable_plane = i9xx_disable_cursor;
16520                 cursor->get_hw_state = i9xx_cursor_get_hw_state;
16521                 cursor->check_plane = i9xx_check_cursor;
16522         }
16523
16524         cursor->cursor.base = ~0;
16525         cursor->cursor.cntl = ~0;
16526
16527         if (IS_I845G(dev_priv) || IS_I865G(dev_priv) || HAS_CUR_FBC(dev_priv))
16528                 cursor->cursor.size = ~0;
16529
16530         ret = drm_universal_plane_init(&dev_priv->drm, &cursor->base,
16531                                        0, &intel_cursor_plane_funcs,
16532                                        intel_cursor_formats,
16533                                        ARRAY_SIZE(intel_cursor_formats),
16534                                        cursor_format_modifiers,
16535                                        DRM_PLANE_TYPE_CURSOR,
16536                                        "cursor %c", pipe_name(pipe));
16537         if (ret)
16538                 goto fail;
16539
16540         if (INTEL_GEN(dev_priv) >= 4)
16541                 drm_plane_create_rotation_property(&cursor->base,
16542                                                    DRM_MODE_ROTATE_0,
16543                                                    DRM_MODE_ROTATE_0 |
16544                                                    DRM_MODE_ROTATE_180);
16545
16546         zpos = RUNTIME_INFO(dev_priv)->num_sprites[pipe] + 1;
16547         drm_plane_create_zpos_immutable_property(&cursor->base, zpos);
16548
16549         if (INTEL_GEN(dev_priv) >= 12)
16550                 drm_plane_enable_fb_damage_clips(&cursor->base);
16551
16552         drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs);
16553
16554         return cursor;
16555
16556 fail:
16557         intel_plane_free(cursor);
16558
16559         return ERR_PTR(ret);
16560 }
16561
16562 #define INTEL_CRTC_FUNCS \
16563         .gamma_set = drm_atomic_helper_legacy_gamma_set, \
16564         .set_config = drm_atomic_helper_set_config, \
16565         .destroy = intel_crtc_destroy, \
16566         .page_flip = drm_atomic_helper_page_flip, \
16567         .atomic_duplicate_state = intel_crtc_duplicate_state, \
16568         .atomic_destroy_state = intel_crtc_destroy_state, \
16569         .set_crc_source = intel_crtc_set_crc_source, \
16570         .verify_crc_source = intel_crtc_verify_crc_source, \
16571         .get_crc_sources = intel_crtc_get_crc_sources
16572
16573 static const struct drm_crtc_funcs bdw_crtc_funcs = {
16574         INTEL_CRTC_FUNCS,
16575
16576         .get_vblank_counter = g4x_get_vblank_counter,
16577         .enable_vblank = bdw_enable_vblank,
16578         .disable_vblank = bdw_disable_vblank,
16579         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
16580 };
16581
16582 static const struct drm_crtc_funcs ilk_crtc_funcs = {
16583         INTEL_CRTC_FUNCS,
16584
16585         .get_vblank_counter = g4x_get_vblank_counter,
16586         .enable_vblank = ilk_enable_vblank,
16587         .disable_vblank = ilk_disable_vblank,
16588         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
16589 };
16590
16591 static const struct drm_crtc_funcs g4x_crtc_funcs = {
16592         INTEL_CRTC_FUNCS,
16593
16594         .get_vblank_counter = g4x_get_vblank_counter,
16595         .enable_vblank = i965_enable_vblank,
16596         .disable_vblank = i965_disable_vblank,
16597         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
16598 };
16599
16600 static const struct drm_crtc_funcs i965_crtc_funcs = {
16601         INTEL_CRTC_FUNCS,
16602
16603         .get_vblank_counter = i915_get_vblank_counter,
16604         .enable_vblank = i965_enable_vblank,
16605         .disable_vblank = i965_disable_vblank,
16606         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
16607 };
16608
16609 static const struct drm_crtc_funcs i915gm_crtc_funcs = {
16610         INTEL_CRTC_FUNCS,
16611
16612         .get_vblank_counter = i915_get_vblank_counter,
16613         .enable_vblank = i915gm_enable_vblank,
16614         .disable_vblank = i915gm_disable_vblank,
16615         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
16616 };
16617
16618 static const struct drm_crtc_funcs i915_crtc_funcs = {
16619         INTEL_CRTC_FUNCS,
16620
16621         .get_vblank_counter = i915_get_vblank_counter,
16622         .enable_vblank = i8xx_enable_vblank,
16623         .disable_vblank = i8xx_disable_vblank,
16624         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
16625 };
16626
16627 static const struct drm_crtc_funcs i8xx_crtc_funcs = {
16628         INTEL_CRTC_FUNCS,
16629
16630         /* no hw vblank counter */
16631         .enable_vblank = i8xx_enable_vblank,
16632         .disable_vblank = i8xx_disable_vblank,
16633         .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
16634 };
16635
16636 static struct intel_crtc *intel_crtc_alloc(void)
16637 {
16638         struct intel_crtc_state *crtc_state;
16639         struct intel_crtc *crtc;
16640
16641         crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
16642         if (!crtc)
16643                 return ERR_PTR(-ENOMEM);
16644
16645         crtc_state = intel_crtc_state_alloc(crtc);
16646         if (!crtc_state) {
16647                 kfree(crtc);
16648                 return ERR_PTR(-ENOMEM);
16649         }
16650
16651         crtc->base.state = &crtc_state->uapi;
16652         crtc->config = crtc_state;
16653
16654         return crtc;
16655 }
16656
16657 static void intel_crtc_free(struct intel_crtc *crtc)
16658 {
16659         intel_crtc_destroy_state(&crtc->base, crtc->base.state);
16660         kfree(crtc);
16661 }
16662
16663 static void intel_plane_possible_crtcs_init(struct drm_i915_private *dev_priv)
16664 {
16665         struct intel_plane *plane;
16666
16667         for_each_intel_plane(&dev_priv->drm, plane) {
16668                 struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv,
16669                                                                   plane->pipe);
16670
16671                 plane->base.possible_crtcs = drm_crtc_mask(&crtc->base);
16672         }
16673 }
16674
16675 static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
16676 {
16677         struct intel_plane *primary, *cursor;
16678         const struct drm_crtc_funcs *funcs;
16679         struct intel_crtc *crtc;
16680         int sprite, ret;
16681
16682         crtc = intel_crtc_alloc();
16683         if (IS_ERR(crtc))
16684                 return PTR_ERR(crtc);
16685
16686         crtc->pipe = pipe;
16687         crtc->num_scalers = RUNTIME_INFO(dev_priv)->num_scalers[pipe];
16688
16689         primary = intel_primary_plane_create(dev_priv, pipe);
16690         if (IS_ERR(primary)) {
16691                 ret = PTR_ERR(primary);
16692                 goto fail;
16693         }
16694         crtc->plane_ids_mask |= BIT(primary->id);
16695
16696         for_each_sprite(dev_priv, pipe, sprite) {
16697                 struct intel_plane *plane;
16698
16699                 plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
16700                 if (IS_ERR(plane)) {
16701                         ret = PTR_ERR(plane);
16702                         goto fail;
16703                 }
16704                 crtc->plane_ids_mask |= BIT(plane->id);
16705         }
16706
16707         cursor = intel_cursor_plane_create(dev_priv, pipe);
16708         if (IS_ERR(cursor)) {
16709                 ret = PTR_ERR(cursor);
16710                 goto fail;
16711         }
16712         crtc->plane_ids_mask |= BIT(cursor->id);
16713
16714         if (HAS_GMCH(dev_priv)) {
16715                 if (IS_CHERRYVIEW(dev_priv) ||
16716                     IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
16717                         funcs = &g4x_crtc_funcs;
16718                 else if (IS_GEN(dev_priv, 4))
16719                         funcs = &i965_crtc_funcs;
16720                 else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv))
16721                         funcs = &i915gm_crtc_funcs;
16722                 else if (IS_GEN(dev_priv, 3))
16723                         funcs = &i915_crtc_funcs;
16724                 else
16725                         funcs = &i8xx_crtc_funcs;
16726         } else {
16727                 if (INTEL_GEN(dev_priv) >= 8)
16728                         funcs = &bdw_crtc_funcs;
16729                 else
16730                         funcs = &ilk_crtc_funcs;
16731         }
16732
16733         ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base,
16734                                         &primary->base, &cursor->base,
16735                                         funcs, "pipe %c", pipe_name(pipe));
16736         if (ret)
16737                 goto fail;
16738
16739         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->pipe_to_crtc_mapping) ||
16740                dev_priv->pipe_to_crtc_mapping[pipe] != NULL);
16741         dev_priv->pipe_to_crtc_mapping[pipe] = crtc;
16742
16743         if (INTEL_GEN(dev_priv) < 9) {
16744                 enum i9xx_plane_id i9xx_plane = primary->i9xx_plane;
16745
16746                 BUG_ON(i9xx_plane >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
16747                        dev_priv->plane_to_crtc_mapping[i9xx_plane] != NULL);
16748                 dev_priv->plane_to_crtc_mapping[i9xx_plane] = crtc;
16749         }
16750
16751         intel_color_init(crtc);
16752
16753         intel_crtc_crc_init(crtc);
16754
16755         drm_WARN_ON(&dev_priv->drm, drm_crtc_index(&crtc->base) != crtc->pipe);
16756
16757         return 0;
16758
16759 fail:
16760         intel_crtc_free(crtc);
16761
16762         return ret;
16763 }
16764
16765 int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
16766                                       struct drm_file *file)
16767 {
16768         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
16769         struct drm_crtc *drmmode_crtc;
16770         struct intel_crtc *crtc;
16771
16772         drmmode_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
16773         if (!drmmode_crtc)
16774                 return -ENOENT;
16775
16776         crtc = to_intel_crtc(drmmode_crtc);
16777         pipe_from_crtc_id->pipe = crtc->pipe;
16778
16779         return 0;
16780 }
16781
16782 static u32 intel_encoder_possible_clones(struct intel_encoder *encoder)
16783 {
16784         struct drm_device *dev = encoder->base.dev;
16785         struct intel_encoder *source_encoder;
16786         u32 possible_clones = 0;
16787
16788         for_each_intel_encoder(dev, source_encoder) {
16789                 if (encoders_cloneable(encoder, source_encoder))
16790                         possible_clones |= drm_encoder_mask(&source_encoder->base);
16791         }
16792
16793         return possible_clones;
16794 }
16795
16796 static u32 intel_encoder_possible_crtcs(struct intel_encoder *encoder)
16797 {
16798         struct drm_device *dev = encoder->base.dev;
16799         struct intel_crtc *crtc;
16800         u32 possible_crtcs = 0;
16801
16802         for_each_intel_crtc(dev, crtc) {
16803                 if (encoder->pipe_mask & BIT(crtc->pipe))
16804                         possible_crtcs |= drm_crtc_mask(&crtc->base);
16805         }
16806
16807         return possible_crtcs;
16808 }
16809
16810 static bool ilk_has_edp_a(struct drm_i915_private *dev_priv)
16811 {
16812         if (!IS_MOBILE(dev_priv))
16813                 return false;
16814
16815         if ((intel_de_read(dev_priv, DP_A) & DP_DETECTED) == 0)
16816                 return false;
16817
16818         if (IS_GEN(dev_priv, 5) && (intel_de_read(dev_priv, FUSE_STRAP) & ILK_eDP_A_DISABLE))
16819                 return false;
16820
16821         return true;
16822 }
16823
16824 static bool intel_ddi_crt_present(struct drm_i915_private *dev_priv)
16825 {
16826         if (INTEL_GEN(dev_priv) >= 9)
16827                 return false;
16828
16829         if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
16830                 return false;
16831
16832         if (HAS_PCH_LPT_H(dev_priv) &&
16833             intel_de_read(dev_priv, SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
16834                 return false;
16835
16836         /* DDI E can't be used if DDI A requires 4 lanes */
16837         if (intel_de_read(dev_priv, DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
16838                 return false;
16839
16840         if (!dev_priv->vbt.int_crt_support)
16841                 return false;
16842
16843         return true;
16844 }
16845
16846 void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv)
16847 {
16848         int pps_num;
16849         int pps_idx;
16850
16851         if (HAS_DDI(dev_priv))
16852                 return;
16853         /*
16854          * This w/a is needed at least on CPT/PPT, but to be sure apply it
16855          * everywhere where registers can be write protected.
16856          */
16857         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
16858                 pps_num = 2;
16859         else
16860                 pps_num = 1;
16861
16862         for (pps_idx = 0; pps_idx < pps_num; pps_idx++) {
16863                 u32 val = intel_de_read(dev_priv, PP_CONTROL(pps_idx));
16864
16865                 val = (val & ~PANEL_UNLOCK_MASK) | PANEL_UNLOCK_REGS;
16866                 intel_de_write(dev_priv, PP_CONTROL(pps_idx), val);
16867         }
16868 }
16869
16870 static void intel_pps_init(struct drm_i915_private *dev_priv)
16871 {
16872         if (HAS_PCH_SPLIT(dev_priv) || IS_GEN9_LP(dev_priv))
16873                 dev_priv->pps_mmio_base = PCH_PPS_BASE;
16874         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
16875                 dev_priv->pps_mmio_base = VLV_PPS_BASE;
16876         else
16877                 dev_priv->pps_mmio_base = PPS_BASE;
16878
16879         intel_pps_unlock_regs_wa(dev_priv);
16880 }
16881
16882 static void intel_setup_outputs(struct drm_i915_private *dev_priv)
16883 {
16884         struct intel_encoder *encoder;
16885         bool dpd_is_edp = false;
16886
16887         intel_pps_init(dev_priv);
16888
16889         if (!HAS_DISPLAY(dev_priv) || !INTEL_DISPLAY_ENABLED(dev_priv))
16890                 return;
16891
16892         if (IS_ROCKETLAKE(dev_priv)) {
16893                 intel_ddi_init(dev_priv, PORT_A);
16894                 intel_ddi_init(dev_priv, PORT_B);
16895                 intel_ddi_init(dev_priv, PORT_D);       /* DDI TC1 */
16896                 intel_ddi_init(dev_priv, PORT_E);       /* DDI TC2 */
16897         } else if (INTEL_GEN(dev_priv) >= 12) {
16898                 intel_ddi_init(dev_priv, PORT_A);
16899                 intel_ddi_init(dev_priv, PORT_B);
16900                 intel_ddi_init(dev_priv, PORT_D);
16901                 intel_ddi_init(dev_priv, PORT_E);
16902                 intel_ddi_init(dev_priv, PORT_F);
16903                 intel_ddi_init(dev_priv, PORT_G);
16904                 intel_ddi_init(dev_priv, PORT_H);
16905                 intel_ddi_init(dev_priv, PORT_I);
16906                 icl_dsi_init(dev_priv);
16907         } else if (IS_ELKHARTLAKE(dev_priv)) {
16908                 intel_ddi_init(dev_priv, PORT_A);
16909                 intel_ddi_init(dev_priv, PORT_B);
16910                 intel_ddi_init(dev_priv, PORT_C);
16911                 intel_ddi_init(dev_priv, PORT_D);
16912                 icl_dsi_init(dev_priv);
16913         } else if (IS_GEN(dev_priv, 11)) {
16914                 intel_ddi_init(dev_priv, PORT_A);
16915                 intel_ddi_init(dev_priv, PORT_B);
16916                 intel_ddi_init(dev_priv, PORT_C);
16917                 intel_ddi_init(dev_priv, PORT_D);
16918                 intel_ddi_init(dev_priv, PORT_E);
16919                 /*
16920                  * On some ICL SKUs port F is not present. No strap bits for
16921                  * this, so rely on VBT.
16922                  * Work around broken VBTs on SKUs known to have no port F.
16923                  */
16924                 if (IS_ICL_WITH_PORT_F(dev_priv) &&
16925                     intel_bios_is_port_present(dev_priv, PORT_F))
16926                         intel_ddi_init(dev_priv, PORT_F);
16927
16928                 icl_dsi_init(dev_priv);
16929         } else if (IS_GEN9_LP(dev_priv)) {
16930                 /*
16931                  * FIXME: Broxton doesn't support port detection via the
16932                  * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to
16933                  * detect the ports.
16934                  */
16935                 intel_ddi_init(dev_priv, PORT_A);
16936                 intel_ddi_init(dev_priv, PORT_B);
16937                 intel_ddi_init(dev_priv, PORT_C);
16938
16939                 vlv_dsi_init(dev_priv);
16940         } else if (HAS_DDI(dev_priv)) {
16941                 int found;
16942
16943                 if (intel_ddi_crt_present(dev_priv))
16944                         intel_crt_init(dev_priv);
16945
16946                 /*
16947                  * Haswell uses DDI functions to detect digital outputs.
16948                  * On SKL pre-D0 the strap isn't connected, so we assume
16949                  * it's there.
16950                  */
16951                 found = intel_de_read(dev_priv, DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
16952                 /* WaIgnoreDDIAStrap: skl */
16953                 if (found || IS_GEN9_BC(dev_priv))
16954                         intel_ddi_init(dev_priv, PORT_A);
16955
16956                 /* DDI B, C, D, and F detection is indicated by the SFUSE_STRAP
16957                  * register */
16958                 found = intel_de_read(dev_priv, SFUSE_STRAP);
16959
16960                 if (found & SFUSE_STRAP_DDIB_DETECTED)
16961                         intel_ddi_init(dev_priv, PORT_B);
16962                 if (found & SFUSE_STRAP_DDIC_DETECTED)
16963                         intel_ddi_init(dev_priv, PORT_C);
16964                 if (found & SFUSE_STRAP_DDID_DETECTED)
16965                         intel_ddi_init(dev_priv, PORT_D);
16966                 if (found & SFUSE_STRAP_DDIF_DETECTED)
16967                         intel_ddi_init(dev_priv, PORT_F);
16968                 /*
16969                  * On SKL we don't have a way to detect DDI-E so we rely on VBT.
16970                  */
16971                 if (IS_GEN9_BC(dev_priv) &&
16972                     intel_bios_is_port_present(dev_priv, PORT_E))
16973                         intel_ddi_init(dev_priv, PORT_E);
16974
16975         } else if (HAS_PCH_SPLIT(dev_priv)) {
16976                 int found;
16977
16978                 /*
16979                  * intel_edp_init_connector() depends on this completing first,
16980                  * to prevent the registration of both eDP and LVDS and the
16981                  * incorrect sharing of the PPS.
16982                  */
16983                 intel_lvds_init(dev_priv);
16984                 intel_crt_init(dev_priv);
16985
16986                 dpd_is_edp = intel_dp_is_port_edp(dev_priv, PORT_D);
16987
16988                 if (ilk_has_edp_a(dev_priv))
16989                         intel_dp_init(dev_priv, DP_A, PORT_A);
16990
16991                 if (intel_de_read(dev_priv, PCH_HDMIB) & SDVO_DETECTED) {
16992                         /* PCH SDVOB multiplex with HDMIB */
16993                         found = intel_sdvo_init(dev_priv, PCH_SDVOB, PORT_B);
16994                         if (!found)
16995                                 intel_hdmi_init(dev_priv, PCH_HDMIB, PORT_B);
16996                         if (!found && (intel_de_read(dev_priv, PCH_DP_B) & DP_DETECTED))
16997                                 intel_dp_init(dev_priv, PCH_DP_B, PORT_B);
16998                 }
16999
17000                 if (intel_de_read(dev_priv, PCH_HDMIC) & SDVO_DETECTED)
17001                         intel_hdmi_init(dev_priv, PCH_HDMIC, PORT_C);
17002
17003                 if (!dpd_is_edp && intel_de_read(dev_priv, PCH_HDMID) & SDVO_DETECTED)
17004                         intel_hdmi_init(dev_priv, PCH_HDMID, PORT_D);
17005
17006                 if (intel_de_read(dev_priv, PCH_DP_C) & DP_DETECTED)
17007                         intel_dp_init(dev_priv, PCH_DP_C, PORT_C);
17008
17009                 if (intel_de_read(dev_priv, PCH_DP_D) & DP_DETECTED)
17010                         intel_dp_init(dev_priv, PCH_DP_D, PORT_D);
17011         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
17012                 bool has_edp, has_port;
17013
17014                 if (IS_VALLEYVIEW(dev_priv) && dev_priv->vbt.int_crt_support)
17015                         intel_crt_init(dev_priv);
17016
17017                 /*
17018                  * The DP_DETECTED bit is the latched state of the DDC
17019                  * SDA pin at boot. However since eDP doesn't require DDC
17020                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
17021                  * eDP ports may have been muxed to an alternate function.
17022                  * Thus we can't rely on the DP_DETECTED bit alone to detect
17023                  * eDP ports. Consult the VBT as well as DP_DETECTED to
17024                  * detect eDP ports.
17025                  *
17026                  * Sadly the straps seem to be missing sometimes even for HDMI
17027                  * ports (eg. on Voyo V3 - CHT x7-Z8700), so check both strap
17028                  * and VBT for the presence of the port. Additionally we can't
17029                  * trust the port type the VBT declares as we've seen at least
17030                  * HDMI ports that the VBT claim are DP or eDP.
17031                  */
17032                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_B);
17033                 has_port = intel_bios_is_port_present(dev_priv, PORT_B);
17034                 if (intel_de_read(dev_priv, VLV_DP_B) & DP_DETECTED || has_port)
17035                         has_edp &= intel_dp_init(dev_priv, VLV_DP_B, PORT_B);
17036                 if ((intel_de_read(dev_priv, VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp)
17037                         intel_hdmi_init(dev_priv, VLV_HDMIB, PORT_B);
17038
17039                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_C);
17040                 has_port = intel_bios_is_port_present(dev_priv, PORT_C);
17041                 if (intel_de_read(dev_priv, VLV_DP_C) & DP_DETECTED || has_port)
17042                         has_edp &= intel_dp_init(dev_priv, VLV_DP_C, PORT_C);
17043                 if ((intel_de_read(dev_priv, VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp)
17044                         intel_hdmi_init(dev_priv, VLV_HDMIC, PORT_C);
17045
17046                 if (IS_CHERRYVIEW(dev_priv)) {
17047                         /*
17048                          * eDP not supported on port D,
17049                          * so no need to worry about it
17050                          */
17051                         has_port = intel_bios_is_port_present(dev_priv, PORT_D);
17052                         if (intel_de_read(dev_priv, CHV_DP_D) & DP_DETECTED || has_port)
17053                                 intel_dp_init(dev_priv, CHV_DP_D, PORT_D);
17054                         if (intel_de_read(dev_priv, CHV_HDMID) & SDVO_DETECTED || has_port)
17055                                 intel_hdmi_init(dev_priv, CHV_HDMID, PORT_D);
17056                 }
17057
17058                 vlv_dsi_init(dev_priv);
17059         } else if (IS_PINEVIEW(dev_priv)) {
17060                 intel_lvds_init(dev_priv);
17061                 intel_crt_init(dev_priv);
17062         } else if (IS_GEN_RANGE(dev_priv, 3, 4)) {
17063                 bool found = false;
17064
17065                 if (IS_MOBILE(dev_priv))
17066                         intel_lvds_init(dev_priv);
17067
17068                 intel_crt_init(dev_priv);
17069
17070                 if (intel_de_read(dev_priv, GEN3_SDVOB) & SDVO_DETECTED) {
17071                         drm_dbg_kms(&dev_priv->drm, "probing SDVOB\n");
17072                         found = intel_sdvo_init(dev_priv, GEN3_SDVOB, PORT_B);
17073                         if (!found && IS_G4X(dev_priv)) {
17074                                 drm_dbg_kms(&dev_priv->drm,
17075                                             "probing HDMI on SDVOB\n");
17076                                 intel_hdmi_init(dev_priv, GEN4_HDMIB, PORT_B);
17077                         }
17078
17079                         if (!found && IS_G4X(dev_priv))
17080                                 intel_dp_init(dev_priv, DP_B, PORT_B);
17081                 }
17082
17083                 /* Before G4X SDVOC doesn't have its own detect register */
17084
17085                 if (intel_de_read(dev_priv, GEN3_SDVOB) & SDVO_DETECTED) {
17086                         drm_dbg_kms(&dev_priv->drm, "probing SDVOC\n");
17087                         found = intel_sdvo_init(dev_priv, GEN3_SDVOC, PORT_C);
17088                 }
17089
17090                 if (!found && (intel_de_read(dev_priv, GEN3_SDVOC) & SDVO_DETECTED)) {
17091
17092                         if (IS_G4X(dev_priv)) {
17093                                 drm_dbg_kms(&dev_priv->drm,
17094                                             "probing HDMI on SDVOC\n");
17095                                 intel_hdmi_init(dev_priv, GEN4_HDMIC, PORT_C);
17096                         }
17097                         if (IS_G4X(dev_priv))
17098                                 intel_dp_init(dev_priv, DP_C, PORT_C);
17099                 }
17100
17101                 if (IS_G4X(dev_priv) && (intel_de_read(dev_priv, DP_D) & DP_DETECTED))
17102                         intel_dp_init(dev_priv, DP_D, PORT_D);
17103
17104                 if (SUPPORTS_TV(dev_priv))
17105                         intel_tv_init(dev_priv);
17106         } else if (IS_GEN(dev_priv, 2)) {
17107                 if (IS_I85X(dev_priv))
17108                         intel_lvds_init(dev_priv);
17109
17110                 intel_crt_init(dev_priv);
17111                 intel_dvo_init(dev_priv);
17112         }
17113
17114         intel_psr_init(dev_priv);
17115
17116         for_each_intel_encoder(&dev_priv->drm, encoder) {
17117                 encoder->base.possible_crtcs =
17118                         intel_encoder_possible_crtcs(encoder);
17119                 encoder->base.possible_clones =
17120                         intel_encoder_possible_clones(encoder);
17121         }
17122
17123         intel_init_pch_refclk(dev_priv);
17124
17125         drm_helper_move_panel_connectors_to_head(&dev_priv->drm);
17126 }
17127
17128 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
17129 {
17130         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
17131
17132         drm_framebuffer_cleanup(fb);
17133         intel_frontbuffer_put(intel_fb->frontbuffer);
17134
17135         kfree(intel_fb);
17136 }
17137
17138 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
17139                                                 struct drm_file *file,
17140                                                 unsigned int *handle)
17141 {
17142         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
17143         struct drm_i915_private *i915 = to_i915(obj->base.dev);
17144
17145         if (obj->userptr.mm) {
17146                 drm_dbg(&i915->drm,
17147                         "attempting to use a userptr for a framebuffer, denied\n");
17148                 return -EINVAL;
17149         }
17150
17151         return drm_gem_handle_create(file, &obj->base, handle);
17152 }
17153
17154 static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
17155                                         struct drm_file *file,
17156                                         unsigned flags, unsigned color,
17157                                         struct drm_clip_rect *clips,
17158                                         unsigned num_clips)
17159 {
17160         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
17161
17162         i915_gem_object_flush_if_display(obj);
17163         intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB);
17164
17165         return 0;
17166 }
17167
17168 static const struct drm_framebuffer_funcs intel_fb_funcs = {
17169         .destroy = intel_user_framebuffer_destroy,
17170         .create_handle = intel_user_framebuffer_create_handle,
17171         .dirty = intel_user_framebuffer_dirty,
17172 };
17173
17174 static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
17175                                   struct drm_i915_gem_object *obj,
17176                                   struct drm_mode_fb_cmd2 *mode_cmd)
17177 {
17178         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
17179         struct drm_framebuffer *fb = &intel_fb->base;
17180         u32 max_stride;
17181         unsigned int tiling, stride;
17182         int ret = -EINVAL;
17183         int i;
17184
17185         intel_fb->frontbuffer = intel_frontbuffer_get(obj);
17186         if (!intel_fb->frontbuffer)
17187                 return -ENOMEM;
17188
17189         i915_gem_object_lock(obj, NULL);
17190         tiling = i915_gem_object_get_tiling(obj);
17191         stride = i915_gem_object_get_stride(obj);
17192         i915_gem_object_unlock(obj);
17193
17194         if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
17195                 /*
17196                  * If there's a fence, enforce that
17197                  * the fb modifier and tiling mode match.
17198                  */
17199                 if (tiling != I915_TILING_NONE &&
17200                     tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
17201                         drm_dbg_kms(&dev_priv->drm,
17202                                     "tiling_mode doesn't match fb modifier\n");
17203                         goto err;
17204                 }
17205         } else {
17206                 if (tiling == I915_TILING_X) {
17207                         mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
17208                 } else if (tiling == I915_TILING_Y) {
17209                         drm_dbg_kms(&dev_priv->drm,
17210                                     "No Y tiling for legacy addfb\n");
17211                         goto err;
17212                 }
17213         }
17214
17215         if (!drm_any_plane_has_format(&dev_priv->drm,
17216                                       mode_cmd->pixel_format,
17217                                       mode_cmd->modifier[0])) {
17218                 struct drm_format_name_buf format_name;
17219
17220                 drm_dbg_kms(&dev_priv->drm,
17221                             "unsupported pixel format %s / modifier 0x%llx\n",
17222                             drm_get_format_name(mode_cmd->pixel_format,
17223                                                 &format_name),
17224                             mode_cmd->modifier[0]);
17225                 goto err;
17226         }
17227
17228         /*
17229          * gen2/3 display engine uses the fence if present,
17230          * so the tiling mode must match the fb modifier exactly.
17231          */
17232         if (INTEL_GEN(dev_priv) < 4 &&
17233             tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
17234                 drm_dbg_kms(&dev_priv->drm,
17235                             "tiling_mode must match fb modifier exactly on gen2/3\n");
17236                 goto err;
17237         }
17238
17239         max_stride = intel_fb_max_stride(dev_priv, mode_cmd->pixel_format,
17240                                          mode_cmd->modifier[0]);
17241         if (mode_cmd->pitches[0] > max_stride) {
17242                 drm_dbg_kms(&dev_priv->drm,
17243                             "%s pitch (%u) must be at most %d\n",
17244                             mode_cmd->modifier[0] != DRM_FORMAT_MOD_LINEAR ?
17245                             "tiled" : "linear",
17246                             mode_cmd->pitches[0], max_stride);
17247                 goto err;
17248         }
17249
17250         /*
17251          * If there's a fence, enforce that
17252          * the fb pitch and fence stride match.
17253          */
17254         if (tiling != I915_TILING_NONE && mode_cmd->pitches[0] != stride) {
17255                 drm_dbg_kms(&dev_priv->drm,
17256                             "pitch (%d) must match tiling stride (%d)\n",
17257                             mode_cmd->pitches[0], stride);
17258                 goto err;
17259         }
17260
17261         /* FIXME need to adjust LINOFF/TILEOFF accordingly. */
17262         if (mode_cmd->offsets[0] != 0) {
17263                 drm_dbg_kms(&dev_priv->drm,
17264                             "plane 0 offset (0x%08x) must be 0\n",
17265                             mode_cmd->offsets[0]);
17266                 goto err;
17267         }
17268
17269         drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd);
17270
17271         for (i = 0; i < fb->format->num_planes; i++) {
17272                 u32 stride_alignment;
17273
17274                 if (mode_cmd->handles[i] != mode_cmd->handles[0]) {
17275                         drm_dbg_kms(&dev_priv->drm, "bad plane %d handle\n",
17276                                     i);
17277                         goto err;
17278                 }
17279
17280                 stride_alignment = intel_fb_stride_alignment(fb, i);
17281                 if (fb->pitches[i] & (stride_alignment - 1)) {
17282                         drm_dbg_kms(&dev_priv->drm,
17283                                     "plane %d pitch (%d) must be at least %u byte aligned\n",
17284                                     i, fb->pitches[i], stride_alignment);
17285                         goto err;
17286                 }
17287
17288                 if (is_gen12_ccs_plane(fb, i)) {
17289                         int ccs_aux_stride = gen12_ccs_aux_stride(fb, i);
17290
17291                         if (fb->pitches[i] != ccs_aux_stride) {
17292                                 drm_dbg_kms(&dev_priv->drm,
17293                                             "ccs aux plane %d pitch (%d) must be %d\n",
17294                                             i,
17295                                             fb->pitches[i], ccs_aux_stride);
17296                                 goto err;
17297                         }
17298                 }
17299
17300                 fb->obj[i] = &obj->base;
17301         }
17302
17303         ret = intel_fill_fb_info(dev_priv, fb);
17304         if (ret)
17305                 goto err;
17306
17307         ret = drm_framebuffer_init(&dev_priv->drm, fb, &intel_fb_funcs);
17308         if (ret) {
17309                 drm_err(&dev_priv->drm, "framebuffer init failed %d\n", ret);
17310                 goto err;
17311         }
17312
17313         return 0;
17314
17315 err:
17316         intel_frontbuffer_put(intel_fb->frontbuffer);
17317         return ret;
17318 }
17319
17320 static struct drm_framebuffer *
17321 intel_user_framebuffer_create(struct drm_device *dev,
17322                               struct drm_file *filp,
17323                               const struct drm_mode_fb_cmd2 *user_mode_cmd)
17324 {
17325         struct drm_framebuffer *fb;
17326         struct drm_i915_gem_object *obj;
17327         struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
17328
17329         obj = i915_gem_object_lookup(filp, mode_cmd.handles[0]);
17330         if (!obj)
17331                 return ERR_PTR(-ENOENT);
17332
17333         fb = intel_framebuffer_create(obj, &mode_cmd);
17334         i915_gem_object_put(obj);
17335
17336         return fb;
17337 }
17338
17339 static enum drm_mode_status
17340 intel_mode_valid(struct drm_device *dev,
17341                  const struct drm_display_mode *mode)
17342 {
17343         struct drm_i915_private *dev_priv = to_i915(dev);
17344         int hdisplay_max, htotal_max;
17345         int vdisplay_max, vtotal_max;
17346
17347         /*
17348          * Can't reject DBLSCAN here because Xorg ddxen can add piles
17349          * of DBLSCAN modes to the output's mode list when they detect
17350          * the scaling mode property on the connector. And they don't
17351          * ask the kernel to validate those modes in any way until
17352          * modeset time at which point the client gets a protocol error.
17353          * So in order to not upset those clients we silently ignore the
17354          * DBLSCAN flag on such connectors. For other connectors we will
17355          * reject modes with the DBLSCAN flag in encoder->compute_config().
17356          * And we always reject DBLSCAN modes in connector->mode_valid()
17357          * as we never want such modes on the connector's mode list.
17358          */
17359
17360         if (mode->vscan > 1)
17361                 return MODE_NO_VSCAN;
17362
17363         if (mode->flags & DRM_MODE_FLAG_HSKEW)
17364                 return MODE_H_ILLEGAL;
17365
17366         if (mode->flags & (DRM_MODE_FLAG_CSYNC |
17367                            DRM_MODE_FLAG_NCSYNC |
17368                            DRM_MODE_FLAG_PCSYNC))
17369                 return MODE_HSYNC;
17370
17371         if (mode->flags & (DRM_MODE_FLAG_BCAST |
17372                            DRM_MODE_FLAG_PIXMUX |
17373                            DRM_MODE_FLAG_CLKDIV2))
17374                 return MODE_BAD;
17375
17376         /* Transcoder timing limits */
17377         if (INTEL_GEN(dev_priv) >= 11) {
17378                 hdisplay_max = 16384;
17379                 vdisplay_max = 8192;
17380                 htotal_max = 16384;
17381                 vtotal_max = 8192;
17382         } else if (INTEL_GEN(dev_priv) >= 9 ||
17383                    IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
17384                 hdisplay_max = 8192; /* FDI max 4096 handled elsewhere */
17385                 vdisplay_max = 4096;
17386                 htotal_max = 8192;
17387                 vtotal_max = 8192;
17388         } else if (INTEL_GEN(dev_priv) >= 3) {
17389                 hdisplay_max = 4096;
17390                 vdisplay_max = 4096;
17391                 htotal_max = 8192;
17392                 vtotal_max = 8192;
17393         } else {
17394                 hdisplay_max = 2048;
17395                 vdisplay_max = 2048;
17396                 htotal_max = 4096;
17397                 vtotal_max = 4096;
17398         }
17399
17400         if (mode->hdisplay > hdisplay_max ||
17401             mode->hsync_start > htotal_max ||
17402             mode->hsync_end > htotal_max ||
17403             mode->htotal > htotal_max)
17404                 return MODE_H_ILLEGAL;
17405
17406         if (mode->vdisplay > vdisplay_max ||
17407             mode->vsync_start > vtotal_max ||
17408             mode->vsync_end > vtotal_max ||
17409             mode->vtotal > vtotal_max)
17410                 return MODE_V_ILLEGAL;
17411
17412         if (INTEL_GEN(dev_priv) >= 5) {
17413                 if (mode->hdisplay < 64 ||
17414                     mode->htotal - mode->hdisplay < 32)
17415                         return MODE_H_ILLEGAL;
17416
17417                 if (mode->vtotal - mode->vdisplay < 5)
17418                         return MODE_V_ILLEGAL;
17419         } else {
17420                 if (mode->htotal - mode->hdisplay < 32)
17421                         return MODE_H_ILLEGAL;
17422
17423                 if (mode->vtotal - mode->vdisplay < 3)
17424                         return MODE_V_ILLEGAL;
17425         }
17426
17427         return MODE_OK;
17428 }
17429
17430 enum drm_mode_status
17431 intel_mode_valid_max_plane_size(struct drm_i915_private *dev_priv,
17432                                 const struct drm_display_mode *mode)
17433 {
17434         int plane_width_max, plane_height_max;
17435
17436         /*
17437          * intel_mode_valid() should be
17438          * sufficient on older platforms.
17439          */
17440         if (INTEL_GEN(dev_priv) < 9)
17441                 return MODE_OK;
17442
17443         /*
17444          * Most people will probably want a fullscreen
17445          * plane so let's not advertize modes that are
17446          * too big for that.
17447          */
17448         if (INTEL_GEN(dev_priv) >= 11) {
17449                 plane_width_max = 5120;
17450                 plane_height_max = 4320;
17451         } else {
17452                 plane_width_max = 5120;
17453                 plane_height_max = 4096;
17454         }
17455
17456         if (mode->hdisplay > plane_width_max)
17457                 return MODE_H_ILLEGAL;
17458
17459         if (mode->vdisplay > plane_height_max)
17460                 return MODE_V_ILLEGAL;
17461
17462         return MODE_OK;
17463 }
17464
17465 static const struct drm_mode_config_funcs intel_mode_funcs = {
17466         .fb_create = intel_user_framebuffer_create,
17467         .get_format_info = intel_get_format_info,
17468         .output_poll_changed = intel_fbdev_output_poll_changed,
17469         .mode_valid = intel_mode_valid,
17470         .atomic_check = intel_atomic_check,
17471         .atomic_commit = intel_atomic_commit,
17472         .atomic_state_alloc = intel_atomic_state_alloc,
17473         .atomic_state_clear = intel_atomic_state_clear,
17474         .atomic_state_free = intel_atomic_state_free,
17475 };
17476
17477 /**
17478  * intel_init_display_hooks - initialize the display modesetting hooks
17479  * @dev_priv: device private
17480  */
17481 void intel_init_display_hooks(struct drm_i915_private *dev_priv)
17482 {
17483         intel_init_cdclk_hooks(dev_priv);
17484
17485         if (INTEL_GEN(dev_priv) >= 9) {
17486                 dev_priv->display.get_pipe_config = hsw_get_pipe_config;
17487                 dev_priv->display.get_initial_plane_config =
17488                         skl_get_initial_plane_config;
17489                 dev_priv->display.crtc_compute_clock = hsw_crtc_compute_clock;
17490                 dev_priv->display.crtc_enable = hsw_crtc_enable;
17491                 dev_priv->display.crtc_disable = hsw_crtc_disable;
17492         } else if (HAS_DDI(dev_priv)) {
17493                 dev_priv->display.get_pipe_config = hsw_get_pipe_config;
17494                 dev_priv->display.get_initial_plane_config =
17495                         i9xx_get_initial_plane_config;
17496                 dev_priv->display.crtc_compute_clock =
17497                         hsw_crtc_compute_clock;
17498                 dev_priv->display.crtc_enable = hsw_crtc_enable;
17499                 dev_priv->display.crtc_disable = hsw_crtc_disable;
17500         } else if (HAS_PCH_SPLIT(dev_priv)) {
17501                 dev_priv->display.get_pipe_config = ilk_get_pipe_config;
17502                 dev_priv->display.get_initial_plane_config =
17503                         i9xx_get_initial_plane_config;
17504                 dev_priv->display.crtc_compute_clock =
17505                         ilk_crtc_compute_clock;
17506                 dev_priv->display.crtc_enable = ilk_crtc_enable;
17507                 dev_priv->display.crtc_disable = ilk_crtc_disable;
17508         } else if (IS_CHERRYVIEW(dev_priv)) {
17509                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
17510                 dev_priv->display.get_initial_plane_config =
17511                         i9xx_get_initial_plane_config;
17512                 dev_priv->display.crtc_compute_clock = chv_crtc_compute_clock;
17513                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
17514                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
17515         } else if (IS_VALLEYVIEW(dev_priv)) {
17516                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
17517                 dev_priv->display.get_initial_plane_config =
17518                         i9xx_get_initial_plane_config;
17519                 dev_priv->display.crtc_compute_clock = vlv_crtc_compute_clock;
17520                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
17521                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
17522         } else if (IS_G4X(dev_priv)) {
17523                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
17524                 dev_priv->display.get_initial_plane_config =
17525                         i9xx_get_initial_plane_config;
17526                 dev_priv->display.crtc_compute_clock = g4x_crtc_compute_clock;
17527                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
17528                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
17529         } else if (IS_PINEVIEW(dev_priv)) {
17530                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
17531                 dev_priv->display.get_initial_plane_config =
17532                         i9xx_get_initial_plane_config;
17533                 dev_priv->display.crtc_compute_clock = pnv_crtc_compute_clock;
17534                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
17535                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
17536         } else if (!IS_GEN(dev_priv, 2)) {
17537                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
17538                 dev_priv->display.get_initial_plane_config =
17539                         i9xx_get_initial_plane_config;
17540                 dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
17541                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
17542                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
17543         } else {
17544                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
17545                 dev_priv->display.get_initial_plane_config =
17546                         i9xx_get_initial_plane_config;
17547                 dev_priv->display.crtc_compute_clock = i8xx_crtc_compute_clock;
17548                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
17549                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
17550         }
17551
17552         if (IS_GEN(dev_priv, 5)) {
17553                 dev_priv->display.fdi_link_train = ilk_fdi_link_train;
17554         } else if (IS_GEN(dev_priv, 6)) {
17555                 dev_priv->display.fdi_link_train = gen6_fdi_link_train;
17556         } else if (IS_IVYBRIDGE(dev_priv)) {
17557                 /* FIXME: detect B0+ stepping and use auto training */
17558                 dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
17559         }
17560
17561         if (INTEL_GEN(dev_priv) >= 9)
17562                 dev_priv->display.commit_modeset_enables = skl_commit_modeset_enables;
17563         else
17564                 dev_priv->display.commit_modeset_enables = intel_commit_modeset_enables;
17565
17566 }
17567
17568 void intel_modeset_init_hw(struct drm_i915_private *i915)
17569 {
17570         struct intel_cdclk_state *cdclk_state =
17571                 to_intel_cdclk_state(i915->cdclk.obj.state);
17572         struct intel_dbuf_state *dbuf_state =
17573                 to_intel_dbuf_state(i915->dbuf.obj.state);
17574
17575         intel_update_cdclk(i915);
17576         intel_dump_cdclk_config(&i915->cdclk.hw, "Current CDCLK");
17577         cdclk_state->logical = cdclk_state->actual = i915->cdclk.hw;
17578
17579         dbuf_state->enabled_slices = i915->dbuf.enabled_slices;
17580 }
17581
17582 static int sanitize_watermarks_add_affected(struct drm_atomic_state *state)
17583 {
17584         struct drm_plane *plane;
17585         struct intel_crtc *crtc;
17586
17587         for_each_intel_crtc(state->dev, crtc) {
17588                 struct intel_crtc_state *crtc_state;
17589
17590                 crtc_state = intel_atomic_get_crtc_state(state, crtc);
17591                 if (IS_ERR(crtc_state))
17592                         return PTR_ERR(crtc_state);
17593
17594                 if (crtc_state->hw.active) {
17595                         /*
17596                          * Preserve the inherited flag to avoid
17597                          * taking the full modeset path.
17598                          */
17599                         crtc_state->inherited = true;
17600                 }
17601         }
17602
17603         drm_for_each_plane(plane, state->dev) {
17604                 struct drm_plane_state *plane_state;
17605
17606                 plane_state = drm_atomic_get_plane_state(state, plane);
17607                 if (IS_ERR(plane_state))
17608                         return PTR_ERR(plane_state);
17609         }
17610
17611         return 0;
17612 }
17613
17614 /*
17615  * Calculate what we think the watermarks should be for the state we've read
17616  * out of the hardware and then immediately program those watermarks so that
17617  * we ensure the hardware settings match our internal state.
17618  *
17619  * We can calculate what we think WM's should be by creating a duplicate of the
17620  * current state (which was constructed during hardware readout) and running it
17621  * through the atomic check code to calculate new watermark values in the
17622  * state object.
17623  */
17624 static void sanitize_watermarks(struct drm_i915_private *dev_priv)
17625 {
17626         struct drm_atomic_state *state;
17627         struct intel_atomic_state *intel_state;
17628         struct intel_crtc *crtc;
17629         struct intel_crtc_state *crtc_state;
17630         struct drm_modeset_acquire_ctx ctx;
17631         int ret;
17632         int i;
17633
17634         /* Only supported on platforms that use atomic watermark design */
17635         if (!dev_priv->display.optimize_watermarks)
17636                 return;
17637
17638         state = drm_atomic_state_alloc(&dev_priv->drm);
17639         if (drm_WARN_ON(&dev_priv->drm, !state))
17640                 return;
17641
17642         intel_state = to_intel_atomic_state(state);
17643
17644         drm_modeset_acquire_init(&ctx, 0);
17645
17646 retry:
17647         state->acquire_ctx = &ctx;
17648
17649         /*
17650          * Hardware readout is the only time we don't want to calculate
17651          * intermediate watermarks (since we don't trust the current
17652          * watermarks).
17653          */
17654         if (!HAS_GMCH(dev_priv))
17655                 intel_state->skip_intermediate_wm = true;
17656
17657         ret = sanitize_watermarks_add_affected(state);
17658         if (ret)
17659                 goto fail;
17660
17661         ret = intel_atomic_check(&dev_priv->drm, state);
17662         if (ret)
17663                 goto fail;
17664
17665         /* Write calculated watermark values back */
17666         for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
17667                 crtc_state->wm.need_postvbl_update = true;
17668                 dev_priv->display.optimize_watermarks(intel_state, crtc);
17669
17670                 to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
17671         }
17672
17673 fail:
17674         if (ret == -EDEADLK) {
17675                 drm_atomic_state_clear(state);
17676                 drm_modeset_backoff(&ctx);
17677                 goto retry;
17678         }
17679
17680         /*
17681          * If we fail here, it means that the hardware appears to be
17682          * programmed in a way that shouldn't be possible, given our
17683          * understanding of watermark requirements.  This might mean a
17684          * mistake in the hardware readout code or a mistake in the
17685          * watermark calculations for a given platform.  Raise a WARN
17686          * so that this is noticeable.
17687          *
17688          * If this actually happens, we'll have to just leave the
17689          * BIOS-programmed watermarks untouched and hope for the best.
17690          */
17691         drm_WARN(&dev_priv->drm, ret,
17692                  "Could not determine valid watermarks for inherited state\n");
17693
17694         drm_atomic_state_put(state);
17695
17696         drm_modeset_drop_locks(&ctx);
17697         drm_modeset_acquire_fini(&ctx);
17698 }
17699
17700 static void intel_update_fdi_pll_freq(struct drm_i915_private *dev_priv)
17701 {
17702         if (IS_GEN(dev_priv, 5)) {
17703                 u32 fdi_pll_clk =
17704                         intel_de_read(dev_priv, FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK;
17705
17706                 dev_priv->fdi_pll_freq = (fdi_pll_clk + 2) * 10000;
17707         } else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) {
17708                 dev_priv->fdi_pll_freq = 270000;
17709         } else {
17710                 return;
17711         }
17712
17713         drm_dbg(&dev_priv->drm, "FDI PLL freq=%d\n", dev_priv->fdi_pll_freq);
17714 }
17715
17716 static int intel_initial_commit(struct drm_device *dev)
17717 {
17718         struct drm_atomic_state *state = NULL;
17719         struct drm_modeset_acquire_ctx ctx;
17720         struct intel_crtc *crtc;
17721         int ret = 0;
17722
17723         state = drm_atomic_state_alloc(dev);
17724         if (!state)
17725                 return -ENOMEM;
17726
17727         drm_modeset_acquire_init(&ctx, 0);
17728
17729 retry:
17730         state->acquire_ctx = &ctx;
17731
17732         for_each_intel_crtc(dev, crtc) {
17733                 struct intel_crtc_state *crtc_state =
17734                         intel_atomic_get_crtc_state(state, crtc);
17735
17736                 if (IS_ERR(crtc_state)) {
17737                         ret = PTR_ERR(crtc_state);
17738                         goto out;
17739                 }
17740
17741                 if (crtc_state->hw.active) {
17742                         /*
17743                          * We've not yet detected sink capabilities
17744                          * (audio,infoframes,etc.) and thus we don't want to
17745                          * force a full state recomputation yet. We want that to
17746                          * happen only for the first real commit from userspace.
17747                          * So preserve the inherited flag for the time being.
17748                          */
17749                         crtc_state->inherited = true;
17750
17751                         ret = drm_atomic_add_affected_planes(state, &crtc->base);
17752                         if (ret)
17753                                 goto out;
17754
17755                         /*
17756                          * FIXME hack to force a LUT update to avoid the
17757                          * plane update forcing the pipe gamma on without
17758                          * having a proper LUT loaded. Remove once we
17759                          * have readout for pipe gamma enable.
17760                          */
17761                         crtc_state->uapi.color_mgmt_changed = true;
17762
17763                         /*
17764                          * FIXME hack to force full modeset when DSC is being
17765                          * used.
17766                          *
17767                          * As long as we do not have full state readout and
17768                          * config comparison of crtc_state->dsc, we have no way
17769                          * to ensure reliable fastset. Remove once we have
17770                          * readout for DSC.
17771                          */
17772                         if (crtc_state->dsc.compression_enable) {
17773                                 ret = drm_atomic_add_affected_connectors(state,
17774                                                                          &crtc->base);
17775                                 if (ret)
17776                                         goto out;
17777                                 crtc_state->uapi.mode_changed = true;
17778                                 drm_dbg_kms(dev, "Force full modeset for DSC\n");
17779                         }
17780                 }
17781         }
17782
17783         ret = drm_atomic_commit(state);
17784
17785 out:
17786         if (ret == -EDEADLK) {
17787                 drm_atomic_state_clear(state);
17788                 drm_modeset_backoff(&ctx);
17789                 goto retry;
17790         }
17791
17792         drm_atomic_state_put(state);
17793
17794         drm_modeset_drop_locks(&ctx);
17795         drm_modeset_acquire_fini(&ctx);
17796
17797         return ret;
17798 }
17799
17800 static void intel_mode_config_init(struct drm_i915_private *i915)
17801 {
17802         struct drm_mode_config *mode_config = &i915->drm.mode_config;
17803
17804         drm_mode_config_init(&i915->drm);
17805         INIT_LIST_HEAD(&i915->global_obj_list);
17806
17807         mode_config->min_width = 0;
17808         mode_config->min_height = 0;
17809
17810         mode_config->preferred_depth = 24;
17811         mode_config->prefer_shadow = 1;
17812
17813         mode_config->allow_fb_modifiers = true;
17814
17815         mode_config->funcs = &intel_mode_funcs;
17816
17817         /*
17818          * Maximum framebuffer dimensions, chosen to match
17819          * the maximum render engine surface size on gen4+.
17820          */
17821         if (INTEL_GEN(i915) >= 7) {
17822                 mode_config->max_width = 16384;
17823                 mode_config->max_height = 16384;
17824         } else if (INTEL_GEN(i915) >= 4) {
17825                 mode_config->max_width = 8192;
17826                 mode_config->max_height = 8192;
17827         } else if (IS_GEN(i915, 3)) {
17828                 mode_config->max_width = 4096;
17829                 mode_config->max_height = 4096;
17830         } else {
17831                 mode_config->max_width = 2048;
17832                 mode_config->max_height = 2048;
17833         }
17834
17835         if (IS_I845G(i915) || IS_I865G(i915)) {
17836                 mode_config->cursor_width = IS_I845G(i915) ? 64 : 512;
17837                 mode_config->cursor_height = 1023;
17838         } else if (IS_I830(i915) || IS_I85X(i915) ||
17839                    IS_I915G(i915) || IS_I915GM(i915)) {
17840                 mode_config->cursor_width = 64;
17841                 mode_config->cursor_height = 64;
17842         } else {
17843                 mode_config->cursor_width = 256;
17844                 mode_config->cursor_height = 256;
17845         }
17846 }
17847
17848 static void intel_mode_config_cleanup(struct drm_i915_private *i915)
17849 {
17850         intel_atomic_global_obj_cleanup(i915);
17851         drm_mode_config_cleanup(&i915->drm);
17852 }
17853
17854 static void plane_config_fini(struct intel_initial_plane_config *plane_config)
17855 {
17856         if (plane_config->fb) {
17857                 struct drm_framebuffer *fb = &plane_config->fb->base;
17858
17859                 /* We may only have the stub and not a full framebuffer */
17860                 if (drm_framebuffer_read_refcount(fb))
17861                         drm_framebuffer_put(fb);
17862                 else
17863                         kfree(fb);
17864         }
17865
17866         if (plane_config->vma)
17867                 i915_vma_put(plane_config->vma);
17868 }
17869
17870 /* part #1: call before irq install */
17871 int intel_modeset_init_noirq(struct drm_i915_private *i915)
17872 {
17873         int ret;
17874
17875         i915->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0);
17876         i915->flip_wq = alloc_workqueue("i915_flip", WQ_HIGHPRI |
17877                                         WQ_UNBOUND, WQ_UNBOUND_MAX_ACTIVE);
17878
17879         intel_mode_config_init(i915);
17880
17881         ret = intel_cdclk_init(i915);
17882         if (ret)
17883                 return ret;
17884
17885         ret = intel_dbuf_init(i915);
17886         if (ret)
17887                 return ret;
17888
17889         ret = intel_bw_init(i915);
17890         if (ret)
17891                 return ret;
17892
17893         init_llist_head(&i915->atomic_helper.free_list);
17894         INIT_WORK(&i915->atomic_helper.free_work,
17895                   intel_atomic_helper_free_state_worker);
17896
17897         intel_init_quirks(i915);
17898
17899         intel_fbc_init(i915);
17900
17901         return 0;
17902 }
17903
17904 /* part #2: call after irq install */
17905 int intel_modeset_init(struct drm_i915_private *i915)
17906 {
17907         struct drm_device *dev = &i915->drm;
17908         enum pipe pipe;
17909         struct intel_crtc *crtc;
17910         int ret;
17911
17912         intel_init_pm(i915);
17913
17914         intel_panel_sanitize_ssc(i915);
17915
17916         intel_gmbus_setup(i915);
17917
17918         drm_dbg_kms(&i915->drm, "%d display pipe%s available.\n",
17919                     INTEL_NUM_PIPES(i915),
17920                     INTEL_NUM_PIPES(i915) > 1 ? "s" : "");
17921
17922         if (HAS_DISPLAY(i915) && INTEL_DISPLAY_ENABLED(i915)) {
17923                 for_each_pipe(i915, pipe) {
17924                         ret = intel_crtc_init(i915, pipe);
17925                         if (ret) {
17926                                 intel_mode_config_cleanup(i915);
17927                                 return ret;
17928                         }
17929                 }
17930         }
17931
17932         intel_plane_possible_crtcs_init(i915);
17933         intel_shared_dpll_init(dev);
17934         intel_update_fdi_pll_freq(i915);
17935
17936         intel_update_czclk(i915);
17937         intel_modeset_init_hw(i915);
17938
17939         intel_hdcp_component_init(i915);
17940
17941         if (i915->max_cdclk_freq == 0)
17942                 intel_update_max_cdclk(i915);
17943
17944         /*
17945          * If the platform has HTI, we need to find out whether it has reserved
17946          * any display resources before we create our display outputs.
17947          */
17948         if (INTEL_INFO(i915)->display.has_hti)
17949                 i915->hti_state = intel_de_read(i915, HDPORT_STATE);
17950
17951         /* Just disable it once at startup */
17952         intel_vga_disable(i915);
17953         intel_setup_outputs(i915);
17954
17955         drm_modeset_lock_all(dev);
17956         intel_modeset_setup_hw_state(dev, dev->mode_config.acquire_ctx);
17957         drm_modeset_unlock_all(dev);
17958
17959         for_each_intel_crtc(dev, crtc) {
17960                 struct intel_initial_plane_config plane_config = {};
17961
17962                 if (!crtc->active)
17963                         continue;
17964
17965                 /*
17966                  * Note that reserving the BIOS fb up front prevents us
17967                  * from stuffing other stolen allocations like the ring
17968                  * on top.  This prevents some ugliness at boot time, and
17969                  * can even allow for smooth boot transitions if the BIOS
17970                  * fb is large enough for the active pipe configuration.
17971                  */
17972                 i915->display.get_initial_plane_config(crtc, &plane_config);
17973
17974                 /*
17975                  * If the fb is shared between multiple heads, we'll
17976                  * just get the first one.
17977                  */
17978                 intel_find_initial_plane_obj(crtc, &plane_config);
17979
17980                 plane_config_fini(&plane_config);
17981         }
17982
17983         /*
17984          * Make sure hardware watermarks really match the state we read out.
17985          * Note that we need to do this after reconstructing the BIOS fb's
17986          * since the watermark calculation done here will use pstate->fb.
17987          */
17988         if (!HAS_GMCH(i915))
17989                 sanitize_watermarks(i915);
17990
17991         /*
17992          * Force all active planes to recompute their states. So that on
17993          * mode_setcrtc after probe, all the intel_plane_state variables
17994          * are already calculated and there is no assert_plane warnings
17995          * during bootup.
17996          */
17997         ret = intel_initial_commit(dev);
17998         if (ret)
17999                 drm_dbg_kms(&i915->drm, "Initial commit in probe failed.\n");
18000
18001         return 0;
18002 }
18003
18004 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
18005 {
18006         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
18007         /* 640x480@60Hz, ~25175 kHz */
18008         struct dpll clock = {
18009                 .m1 = 18,
18010                 .m2 = 7,
18011                 .p1 = 13,
18012                 .p2 = 4,
18013                 .n = 2,
18014         };
18015         u32 dpll, fp;
18016         int i;
18017
18018         drm_WARN_ON(&dev_priv->drm,
18019                     i9xx_calc_dpll_params(48000, &clock) != 25154);
18020
18021         drm_dbg_kms(&dev_priv->drm,
18022                     "enabling pipe %c due to force quirk (vco=%d dot=%d)\n",
18023                     pipe_name(pipe), clock.vco, clock.dot);
18024
18025         fp = i9xx_dpll_compute_fp(&clock);
18026         dpll = DPLL_DVO_2X_MODE |
18027                 DPLL_VGA_MODE_DIS |
18028                 ((clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT) |
18029                 PLL_P2_DIVIDE_BY_4 |
18030                 PLL_REF_INPUT_DREFCLK |
18031                 DPLL_VCO_ENABLE;
18032
18033         intel_de_write(dev_priv, FP0(pipe), fp);
18034         intel_de_write(dev_priv, FP1(pipe), fp);
18035
18036         intel_de_write(dev_priv, HTOTAL(pipe), (640 - 1) | ((800 - 1) << 16));
18037         intel_de_write(dev_priv, HBLANK(pipe), (640 - 1) | ((800 - 1) << 16));
18038         intel_de_write(dev_priv, HSYNC(pipe), (656 - 1) | ((752 - 1) << 16));
18039         intel_de_write(dev_priv, VTOTAL(pipe), (480 - 1) | ((525 - 1) << 16));
18040         intel_de_write(dev_priv, VBLANK(pipe), (480 - 1) | ((525 - 1) << 16));
18041         intel_de_write(dev_priv, VSYNC(pipe), (490 - 1) | ((492 - 1) << 16));
18042         intel_de_write(dev_priv, PIPESRC(pipe), ((640 - 1) << 16) | (480 - 1));
18043
18044         /*
18045          * Apparently we need to have VGA mode enabled prior to changing
18046          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
18047          * dividers, even though the register value does change.
18048          */
18049         intel_de_write(dev_priv, DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS);
18050         intel_de_write(dev_priv, DPLL(pipe), dpll);
18051
18052         /* Wait for the clocks to stabilize. */
18053         intel_de_posting_read(dev_priv, DPLL(pipe));
18054         udelay(150);
18055
18056         /* The pixel multiplier can only be updated once the
18057          * DPLL is enabled and the clocks are stable.
18058          *
18059          * So write it again.
18060          */
18061         intel_de_write(dev_priv, DPLL(pipe), dpll);
18062
18063         /* We do this three times for luck */
18064         for (i = 0; i < 3 ; i++) {
18065                 intel_de_write(dev_priv, DPLL(pipe), dpll);
18066                 intel_de_posting_read(dev_priv, DPLL(pipe));
18067                 udelay(150); /* wait for warmup */
18068         }
18069
18070         intel_de_write(dev_priv, PIPECONF(pipe),
18071                        PIPECONF_ENABLE | PIPECONF_PROGRESSIVE);
18072         intel_de_posting_read(dev_priv, PIPECONF(pipe));
18073
18074         intel_wait_for_pipe_scanline_moving(crtc);
18075 }
18076
18077 void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
18078 {
18079         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
18080
18081         drm_dbg_kms(&dev_priv->drm, "disabling pipe %c due to force quirk\n",
18082                     pipe_name(pipe));
18083
18084         drm_WARN_ON(&dev_priv->drm,
18085                     intel_de_read(dev_priv, DSPCNTR(PLANE_A)) &
18086                     DISPLAY_PLANE_ENABLE);
18087         drm_WARN_ON(&dev_priv->drm,
18088                     intel_de_read(dev_priv, DSPCNTR(PLANE_B)) &
18089                     DISPLAY_PLANE_ENABLE);
18090         drm_WARN_ON(&dev_priv->drm,
18091                     intel_de_read(dev_priv, DSPCNTR(PLANE_C)) &
18092                     DISPLAY_PLANE_ENABLE);
18093         drm_WARN_ON(&dev_priv->drm,
18094                     intel_de_read(dev_priv, CURCNTR(PIPE_A)) & MCURSOR_MODE);
18095         drm_WARN_ON(&dev_priv->drm,
18096                     intel_de_read(dev_priv, CURCNTR(PIPE_B)) & MCURSOR_MODE);
18097
18098         intel_de_write(dev_priv, PIPECONF(pipe), 0);
18099         intel_de_posting_read(dev_priv, PIPECONF(pipe));
18100
18101         intel_wait_for_pipe_scanline_stopped(crtc);
18102
18103         intel_de_write(dev_priv, DPLL(pipe), DPLL_VGA_MODE_DIS);
18104         intel_de_posting_read(dev_priv, DPLL(pipe));
18105 }
18106
18107 static void
18108 intel_sanitize_plane_mapping(struct drm_i915_private *dev_priv)
18109 {
18110         struct intel_crtc *crtc;
18111
18112         if (INTEL_GEN(dev_priv) >= 4)
18113                 return;
18114
18115         for_each_intel_crtc(&dev_priv->drm, crtc) {
18116                 struct intel_plane *plane =
18117                         to_intel_plane(crtc->base.primary);
18118                 struct intel_crtc *plane_crtc;
18119                 enum pipe pipe;
18120
18121                 if (!plane->get_hw_state(plane, &pipe))
18122                         continue;
18123
18124                 if (pipe == crtc->pipe)
18125                         continue;
18126
18127                 drm_dbg_kms(&dev_priv->drm,
18128                             "[PLANE:%d:%s] attached to the wrong pipe, disabling plane\n",
18129                             plane->base.base.id, plane->base.name);
18130
18131                 plane_crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
18132                 intel_plane_disable_noatomic(plane_crtc, plane);
18133         }
18134 }
18135
18136 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
18137 {
18138         struct drm_device *dev = crtc->base.dev;
18139         struct intel_encoder *encoder;
18140
18141         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
18142                 return true;
18143
18144         return false;
18145 }
18146
18147 static struct intel_connector *intel_encoder_find_connector(struct intel_encoder *encoder)
18148 {
18149         struct drm_device *dev = encoder->base.dev;
18150         struct intel_connector *connector;
18151
18152         for_each_connector_on_encoder(dev, &encoder->base, connector)
18153                 return connector;
18154
18155         return NULL;
18156 }
18157
18158 static bool has_pch_trancoder(struct drm_i915_private *dev_priv,
18159                               enum pipe pch_transcoder)
18160 {
18161         return HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
18162                 (HAS_PCH_LPT_H(dev_priv) && pch_transcoder == PIPE_A);
18163 }
18164
18165 static void intel_sanitize_frame_start_delay(const struct intel_crtc_state *crtc_state)
18166 {
18167         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
18168         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
18169         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
18170
18171         if (INTEL_GEN(dev_priv) >= 9 ||
18172             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
18173                 i915_reg_t reg = CHICKEN_TRANS(cpu_transcoder);
18174                 u32 val;
18175
18176                 if (transcoder_is_dsi(cpu_transcoder))
18177                         return;
18178
18179                 val = intel_de_read(dev_priv, reg);
18180                 val &= ~HSW_FRAME_START_DELAY_MASK;
18181                 val |= HSW_FRAME_START_DELAY(0);
18182                 intel_de_write(dev_priv, reg, val);
18183         } else {
18184                 i915_reg_t reg = PIPECONF(cpu_transcoder);
18185                 u32 val;
18186
18187                 val = intel_de_read(dev_priv, reg);
18188                 val &= ~PIPECONF_FRAME_START_DELAY_MASK;
18189                 val |= PIPECONF_FRAME_START_DELAY(0);
18190                 intel_de_write(dev_priv, reg, val);
18191         }
18192
18193         if (!crtc_state->has_pch_encoder)
18194                 return;
18195
18196         if (HAS_PCH_IBX(dev_priv)) {
18197                 i915_reg_t reg = PCH_TRANSCONF(crtc->pipe);
18198                 u32 val;
18199
18200                 val = intel_de_read(dev_priv, reg);
18201                 val &= ~TRANS_FRAME_START_DELAY_MASK;
18202                 val |= TRANS_FRAME_START_DELAY(0);
18203                 intel_de_write(dev_priv, reg, val);
18204         } else {
18205                 enum pipe pch_transcoder = intel_crtc_pch_transcoder(crtc);
18206                 i915_reg_t reg = TRANS_CHICKEN2(pch_transcoder);
18207                 u32 val;
18208
18209                 val = intel_de_read(dev_priv, reg);
18210                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
18211                 val |= TRANS_CHICKEN2_FRAME_START_DELAY(0);
18212                 intel_de_write(dev_priv, reg, val);
18213         }
18214 }
18215
18216 static void intel_sanitize_crtc(struct intel_crtc *crtc,
18217                                 struct drm_modeset_acquire_ctx *ctx)
18218 {
18219         struct drm_device *dev = crtc->base.dev;
18220         struct drm_i915_private *dev_priv = to_i915(dev);
18221         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
18222
18223         if (crtc_state->hw.active) {
18224                 struct intel_plane *plane;
18225
18226                 /* Clear any frame start delays used for debugging left by the BIOS */
18227                 intel_sanitize_frame_start_delay(crtc_state);
18228
18229                 /* Disable everything but the primary plane */
18230                 for_each_intel_plane_on_crtc(dev, crtc, plane) {
18231                         const struct intel_plane_state *plane_state =
18232                                 to_intel_plane_state(plane->base.state);
18233
18234                         if (plane_state->uapi.visible &&
18235                             plane->base.type != DRM_PLANE_TYPE_PRIMARY)
18236                                 intel_plane_disable_noatomic(crtc, plane);
18237                 }
18238
18239                 /*
18240                  * Disable any background color set by the BIOS, but enable the
18241                  * gamma and CSC to match how we program our planes.
18242                  */
18243                 if (INTEL_GEN(dev_priv) >= 9)
18244                         intel_de_write(dev_priv, SKL_BOTTOM_COLOR(crtc->pipe),
18245                                        SKL_BOTTOM_COLOR_GAMMA_ENABLE | SKL_BOTTOM_COLOR_CSC_ENABLE);
18246         }
18247
18248         /* Adjust the state of the output pipe according to whether we
18249          * have active connectors/encoders. */
18250         if (crtc_state->hw.active && !intel_crtc_has_encoders(crtc))
18251                 intel_crtc_disable_noatomic(crtc, ctx);
18252
18253         if (crtc_state->hw.active || HAS_GMCH(dev_priv)) {
18254                 /*
18255                  * We start out with underrun reporting disabled to avoid races.
18256                  * For correct bookkeeping mark this on active crtcs.
18257                  *
18258                  * Also on gmch platforms we dont have any hardware bits to
18259                  * disable the underrun reporting. Which means we need to start
18260                  * out with underrun reporting disabled also on inactive pipes,
18261                  * since otherwise we'll complain about the garbage we read when
18262                  * e.g. coming up after runtime pm.
18263                  *
18264                  * No protection against concurrent access is required - at
18265                  * worst a fifo underrun happens which also sets this to false.
18266                  */
18267                 crtc->cpu_fifo_underrun_disabled = true;
18268                 /*
18269                  * We track the PCH trancoder underrun reporting state
18270                  * within the crtc. With crtc for pipe A housing the underrun
18271                  * reporting state for PCH transcoder A, crtc for pipe B housing
18272                  * it for PCH transcoder B, etc. LPT-H has only PCH transcoder A,
18273                  * and marking underrun reporting as disabled for the non-existing
18274                  * PCH transcoders B and C would prevent enabling the south
18275                  * error interrupt (see cpt_can_enable_serr_int()).
18276                  */
18277                 if (has_pch_trancoder(dev_priv, crtc->pipe))
18278                         crtc->pch_fifo_underrun_disabled = true;
18279         }
18280 }
18281
18282 static bool has_bogus_dpll_config(const struct intel_crtc_state *crtc_state)
18283 {
18284         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
18285
18286         /*
18287          * Some SNB BIOSen (eg. ASUS K53SV) are known to misprogram
18288          * the hardware when a high res displays plugged in. DPLL P
18289          * divider is zero, and the pipe timings are bonkers. We'll
18290          * try to disable everything in that case.
18291          *
18292          * FIXME would be nice to be able to sanitize this state
18293          * without several WARNs, but for now let's take the easy
18294          * road.
18295          */
18296         return IS_GEN(dev_priv, 6) &&
18297                 crtc_state->hw.active &&
18298                 crtc_state->shared_dpll &&
18299                 crtc_state->port_clock == 0;
18300 }
18301
18302 static void intel_sanitize_encoder(struct intel_encoder *encoder)
18303 {
18304         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
18305         struct intel_connector *connector;
18306         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
18307         struct intel_crtc_state *crtc_state = crtc ?
18308                 to_intel_crtc_state(crtc->base.state) : NULL;
18309
18310         /* We need to check both for a crtc link (meaning that the
18311          * encoder is active and trying to read from a pipe) and the
18312          * pipe itself being active. */
18313         bool has_active_crtc = crtc_state &&
18314                 crtc_state->hw.active;
18315
18316         if (crtc_state && has_bogus_dpll_config(crtc_state)) {
18317                 drm_dbg_kms(&dev_priv->drm,
18318                             "BIOS has misprogrammed the hardware. Disabling pipe %c\n",
18319                             pipe_name(crtc->pipe));
18320                 has_active_crtc = false;
18321         }
18322
18323         connector = intel_encoder_find_connector(encoder);
18324         if (connector && !has_active_crtc) {
18325                 drm_dbg_kms(&dev_priv->drm,
18326                             "[ENCODER:%d:%s] has active connectors but no active pipe!\n",
18327                             encoder->base.base.id,
18328                             encoder->base.name);
18329
18330                 /* Connector is active, but has no active pipe. This is
18331                  * fallout from our resume register restoring. Disable
18332                  * the encoder manually again. */
18333                 if (crtc_state) {
18334                         struct drm_encoder *best_encoder;
18335
18336                         drm_dbg_kms(&dev_priv->drm,
18337                                     "[ENCODER:%d:%s] manually disabled\n",
18338                                     encoder->base.base.id,
18339                                     encoder->base.name);
18340
18341                         /* avoid oopsing in case the hooks consult best_encoder */
18342                         best_encoder = connector->base.state->best_encoder;
18343                         connector->base.state->best_encoder = &encoder->base;
18344
18345                         /* FIXME NULL atomic state passed! */
18346                         if (encoder->disable)
18347                                 encoder->disable(NULL, encoder, crtc_state,
18348                                                  connector->base.state);
18349                         if (encoder->post_disable)
18350                                 encoder->post_disable(NULL, encoder, crtc_state,
18351                                                       connector->base.state);
18352
18353                         connector->base.state->best_encoder = best_encoder;
18354                 }
18355                 encoder->base.crtc = NULL;
18356
18357                 /* Inconsistent output/port/pipe state happens presumably due to
18358                  * a bug in one of the get_hw_state functions. Or someplace else
18359                  * in our code, like the register restore mess on resume. Clamp
18360                  * things to off as a safer default. */
18361
18362                 connector->base.dpms = DRM_MODE_DPMS_OFF;
18363                 connector->base.encoder = NULL;
18364         }
18365
18366         /* notify opregion of the sanitized encoder state */
18367         intel_opregion_notify_encoder(encoder, connector && has_active_crtc);
18368
18369         if (INTEL_GEN(dev_priv) >= 11)
18370                 icl_sanitize_encoder_pll_mapping(encoder);
18371 }
18372
18373 /* FIXME read out full plane state for all planes */
18374 static void readout_plane_state(struct drm_i915_private *dev_priv)
18375 {
18376         struct intel_plane *plane;
18377         struct intel_crtc *crtc;
18378
18379         for_each_intel_plane(&dev_priv->drm, plane) {
18380                 struct intel_plane_state *plane_state =
18381                         to_intel_plane_state(plane->base.state);
18382                 struct intel_crtc_state *crtc_state;
18383                 enum pipe pipe = PIPE_A;
18384                 bool visible;
18385
18386                 visible = plane->get_hw_state(plane, &pipe);
18387
18388                 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
18389                 crtc_state = to_intel_crtc_state(crtc->base.state);
18390
18391                 intel_set_plane_visible(crtc_state, plane_state, visible);
18392
18393                 drm_dbg_kms(&dev_priv->drm,
18394                             "[PLANE:%d:%s] hw state readout: %s, pipe %c\n",
18395                             plane->base.base.id, plane->base.name,
18396                             enableddisabled(visible), pipe_name(pipe));
18397         }
18398
18399         for_each_intel_crtc(&dev_priv->drm, crtc) {
18400                 struct intel_crtc_state *crtc_state =
18401                         to_intel_crtc_state(crtc->base.state);
18402
18403                 fixup_active_planes(crtc_state);
18404         }
18405 }
18406
18407 static void intel_modeset_readout_hw_state(struct drm_device *dev)
18408 {
18409         struct drm_i915_private *dev_priv = to_i915(dev);
18410         struct intel_cdclk_state *cdclk_state =
18411                 to_intel_cdclk_state(dev_priv->cdclk.obj.state);
18412         struct intel_dbuf_state *dbuf_state =
18413                 to_intel_dbuf_state(dev_priv->dbuf.obj.state);
18414         enum pipe pipe;
18415         struct intel_crtc *crtc;
18416         struct intel_encoder *encoder;
18417         struct intel_connector *connector;
18418         struct drm_connector_list_iter conn_iter;
18419         u8 active_pipes = 0;
18420
18421         for_each_intel_crtc(dev, crtc) {
18422                 struct intel_crtc_state *crtc_state =
18423                         to_intel_crtc_state(crtc->base.state);
18424
18425                 __drm_atomic_helper_crtc_destroy_state(&crtc_state->uapi);
18426                 intel_crtc_free_hw_state(crtc_state);
18427                 intel_crtc_state_reset(crtc_state, crtc);
18428
18429                 crtc_state->hw.active = crtc_state->hw.enable =
18430                         dev_priv->display.get_pipe_config(crtc, crtc_state);
18431
18432                 crtc->base.enabled = crtc_state->hw.enable;
18433                 crtc->active = crtc_state->hw.active;
18434
18435                 if (crtc_state->hw.active)
18436                         active_pipes |= BIT(crtc->pipe);
18437
18438                 drm_dbg_kms(&dev_priv->drm,
18439                             "[CRTC:%d:%s] hw state readout: %s\n",
18440                             crtc->base.base.id, crtc->base.name,
18441                             enableddisabled(crtc_state->hw.active));
18442         }
18443
18444         dev_priv->active_pipes = cdclk_state->active_pipes =
18445                 dbuf_state->active_pipes = active_pipes;
18446
18447         readout_plane_state(dev_priv);
18448
18449         intel_dpll_readout_hw_state(dev_priv);
18450
18451         for_each_intel_encoder(dev, encoder) {
18452                 pipe = 0;
18453
18454                 if (encoder->get_hw_state(encoder, &pipe)) {
18455                         struct intel_crtc_state *crtc_state;
18456
18457                         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
18458                         crtc_state = to_intel_crtc_state(crtc->base.state);
18459
18460                         encoder->base.crtc = &crtc->base;
18461                         encoder->get_config(encoder, crtc_state);
18462                 } else {
18463                         encoder->base.crtc = NULL;
18464                 }
18465
18466                 drm_dbg_kms(&dev_priv->drm,
18467                             "[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
18468                             encoder->base.base.id, encoder->base.name,
18469                             enableddisabled(encoder->base.crtc),
18470                             pipe_name(pipe));
18471         }
18472
18473         drm_connector_list_iter_begin(dev, &conn_iter);
18474         for_each_intel_connector_iter(connector, &conn_iter) {
18475                 if (connector->get_hw_state(connector)) {
18476                         struct intel_crtc_state *crtc_state;
18477                         struct intel_crtc *crtc;
18478
18479                         connector->base.dpms = DRM_MODE_DPMS_ON;
18480
18481                         encoder = intel_attached_encoder(connector);
18482                         connector->base.encoder = &encoder->base;
18483
18484                         crtc = to_intel_crtc(encoder->base.crtc);
18485                         crtc_state = crtc ? to_intel_crtc_state(crtc->base.state) : NULL;
18486
18487                         if (crtc_state && crtc_state->hw.active) {
18488                                 /*
18489                                  * This has to be done during hardware readout
18490                                  * because anything calling .crtc_disable may
18491                                  * rely on the connector_mask being accurate.
18492                                  */
18493                                 crtc_state->uapi.connector_mask |=
18494                                         drm_connector_mask(&connector->base);
18495                                 crtc_state->uapi.encoder_mask |=
18496                                         drm_encoder_mask(&encoder->base);
18497                         }
18498                 } else {
18499                         connector->base.dpms = DRM_MODE_DPMS_OFF;
18500                         connector->base.encoder = NULL;
18501                 }
18502                 drm_dbg_kms(&dev_priv->drm,
18503                             "[CONNECTOR:%d:%s] hw state readout: %s\n",
18504                             connector->base.base.id, connector->base.name,
18505                             enableddisabled(connector->base.encoder));
18506         }
18507         drm_connector_list_iter_end(&conn_iter);
18508
18509         for_each_intel_crtc(dev, crtc) {
18510                 struct intel_bw_state *bw_state =
18511                         to_intel_bw_state(dev_priv->bw_obj.state);
18512                 struct intel_crtc_state *crtc_state =
18513                         to_intel_crtc_state(crtc->base.state);
18514                 struct intel_plane *plane;
18515                 int min_cdclk = 0;
18516
18517                 if (crtc_state->hw.active) {
18518                         struct drm_display_mode *mode = &crtc_state->hw.mode;
18519
18520                         intel_mode_from_pipe_config(&crtc_state->hw.adjusted_mode,
18521                                                     crtc_state);
18522
18523                         *mode = crtc_state->hw.adjusted_mode;
18524                         mode->hdisplay = crtc_state->pipe_src_w;
18525                         mode->vdisplay = crtc_state->pipe_src_h;
18526
18527                         /*
18528                          * The initial mode needs to be set in order to keep
18529                          * the atomic core happy. It wants a valid mode if the
18530                          * crtc's enabled, so we do the above call.
18531                          *
18532                          * But we don't set all the derived state fully, hence
18533                          * set a flag to indicate that a full recalculation is
18534                          * needed on the next commit.
18535                          */
18536                         crtc_state->inherited = true;
18537
18538                         intel_crtc_compute_pixel_rate(crtc_state);
18539
18540                         intel_crtc_update_active_timings(crtc_state);
18541
18542                         intel_crtc_copy_hw_to_uapi_state(crtc_state);
18543                 }
18544
18545                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
18546                         const struct intel_plane_state *plane_state =
18547                                 to_intel_plane_state(plane->base.state);
18548
18549                         /*
18550                          * FIXME don't have the fb yet, so can't
18551                          * use intel_plane_data_rate() :(
18552                          */
18553                         if (plane_state->uapi.visible)
18554                                 crtc_state->data_rate[plane->id] =
18555                                         4 * crtc_state->pixel_rate;
18556                         /*
18557                          * FIXME don't have the fb yet, so can't
18558                          * use plane->min_cdclk() :(
18559                          */
18560                         if (plane_state->uapi.visible && plane->min_cdclk) {
18561                                 if (crtc_state->double_wide ||
18562                                     INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
18563                                         crtc_state->min_cdclk[plane->id] =
18564                                                 DIV_ROUND_UP(crtc_state->pixel_rate, 2);
18565                                 else
18566                                         crtc_state->min_cdclk[plane->id] =
18567                                                 crtc_state->pixel_rate;
18568                         }
18569                         drm_dbg_kms(&dev_priv->drm,
18570                                     "[PLANE:%d:%s] min_cdclk %d kHz\n",
18571                                     plane->base.base.id, plane->base.name,
18572                                     crtc_state->min_cdclk[plane->id]);
18573                 }
18574
18575                 if (crtc_state->hw.active) {
18576                         min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
18577                         if (drm_WARN_ON(dev, min_cdclk < 0))
18578                                 min_cdclk = 0;
18579                 }
18580
18581                 cdclk_state->min_cdclk[crtc->pipe] = min_cdclk;
18582                 cdclk_state->min_voltage_level[crtc->pipe] =
18583                         crtc_state->min_voltage_level;
18584
18585                 intel_bw_crtc_update(bw_state, crtc_state);
18586
18587                 intel_pipe_config_sanity_check(dev_priv, crtc_state);
18588         }
18589 }
18590
18591 static void
18592 get_encoder_power_domains(struct drm_i915_private *dev_priv)
18593 {
18594         struct intel_encoder *encoder;
18595
18596         for_each_intel_encoder(&dev_priv->drm, encoder) {
18597                 struct intel_crtc_state *crtc_state;
18598
18599                 if (!encoder->get_power_domains)
18600                         continue;
18601
18602                 /*
18603                  * MST-primary and inactive encoders don't have a crtc state
18604                  * and neither of these require any power domain references.
18605                  */
18606                 if (!encoder->base.crtc)
18607                         continue;
18608
18609                 crtc_state = to_intel_crtc_state(encoder->base.crtc->state);
18610                 encoder->get_power_domains(encoder, crtc_state);
18611         }
18612 }
18613
18614 static void intel_early_display_was(struct drm_i915_private *dev_priv)
18615 {
18616         /*
18617          * Display WA #1185 WaDisableDARBFClkGating:cnl,glk,icl,ehl,tgl
18618          * Also known as Wa_14010480278.
18619          */
18620         if (IS_GEN_RANGE(dev_priv, 10, 12) || IS_GEMINILAKE(dev_priv))
18621                 intel_de_write(dev_priv, GEN9_CLKGATE_DIS_0,
18622                                intel_de_read(dev_priv, GEN9_CLKGATE_DIS_0) | DARBF_GATING_DIS);
18623
18624         if (IS_HASWELL(dev_priv)) {
18625                 /*
18626                  * WaRsPkgCStateDisplayPMReq:hsw
18627                  * System hang if this isn't done before disabling all planes!
18628                  */
18629                 intel_de_write(dev_priv, CHICKEN_PAR1_1,
18630                                intel_de_read(dev_priv, CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
18631         }
18632 }
18633
18634 static void ibx_sanitize_pch_hdmi_port(struct drm_i915_private *dev_priv,
18635                                        enum port port, i915_reg_t hdmi_reg)
18636 {
18637         u32 val = intel_de_read(dev_priv, hdmi_reg);
18638
18639         if (val & SDVO_ENABLE ||
18640             (val & SDVO_PIPE_SEL_MASK) == SDVO_PIPE_SEL(PIPE_A))
18641                 return;
18642
18643         drm_dbg_kms(&dev_priv->drm,
18644                     "Sanitizing transcoder select for HDMI %c\n",
18645                     port_name(port));
18646
18647         val &= ~SDVO_PIPE_SEL_MASK;
18648         val |= SDVO_PIPE_SEL(PIPE_A);
18649
18650         intel_de_write(dev_priv, hdmi_reg, val);
18651 }
18652
18653 static void ibx_sanitize_pch_dp_port(struct drm_i915_private *dev_priv,
18654                                      enum port port, i915_reg_t dp_reg)
18655 {
18656         u32 val = intel_de_read(dev_priv, dp_reg);
18657
18658         if (val & DP_PORT_EN ||
18659             (val & DP_PIPE_SEL_MASK) == DP_PIPE_SEL(PIPE_A))
18660                 return;
18661
18662         drm_dbg_kms(&dev_priv->drm,
18663                     "Sanitizing transcoder select for DP %c\n",
18664                     port_name(port));
18665
18666         val &= ~DP_PIPE_SEL_MASK;
18667         val |= DP_PIPE_SEL(PIPE_A);
18668
18669         intel_de_write(dev_priv, dp_reg, val);
18670 }
18671
18672 static void ibx_sanitize_pch_ports(struct drm_i915_private *dev_priv)
18673 {
18674         /*
18675          * The BIOS may select transcoder B on some of the PCH
18676          * ports even it doesn't enable the port. This would trip
18677          * assert_pch_dp_disabled() and assert_pch_hdmi_disabled().
18678          * Sanitize the transcoder select bits to prevent that. We
18679          * assume that the BIOS never actually enabled the port,
18680          * because if it did we'd actually have to toggle the port
18681          * on and back off to make the transcoder A select stick
18682          * (see. intel_dp_link_down(), intel_disable_hdmi(),
18683          * intel_disable_sdvo()).
18684          */
18685         ibx_sanitize_pch_dp_port(dev_priv, PORT_B, PCH_DP_B);
18686         ibx_sanitize_pch_dp_port(dev_priv, PORT_C, PCH_DP_C);
18687         ibx_sanitize_pch_dp_port(dev_priv, PORT_D, PCH_DP_D);
18688
18689         /* PCH SDVOB multiplex with HDMIB */
18690         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_B, PCH_HDMIB);
18691         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_C, PCH_HDMIC);
18692         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_D, PCH_HDMID);
18693 }
18694
18695 /* Scan out the current hw modeset state,
18696  * and sanitizes it to the current state
18697  */
18698 static void
18699 intel_modeset_setup_hw_state(struct drm_device *dev,
18700                              struct drm_modeset_acquire_ctx *ctx)
18701 {
18702         struct drm_i915_private *dev_priv = to_i915(dev);
18703         struct intel_encoder *encoder;
18704         struct intel_crtc *crtc;
18705         intel_wakeref_t wakeref;
18706
18707         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
18708
18709         intel_early_display_was(dev_priv);
18710         intel_modeset_readout_hw_state(dev);
18711
18712         /* HW state is read out, now we need to sanitize this mess. */
18713
18714         /* Sanitize the TypeC port mode upfront, encoders depend on this */
18715         for_each_intel_encoder(dev, encoder) {
18716                 enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
18717
18718                 /* We need to sanitize only the MST primary port. */
18719                 if (encoder->type != INTEL_OUTPUT_DP_MST &&
18720                     intel_phy_is_tc(dev_priv, phy))
18721                         intel_tc_port_sanitize(enc_to_dig_port(encoder));
18722         }
18723
18724         get_encoder_power_domains(dev_priv);
18725
18726         if (HAS_PCH_IBX(dev_priv))
18727                 ibx_sanitize_pch_ports(dev_priv);
18728
18729         /*
18730          * intel_sanitize_plane_mapping() may need to do vblank
18731          * waits, so we need vblank interrupts restored beforehand.
18732          */
18733         for_each_intel_crtc(&dev_priv->drm, crtc) {
18734                 struct intel_crtc_state *crtc_state =
18735                         to_intel_crtc_state(crtc->base.state);
18736
18737                 drm_crtc_vblank_reset(&crtc->base);
18738
18739                 if (crtc_state->hw.active)
18740                         intel_crtc_vblank_on(crtc_state);
18741         }
18742
18743         intel_sanitize_plane_mapping(dev_priv);
18744
18745         for_each_intel_encoder(dev, encoder)
18746                 intel_sanitize_encoder(encoder);
18747
18748         for_each_intel_crtc(&dev_priv->drm, crtc) {
18749                 struct intel_crtc_state *crtc_state =
18750                         to_intel_crtc_state(crtc->base.state);
18751
18752                 intel_sanitize_crtc(crtc, ctx);
18753                 intel_dump_pipe_config(crtc_state, NULL, "[setup_hw_state]");
18754         }
18755
18756         intel_modeset_update_connector_atomic_state(dev);
18757
18758         intel_dpll_sanitize_state(dev_priv);
18759
18760         if (IS_G4X(dev_priv)) {
18761                 g4x_wm_get_hw_state(dev_priv);
18762                 g4x_wm_sanitize(dev_priv);
18763         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
18764                 vlv_wm_get_hw_state(dev_priv);
18765                 vlv_wm_sanitize(dev_priv);
18766         } else if (INTEL_GEN(dev_priv) >= 9) {
18767                 skl_wm_get_hw_state(dev_priv);
18768         } else if (HAS_PCH_SPLIT(dev_priv)) {
18769                 ilk_wm_get_hw_state(dev_priv);
18770         }
18771
18772         for_each_intel_crtc(dev, crtc) {
18773                 struct intel_crtc_state *crtc_state =
18774                         to_intel_crtc_state(crtc->base.state);
18775                 u64 put_domains;
18776
18777                 put_domains = modeset_get_crtc_power_domains(crtc_state);
18778                 if (drm_WARN_ON(dev, put_domains))
18779                         modeset_put_power_domains(dev_priv, put_domains);
18780         }
18781
18782         intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref);
18783 }
18784
18785 void intel_display_resume(struct drm_device *dev)
18786 {
18787         struct drm_i915_private *dev_priv = to_i915(dev);
18788         struct drm_atomic_state *state = dev_priv->modeset_restore_state;
18789         struct drm_modeset_acquire_ctx ctx;
18790         int ret;
18791
18792         dev_priv->modeset_restore_state = NULL;
18793         if (state)
18794                 state->acquire_ctx = &ctx;
18795
18796         drm_modeset_acquire_init(&ctx, 0);
18797
18798         while (1) {
18799                 ret = drm_modeset_lock_all_ctx(dev, &ctx);
18800                 if (ret != -EDEADLK)
18801                         break;
18802
18803                 drm_modeset_backoff(&ctx);
18804         }
18805
18806         if (!ret)
18807                 ret = __intel_display_resume(dev, state, &ctx);
18808
18809         intel_enable_ipc(dev_priv);
18810         drm_modeset_drop_locks(&ctx);
18811         drm_modeset_acquire_fini(&ctx);
18812
18813         if (ret)
18814                 drm_err(&dev_priv->drm,
18815                         "Restoring old state failed with %i\n", ret);
18816         if (state)
18817                 drm_atomic_state_put(state);
18818 }
18819
18820 static void intel_hpd_poll_fini(struct drm_i915_private *i915)
18821 {
18822         struct intel_connector *connector;
18823         struct drm_connector_list_iter conn_iter;
18824
18825         /* Kill all the work that may have been queued by hpd. */
18826         drm_connector_list_iter_begin(&i915->drm, &conn_iter);
18827         for_each_intel_connector_iter(connector, &conn_iter) {
18828                 if (connector->modeset_retry_work.func)
18829                         cancel_work_sync(&connector->modeset_retry_work);
18830                 if (connector->hdcp.shim) {
18831                         cancel_delayed_work_sync(&connector->hdcp.check_work);
18832                         cancel_work_sync(&connector->hdcp.prop_work);
18833                 }
18834         }
18835         drm_connector_list_iter_end(&conn_iter);
18836 }
18837
18838 /* part #1: call before irq uninstall */
18839 void intel_modeset_driver_remove(struct drm_i915_private *i915)
18840 {
18841         flush_workqueue(i915->flip_wq);
18842         flush_workqueue(i915->modeset_wq);
18843
18844         flush_work(&i915->atomic_helper.free_work);
18845         drm_WARN_ON(&i915->drm, !llist_empty(&i915->atomic_helper.free_list));
18846 }
18847
18848 /* part #2: call after irq uninstall */
18849 void intel_modeset_driver_remove_noirq(struct drm_i915_private *i915)
18850 {
18851         /*
18852          * Due to the hpd irq storm handling the hotplug work can re-arm the
18853          * poll handlers. Hence disable polling after hpd handling is shut down.
18854          */
18855         intel_hpd_poll_fini(i915);
18856
18857         /*
18858          * MST topology needs to be suspended so we don't have any calls to
18859          * fbdev after it's finalized. MST will be destroyed later as part of
18860          * drm_mode_config_cleanup()
18861          */
18862         intel_dp_mst_suspend(i915);
18863
18864         /* poll work can call into fbdev, hence clean that up afterwards */
18865         intel_fbdev_fini(i915);
18866
18867         intel_unregister_dsm_handler();
18868
18869         intel_fbc_global_disable(i915);
18870
18871         /* flush any delayed tasks or pending work */
18872         flush_scheduled_work();
18873
18874         intel_hdcp_component_fini(i915);
18875
18876         intel_mode_config_cleanup(i915);
18877
18878         intel_overlay_cleanup(i915);
18879
18880         intel_gmbus_teardown(i915);
18881
18882         destroy_workqueue(i915->flip_wq);
18883         destroy_workqueue(i915->modeset_wq);
18884
18885         intel_fbc_cleanup_cfb(i915);
18886 }
18887
18888 #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
18889
18890 struct intel_display_error_state {
18891
18892         u32 power_well_driver;
18893
18894         struct intel_cursor_error_state {
18895                 u32 control;
18896                 u32 position;
18897                 u32 base;
18898                 u32 size;
18899         } cursor[I915_MAX_PIPES];
18900
18901         struct intel_pipe_error_state {
18902                 bool power_domain_on;
18903                 u32 source;
18904                 u32 stat;
18905         } pipe[I915_MAX_PIPES];
18906
18907         struct intel_plane_error_state {
18908                 u32 control;
18909                 u32 stride;
18910                 u32 size;
18911                 u32 pos;
18912                 u32 addr;
18913                 u32 surface;
18914                 u32 tile_offset;
18915         } plane[I915_MAX_PIPES];
18916
18917         struct intel_transcoder_error_state {
18918                 bool available;
18919                 bool power_domain_on;
18920                 enum transcoder cpu_transcoder;
18921
18922                 u32 conf;
18923
18924                 u32 htotal;
18925                 u32 hblank;
18926                 u32 hsync;
18927                 u32 vtotal;
18928                 u32 vblank;
18929                 u32 vsync;
18930         } transcoder[5];
18931 };
18932
18933 struct intel_display_error_state *
18934 intel_display_capture_error_state(struct drm_i915_private *dev_priv)
18935 {
18936         struct intel_display_error_state *error;
18937         int transcoders[] = {
18938                 TRANSCODER_A,
18939                 TRANSCODER_B,
18940                 TRANSCODER_C,
18941                 TRANSCODER_D,
18942                 TRANSCODER_EDP,
18943         };
18944         int i;
18945
18946         BUILD_BUG_ON(ARRAY_SIZE(transcoders) != ARRAY_SIZE(error->transcoder));
18947
18948         if (!HAS_DISPLAY(dev_priv) || !INTEL_DISPLAY_ENABLED(dev_priv))
18949                 return NULL;
18950
18951         error = kzalloc(sizeof(*error), GFP_ATOMIC);
18952         if (error == NULL)
18953                 return NULL;
18954
18955         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
18956                 error->power_well_driver = intel_de_read(dev_priv,
18957                                                          HSW_PWR_WELL_CTL2);
18958
18959         for_each_pipe(dev_priv, i) {
18960                 error->pipe[i].power_domain_on =
18961                         __intel_display_power_is_enabled(dev_priv,
18962                                                          POWER_DOMAIN_PIPE(i));
18963                 if (!error->pipe[i].power_domain_on)
18964                         continue;
18965
18966                 error->cursor[i].control = intel_de_read(dev_priv, CURCNTR(i));
18967                 error->cursor[i].position = intel_de_read(dev_priv, CURPOS(i));
18968                 error->cursor[i].base = intel_de_read(dev_priv, CURBASE(i));
18969
18970                 error->plane[i].control = intel_de_read(dev_priv, DSPCNTR(i));
18971                 error->plane[i].stride = intel_de_read(dev_priv, DSPSTRIDE(i));
18972                 if (INTEL_GEN(dev_priv) <= 3) {
18973                         error->plane[i].size = intel_de_read(dev_priv,
18974                                                              DSPSIZE(i));
18975                         error->plane[i].pos = intel_de_read(dev_priv,
18976                                                             DSPPOS(i));
18977                 }
18978                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
18979                         error->plane[i].addr = intel_de_read(dev_priv,
18980                                                              DSPADDR(i));
18981                 if (INTEL_GEN(dev_priv) >= 4) {
18982                         error->plane[i].surface = intel_de_read(dev_priv,
18983                                                                 DSPSURF(i));
18984                         error->plane[i].tile_offset = intel_de_read(dev_priv,
18985                                                                     DSPTILEOFF(i));
18986                 }
18987
18988                 error->pipe[i].source = intel_de_read(dev_priv, PIPESRC(i));
18989
18990                 if (HAS_GMCH(dev_priv))
18991                         error->pipe[i].stat = intel_de_read(dev_priv,
18992                                                             PIPESTAT(i));
18993         }
18994
18995         for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
18996                 enum transcoder cpu_transcoder = transcoders[i];
18997
18998                 if (!HAS_TRANSCODER(dev_priv, cpu_transcoder))
18999                         continue;
19000
19001                 error->transcoder[i].available = true;
19002                 error->transcoder[i].power_domain_on =
19003                         __intel_display_power_is_enabled(dev_priv,
19004                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder));
19005                 if (!error->transcoder[i].power_domain_on)
19006                         continue;
19007
19008                 error->transcoder[i].cpu_transcoder = cpu_transcoder;
19009
19010                 error->transcoder[i].conf = intel_de_read(dev_priv,
19011                                                           PIPECONF(cpu_transcoder));
19012                 error->transcoder[i].htotal = intel_de_read(dev_priv,
19013                                                             HTOTAL(cpu_transcoder));
19014                 error->transcoder[i].hblank = intel_de_read(dev_priv,
19015                                                             HBLANK(cpu_transcoder));
19016                 error->transcoder[i].hsync = intel_de_read(dev_priv,
19017                                                            HSYNC(cpu_transcoder));
19018                 error->transcoder[i].vtotal = intel_de_read(dev_priv,
19019                                                             VTOTAL(cpu_transcoder));
19020                 error->transcoder[i].vblank = intel_de_read(dev_priv,
19021                                                             VBLANK(cpu_transcoder));
19022                 error->transcoder[i].vsync = intel_de_read(dev_priv,
19023                                                            VSYNC(cpu_transcoder));
19024         }
19025
19026         return error;
19027 }
19028
19029 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
19030
19031 void
19032 intel_display_print_error_state(struct drm_i915_error_state_buf *m,
19033                                 struct intel_display_error_state *error)
19034 {
19035         struct drm_i915_private *dev_priv = m->i915;
19036         int i;
19037
19038         if (!error)
19039                 return;
19040
19041         err_printf(m, "Num Pipes: %d\n", INTEL_NUM_PIPES(dev_priv));
19042         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
19043                 err_printf(m, "PWR_WELL_CTL2: %08x\n",
19044                            error->power_well_driver);
19045         for_each_pipe(dev_priv, i) {
19046                 err_printf(m, "Pipe [%d]:\n", i);
19047                 err_printf(m, "  Power: %s\n",
19048                            onoff(error->pipe[i].power_domain_on));
19049                 err_printf(m, "  SRC: %08x\n", error->pipe[i].source);
19050                 err_printf(m, "  STAT: %08x\n", error->pipe[i].stat);
19051
19052                 err_printf(m, "Plane [%d]:\n", i);
19053                 err_printf(m, "  CNTR: %08x\n", error->plane[i].control);
19054                 err_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
19055                 if (INTEL_GEN(dev_priv) <= 3) {
19056                         err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
19057                         err_printf(m, "  POS: %08x\n", error->plane[i].pos);
19058                 }
19059                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
19060                         err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
19061                 if (INTEL_GEN(dev_priv) >= 4) {
19062                         err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
19063                         err_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
19064                 }
19065
19066                 err_printf(m, "Cursor [%d]:\n", i);
19067                 err_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
19068                 err_printf(m, "  POS: %08x\n", error->cursor[i].position);
19069                 err_printf(m, "  BASE: %08x\n", error->cursor[i].base);
19070         }
19071
19072         for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
19073                 if (!error->transcoder[i].available)
19074                         continue;
19075
19076                 err_printf(m, "CPU transcoder: %s\n",
19077                            transcoder_name(error->transcoder[i].cpu_transcoder));
19078                 err_printf(m, "  Power: %s\n",
19079                            onoff(error->transcoder[i].power_domain_on));
19080                 err_printf(m, "  CONF: %08x\n", error->transcoder[i].conf);
19081                 err_printf(m, "  HTOTAL: %08x\n", error->transcoder[i].htotal);
19082                 err_printf(m, "  HBLANK: %08x\n", error->transcoder[i].hblank);
19083                 err_printf(m, "  HSYNC: %08x\n", error->transcoder[i].hsync);
19084                 err_printf(m, "  VTOTAL: %08x\n", error->transcoder[i].vtotal);
19085                 err_printf(m, "  VBLANK: %08x\n", error->transcoder[i].vblank);
19086                 err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
19087         }
19088 }
19089
19090 #endif