drivers/gpu/drm/i915/display/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/i2c.h>
  28 #include <linux/input.h>
  29 #include <linux/intel-iommu.h>
  30 #include <linux/kernel.h>
  31 #include <linux/module.h>
  32 #include <linux/dma-resv.h>
  33 #include <linux/slab.h>
  34
  35 #include <drm/drm_atomic.h>
  36 #include <drm/drm_atomic_helper.h>
  37 #include <drm/drm_atomic_uapi.h>
  38 #include <drm/drm_dp_helper.h>
  39 #include <drm/drm_edid.h>
  40 #include <drm/drm_fourcc.h>
  41 #include <drm/drm_plane_helper.h>
  42 #include <drm/drm_probe_helper.h>
  43 #include <drm/drm_rect.h>
  44 #include <drm/i915_drm.h>
  45
  46 #include "display/intel_crt.h"
  47 #include "display/intel_ddi.h"
  48 #include "display/intel_dp.h"
  49 #include "display/intel_dp_mst.h"
  50 #include "display/intel_dsi.h"
  51 #include "display/intel_dvo.h"
  52 #include "display/intel_gmbus.h"
  53 #include "display/intel_hdmi.h"
  54 #include "display/intel_lvds.h"
  55 #include "display/intel_sdvo.h"
  56 #include "display/intel_tv.h"
  57 #include "display/intel_vdsc.h"
  58
  59 #include "gt/intel_rps.h"
  60
  61 #include "i915_drv.h"
  62 #include "i915_trace.h"
  63 #include "intel_acpi.h"
  64 #include "intel_atomic.h"
  65 #include "intel_atomic_plane.h"
  66 #include "intel_bw.h"
  67 #include "intel_cdclk.h"
  68 #include "intel_color.h"
  69 #include "intel_display_types.h"
  70 #include "intel_dp_link_training.h"
  71 #include "intel_fbc.h"
  72 #include "intel_fbdev.h"
  73 #include "intel_fifo_underrun.h"
  74 #include "intel_frontbuffer.h"
  75 #include "intel_hdcp.h"
  76 #include "intel_hotplug.h"
  77 #include "intel_overlay.h"
  78 #include "intel_pipe_crc.h"
  79 #include "intel_pm.h"
  80 #include "intel_psr.h"
  81 #include "intel_quirks.h"
  82 #include "intel_sideband.h"
  83 #include "intel_sprite.h"
  84 #include "intel_tc.h"
  85 #include "intel_vga.h"
  86
  87 /* Primary plane formats for gen <= 3 */
  88 static const u32 i8xx_primary_formats[] = {
  89         DRM_FORMAT_C8,
  90         DRM_FORMAT_XRGB1555,
  91         DRM_FORMAT_RGB565,
  92         DRM_FORMAT_XRGB8888,
  93 };
  94
  95 /* Primary plane formats for ivb (no fp16 due to hw issue) */
  96 static const u32 ivb_primary_formats[] = {
  97         DRM_FORMAT_C8,
  98         DRM_FORMAT_RGB565,
  99         DRM_FORMAT_XRGB8888,
 100         DRM_FORMAT_XBGR8888,
 101         DRM_FORMAT_XRGB2101010,
 102         DRM_FORMAT_XBGR2101010,
 103 };
 104
 105 /* Primary plane formats for gen >= 4, except ivb */
 106 static const u32 i965_primary_formats[] = {
 107         DRM_FORMAT_C8,
 108         DRM_FORMAT_RGB565,
 109         DRM_FORMAT_XRGB8888,
 110         DRM_FORMAT_XBGR8888,
 111         DRM_FORMAT_XRGB2101010,
 112         DRM_FORMAT_XBGR2101010,
 113         DRM_FORMAT_XBGR16161616F,
 114 };
 115
 116 /* Primary plane formats for vlv/chv */
 117 static const u32 vlv_primary_formats[] = {
 118         DRM_FORMAT_C8,
 119         DRM_FORMAT_RGB565,
 120         DRM_FORMAT_XRGB8888,
 121         DRM_FORMAT_XBGR8888,
 122         DRM_FORMAT_ARGB8888,
 123         DRM_FORMAT_ABGR8888,
 124         DRM_FORMAT_XRGB2101010,
 125         DRM_FORMAT_XBGR2101010,
 126         DRM_FORMAT_ARGB2101010,
 127         DRM_FORMAT_ABGR2101010,
 128         DRM_FORMAT_XBGR16161616F,
 129 };
 130
 131 static const u64 i9xx_format_modifiers[] = {
 132         I915_FORMAT_MOD_X_TILED,
 133         DRM_FORMAT_MOD_LINEAR,
 134         DRM_FORMAT_MOD_INVALID
 135 };
 136
 137 /* Cursor formats */
 138 static const u32 intel_cursor_formats[] = {
 139         DRM_FORMAT_ARGB8888,
 140 };
 141
 142 static const u64 cursor_format_modifiers[] = {
 143         DRM_FORMAT_MOD_LINEAR,
 144         DRM_FORMAT_MOD_INVALID
 145 };
 146
 147 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
 148                                 struct intel_crtc_state *pipe_config);
 149 static void ilk_pch_clock_get(struct intel_crtc *crtc,
 150                               struct intel_crtc_state *pipe_config);
 151
 152 static int intel_framebuffer_init(struct intel_framebuffer *ifb,
 153                                   struct drm_i915_gem_object *obj,
 154                                   struct drm_mode_fb_cmd2 *mode_cmd);
 155 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state);
 156 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state);
 157 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
 158                                          const struct intel_link_m_n *m_n,
 159                                          const struct intel_link_m_n *m2_n2);
 160 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state);
 161 static void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state);
 162 static void hsw_set_pipeconf(const struct intel_crtc_state *crtc_state);
 163 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state);
 164 static void vlv_prepare_pll(struct intel_crtc *crtc,
 165                             const struct intel_crtc_state *pipe_config);
 166 static void chv_prepare_pll(struct intel_crtc *crtc,
 167                             const struct intel_crtc_state *pipe_config);
 168 static void skl_pfit_enable(const struct intel_crtc_state *crtc_state);
 169 static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state);
 170 static void intel_modeset_setup_hw_state(struct drm_device *dev,
 171                                          struct drm_modeset_acquire_ctx *ctx);
 172 static struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc);
 173
 174 struct intel_limit {
 175         struct {
 176                 int min, max;
 177         } dot, vco, n, m, m1, m2, p, p1;
 178
 179         struct {
 180                 int dot_limit;
 181                 int p2_slow, p2_fast;
 182         } p2;
 183 };
 184
 185 /* returns HPLL frequency in kHz */
 186 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv)
 187 {
 188         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 189
 190         /* Obtain SKU information */
 191         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
 192                 CCK_FUSE_HPLL_FREQ_MASK;
 193
 194         return vco_freq[hpll_freq] * 1000;
 195 }
 196
 197 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
 198                       const char *name, u32 reg, int ref_freq)
 199 {
 200         u32 val;
 201         int divider;
 202
 203         val = vlv_cck_read(dev_priv, reg);
 204         divider = val & CCK_FREQUENCY_VALUES;
 205
 206         WARN((val & CCK_FREQUENCY_STATUS) !=
 207              (divider << CCK_FREQUENCY_STATUS_SHIFT),
 208              "%s change in progress\n", name);
 209
 210         return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1);
 211 }
 212
 213 int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 214                            const char *name, u32 reg)
 215 {
 216         int hpll;
 217
 218         vlv_cck_get(dev_priv);
 219
 220         if (dev_priv->hpll_freq == 0)
 221                 dev_priv->hpll_freq = vlv_get_hpll_vco(dev_priv);
 222
 223         hpll = vlv_get_cck_clock(dev_priv, name, reg, dev_priv->hpll_freq);
 224
 225         vlv_cck_put(dev_priv);
 226
 227         return hpll;
 228 }
 229
 230 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 231 {
 232         if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
 233                 return;
 234
 235         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
 236                                                       CCK_CZ_CLOCK_CONTROL);
 237
 238         DRM_DEBUG_DRIVER("CZ clock rate: %d kHz\n", dev_priv->czclk_freq);
 239 }
 240
 241 static inline u32 /* units of 100MHz */
 242 intel_fdi_link_freq(struct drm_i915_private *dev_priv,
 243                     const struct intel_crtc_state *pipe_config)
 244 {
 245         if (HAS_DDI(dev_priv))
 246                 return pipe_config->port_clock; /* SPLL */
 247         else
 248                 return dev_priv->fdi_pll_freq;
 249 }
 250
 251 static const struct intel_limit intel_limits_i8xx_dac = {
 252         .dot = { .min = 25000, .max = 350000 },
 253         .vco = { .min = 908000, .max = 1512000 },
 254         .n = { .min = 2, .max = 16 },
 255         .m = { .min = 96, .max = 140 },
 256         .m1 = { .min = 18, .max = 26 },
 257         .m2 = { .min = 6, .max = 16 },
 258         .p = { .min = 4, .max = 128 },
 259         .p1 = { .min = 2, .max = 33 },
 260         .p2 = { .dot_limit = 165000,
 261                 .p2_slow = 4, .p2_fast = 2 },
 262 };
 263
 264 static const struct intel_limit intel_limits_i8xx_dvo = {
 265         .dot = { .min = 25000, .max = 350000 },
 266         .vco = { .min = 908000, .max = 1512000 },
 267         .n = { .min = 2, .max = 16 },
 268         .m = { .min = 96, .max = 140 },
 269         .m1 = { .min = 18, .max = 26 },
 270         .m2 = { .min = 6, .max = 16 },
 271         .p = { .min = 4, .max = 128 },
 272         .p1 = { .min = 2, .max = 33 },
 273         .p2 = { .dot_limit = 165000,
 274                 .p2_slow = 4, .p2_fast = 4 },
 275 };
 276
 277 static const struct intel_limit intel_limits_i8xx_lvds = {
 278         .dot = { .min = 25000, .max = 350000 },
 279         .vco = { .min = 908000, .max = 1512000 },
 280         .n = { .min = 2, .max = 16 },
 281         .m = { .min = 96, .max = 140 },
 282         .m1 = { .min = 18, .max = 26 },
 283         .m2 = { .min = 6, .max = 16 },
 284         .p = { .min = 4, .max = 128 },
 285         .p1 = { .min = 1, .max = 6 },
 286         .p2 = { .dot_limit = 165000,
 287                 .p2_slow = 14, .p2_fast = 7 },
 288 };
 289
 290 static const struct intel_limit intel_limits_i9xx_sdvo = {
 291         .dot = { .min = 20000, .max = 400000 },
 292         .vco = { .min = 1400000, .max = 2800000 },
 293         .n = { .min = 1, .max = 6 },
 294         .m = { .min = 70, .max = 120 },
 295         .m1 = { .min = 8, .max = 18 },
 296         .m2 = { .min = 3, .max = 7 },
 297         .p = { .min = 5, .max = 80 },
 298         .p1 = { .min = 1, .max = 8 },
 299         .p2 = { .dot_limit = 200000,
 300                 .p2_slow = 10, .p2_fast = 5 },
 301 };
 302
 303 static const struct intel_limit intel_limits_i9xx_lvds = {
 304         .dot = { .min = 20000, .max = 400000 },
 305         .vco = { .min = 1400000, .max = 2800000 },
 306         .n = { .min = 1, .max = 6 },
 307         .m = { .min = 70, .max = 120 },
 308         .m1 = { .min = 8, .max = 18 },
 309         .m2 = { .min = 3, .max = 7 },
 310         .p = { .min = 7, .max = 98 },
 311         .p1 = { .min = 1, .max = 8 },
 312         .p2 = { .dot_limit = 112000,
 313                 .p2_slow = 14, .p2_fast = 7 },
 314 };
 315
 316
 317 static const struct intel_limit intel_limits_g4x_sdvo = {
 318         .dot = { .min = 25000, .max = 270000 },
 319         .vco = { .min = 1750000, .max = 3500000},
 320         .n = { .min = 1, .max = 4 },
 321         .m = { .min = 104, .max = 138 },
 322         .m1 = { .min = 17, .max = 23 },
 323         .m2 = { .min = 5, .max = 11 },
 324         .p = { .min = 10, .max = 30 },
 325         .p1 = { .min = 1, .max = 3},
 326         .p2 = { .dot_limit = 270000,
 327                 .p2_slow = 10,
 328                 .p2_fast = 10
 329         },
 330 };
 331
 332 static const struct intel_limit intel_limits_g4x_hdmi = {
 333         .dot = { .min = 22000, .max = 400000 },
 334         .vco = { .min = 1750000, .max = 3500000},
 335         .n = { .min = 1, .max = 4 },
 336         .m = { .min = 104, .max = 138 },
 337         .m1 = { .min = 16, .max = 23 },
 338         .m2 = { .min = 5, .max = 11 },
 339         .p = { .min = 5, .max = 80 },
 340         .p1 = { .min = 1, .max = 8},
 341         .p2 = { .dot_limit = 165000,
 342                 .p2_slow = 10, .p2_fast = 5 },
 343 };
 344
 345 static const struct intel_limit intel_limits_g4x_single_channel_lvds = {
 346         .dot = { .min = 20000, .max = 115000 },
 347         .vco = { .min = 1750000, .max = 3500000 },
 348         .n = { .min = 1, .max = 3 },
 349         .m = { .min = 104, .max = 138 },
 350         .m1 = { .min = 17, .max = 23 },
 351         .m2 = { .min = 5, .max = 11 },
 352         .p = { .min = 28, .max = 112 },
 353         .p1 = { .min = 2, .max = 8 },
 354         .p2 = { .dot_limit = 0,
 355                 .p2_slow = 14, .p2_fast = 14
 356         },
 357 };
 358
 359 static const struct intel_limit intel_limits_g4x_dual_channel_lvds = {
 360         .dot = { .min = 80000, .max = 224000 },
 361         .vco = { .min = 1750000, .max = 3500000 },
 362         .n = { .min = 1, .max = 3 },
 363         .m = { .min = 104, .max = 138 },
 364         .m1 = { .min = 17, .max = 23 },
 365         .m2 = { .min = 5, .max = 11 },
 366         .p = { .min = 14, .max = 42 },
 367         .p1 = { .min = 2, .max = 6 },
 368         .p2 = { .dot_limit = 0,
 369                 .p2_slow = 7, .p2_fast = 7
 370         },
 371 };
 372
 373 static const struct intel_limit pnv_limits_sdvo = {
 374         .dot = { .min = 20000, .max = 400000},
 375         .vco = { .min = 1700000, .max = 3500000 },
 376         /* Pineview's Ncounter is a ring counter */
 377         .n = { .min = 3, .max = 6 },
 378         .m = { .min = 2, .max = 256 },
 379         /* Pineview only has one combined m divider, which we treat as m2. */
 380         .m1 = { .min = 0, .max = 0 },
 381         .m2 = { .min = 0, .max = 254 },
 382         .p = { .min = 5, .max = 80 },
 383         .p1 = { .min = 1, .max = 8 },
 384         .p2 = { .dot_limit = 200000,
 385                 .p2_slow = 10, .p2_fast = 5 },
 386 };
 387
 388 static const struct intel_limit pnv_limits_lvds = {
 389         .dot = { .min = 20000, .max = 400000 },
 390         .vco = { .min = 1700000, .max = 3500000 },
 391         .n = { .min = 3, .max = 6 },
 392         .m = { .min = 2, .max = 256 },
 393         .m1 = { .min = 0, .max = 0 },
 394         .m2 = { .min = 0, .max = 254 },
 395         .p = { .min = 7, .max = 112 },
 396         .p1 = { .min = 1, .max = 8 },
 397         .p2 = { .dot_limit = 112000,
 398                 .p2_slow = 14, .p2_fast = 14 },
 399 };
 400
 401 /* Ironlake / Sandybridge
 402  *
 403  * We calculate clock using (register_value + 2) for N/M1/M2, so here
 404  * the range value for them is (actual_value - 2).
 405  */
 406 static const struct intel_limit ilk_limits_dac = {
 407         .dot = { .min = 25000, .max = 350000 },
 408         .vco = { .min = 1760000, .max = 3510000 },
 409         .n = { .min = 1, .max = 5 },
 410         .m = { .min = 79, .max = 127 },
 411         .m1 = { .min = 12, .max = 22 },
 412         .m2 = { .min = 5, .max = 9 },
 413         .p = { .min = 5, .max = 80 },
 414         .p1 = { .min = 1, .max = 8 },
 415         .p2 = { .dot_limit = 225000,
 416                 .p2_slow = 10, .p2_fast = 5 },
 417 };
 418
 419 static const struct intel_limit ilk_limits_single_lvds = {
 420         .dot = { .min = 25000, .max = 350000 },
 421         .vco = { .min = 1760000, .max = 3510000 },
 422         .n = { .min = 1, .max = 3 },
 423         .m = { .min = 79, .max = 118 },
 424         .m1 = { .min = 12, .max = 22 },
 425         .m2 = { .min = 5, .max = 9 },
 426         .p = { .min = 28, .max = 112 },
 427         .p1 = { .min = 2, .max = 8 },
 428         .p2 = { .dot_limit = 225000,
 429                 .p2_slow = 14, .p2_fast = 14 },
 430 };
 431
 432 static const struct intel_limit ilk_limits_dual_lvds = {
 433         .dot = { .min = 25000, .max = 350000 },
 434         .vco = { .min = 1760000, .max = 3510000 },
 435         .n = { .min = 1, .max = 3 },
 436         .m = { .min = 79, .max = 127 },
 437         .m1 = { .min = 12, .max = 22 },
 438         .m2 = { .min = 5, .max = 9 },
 439         .p = { .min = 14, .max = 56 },
 440         .p1 = { .min = 2, .max = 8 },
 441         .p2 = { .dot_limit = 225000,
 442                 .p2_slow = 7, .p2_fast = 7 },
 443 };
 444
 445 /* LVDS 100mhz refclk limits. */
 446 static const struct intel_limit ilk_limits_single_lvds_100m = {
 447         .dot = { .min = 25000, .max = 350000 },
 448         .vco = { .min = 1760000, .max = 3510000 },
 449         .n = { .min = 1, .max = 2 },
 450         .m = { .min = 79, .max = 126 },
 451         .m1 = { .min = 12, .max = 22 },
 452         .m2 = { .min = 5, .max = 9 },
 453         .p = { .min = 28, .max = 112 },
 454         .p1 = { .min = 2, .max = 8 },
 455         .p2 = { .dot_limit = 225000,
 456                 .p2_slow = 14, .p2_fast = 14 },
 457 };
 458
 459 static const struct intel_limit ilk_limits_dual_lvds_100m = {
 460         .dot = { .min = 25000, .max = 350000 },
 461         .vco = { .min = 1760000, .max = 3510000 },
 462         .n = { .min = 1, .max = 3 },
 463         .m = { .min = 79, .max = 126 },
 464         .m1 = { .min = 12, .max = 22 },
 465         .m2 = { .min = 5, .max = 9 },
 466         .p = { .min = 14, .max = 42 },
 467         .p1 = { .min = 2, .max = 6 },
 468         .p2 = { .dot_limit = 225000,
 469                 .p2_slow = 7, .p2_fast = 7 },
 470 };
 471
 472 static const struct intel_limit intel_limits_vlv = {
 473          /*
 474           * These are the data rate limits (measured in fast clocks)
 475           * since those are the strictest limits we have. The fast
 476           * clock and actual rate limits are more relaxed, so checking
 477           * them would make no difference.
 478           */
 479         .dot = { .min = 25000 * 5, .max = 270000 * 5 },
 480         .vco = { .min = 4000000, .max = 6000000 },
 481         .n = { .min = 1, .max = 7 },
 482         .m1 = { .min = 2, .max = 3 },
 483         .m2 = { .min = 11, .max = 156 },
 484         .p1 = { .min = 2, .max = 3 },
 485         .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */
 486 };
 487
 488 static const struct intel_limit intel_limits_chv = {
 489         /*
 490          * These are the data rate limits (measured in fast clocks)
 491          * since those are the strictest limits we have.  The fast
 492          * clock and actual rate limits are more relaxed, so checking
 493          * them would make no difference.
 494          */
 495         .dot = { .min = 25000 * 5, .max = 540000 * 5},
 496         .vco = { .min = 4800000, .max = 6480000 },
 497         .n = { .min = 1, .max = 1 },
 498         .m1 = { .min = 2, .max = 2 },
 499         .m2 = { .min = 24 << 22, .max = 175 << 22 },
 500         .p1 = { .min = 2, .max = 4 },
 501         .p2 = { .p2_slow = 1, .p2_fast = 14 },
 502 };
 503
 504 static const struct intel_limit intel_limits_bxt = {
 505         /* FIXME: find real dot limits */
 506         .dot = { .min = 0, .max = INT_MAX },
 507         .vco = { .min = 4800000, .max = 6700000 },
 508         .n = { .min = 1, .max = 1 },
 509         .m1 = { .min = 2, .max = 2 },
 510         /* FIXME: find real m2 limits */
 511         .m2 = { .min = 2 << 22, .max = 255 << 22 },
 512         .p1 = { .min = 2, .max = 4 },
 513         .p2 = { .p2_slow = 1, .p2_fast = 20 },
 514 };
 515
 516 /* WA Display #0827: Gen9:all */
 517 static void
 518 skl_wa_827(struct drm_i915_private *dev_priv, enum pipe pipe, bool enable)
 519 {
 520         if (enable)
 521                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 522                            I915_READ(CLKGATE_DIS_PSL(pipe)) |
 523                            DUPS1_GATING_DIS | DUPS2_GATING_DIS);
 524         else
 525                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 526                            I915_READ(CLKGATE_DIS_PSL(pipe)) &
 527                            ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS));
 528 }
 529
 530 /* Wa_2006604312:icl */
 531 static void
 532 icl_wa_scalerclkgating(struct drm_i915_private *dev_priv, enum pipe pipe,
 533                        bool enable)
 534 {
 535         if (enable)
 536                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 537                            I915_READ(CLKGATE_DIS_PSL(pipe)) | DPFR_GATING_DIS);
 538         else
 539                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 540                            I915_READ(CLKGATE_DIS_PSL(pipe)) & ~DPFR_GATING_DIS);
 541 }
 542
 543 static bool
 544 needs_modeset(const struct intel_crtc_state *state)
 545 {
 546         return drm_atomic_crtc_needs_modeset(&state->uapi);
 547 }
 548
 549 bool
 550 is_trans_port_sync_mode(const struct intel_crtc_state *crtc_state)
 551 {
 552         return (crtc_state->master_transcoder != INVALID_TRANSCODER ||
 553                 crtc_state->sync_mode_slaves_mask);
 554 }
 555
 556 static bool
 557 is_trans_port_sync_master(const struct intel_crtc_state *crtc_state)
 558 {
 559         return (crtc_state->master_transcoder == INVALID_TRANSCODER &&
 560                 crtc_state->sync_mode_slaves_mask);
 561 }
 562
 563 static bool
 564 is_trans_port_sync_slave(const struct intel_crtc_state *crtc_state)
 565 {
 566         return crtc_state->master_transcoder != INVALID_TRANSCODER;
 567 }
 568
 569 /*
 570  * Platform specific helpers to calculate the port PLL loopback- (clock.m),
 571  * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast
 572  * (clock.dot) clock rates. This fast dot clock is fed to the port's IO logic.
 573  * The helpers' return value is the rate of the clock that is fed to the
 574  * display engine's pipe which can be the above fast dot clock rate or a
 575  * divided-down version of it.
 576  */
 577 /* m1 is reserved as 0 in Pineview, n is a ring counter */
 578 static int pnv_calc_dpll_params(int refclk, struct dpll *clock)
 579 {
 580         clock->m = clock->m2 + 2;
 581         clock->p = clock->p1 * clock->p2;
 582         if (WARN_ON(clock->n == 0 || clock->p == 0))
 583                 return 0;
 584         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 585         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 586
 587         return clock->dot;
 588 }
 589
 590 static u32 i9xx_dpll_compute_m(struct dpll *dpll)
 591 {
 592         return 5 * (dpll->m1 + 2) + (dpll->m2 + 2);
 593 }
 594
 595 static int i9xx_calc_dpll_params(int refclk, struct dpll *clock)
 596 {
 597         clock->m = i9xx_dpll_compute_m(clock);
 598         clock->p = clock->p1 * clock->p2;
 599         if (WARN_ON(clock->n + 2 == 0 || clock->p == 0))
 600                 return 0;
 601         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n + 2);
 602         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 603
 604         return clock->dot;
 605 }
 606
 607 static int vlv_calc_dpll_params(int refclk, struct dpll *clock)
 608 {
 609         clock->m = clock->m1 * clock->m2;
 610         clock->p = clock->p1 * clock->p2;
 611         if (WARN_ON(clock->n == 0 || clock->p == 0))
 612                 return 0;
 613         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 614         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 615
 616         return clock->dot / 5;
 617 }
 618
 619 int chv_calc_dpll_params(int refclk, struct dpll *clock)
 620 {
 621         clock->m = clock->m1 * clock->m2;
 622         clock->p = clock->p1 * clock->p2;
 623         if (WARN_ON(clock->n == 0 || clock->p == 0))
 624                 return 0;
 625         clock->vco = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(refclk, clock->m),
 626                                            clock->n << 22);
 627         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 628
 629         return clock->dot / 5;
 630 }
 631
 632 #define INTELPllInvalid(s)   do { /* DRM_DEBUG(s); */ return false; } while (0)
 633
 634 /*
 635  * Returns whether the given set of divisors are valid for a given refclk with
 636  * the given connectors.
 637  */
 638 static bool intel_PLL_is_valid(struct drm_i915_private *dev_priv,
 639                                const struct intel_limit *limit,
 640                                const struct dpll *clock)
 641 {
 642         if (clock->n   < limit->n.min   || limit->n.max   < clock->n)
 643                 INTELPllInvalid("n out of range\n");
 644         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
 645                 INTELPllInvalid("p1 out of range\n");
 646         if (clock->m2  < limit->m2.min  || limit->m2.max  < clock->m2)
 647                 INTELPllInvalid("m2 out of range\n");
 648         if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
 649                 INTELPllInvalid("m1 out of range\n");
 650
 651         if (!IS_PINEVIEW(dev_priv) && !IS_VALLEYVIEW(dev_priv) &&
 652             !IS_CHERRYVIEW(dev_priv) && !IS_GEN9_LP(dev_priv))
 653                 if (clock->m1 <= clock->m2)
 654                         INTELPllInvalid("m1 <= m2\n");
 655
 656         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
 657             !IS_GEN9_LP(dev_priv)) {
 658                 if (clock->p < limit->p.min || limit->p.max < clock->p)
 659                         INTELPllInvalid("p out of range\n");
 660                 if (clock->m < limit->m.min || limit->m.max < clock->m)
 661                         INTELPllInvalid("m out of range\n");
 662         }
 663
 664         if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
 665                 INTELPllInvalid("vco out of range\n");
 666         /* XXX: We may need to be checking "Dot clock" depending on the multiplier,
 667          * connector, etc., rather than just a single range.
 668          */
 669         if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
 670                 INTELPllInvalid("dot out of range\n");
 671
 672         return true;
 673 }
 674
 675 static int
 676 i9xx_select_p2_div(const struct intel_limit *limit,
 677                    const struct intel_crtc_state *crtc_state,
 678                    int target)
 679 {
 680         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 681
 682         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 683                 /*
 684                  * For LVDS just rely on its current settings for dual-channel.
 685                  * We haven't figured out how to reliably set up different
 686                  * single/dual channel state, if we even can.
 687                  */
 688                 if (intel_is_dual_link_lvds(dev_priv))
 689                         return limit->p2.p2_fast;
 690                 else
 691                         return limit->p2.p2_slow;
 692         } else {
 693                 if (target < limit->p2.dot_limit)
 694                         return limit->p2.p2_slow;
 695                 else
 696                         return limit->p2.p2_fast;
 697         }
 698 }
 699
 700 /*
 701  * Returns a set of divisors for the desired target clock with the given
 702  * refclk, or FALSE.  The returned values represent the clock equation:
 703  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 704  *
 705  * Target and reference clocks are specified in kHz.
 706  *
 707  * If match_clock is provided, then best_clock P divider must match the P
 708  * divider from @match_clock used for LVDS downclocking.
 709  */
 710 static bool
 711 i9xx_find_best_dpll(const struct intel_limit *limit,
 712                     struct intel_crtc_state *crtc_state,
 713                     int target, int refclk, struct dpll *match_clock,
 714                     struct dpll *best_clock)
 715 {
 716         struct drm_device *dev = crtc_state->uapi.crtc->dev;
 717         struct dpll clock;
 718         int err = target;
 719
 720         memset(best_clock, 0, sizeof(*best_clock));
 721
 722         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 723
 724         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 725              clock.m1++) {
 726                 for (clock.m2 = limit->m2.min;
 727                      clock.m2 <= limit->m2.max; clock.m2++) {
 728                         if (clock.m2 >= clock.m1)
 729                                 break;
 730                         for (clock.n = limit->n.min;
 731                              clock.n <= limit->n.max; clock.n++) {
 732                                 for (clock.p1 = limit->p1.min;
 733                                         clock.p1 <= limit->p1.max; clock.p1++) {
 734                                         int this_err;
 735
 736                                         i9xx_calc_dpll_params(refclk, &clock);
 737                                         if (!intel_PLL_is_valid(to_i915(dev),
 738                                                                 limit,
 739                                                                 &clock))
 740                                                 continue;
 741                                         if (match_clock &&
 742                                             clock.p != match_clock->p)
 743                                                 continue;
 744
 745                                         this_err = abs(clock.dot - target);
 746                                         if (this_err < err) {
 747                                                 *best_clock = clock;
 748                                                 err = this_err;
 749                                         }
 750                                 }
 751                         }
 752                 }
 753         }
 754
 755         return (err != target);
 756 }
 757
 758 /*
 759  * Returns a set of divisors for the desired target clock with the given
 760  * refclk, or FALSE.  The returned values represent the clock equation:
 761  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 762  *
 763  * Target and reference clocks are specified in kHz.
 764  *
 765  * If match_clock is provided, then best_clock P divider must match the P
 766  * divider from @match_clock used for LVDS downclocking.
 767  */
 768 static bool
 769 pnv_find_best_dpll(const struct intel_limit *limit,
 770                    struct intel_crtc_state *crtc_state,
 771                    int target, int refclk, struct dpll *match_clock,
 772                    struct dpll *best_clock)
 773 {
 774         struct drm_device *dev = crtc_state->uapi.crtc->dev;
 775         struct dpll clock;
 776         int err = target;
 777
 778         memset(best_clock, 0, sizeof(*best_clock));
 779
 780         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 781
 782         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 783              clock.m1++) {
 784                 for (clock.m2 = limit->m2.min;
 785                      clock.m2 <= limit->m2.max; clock.m2++) {
 786                         for (clock.n = limit->n.min;
 787                              clock.n <= limit->n.max; clock.n++) {
 788                                 for (clock.p1 = limit->p1.min;
 789                                         clock.p1 <= limit->p1.max; clock.p1++) {
 790                                         int this_err;
 791
 792                                         pnv_calc_dpll_params(refclk, &clock);
 793                                         if (!intel_PLL_is_valid(to_i915(dev),
 794                                                                 limit,
 795                                                                 &clock))
 796                                                 continue;
 797                                         if (match_clock &&
 798                                             clock.p != match_clock->p)
 799                                                 continue;
 800
 801                                         this_err = abs(clock.dot - target);
 802                                         if (this_err < err) {
 803                                                 *best_clock = clock;
 804                                                 err = this_err;
 805                                         }
 806                                 }
 807                         }
 808                 }
 809         }
 810
 811         return (err != target);
 812 }
 813
 814 /*
 815  * Returns a set of divisors for the desired target clock with the given
 816  * refclk, or FALSE.  The returned values represent the clock equation:
 817  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 818  *
 819  * Target and reference clocks are specified in kHz.
 820  *
 821  * If match_clock is provided, then best_clock P divider must match the P
 822  * divider from @match_clock used for LVDS downclocking.
 823  */
 824 static bool
 825 g4x_find_best_dpll(const struct intel_limit *limit,
 826                    struct intel_crtc_state *crtc_state,
 827                    int target, int refclk, struct dpll *match_clock,
 828                    struct dpll *best_clock)
 829 {
 830         struct drm_device *dev = crtc_state->uapi.crtc->dev;
 831         struct dpll clock;
 832         int max_n;
 833         bool found = false;
 834         /* approximately equals target * 0.00585 */
 835         int err_most = (target >> 8) + (target >> 9);
 836
 837         memset(best_clock, 0, sizeof(*best_clock));
 838
 839         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 840
 841         max_n = limit->n.max;
 842         /* based on hardware requirement, prefer smaller n to precision */
 843         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 844                 /* based on hardware requirement, prefere larger m1,m2 */
 845                 for (clock.m1 = limit->m1.max;
 846                      clock.m1 >= limit->m1.min; clock.m1--) {
 847                         for (clock.m2 = limit->m2.max;
 848                              clock.m2 >= limit->m2.min; clock.m2--) {
 849                                 for (clock.p1 = limit->p1.max;
 850                                      clock.p1 >= limit->p1.min; clock.p1--) {
 851                                         int this_err;
 852
 853                                         i9xx_calc_dpll_params(refclk, &clock);
 854                                         if (!intel_PLL_is_valid(to_i915(dev),
 855                                                                 limit,
 856                                                                 &clock))
 857                                                 continue;
 858
 859                                         this_err = abs(clock.dot - target);
 860                                         if (this_err < err_most) {
 861                                                 *best_clock = clock;
 862                                                 err_most = this_err;
 863                                                 max_n = clock.n;
 864                                                 found = true;
 865                                         }
 866                                 }
 867                         }
 868                 }
 869         }
 870         return found;
 871 }
 872
 873 /*
 874  * Check if the calculated PLL configuration is more optimal compared to the
 875  * best configuration and error found so far. Return the calculated error.
 876  */
 877 static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
 878                                const struct dpll *calculated_clock,
 879                                const struct dpll *best_clock,
 880                                unsigned int best_error_ppm,
 881                                unsigned int *error_ppm)
 882 {
 883         /*
 884          * For CHV ignore the error and consider only the P value.
 885          * Prefer a bigger P value based on HW requirements.
 886          */
 887         if (IS_CHERRYVIEW(to_i915(dev))) {
 888                 *error_ppm = 0;
 889
 890                 return calculated_clock->p > best_clock->p;
 891         }
 892
 893         if (WARN_ON_ONCE(!target_freq))
 894                 return false;
 895
 896         *error_ppm = div_u64(1000000ULL *
 897                                 abs(target_freq - calculated_clock->dot),
 898                              target_freq);
 899         /*
 900          * Prefer a better P value over a better (smaller) error if the error
 901          * is small. Ensure this preference for future configurations too by
 902          * setting the error to 0.
 903          */
 904         if (*error_ppm < 100 && calculated_clock->p > best_clock->p) {
 905                 *error_ppm = 0;
 906
 907                 return true;
 908         }
 909
 910         return *error_ppm + 10 < best_error_ppm;
 911 }
 912
 913 /*
 914  * Returns a set of divisors for the desired target clock with the given
 915  * refclk, or FALSE.  The returned values represent the clock equation:
 916  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 917  */
 918 static bool
 919 vlv_find_best_dpll(const struct intel_limit *limit,
 920                    struct intel_crtc_state *crtc_state,
 921                    int target, int refclk, struct dpll *match_clock,
 922                    struct dpll *best_clock)
 923 {
 924         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 925         struct drm_device *dev = crtc->base.dev;
 926         struct dpll clock;
 927         unsigned int bestppm = 1000000;
 928         /* min update 19.2 MHz */
 929         int max_n = min(limit->n.max, refclk / 19200);
 930         bool found = false;
 931
 932         target *= 5; /* fast clock */
 933
 934         memset(best_clock, 0, sizeof(*best_clock));
 935
 936         /* based on hardware requirement, prefer smaller n to precision */
 937         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 938                 for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 939                         for (clock.p2 = limit->p2.p2_fast; clock.p2 >= limit->p2.p2_slow;
 940                              clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 941                                 clock.p = clock.p1 * clock.p2;
 942                                 /* based on hardware requirement, prefer bigger m1,m2 values */
 943                                 for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) {
 944                                         unsigned int ppm;
 945
 946                                         clock.m2 = DIV_ROUND_CLOSEST(target * clock.p * clock.n,
 947                                                                      refclk * clock.m1);
 948
 949                                         vlv_calc_dpll_params(refclk, &clock);
 950
 951                                         if (!intel_PLL_is_valid(to_i915(dev),
 952                                                                 limit,
 953                                                                 &clock))
 954                                                 continue;
 955
 956                                         if (!vlv_PLL_is_optimal(dev, target,
 957                                                                 &clock,
 958                                                                 best_clock,
 959                                                                 bestppm, &ppm))
 960                                                 continue;
 961
 962                                         *best_clock = clock;
 963                                         bestppm = ppm;
 964                                         found = true;
 965                                 }
 966                         }
 967                 }
 968         }
 969
 970         return found;
 971 }
 972
 973 /*
 974  * Returns a set of divisors for the desired target clock with the given
 975  * refclk, or FALSE.  The returned values represent the clock equation:
 976  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 977  */
 978 static bool
 979 chv_find_best_dpll(const struct intel_limit *limit,
 980                    struct intel_crtc_state *crtc_state,
 981                    int target, int refclk, struct dpll *match_clock,
 982                    struct dpll *best_clock)
 983 {
 984         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 985         struct drm_device *dev = crtc->base.dev;
 986         unsigned int best_error_ppm;
 987         struct dpll clock;
 988         u64 m2;
 989         int found = false;
 990
 991         memset(best_clock, 0, sizeof(*best_clock));
 992         best_error_ppm = 1000000;
 993
 994         /*
 995          * Based on hardware doc, the n always set to 1, and m1 always
 996          * set to 2.  If requires to support 200Mhz refclk, we need to
 997          * revisit this because n may not 1 anymore.
 998          */
 999         clock.n = 1, clock.m1 = 2;
1000         target *= 5;    /* fast clock */
1001
1002         for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
1003                 for (clock.p2 = limit->p2.p2_fast;
1004                                 clock.p2 >= limit->p2.p2_slow;
1005                                 clock.p2 -= clock.p2 > 10 ? 2 : 1) {
1006                         unsigned int error_ppm;
1007
1008                         clock.p = clock.p1 * clock.p2;
1009
1010                         m2 = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(target, clock.p * clock.n) << 22,
1011                                                    refclk * clock.m1);
1012
1013                         if (m2 > INT_MAX/clock.m1)
1014                                 continue;
1015
1016                         clock.m2 = m2;
1017
1018                         chv_calc_dpll_params(refclk, &clock);
1019
1020                         if (!intel_PLL_is_valid(to_i915(dev), limit, &clock))
1021                                 continue;
1022
1023                         if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock,
1024                                                 best_error_ppm, &error_ppm))
1025                                 continue;
1026
1027                         *best_clock = clock;
1028                         best_error_ppm = error_ppm;
1029                         found = true;
1030                 }
1031         }
1032
1033         return found;
1034 }
1035
1036 bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state,
1037                         struct dpll *best_clock)
1038 {
1039         int refclk = 100000;
1040         const struct intel_limit *limit = &intel_limits_bxt;
1041
1042         return chv_find_best_dpll(limit, crtc_state,
1043                                   crtc_state->port_clock, refclk,
1044                                   NULL, best_clock);
1045 }
1046
1047 static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv,
1048                                     enum pipe pipe)
1049 {
1050         i915_reg_t reg = PIPEDSL(pipe);
1051         u32 line1, line2;
1052         u32 line_mask;
1053
1054         if (IS_GEN(dev_priv, 2))
1055                 line_mask = DSL_LINEMASK_GEN2;
1056         else
1057                 line_mask = DSL_LINEMASK_GEN3;
1058
1059         line1 = I915_READ(reg) & line_mask;
1060         msleep(5);
1061         line2 = I915_READ(reg) & line_mask;
1062
1063         return line1 != line2;
1064 }
1065
1066 static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state)
1067 {
1068         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1069         enum pipe pipe = crtc->pipe;
1070
1071         /* Wait for the display line to settle/start moving */
1072         if (wait_for(pipe_scanline_is_moving(dev_priv, pipe) == state, 100))
1073                 DRM_ERROR("pipe %c scanline %s wait timed out\n",
1074                           pipe_name(pipe), onoff(state));
1075 }
1076
1077 static void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc)
1078 {
1079         wait_for_pipe_scanline_moving(crtc, false);
1080 }
1081
1082 static void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
1083 {
1084         wait_for_pipe_scanline_moving(crtc, true);
1085 }
1086
1087 static void
1088 intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
1089 {
1090         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
1091         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1092
1093         if (INTEL_GEN(dev_priv) >= 4) {
1094                 enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1095                 i915_reg_t reg = PIPECONF(cpu_transcoder);
1096
1097                 /* Wait for the Pipe State to go off */
1098                 if (intel_de_wait_for_clear(dev_priv, reg,
1099                                             I965_PIPECONF_ACTIVE, 100))
1100                         WARN(1, "pipe_off wait timed out\n");
1101         } else {
1102                 intel_wait_for_pipe_scanline_stopped(crtc);
1103         }
1104 }
1105
1106 /* Only for pre-ILK configs */
1107 void assert_pll(struct drm_i915_private *dev_priv,
1108                 enum pipe pipe, bool state)
1109 {
1110         u32 val;
1111         bool cur_state;
1112
1113         val = I915_READ(DPLL(pipe));
1114         cur_state = !!(val & DPLL_VCO_ENABLE);
1115         I915_STATE_WARN(cur_state != state,
1116              "PLL state assertion failure (expected %s, current %s)\n",
1117                         onoff(state), onoff(cur_state));
1118 }
1119
1120 /* XXX: the dsi pll is shared between MIPI DSI ports */
1121 void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state)
1122 {
1123         u32 val;
1124         bool cur_state;
1125
1126         vlv_cck_get(dev_priv);
1127         val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL);
1128         vlv_cck_put(dev_priv);
1129
1130         cur_state = val & DSI_PLL_VCO_EN;
1131         I915_STATE_WARN(cur_state != state,
1132              "DSI PLL state assertion failure (expected %s, current %s)\n",
1133                         onoff(state), onoff(cur_state));
1134 }
1135
1136 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
1137                           enum pipe pipe, bool state)
1138 {
1139         bool cur_state;
1140
1141         if (HAS_DDI(dev_priv)) {
1142                 /*
1143                  * DDI does not have a specific FDI_TX register.
1144                  *
1145                  * FDI is never fed from EDP transcoder
1146                  * so pipe->transcoder cast is fine here.
1147                  */
1148                 enum transcoder cpu_transcoder = (enum transcoder)pipe;
1149                 u32 val = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
1150                 cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
1151         } else {
1152                 u32 val = I915_READ(FDI_TX_CTL(pipe));
1153                 cur_state = !!(val & FDI_TX_ENABLE);
1154         }
1155         I915_STATE_WARN(cur_state != state,
1156              "FDI TX state assertion failure (expected %s, current %s)\n",
1157                         onoff(state), onoff(cur_state));
1158 }
1159 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
1160 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
1161
1162 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
1163                           enum pipe pipe, bool state)
1164 {
1165         u32 val;
1166         bool cur_state;
1167
1168         val = I915_READ(FDI_RX_CTL(pipe));
1169         cur_state = !!(val & FDI_RX_ENABLE);
1170         I915_STATE_WARN(cur_state != state,
1171              "FDI RX state assertion failure (expected %s, current %s)\n",
1172                         onoff(state), onoff(cur_state));
1173 }
1174 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
1175 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
1176
1177 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
1178                                       enum pipe pipe)
1179 {
1180         u32 val;
1181
1182         /* ILK FDI PLL is always enabled */
1183         if (IS_GEN(dev_priv, 5))
1184                 return;
1185
1186         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
1187         if (HAS_DDI(dev_priv))
1188                 return;
1189
1190         val = I915_READ(FDI_TX_CTL(pipe));
1191         I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
1192 }
1193
1194 void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
1195                        enum pipe pipe, bool state)
1196 {
1197         u32 val;
1198         bool cur_state;
1199
1200         val = I915_READ(FDI_RX_CTL(pipe));
1201         cur_state = !!(val & FDI_RX_PLL_ENABLE);
1202         I915_STATE_WARN(cur_state != state,
1203              "FDI RX PLL assertion failure (expected %s, current %s)\n",
1204                         onoff(state), onoff(cur_state));
1205 }
1206
1207 void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe)
1208 {
1209         i915_reg_t pp_reg;
1210         u32 val;
1211         enum pipe panel_pipe = INVALID_PIPE;
1212         bool locked = true;
1213
1214         if (WARN_ON(HAS_DDI(dev_priv)))
1215                 return;
1216
1217         if (HAS_PCH_SPLIT(dev_priv)) {
1218                 u32 port_sel;
1219
1220                 pp_reg = PP_CONTROL(0);
1221                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1222
1223                 switch (port_sel) {
1224                 case PANEL_PORT_SELECT_LVDS:
1225                         intel_lvds_port_enabled(dev_priv, PCH_LVDS, &panel_pipe);
1226                         break;
1227                 case PANEL_PORT_SELECT_DPA:
1228                         intel_dp_port_enabled(dev_priv, DP_A, PORT_A, &panel_pipe);
1229                         break;
1230                 case PANEL_PORT_SELECT_DPC:
1231                         intel_dp_port_enabled(dev_priv, PCH_DP_C, PORT_C, &panel_pipe);
1232                         break;
1233                 case PANEL_PORT_SELECT_DPD:
1234                         intel_dp_port_enabled(dev_priv, PCH_DP_D, PORT_D, &panel_pipe);
1235                         break;
1236                 default:
1237                         MISSING_CASE(port_sel);
1238                         break;
1239                 }
1240         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1241                 /* presumably write lock depends on pipe, not port select */
1242                 pp_reg = PP_CONTROL(pipe);
1243                 panel_pipe = pipe;
1244         } else {
1245                 u32 port_sel;
1246
1247                 pp_reg = PP_CONTROL(0);
1248                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1249
1250                 WARN_ON(port_sel != PANEL_PORT_SELECT_LVDS);
1251                 intel_lvds_port_enabled(dev_priv, LVDS, &panel_pipe);
1252         }
1253
1254         val = I915_READ(pp_reg);
1255         if (!(val & PANEL_POWER_ON) ||
1256             ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
1257                 locked = false;
1258
1259         I915_STATE_WARN(panel_pipe == pipe && locked,
1260              "panel assertion failure, pipe %c regs locked\n",
1261              pipe_name(pipe));
1262 }
1263
1264 void assert_pipe(struct drm_i915_private *dev_priv,
1265                  enum transcoder cpu_transcoder, bool state)
1266 {
1267         bool cur_state;
1268         enum intel_display_power_domain power_domain;
1269         intel_wakeref_t wakeref;
1270
1271         /* we keep both pipes enabled on 830 */
1272         if (IS_I830(dev_priv))
1273                 state = true;
1274
1275         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
1276         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
1277         if (wakeref) {
1278                 u32 val = I915_READ(PIPECONF(cpu_transcoder));
1279                 cur_state = !!(val & PIPECONF_ENABLE);
1280
1281                 intel_display_power_put(dev_priv, power_domain, wakeref);
1282         } else {
1283                 cur_state = false;
1284         }
1285
1286         I915_STATE_WARN(cur_state != state,
1287                         "transcoder %s assertion failure (expected %s, current %s)\n",
1288                         transcoder_name(cpu_transcoder),
1289                         onoff(state), onoff(cur_state));
1290 }
1291
1292 static void assert_plane(struct intel_plane *plane, bool state)
1293 {
1294         enum pipe pipe;
1295         bool cur_state;
1296
1297         cur_state = plane->get_hw_state(plane, &pipe);
1298
1299         I915_STATE_WARN(cur_state != state,
1300                         "%s assertion failure (expected %s, current %s)\n",
1301                         plane->base.name, onoff(state), onoff(cur_state));
1302 }
1303
1304 #define assert_plane_enabled(p) assert_plane(p, true)
1305 #define assert_plane_disabled(p) assert_plane(p, false)
1306
1307 static void assert_planes_disabled(struct intel_crtc *crtc)
1308 {
1309         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1310         struct intel_plane *plane;
1311
1312         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
1313                 assert_plane_disabled(plane);
1314 }
1315
1316 static void assert_vblank_disabled(struct drm_crtc *crtc)
1317 {
1318         if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
1319                 drm_crtc_vblank_put(crtc);
1320 }
1321
1322 void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
1323                                     enum pipe pipe)
1324 {
1325         u32 val;
1326         bool enabled;
1327
1328         val = I915_READ(PCH_TRANSCONF(pipe));
1329         enabled = !!(val & TRANS_ENABLE);
1330         I915_STATE_WARN(enabled,
1331              "transcoder assertion failed, should be off on pipe %c but is still active\n",
1332              pipe_name(pipe));
1333 }
1334
1335 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
1336                                    enum pipe pipe, enum port port,
1337                                    i915_reg_t dp_reg)
1338 {
1339         enum pipe port_pipe;
1340         bool state;
1341
1342         state = intel_dp_port_enabled(dev_priv, dp_reg, port, &port_pipe);
1343
1344         I915_STATE_WARN(state && port_pipe == pipe,
1345                         "PCH DP %c enabled on transcoder %c, should be disabled\n",
1346                         port_name(port), pipe_name(pipe));
1347
1348         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1349                         "IBX PCH DP %c still using transcoder B\n",
1350                         port_name(port));
1351 }
1352
1353 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
1354                                      enum pipe pipe, enum port port,
1355                                      i915_reg_t hdmi_reg)
1356 {
1357         enum pipe port_pipe;
1358         bool state;
1359
1360         state = intel_sdvo_port_enabled(dev_priv, hdmi_reg, &port_pipe);
1361
1362         I915_STATE_WARN(state && port_pipe == pipe,
1363                         "PCH HDMI %c enabled on transcoder %c, should be disabled\n",
1364                         port_name(port), pipe_name(pipe));
1365
1366         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1367                         "IBX PCH HDMI %c still using transcoder B\n",
1368                         port_name(port));
1369 }
1370
1371 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
1372                                       enum pipe pipe)
1373 {
1374         enum pipe port_pipe;
1375
1376         assert_pch_dp_disabled(dev_priv, pipe, PORT_B, PCH_DP_B);
1377         assert_pch_dp_disabled(dev_priv, pipe, PORT_C, PCH_DP_C);
1378         assert_pch_dp_disabled(dev_priv, pipe, PORT_D, PCH_DP_D);
1379
1380         I915_STATE_WARN(intel_crt_port_enabled(dev_priv, PCH_ADPA, &port_pipe) &&
1381                         port_pipe == pipe,
1382                         "PCH VGA enabled on transcoder %c, should be disabled\n",
1383                         pipe_name(pipe));
1384
1385         I915_STATE_WARN(intel_lvds_port_enabled(dev_priv, PCH_LVDS, &port_pipe) &&
1386                         port_pipe == pipe,
1387                         "PCH LVDS enabled on transcoder %c, should be disabled\n",
1388                         pipe_name(pipe));
1389
1390         /* PCH SDVOB multiplex with HDMIB */
1391         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_B, PCH_HDMIB);
1392         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_C, PCH_HDMIC);
1393         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_D, PCH_HDMID);
1394 }
1395
1396 static void _vlv_enable_pll(struct intel_crtc *crtc,
1397                             const struct intel_crtc_state *pipe_config)
1398 {
1399         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1400         enum pipe pipe = crtc->pipe;
1401
1402         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1403         POSTING_READ(DPLL(pipe));
1404         udelay(150);
1405
1406         if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1))
1407                 DRM_ERROR("DPLL %d failed to lock\n", pipe);
1408 }
1409
1410 static void vlv_enable_pll(struct intel_crtc *crtc,
1411                            const struct intel_crtc_state *pipe_config)
1412 {
1413         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1414         enum pipe pipe = crtc->pipe;
1415
1416         assert_pipe_disabled(dev_priv, pipe_config->cpu_transcoder);
1417
1418         /* PLL is protected by panel, make sure we can write it */
1419         assert_panel_unlocked(dev_priv, pipe);
1420
1421         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1422                 _vlv_enable_pll(crtc, pipe_config);
1423
1424         I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1425         POSTING_READ(DPLL_MD(pipe));
1426 }
1427
1428
1429 static void _chv_enable_pll(struct intel_crtc *crtc,
1430                             const struct intel_crtc_state *pipe_config)
1431 {
1432         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1433         enum pipe pipe = crtc->pipe;
1434         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1435         u32 tmp;
1436
1437         vlv_dpio_get(dev_priv);
1438
1439         /* Enable back the 10bit clock to display controller */
1440         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1441         tmp |= DPIO_DCLKP_EN;
1442         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp);
1443
1444         vlv_dpio_put(dev_priv);
1445
1446         /*
1447          * Need to wait > 100ns between dclkp clock enable bit and PLL enable.
1448          */
1449         udelay(1);
1450
1451         /* Enable PLL */
1452         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1453
1454         /* Check PLL is locked */
1455         if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1))
1456                 DRM_ERROR("PLL %d failed to lock\n", pipe);
1457 }
1458
1459 static void chv_enable_pll(struct intel_crtc *crtc,
1460                            const struct intel_crtc_state *pipe_config)
1461 {
1462         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1463         enum pipe pipe = crtc->pipe;
1464
1465         assert_pipe_disabled(dev_priv, pipe_config->cpu_transcoder);
1466
1467         /* PLL is protected by panel, make sure we can write it */
1468         assert_panel_unlocked(dev_priv, pipe);
1469
1470         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1471                 _chv_enable_pll(crtc, pipe_config);
1472
1473         if (pipe != PIPE_A) {
1474                 /*
1475                  * WaPixelRepeatModeFixForC0:chv
1476                  *
1477                  * DPLLCMD is AWOL. Use chicken bits to propagate
1478                  * the value from DPLLBMD to either pipe B or C.
1479                  */
1480                 I915_WRITE(CBR4_VLV, CBR_DPLLBMD_PIPE(pipe));
1481                 I915_WRITE(DPLL_MD(PIPE_B), pipe_config->dpll_hw_state.dpll_md);
1482                 I915_WRITE(CBR4_VLV, 0);
1483                 dev_priv->chv_dpll_md[pipe] = pipe_config->dpll_hw_state.dpll_md;
1484
1485                 /*
1486                  * DPLLB VGA mode also seems to cause problems.
1487                  * We should always have it disabled.
1488                  */
1489                 WARN_ON((I915_READ(DPLL(PIPE_B)) & DPLL_VGA_MODE_DIS) == 0);
1490         } else {
1491                 I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1492                 POSTING_READ(DPLL_MD(pipe));
1493         }
1494 }
1495
1496 static bool i9xx_has_pps(struct drm_i915_private *dev_priv)
1497 {
1498         if (IS_I830(dev_priv))
1499                 return false;
1500
1501         return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
1502 }
1503
1504 static void i9xx_enable_pll(struct intel_crtc *crtc,
1505                             const struct intel_crtc_state *crtc_state)
1506 {
1507         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1508         i915_reg_t reg = DPLL(crtc->pipe);
1509         u32 dpll = crtc_state->dpll_hw_state.dpll;
1510         int i;
1511
1512         assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder);
1513
1514         /* PLL is protected by panel, make sure we can write it */
1515         if (i9xx_has_pps(dev_priv))
1516                 assert_panel_unlocked(dev_priv, crtc->pipe);
1517
1518         /*
1519          * Apparently we need to have VGA mode enabled prior to changing
1520          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
1521          * dividers, even though the register value does change.
1522          */
1523         I915_WRITE(reg, dpll & ~DPLL_VGA_MODE_DIS);
1524         I915_WRITE(reg, dpll);
1525
1526         /* Wait for the clocks to stabilize. */
1527         POSTING_READ(reg);
1528         udelay(150);
1529
1530         if (INTEL_GEN(dev_priv) >= 4) {
1531                 I915_WRITE(DPLL_MD(crtc->pipe),
1532                            crtc_state->dpll_hw_state.dpll_md);
1533         } else {
1534                 /* The pixel multiplier can only be updated once the
1535                  * DPLL is enabled and the clocks are stable.
1536                  *
1537                  * So write it again.
1538                  */
1539                 I915_WRITE(reg, dpll);
1540         }
1541
1542         /* We do this three times for luck */
1543         for (i = 0; i < 3; i++) {
1544                 I915_WRITE(reg, dpll);
1545                 POSTING_READ(reg);
1546                 udelay(150); /* wait for warmup */
1547         }
1548 }
1549
1550 static void i9xx_disable_pll(const struct intel_crtc_state *crtc_state)
1551 {
1552         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1553         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1554         enum pipe pipe = crtc->pipe;
1555
1556         /* Don't disable pipe or pipe PLLs if needed */
1557         if (IS_I830(dev_priv))
1558                 return;
1559
1560         /* Make sure the pipe isn't still relying on us */
1561         assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder);
1562
1563         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
1564         POSTING_READ(DPLL(pipe));
1565 }
1566
1567 static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1568 {
1569         u32 val;
1570
1571         /* Make sure the pipe isn't still relying on us */
1572         assert_pipe_disabled(dev_priv, (enum transcoder)pipe);
1573
1574         val = DPLL_INTEGRATED_REF_CLK_VLV |
1575                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1576         if (pipe != PIPE_A)
1577                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1578
1579         I915_WRITE(DPLL(pipe), val);
1580         POSTING_READ(DPLL(pipe));
1581 }
1582
1583 static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1584 {
1585         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1586         u32 val;
1587
1588         /* Make sure the pipe isn't still relying on us */
1589         assert_pipe_disabled(dev_priv, (enum transcoder)pipe);
1590
1591         val = DPLL_SSC_REF_CLK_CHV |
1592                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1593         if (pipe != PIPE_A)
1594                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1595
1596         I915_WRITE(DPLL(pipe), val);
1597         POSTING_READ(DPLL(pipe));
1598
1599         vlv_dpio_get(dev_priv);
1600
1601         /* Disable 10bit clock to display controller */
1602         val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1603         val &= ~DPIO_DCLKP_EN;
1604         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
1605
1606         vlv_dpio_put(dev_priv);
1607 }
1608
1609 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
1610                          struct intel_digital_port *dport,
1611                          unsigned int expected_mask)
1612 {
1613         u32 port_mask;
1614         i915_reg_t dpll_reg;
1615
1616         switch (dport->base.port) {
1617         case PORT_B:
1618                 port_mask = DPLL_PORTB_READY_MASK;
1619                 dpll_reg = DPLL(0);
1620                 break;
1621         case PORT_C:
1622                 port_mask = DPLL_PORTC_READY_MASK;
1623                 dpll_reg = DPLL(0);
1624                 expected_mask <<= 4;
1625                 break;
1626         case PORT_D:
1627                 port_mask = DPLL_PORTD_READY_MASK;
1628                 dpll_reg = DPIO_PHY_STATUS;
1629                 break;
1630         default:
1631                 BUG();
1632         }
1633
1634         if (intel_de_wait_for_register(dev_priv, dpll_reg,
1635                                        port_mask, expected_mask, 1000))
1636                 WARN(1, "timed out waiting for [ENCODER:%d:%s] port ready: got 0x%x, expected 0x%x\n",
1637                      dport->base.base.base.id, dport->base.base.name,
1638                      I915_READ(dpll_reg) & port_mask, expected_mask);
1639 }
1640
1641 static void ilk_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
1642 {
1643         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1644         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1645         enum pipe pipe = crtc->pipe;
1646         i915_reg_t reg;
1647         u32 val, pipeconf_val;
1648
1649         /* Make sure PCH DPLL is enabled */
1650         assert_shared_dpll_enabled(dev_priv, crtc_state->shared_dpll);
1651
1652         /* FDI must be feeding us bits for PCH ports */
1653         assert_fdi_tx_enabled(dev_priv, pipe);
1654         assert_fdi_rx_enabled(dev_priv, pipe);
1655
1656         if (HAS_PCH_CPT(dev_priv)) {
1657                 reg = TRANS_CHICKEN2(pipe);
1658                 val = I915_READ(reg);
1659                 /*
1660                  * Workaround: Set the timing override bit
1661                  * before enabling the pch transcoder.
1662                  */
1663                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1664                 /* Configure frame start delay to match the CPU */
1665                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
1666                 val |= TRANS_CHICKEN2_FRAME_START_DELAY(0);
1667                 I915_WRITE(reg, val);
1668         }
1669
1670         reg = PCH_TRANSCONF(pipe);
1671         val = I915_READ(reg);
1672         pipeconf_val = I915_READ(PIPECONF(pipe));
1673
1674         if (HAS_PCH_IBX(dev_priv)) {
1675                 /* Configure frame start delay to match the CPU */
1676                 val &= ~TRANS_FRAME_START_DELAY_MASK;
1677                 val |= TRANS_FRAME_START_DELAY(0);
1678
1679                 /*
1680                  * Make the BPC in transcoder be consistent with
1681                  * that in pipeconf reg. For HDMI we must use 8bpc
1682                  * here for both 8bpc and 12bpc.
1683                  */
1684                 val &= ~PIPECONF_BPC_MASK;
1685                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
1686                         val |= PIPECONF_8BPC;
1687                 else
1688                         val |= pipeconf_val & PIPECONF_BPC_MASK;
1689         }
1690
1691         val &= ~TRANS_INTERLACE_MASK;
1692         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK) {
1693                 if (HAS_PCH_IBX(dev_priv) &&
1694                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
1695                         val |= TRANS_LEGACY_INTERLACED_ILK;
1696                 else
1697                         val |= TRANS_INTERLACED;
1698         } else {
1699                 val |= TRANS_PROGRESSIVE;
1700         }
1701
1702         I915_WRITE(reg, val | TRANS_ENABLE);
1703         if (intel_de_wait_for_set(dev_priv, reg, TRANS_STATE_ENABLE, 100))
1704                 DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe));
1705 }
1706
1707 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1708                                       enum transcoder cpu_transcoder)
1709 {
1710         u32 val, pipeconf_val;
1711
1712         /* FDI must be feeding us bits for PCH ports */
1713         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
1714         assert_fdi_rx_enabled(dev_priv, PIPE_A);
1715
1716         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1717         /* Workaround: set timing override bit. */
1718         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1719         /* Configure frame start delay to match the CPU */
1720         val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
1721         val |= TRANS_CHICKEN2_FRAME_START_DELAY(0);
1722         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1723
1724         val = TRANS_ENABLE;
1725         pipeconf_val = I915_READ(PIPECONF(cpu_transcoder));
1726
1727         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
1728             PIPECONF_INTERLACED_ILK)
1729                 val |= TRANS_INTERLACED;
1730         else
1731                 val |= TRANS_PROGRESSIVE;
1732
1733         I915_WRITE(LPT_TRANSCONF, val);
1734         if (intel_de_wait_for_set(dev_priv, LPT_TRANSCONF,
1735                                   TRANS_STATE_ENABLE, 100))
1736                 DRM_ERROR("Failed to enable PCH transcoder\n");
1737 }
1738
1739 static void ilk_disable_pch_transcoder(struct drm_i915_private *dev_priv,
1740                                        enum pipe pipe)
1741 {
1742         i915_reg_t reg;
1743         u32 val;
1744
1745         /* FDI relies on the transcoder */
1746         assert_fdi_tx_disabled(dev_priv, pipe);
1747         assert_fdi_rx_disabled(dev_priv, pipe);
1748
1749         /* Ports must be off as well */
1750         assert_pch_ports_disabled(dev_priv, pipe);
1751
1752         reg = PCH_TRANSCONF(pipe);
1753         val = I915_READ(reg);
1754         val &= ~TRANS_ENABLE;
1755         I915_WRITE(reg, val);
1756         /* wait for PCH transcoder off, transcoder state */
1757         if (intel_de_wait_for_clear(dev_priv, reg, TRANS_STATE_ENABLE, 50))
1758                 DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
1759
1760         if (HAS_PCH_CPT(dev_priv)) {
1761                 /* Workaround: Clear the timing override chicken bit again. */
1762                 reg = TRANS_CHICKEN2(pipe);
1763                 val = I915_READ(reg);
1764                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1765                 I915_WRITE(reg, val);
1766         }
1767 }
1768
1769 void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
1770 {
1771         u32 val;
1772
1773         val = I915_READ(LPT_TRANSCONF);
1774         val &= ~TRANS_ENABLE;
1775         I915_WRITE(LPT_TRANSCONF, val);
1776         /* wait for PCH transcoder off, transcoder state */
1777         if (intel_de_wait_for_clear(dev_priv, LPT_TRANSCONF,
1778                                     TRANS_STATE_ENABLE, 50))
1779                 DRM_ERROR("Failed to disable PCH transcoder\n");
1780
1781         /* Workaround: clear timing override bit. */
1782         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1783         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1784         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1785 }
1786
1787 enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
1788 {
1789         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1790
1791         if (HAS_PCH_LPT(dev_priv))
1792                 return PIPE_A;
1793         else
1794                 return crtc->pipe;
1795 }
1796
1797 static u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
1798 {
1799         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1800
1801         /*
1802          * On i965gm the hardware frame counter reads
1803          * zero when the TV encoder is enabled :(
1804          */
1805         if (IS_I965GM(dev_priv) &&
1806             (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
1807                 return 0;
1808
1809         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1810                 return 0xffffffff; /* full 32 bit counter */
1811         else if (INTEL_GEN(dev_priv) >= 3)
1812                 return 0xffffff; /* only 24 bits of frame count */
1813         else
1814                 return 0; /* Gen2 doesn't have a hardware frame counter */
1815 }
1816
1817 static void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
1818 {
1819         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1820
1821         assert_vblank_disabled(&crtc->base);
1822         drm_crtc_set_max_vblank_count(&crtc->base,
1823                                       intel_crtc_max_vblank_count(crtc_state));
1824         drm_crtc_vblank_on(&crtc->base);
1825 }
1826
1827 void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
1828 {
1829         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1830
1831         drm_crtc_vblank_off(&crtc->base);
1832         assert_vblank_disabled(&crtc->base);
1833 }
1834
1835 static void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state)
1836 {
1837         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
1838         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1839         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
1840         enum pipe pipe = crtc->pipe;
1841         i915_reg_t reg;
1842         u32 val;
1843
1844         DRM_DEBUG_KMS("enabling pipe %c\n", pipe_name(pipe));
1845
1846         assert_planes_disabled(crtc);
1847
1848         /*
1849          * A pipe without a PLL won't actually be able to drive bits from
1850          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
1851          * need the check.
1852          */
1853         if (HAS_GMCH(dev_priv)) {
1854                 if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
1855                         assert_dsi_pll_enabled(dev_priv);
1856                 else
1857                         assert_pll_enabled(dev_priv, pipe);
1858         } else {
1859                 if (new_crtc_state->has_pch_encoder) {
1860                         /* if driving the PCH, we need FDI enabled */
1861                         assert_fdi_rx_pll_enabled(dev_priv,
1862                                                   intel_crtc_pch_transcoder(crtc));
1863                         assert_fdi_tx_pll_enabled(dev_priv,
1864                                                   (enum pipe) cpu_transcoder);
1865                 }
1866                 /* FIXME: assert CPU port conditions for SNB+ */
1867         }
1868
1869         trace_intel_pipe_enable(crtc);
1870
1871         reg = PIPECONF(cpu_transcoder);
1872         val = I915_READ(reg);
1873         if (val & PIPECONF_ENABLE) {
1874                 /* we keep both pipes enabled on 830 */
1875                 WARN_ON(!IS_I830(dev_priv));
1876                 return;
1877         }
1878
1879         I915_WRITE(reg, val | PIPECONF_ENABLE);
1880         POSTING_READ(reg);
1881
1882         /*
1883          * Until the pipe starts PIPEDSL reads will return a stale value,
1884          * which causes an apparent vblank timestamp jump when PIPEDSL
1885          * resets to its proper value. That also messes up the frame count
1886          * when it's derived from the timestamps. So let's wait for the
1887          * pipe to start properly before we call drm_crtc_vblank_on()
1888          */
1889         if (intel_crtc_max_vblank_count(new_crtc_state) == 0)
1890                 intel_wait_for_pipe_scanline_moving(crtc);
1891 }
1892
1893 void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state)
1894 {
1895         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
1896         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1897         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1898         enum pipe pipe = crtc->pipe;
1899         i915_reg_t reg;
1900         u32 val;
1901
1902         DRM_DEBUG_KMS("disabling pipe %c\n", pipe_name(pipe));
1903
1904         /*
1905          * Make sure planes won't keep trying to pump pixels to us,
1906          * or we might hang the display.
1907          */
1908         assert_planes_disabled(crtc);
1909
1910         trace_intel_pipe_disable(crtc);
1911
1912         reg = PIPECONF(cpu_transcoder);
1913         val = I915_READ(reg);
1914         if ((val & PIPECONF_ENABLE) == 0)
1915                 return;
1916
1917         /*
1918          * Double wide has implications for planes
1919          * so best keep it disabled when not needed.
1920          */
1921         if (old_crtc_state->double_wide)
1922                 val &= ~PIPECONF_DOUBLE_WIDE;
1923
1924         /* Don't disable pipe or pipe PLLs if needed */
1925         if (!IS_I830(dev_priv))
1926                 val &= ~PIPECONF_ENABLE;
1927
1928         I915_WRITE(reg, val);
1929         if ((val & PIPECONF_ENABLE) == 0)
1930                 intel_wait_for_pipe_off(old_crtc_state);
1931 }
1932
1933 static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv)
1934 {
1935         return IS_GEN(dev_priv, 2) ? 2048 : 4096;
1936 }
1937
1938 static bool is_ccs_plane(const struct drm_framebuffer *fb, int plane)
1939 {
1940         if (!is_ccs_modifier(fb->modifier))
1941                 return false;
1942
1943         return plane >= fb->format->num_planes / 2;
1944 }
1945
1946 static bool is_gen12_ccs_modifier(u64 modifier)
1947 {
1948         return modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS;
1949 }
1950
1951 static bool is_gen12_ccs_plane(const struct drm_framebuffer *fb, int plane)
1952 {
1953         return is_gen12_ccs_modifier(fb->modifier) && is_ccs_plane(fb, plane);
1954 }
1955
1956 static bool is_aux_plane(const struct drm_framebuffer *fb, int plane)
1957 {
1958         if (is_ccs_modifier(fb->modifier))
1959                 return is_ccs_plane(fb, plane);
1960
1961         return plane == 1;
1962 }
1963
1964 static int main_to_ccs_plane(const struct drm_framebuffer *fb, int main_plane)
1965 {
1966         WARN_ON(!is_ccs_modifier(fb->modifier) ||
1967                 (main_plane && main_plane >= fb->format->num_planes / 2));
1968
1969         return fb->format->num_planes / 2 + main_plane;
1970 }
1971
1972 static int ccs_to_main_plane(const struct drm_framebuffer *fb, int ccs_plane)
1973 {
1974         WARN_ON(!is_ccs_modifier(fb->modifier) ||
1975                 ccs_plane < fb->format->num_planes / 2);
1976
1977         return ccs_plane - fb->format->num_planes / 2;
1978 }
1979
1980 /* Return either the main plane's CCS or - if not a CCS FB - UV plane */
1981 static int
1982 intel_main_to_aux_plane(const struct drm_framebuffer *fb, int main_plane)
1983 {
1984         if (is_ccs_modifier(fb->modifier))
1985                 return main_to_ccs_plane(fb, main_plane);
1986
1987         return 1;
1988 }
1989
1990 bool
1991 intel_format_info_is_yuv_semiplanar(const struct drm_format_info *info,
1992                                     uint64_t modifier)
1993 {
1994         return info->is_yuv &&
1995                info->num_planes == (is_ccs_modifier(modifier) ? 4 : 2);
1996 }
1997
1998 static unsigned int
1999 intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane)
2000 {
2001         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2002         unsigned int cpp = fb->format->cpp[color_plane];
2003
2004         switch (fb->modifier) {
2005         case DRM_FORMAT_MOD_LINEAR:
2006                 return intel_tile_size(dev_priv);
2007         case I915_FORMAT_MOD_X_TILED:
2008                 if (IS_GEN(dev_priv, 2))
2009                         return 128;
2010                 else
2011                         return 512;
2012         case I915_FORMAT_MOD_Y_TILED_CCS:
2013                 if (is_ccs_plane(fb, color_plane))
2014                         return 128;
2015                 /* fall through */
2016         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
2017                 if (is_ccs_plane(fb, color_plane))
2018                         return 64;
2019                 /* fall through */
2020         case I915_FORMAT_MOD_Y_TILED:
2021                 if (IS_GEN(dev_priv, 2) || HAS_128_BYTE_Y_TILING(dev_priv))
2022                         return 128;
2023                 else
2024                         return 512;
2025         case I915_FORMAT_MOD_Yf_TILED_CCS:
2026                 if (is_ccs_plane(fb, color_plane))
2027                         return 128;
2028                 /* fall through */
2029         case I915_FORMAT_MOD_Yf_TILED:
2030                 switch (cpp) {
2031                 case 1:
2032                         return 64;
2033                 case 2:
2034                 case 4:
2035                         return 128;
2036                 case 8:
2037                 case 16:
2038                         return 256;
2039                 default:
2040                         MISSING_CASE(cpp);
2041                         return cpp;
2042                 }
2043                 break;
2044         default:
2045                 MISSING_CASE(fb->modifier);
2046                 return cpp;
2047         }
2048 }
2049
2050 static unsigned int
2051 intel_tile_height(const struct drm_framebuffer *fb, int color_plane)
2052 {
2053         if (is_gen12_ccs_plane(fb, color_plane))
2054                 return 1;
2055
2056         return intel_tile_size(to_i915(fb->dev)) /
2057                 intel_tile_width_bytes(fb, color_plane);
2058 }
2059
2060 /* Return the tile dimensions in pixel units */
2061 static void intel_tile_dims(const struct drm_framebuffer *fb, int color_plane,
2062                             unsigned int *tile_width,
2063                             unsigned int *tile_height)
2064 {
2065         unsigned int tile_width_bytes = intel_tile_width_bytes(fb, color_plane);
2066         unsigned int cpp = fb->format->cpp[color_plane];
2067
2068         *tile_width = tile_width_bytes / cpp;
2069         *tile_height = intel_tile_height(fb, color_plane);
2070 }
2071
2072 unsigned int
2073 intel_fb_align_height(const struct drm_framebuffer *fb,
2074                       int color_plane, unsigned int height)
2075 {
2076         unsigned int tile_height = intel_tile_height(fb, color_plane);
2077
2078         return ALIGN(height, tile_height);
2079 }
2080
2081 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info)
2082 {
2083         unsigned int size = 0;
2084         int i;
2085
2086         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
2087                 size += rot_info->plane[i].width * rot_info->plane[i].height;
2088
2089         return size;
2090 }
2091
2092 unsigned int intel_remapped_info_size(const struct intel_remapped_info *rem_info)
2093 {
2094         unsigned int size = 0;
2095         int i;
2096
2097         for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++)
2098                 size += rem_info->plane[i].width * rem_info->plane[i].height;
2099
2100         return size;
2101 }
2102
2103 static void
2104 intel_fill_fb_ggtt_view(struct i915_ggtt_view *view,
2105                         const struct drm_framebuffer *fb,
2106                         unsigned int rotation)
2107 {
2108         view->type = I915_GGTT_VIEW_NORMAL;
2109         if (drm_rotation_90_or_270(rotation)) {
2110                 view->type = I915_GGTT_VIEW_ROTATED;
2111                 view->rotated = to_intel_framebuffer(fb)->rot_info;
2112         }
2113 }
2114
2115 static unsigned int intel_cursor_alignment(const struct drm_i915_private *dev_priv)
2116 {
2117         if (IS_I830(dev_priv))
2118                 return 16 * 1024;
2119         else if (IS_I85X(dev_priv))
2120                 return 256;
2121         else if (IS_I845G(dev_priv) || IS_I865G(dev_priv))
2122                 return 32;
2123         else
2124                 return 4 * 1024;
2125 }
2126
2127 static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv)
2128 {
2129         if (INTEL_GEN(dev_priv) >= 9)
2130                 return 256 * 1024;
2131         else if (IS_I965G(dev_priv) || IS_I965GM(dev_priv) ||
2132                  IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2133                 return 128 * 1024;
2134         else if (INTEL_GEN(dev_priv) >= 4)
2135                 return 4 * 1024;
2136         else
2137                 return 0;
2138 }
2139
2140 static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb,
2141                                          int color_plane)
2142 {
2143         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2144
2145         /* AUX_DIST needs only 4K alignment */
2146         if (is_aux_plane(fb, color_plane))
2147                 return 4096;
2148
2149         switch (fb->modifier) {
2150         case DRM_FORMAT_MOD_LINEAR:
2151                 return intel_linear_alignment(dev_priv);
2152         case I915_FORMAT_MOD_X_TILED:
2153                 if (INTEL_GEN(dev_priv) >= 9)
2154                         return 256 * 1024;
2155                 return 0;
2156         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
2157                 return 16 * 1024;
2158         case I915_FORMAT_MOD_Y_TILED_CCS:
2159         case I915_FORMAT_MOD_Yf_TILED_CCS:
2160         case I915_FORMAT_MOD_Y_TILED:
2161         case I915_FORMAT_MOD_Yf_TILED:
2162                 return 1 * 1024 * 1024;
2163         default:
2164                 MISSING_CASE(fb->modifier);
2165                 return 0;
2166         }
2167 }
2168
2169 static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
2170 {
2171         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2172         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2173
2174         return INTEL_GEN(dev_priv) < 4 ||
2175                 (plane->has_fbc &&
2176                  plane_state->view.type == I915_GGTT_VIEW_NORMAL);
2177 }
2178
2179 struct i915_vma *
2180 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
2181                            const struct i915_ggtt_view *view,
2182                            bool uses_fence,
2183                            unsigned long *out_flags)
2184 {
2185         struct drm_device *dev = fb->dev;
2186         struct drm_i915_private *dev_priv = to_i915(dev);
2187         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2188         intel_wakeref_t wakeref;
2189         struct i915_vma *vma;
2190         unsigned int pinctl;
2191         u32 alignment;
2192
2193         if (WARN_ON(!i915_gem_object_is_framebuffer(obj)))
2194                 return ERR_PTR(-EINVAL);
2195
2196         alignment = intel_surf_alignment(fb, 0);
2197
2198         /* Note that the w/a also requires 64 PTE of padding following the
2199          * bo. We currently fill all unused PTE with the shadow page and so
2200          * we should always have valid PTE following the scanout preventing
2201          * the VT-d warning.
2202          */
2203         if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024)
2204                 alignment = 256 * 1024;
2205
2206         /*
2207          * Global gtt pte registers are special registers which actually forward
2208          * writes to a chunk of system memory. Which means that there is no risk
2209          * that the register values disappear as soon as we call
2210          * intel_runtime_pm_put(), so it is correct to wrap only the
2211          * pin/unpin/fence and not more.
2212          */
2213         wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
2214
2215         atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
2216
2217         /*
2218          * Valleyview is definitely limited to scanning out the first
2219          * 512MiB. Lets presume this behaviour was inherited from the
2220          * g4x display engine and that all earlier gen are similarly
2221          * limited. Testing suggests that it is a little more
2222          * complicated than this. For example, Cherryview appears quite
2223          * happy to scanout from anywhere within its global aperture.
2224          */
2225         pinctl = 0;
2226         if (HAS_GMCH(dev_priv))
2227                 pinctl |= PIN_MAPPABLE;
2228
2229         vma = i915_gem_object_pin_to_display_plane(obj,
2230                                                    alignment, view, pinctl);
2231         if (IS_ERR(vma))
2232                 goto err;
2233
2234         if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
2235                 int ret;
2236
2237                 /*
2238                  * Install a fence for tiled scan-out. Pre-i965 always needs a
2239                  * fence, whereas 965+ only requires a fence if using
2240                  * framebuffer compression.  For simplicity, we always, when
2241                  * possible, install a fence as the cost is not that onerous.
2242                  *
2243                  * If we fail to fence the tiled scanout, then either the
2244                  * modeset will reject the change (which is highly unlikely as
2245                  * the affected systems, all but one, do not have unmappable
2246                  * space) or we will not be able to enable full powersaving
2247                  * techniques (also likely not to apply due to various limits
2248                  * FBC and the like impose on the size of the buffer, which
2249                  * presumably we violated anyway with this unmappable buffer).
2250                  * Anyway, it is presumably better to stumble onwards with
2251                  * something and try to run the system in a "less than optimal"
2252                  * mode that matches the user configuration.
2253                  */
2254                 ret = i915_vma_pin_fence(vma);
2255                 if (ret != 0 && INTEL_GEN(dev_priv) < 4) {
2256                         i915_gem_object_unpin_from_display_plane(vma);
2257                         vma = ERR_PTR(ret);
2258                         goto err;
2259                 }
2260
2261                 if (ret == 0 && vma->fence)
2262                         *out_flags |= PLANE_HAS_FENCE;
2263         }
2264
2265         i915_vma_get(vma);
2266 err:
2267         atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
2268         intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
2269         return vma;
2270 }
2271
2272 void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
2273 {
2274         i915_gem_object_lock(vma->obj);
2275         if (flags & PLANE_HAS_FENCE)
2276                 i915_vma_unpin_fence(vma);
2277         i915_gem_object_unpin_from_display_plane(vma);
2278         i915_gem_object_unlock(vma->obj);
2279
2280         i915_vma_put(vma);
2281 }
2282
2283 static int intel_fb_pitch(const struct drm_framebuffer *fb, int color_plane,
2284                           unsigned int rotation)
2285 {
2286         if (drm_rotation_90_or_270(rotation))
2287                 return to_intel_framebuffer(fb)->rotated[color_plane].pitch;
2288         else
2289                 return fb->pitches[color_plane];
2290 }
2291
2292 /*
2293  * Convert the x/y offsets into a linear offset.
2294  * Only valid with 0/180 degree rotation, which is fine since linear
2295  * offset is only used with linear buffers on pre-hsw and tiled buffers
2296  * with gen2/3, and 90/270 degree rotations isn't supported on any of them.
2297  */
2298 u32 intel_fb_xy_to_linear(int x, int y,
2299                           const struct intel_plane_state *state,
2300                           int color_plane)
2301 {
2302         const struct drm_framebuffer *fb = state->hw.fb;
2303         unsigned int cpp = fb->format->cpp[color_plane];
2304         unsigned int pitch = state->color_plane[color_plane].stride;
2305
2306         return y * pitch + x * cpp;
2307 }
2308
2309 /*
2310  * Add the x/y offsets derived from fb->offsets[] to the user
2311  * specified plane src x/y offsets. The resulting x/y offsets
2312  * specify the start of scanout from the beginning of the gtt mapping.
2313  */
2314 void intel_add_fb_offsets(int *x, int *y,
2315                           const struct intel_plane_state *state,
2316                           int color_plane)
2317
2318 {
2319         *x += state->color_plane[color_plane].x;
2320         *y += state->color_plane[color_plane].y;
2321 }
2322
2323 static u32 intel_adjust_tile_offset(int *x, int *y,
2324                                     unsigned int tile_width,
2325                                     unsigned int tile_height,
2326                                     unsigned int tile_size,
2327                                     unsigned int pitch_tiles,
2328                                     u32 old_offset,
2329                                     u32 new_offset)
2330 {
2331         unsigned int pitch_pixels = pitch_tiles * tile_width;
2332         unsigned int tiles;
2333
2334         WARN_ON(old_offset & (tile_size - 1));
2335         WARN_ON(new_offset & (tile_size - 1));
2336         WARN_ON(new_offset > old_offset);
2337
2338         tiles = (old_offset - new_offset) / tile_size;
2339
2340         *y += tiles / pitch_tiles * tile_height;
2341         *x += tiles % pitch_tiles * tile_width;
2342
2343         /* minimize x in case it got needlessly big */
2344         *y += *x / pitch_pixels * tile_height;
2345         *x %= pitch_pixels;
2346
2347         return new_offset;
2348 }
2349
2350 static bool is_surface_linear(const struct drm_framebuffer *fb, int color_plane)
2351 {
2352         return fb->modifier == DRM_FORMAT_MOD_LINEAR ||
2353                is_gen12_ccs_plane(fb, color_plane);
2354 }
2355
2356 static u32 intel_adjust_aligned_offset(int *x, int *y,
2357                                        const struct drm_framebuffer *fb,
2358                                        int color_plane,
2359                                        unsigned int rotation,
2360                                        unsigned int pitch,
2361                                        u32 old_offset, u32 new_offset)
2362 {
2363         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2364         unsigned int cpp = fb->format->cpp[color_plane];
2365
2366         WARN_ON(new_offset > old_offset);
2367
2368         if (!is_surface_linear(fb, color_plane)) {
2369                 unsigned int tile_size, tile_width, tile_height;
2370                 unsigned int pitch_tiles;
2371
2372                 tile_size = intel_tile_size(dev_priv);
2373                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2374
2375                 if (drm_rotation_90_or_270(rotation)) {
2376                         pitch_tiles = pitch / tile_height;
2377                         swap(tile_width, tile_height);
2378                 } else {
2379                         pitch_tiles = pitch / (tile_width * cpp);
2380                 }
2381
2382                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2383                                          tile_size, pitch_tiles,
2384                                          old_offset, new_offset);
2385         } else {
2386                 old_offset += *y * pitch + *x * cpp;
2387
2388                 *y = (old_offset - new_offset) / pitch;
2389                 *x = ((old_offset - new_offset) - *y * pitch) / cpp;
2390         }
2391
2392         return new_offset;
2393 }
2394
2395 /*
2396  * Adjust the tile offset by moving the difference into
2397  * the x/y offsets.
2398  */
2399 static u32 intel_plane_adjust_aligned_offset(int *x, int *y,
2400                                              const struct intel_plane_state *state,
2401                                              int color_plane,
2402                                              u32 old_offset, u32 new_offset)
2403 {
2404         return intel_adjust_aligned_offset(x, y, state->hw.fb, color_plane,
2405                                            state->hw.rotation,
2406                                            state->color_plane[color_plane].stride,
2407                                            old_offset, new_offset);
2408 }
2409
2410 /*
2411  * Computes the aligned offset to the base tile and adjusts
2412  * x, y. bytes per pixel is assumed to be a power-of-two.
2413  *
2414  * In the 90/270 rotated case, x and y are assumed
2415  * to be already rotated to match the rotated GTT view, and
2416  * pitch is the tile_height aligned framebuffer height.
2417  *
2418  * This function is used when computing the derived information
2419  * under intel_framebuffer, so using any of that information
2420  * here is not allowed. Anything under drm_framebuffer can be
2421  * used. This is why the user has to pass in the pitch since it
2422  * is specified in the rotated orientation.
2423  */
2424 static u32 intel_compute_aligned_offset(struct drm_i915_private *dev_priv,
2425                                         int *x, int *y,
2426                                         const struct drm_framebuffer *fb,
2427                                         int color_plane,
2428                                         unsigned int pitch,
2429                                         unsigned int rotation,
2430                                         u32 alignment)
2431 {
2432         unsigned int cpp = fb->format->cpp[color_plane];
2433         u32 offset, offset_aligned;
2434
2435         if (alignment)
2436                 alignment--;
2437
2438         if (!is_surface_linear(fb, color_plane)) {
2439                 unsigned int tile_size, tile_width, tile_height;
2440                 unsigned int tile_rows, tiles, pitch_tiles;
2441
2442                 tile_size = intel_tile_size(dev_priv);
2443                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2444
2445                 if (drm_rotation_90_or_270(rotation)) {
2446                         pitch_tiles = pitch / tile_height;
2447                         swap(tile_width, tile_height);
2448                 } else {
2449                         pitch_tiles = pitch / (tile_width * cpp);
2450                 }
2451
2452                 tile_rows = *y / tile_height;
2453                 *y %= tile_height;
2454
2455                 tiles = *x / tile_width;
2456                 *x %= tile_width;
2457
2458                 offset = (tile_rows * pitch_tiles + tiles) * tile_size;
2459                 offset_aligned = offset & ~alignment;
2460
2461                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2462                                          tile_size, pitch_tiles,
2463                                          offset, offset_aligned);
2464         } else {
2465                 offset = *y * pitch + *x * cpp;
2466                 offset_aligned = offset & ~alignment;
2467
2468                 *y = (offset & alignment) / pitch;
2469                 *x = ((offset & alignment) - *y * pitch) / cpp;
2470         }
2471
2472         return offset_aligned;
2473 }
2474
2475 static u32 intel_plane_compute_aligned_offset(int *x, int *y,
2476                                               const struct intel_plane_state *state,
2477                                               int color_plane)
2478 {
2479         struct intel_plane *intel_plane = to_intel_plane(state->uapi.plane);
2480         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
2481         const struct drm_framebuffer *fb = state->hw.fb;
2482         unsigned int rotation = state->hw.rotation;
2483         int pitch = state->color_plane[color_plane].stride;
2484         u32 alignment;
2485
2486         if (intel_plane->id == PLANE_CURSOR)
2487                 alignment = intel_cursor_alignment(dev_priv);
2488         else
2489                 alignment = intel_surf_alignment(fb, color_plane);
2490
2491         return intel_compute_aligned_offset(dev_priv, x, y, fb, color_plane,
2492                                             pitch, rotation, alignment);
2493 }
2494
2495 /* Convert the fb->offset[] into x/y offsets */
2496 static int intel_fb_offset_to_xy(int *x, int *y,
2497                                  const struct drm_framebuffer *fb,
2498                                  int color_plane)
2499 {
2500         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2501         unsigned int height;
2502
2503         if (fb->modifier != DRM_FORMAT_MOD_LINEAR &&
2504             fb->offsets[color_plane] % intel_tile_size(dev_priv)) {
2505                 DRM_DEBUG_KMS("Misaligned offset 0x%08x for color plane %d\n",
2506                               fb->offsets[color_plane], color_plane);
2507                 return -EINVAL;
2508         }
2509
2510         height = drm_framebuffer_plane_height(fb->height, fb, color_plane);
2511         height = ALIGN(height, intel_tile_height(fb, color_plane));
2512
2513         /* Catch potential overflows early */
2514         if (add_overflows_t(u32, mul_u32_u32(height, fb->pitches[color_plane]),
2515                             fb->offsets[color_plane])) {
2516                 DRM_DEBUG_KMS("Bad offset 0x%08x or pitch %d for color plane %d\n",
2517                               fb->offsets[color_plane], fb->pitches[color_plane],
2518                               color_plane);
2519                 return -ERANGE;
2520         }
2521
2522         *x = 0;
2523         *y = 0;
2524
2525         intel_adjust_aligned_offset(x, y,
2526                                     fb, color_plane, DRM_MODE_ROTATE_0,
2527                                     fb->pitches[color_plane],
2528                                     fb->offsets[color_plane], 0);
2529
2530         return 0;
2531 }
2532
2533 static unsigned int intel_fb_modifier_to_tiling(u64 fb_modifier)
2534 {
2535         switch (fb_modifier) {
2536         case I915_FORMAT_MOD_X_TILED:
2537                 return I915_TILING_X;
2538         case I915_FORMAT_MOD_Y_TILED:
2539         case I915_FORMAT_MOD_Y_TILED_CCS:
2540         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
2541                 return I915_TILING_Y;
2542         default:
2543                 return I915_TILING_NONE;
2544         }
2545 }
2546
2547 /*
2548  * From the Sky Lake PRM:
2549  * "The Color Control Surface (CCS) contains the compression status of
2550  *  the cache-line pairs. The compression state of the cache-line pair
2551  *  is specified by 2 bits in the CCS. Each CCS cache-line represents
2552  *  an area on the main surface of 16 x16 sets of 128 byte Y-tiled
2553  *  cache-line-pairs. CCS is always Y tiled."
2554  *
2555  * Since cache line pairs refers to horizontally adjacent cache lines,
2556  * each cache line in the CCS corresponds to an area of 32x16 cache
2557  * lines on the main surface. Since each pixel is 4 bytes, this gives
2558  * us a ratio of one byte in the CCS for each 8x16 pixels in the
2559  * main surface.
2560  */
2561 static const struct drm_format_info skl_ccs_formats[] = {
2562         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
2563           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2564         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
2565           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2566         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
2567           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
2568         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
2569           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
2570 };
2571
2572 /*
2573  * Gen-12 compression uses 4 bits of CCS data for each cache line pair in the
2574  * main surface. And each 64B CCS cache line represents an area of 4x1 Y-tiles
2575  * in the main surface. With 4 byte pixels and each Y-tile having dimensions of
2576  * 32x32 pixels, the ratio turns out to 1B in the CCS for every 2x32 pixels in
2577  * the main surface.
2578  */
2579 static const struct drm_format_info gen12_ccs_formats[] = {
2580         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
2581           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2582           .hsub = 1, .vsub = 1, },
2583         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
2584           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2585           .hsub = 1, .vsub = 1, },
2586         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
2587           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2588           .hsub = 1, .vsub = 1, .has_alpha = true },
2589         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
2590           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2591           .hsub = 1, .vsub = 1, .has_alpha = true },
2592 };
2593
2594 static const struct drm_format_info *
2595 lookup_format_info(const struct drm_format_info formats[],
2596                    int num_formats, u32 format)
2597 {
2598         int i;
2599
2600         for (i = 0; i < num_formats; i++) {
2601                 if (formats[i].format == format)
2602                         return &formats[i];
2603         }
2604
2605         return NULL;
2606 }
2607
2608 static const struct drm_format_info *
2609 intel_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
2610 {
2611         switch (cmd->modifier[0]) {
2612         case I915_FORMAT_MOD_Y_TILED_CCS:
2613         case I915_FORMAT_MOD_Yf_TILED_CCS:
2614                 return lookup_format_info(skl_ccs_formats,
2615                                           ARRAY_SIZE(skl_ccs_formats),
2616                                           cmd->pixel_format);
2617         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
2618                 return lookup_format_info(gen12_ccs_formats,
2619                                           ARRAY_SIZE(gen12_ccs_formats),
2620                                           cmd->pixel_format);
2621         default:
2622                 return NULL;
2623         }
2624 }
2625
2626 bool is_ccs_modifier(u64 modifier)
2627 {
2628         return modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS ||
2629                modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
2630                modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
2631 }
2632
2633 static int gen12_ccs_aux_stride(struct drm_framebuffer *fb, int ccs_plane)
2634 {
2635         return DIV_ROUND_UP(fb->pitches[ccs_to_main_plane(fb, ccs_plane)],
2636                             512) * 64;
2637 }
2638
2639 u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv,
2640                               u32 pixel_format, u64 modifier)
2641 {
2642         struct intel_crtc *crtc;
2643         struct intel_plane *plane;
2644
2645         /*
2646          * We assume the primary plane for pipe A has
2647          * the highest stride limits of them all.
2648          */
2649         crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A);
2650         if (!crtc)
2651                 return 0;
2652
2653         plane = to_intel_plane(crtc->base.primary);
2654
2655         return plane->max_stride(plane, pixel_format, modifier,
2656                                  DRM_MODE_ROTATE_0);
2657 }
2658
2659 static
2660 u32 intel_fb_max_stride(struct drm_i915_private *dev_priv,
2661                         u32 pixel_format, u64 modifier)
2662 {
2663         /*
2664          * Arbitrary limit for gen4+ chosen to match the
2665          * render engine max stride.
2666          *
2667          * The new CCS hash mode makes remapping impossible
2668          */
2669         if (!is_ccs_modifier(modifier)) {
2670                 if (INTEL_GEN(dev_priv) >= 7)
2671                         return 256*1024;
2672                 else if (INTEL_GEN(dev_priv) >= 4)
2673                         return 128*1024;
2674         }
2675
2676         return intel_plane_fb_max_stride(dev_priv, pixel_format, modifier);
2677 }
2678
2679 static u32
2680 intel_fb_stride_alignment(const struct drm_framebuffer *fb, int color_plane)
2681 {
2682         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2683         u32 tile_width;
2684
2685         if (is_surface_linear(fb, color_plane)) {
2686                 u32 max_stride = intel_plane_fb_max_stride(dev_priv,
2687                                                            fb->format->format,
2688                                                            fb->modifier);
2689
2690                 /*
2691                  * To make remapping with linear generally feasible
2692                  * we need the stride to be page aligned.
2693                  */
2694                 if (fb->pitches[color_plane] > max_stride &&
2695                     !is_ccs_modifier(fb->modifier))
2696                         return intel_tile_size(dev_priv);
2697                 else
2698                         return 64;
2699         }
2700
2701         tile_width = intel_tile_width_bytes(fb, color_plane);
2702         if (is_ccs_modifier(fb->modifier) && color_plane == 0) {
2703                 /*
2704                  * Display WA #0531: skl,bxt,kbl,glk
2705                  *
2706                  * Render decompression and plane width > 3840
2707                  * combined with horizontal panning requires the
2708                  * plane stride to be a multiple of 4. We'll just
2709                  * require the entire fb to accommodate that to avoid
2710                  * potential runtime errors at plane configuration time.
2711                  */
2712                 if (IS_GEN(dev_priv, 9) && fb->width > 3840)
2713                         tile_width *= 4;
2714                 /*
2715                  * The main surface pitch must be padded to a multiple of four
2716                  * tile widths.
2717                  */
2718                 else if (INTEL_GEN(dev_priv) >= 12)
2719                         tile_width *= 4;
2720         }
2721         return tile_width;
2722 }
2723
2724 bool intel_plane_can_remap(const struct intel_plane_state *plane_state)
2725 {
2726         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2727         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2728         const struct drm_framebuffer *fb = plane_state->hw.fb;
2729         int i;
2730
2731         /* We don't want to deal with remapping with cursors */
2732         if (plane->id == PLANE_CURSOR)
2733                 return false;
2734
2735         /*
2736          * The display engine limits already match/exceed the
2737          * render engine limits, so not much point in remapping.
2738          * Would also need to deal with the fence POT alignment
2739          * and gen2 2KiB GTT tile size.
2740          */
2741         if (INTEL_GEN(dev_priv) < 4)
2742                 return false;
2743
2744         /*
2745          * The new CCS hash mode isn't compatible with remapping as
2746          * the virtual address of the pages affects the compressed data.
2747          */
2748         if (is_ccs_modifier(fb->modifier))
2749                 return false;
2750
2751         /* Linear needs a page aligned stride for remapping */
2752         if (fb->modifier == DRM_FORMAT_MOD_LINEAR) {
2753                 unsigned int alignment = intel_tile_size(dev_priv) - 1;
2754
2755                 for (i = 0; i < fb->format->num_planes; i++) {
2756                         if (fb->pitches[i] & alignment)
2757                                 return false;
2758                 }
2759         }
2760
2761         return true;
2762 }
2763
2764 static bool intel_plane_needs_remap(const struct intel_plane_state *plane_state)
2765 {
2766         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2767         const struct drm_framebuffer *fb = plane_state->hw.fb;
2768         unsigned int rotation = plane_state->hw.rotation;
2769         u32 stride, max_stride;
2770
2771         /*
2772          * No remapping for invisible planes since we don't have
2773          * an actual source viewport to remap.
2774          */
2775         if (!plane_state->uapi.visible)
2776                 return false;
2777
2778         if (!intel_plane_can_remap(plane_state))
2779                 return false;
2780
2781         /*
2782          * FIXME: aux plane limits on gen9+ are
2783          * unclear in Bspec, for now no checking.
2784          */
2785         stride = intel_fb_pitch(fb, 0, rotation);
2786         max_stride = plane->max_stride(plane, fb->format->format,
2787                                        fb->modifier, rotation);
2788
2789         return stride > max_stride;
2790 }
2791
2792 static void
2793 intel_fb_plane_get_subsampling(int *hsub, int *vsub,
2794                                const struct drm_framebuffer *fb,
2795                                int color_plane)
2796 {
2797         int main_plane;
2798
2799         if (color_plane == 0) {
2800                 *hsub = 1;
2801                 *vsub = 1;
2802
2803                 return;
2804         }
2805
2806         /*
2807          * TODO: Deduct the subsampling from the char block for all CCS
2808          * formats and planes.
2809          */
2810         if (!is_gen12_ccs_plane(fb, color_plane)) {
2811                 *hsub = fb->format->hsub;
2812                 *vsub = fb->format->vsub;
2813
2814                 return;
2815         }
2816
2817         main_plane = ccs_to_main_plane(fb, color_plane);
2818         *hsub = drm_format_info_block_width(fb->format, color_plane) /
2819                 drm_format_info_block_width(fb->format, main_plane);
2820
2821         /*
2822          * The min stride check in the core framebuffer_check() function
2823          * assumes that format->hsub applies to every plane except for the
2824          * first plane. That's incorrect for the CCS AUX plane of the first
2825          * plane, but for the above check to pass we must define the block
2826          * width with that subsampling applied to it. Adjust the width here
2827          * accordingly, so we can calculate the actual subsampling factor.
2828          */
2829         if (main_plane == 0)
2830                 *hsub *= fb->format->hsub;
2831
2832         *vsub = 32;
2833 }
2834 static int
2835 intel_fb_check_ccs_xy(struct drm_framebuffer *fb, int ccs_plane, int x, int y)
2836 {
2837         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2838         int main_plane;
2839         int hsub, vsub;
2840         int tile_width, tile_height;
2841         int ccs_x, ccs_y;
2842         int main_x, main_y;
2843
2844         if (!is_ccs_plane(fb, ccs_plane))
2845                 return 0;
2846
2847         intel_tile_dims(fb, ccs_plane, &tile_width, &tile_height);
2848         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, ccs_plane);
2849
2850         tile_width *= hsub;
2851         tile_height *= vsub;
2852
2853         ccs_x = (x * hsub) % tile_width;
2854         ccs_y = (y * vsub) % tile_height;
2855
2856         main_plane = ccs_to_main_plane(fb, ccs_plane);
2857         main_x = intel_fb->normal[main_plane].x % tile_width;
2858         main_y = intel_fb->normal[main_plane].y % tile_height;
2859
2860         /*
2861          * CCS doesn't have its own x/y offset register, so the intra CCS tile
2862          * x/y offsets must match between CCS and the main surface.
2863          */
2864         if (main_x != ccs_x || main_y != ccs_y) {
2865                 DRM_DEBUG_KMS("Bad CCS x/y (main %d,%d ccs %d,%d) full (main %d,%d ccs %d,%d)\n",
2866                               main_x, main_y,
2867                               ccs_x, ccs_y,
2868                               intel_fb->normal[main_plane].x,
2869                               intel_fb->normal[main_plane].y,
2870                               x, y);
2871                 return -EINVAL;
2872         }
2873
2874         return 0;
2875 }
2876
2877 static void
2878 intel_fb_plane_dims(int *w, int *h, struct drm_framebuffer *fb, int color_plane)
2879 {
2880         int hsub, vsub;
2881
2882         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, color_plane);
2883         *w = fb->width / hsub;
2884         *h = fb->height / vsub;
2885 }
2886
2887 /*
2888  * Setup the rotated view for an FB plane and return the size the GTT mapping
2889  * requires for this view.
2890  */
2891 static u32
2892 setup_fb_rotation(int plane, const struct intel_remapped_plane_info *plane_info,
2893                   u32 gtt_offset_rotated, int x, int y,
2894                   unsigned int width, unsigned int height,
2895                   unsigned int tile_size,
2896                   unsigned int tile_width, unsigned int tile_height,
2897                   struct drm_framebuffer *fb)
2898 {
2899         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2900         struct intel_rotation_info *rot_info = &intel_fb->rot_info;
2901         unsigned int pitch_tiles;
2902         struct drm_rect r;
2903
2904         /* Y or Yf modifiers required for 90/270 rotation */
2905         if (fb->modifier != I915_FORMAT_MOD_Y_TILED &&
2906             fb->modifier != I915_FORMAT_MOD_Yf_TILED)
2907                 return 0;
2908
2909         if (WARN_ON(plane >= ARRAY_SIZE(rot_info->plane)))
2910                 return 0;
2911
2912         rot_info->plane[plane] = *plane_info;
2913
2914         intel_fb->rotated[plane].pitch = plane_info->height * tile_height;
2915
2916         /* rotate the x/y offsets to match the GTT view */
2917         drm_rect_init(&r, x, y, width, height);
2918         drm_rect_rotate(&r,
2919                         plane_info->width * tile_width,
2920                         plane_info->height * tile_height,
2921                         DRM_MODE_ROTATE_270);
2922         x = r.x1;
2923         y = r.y1;
2924
2925         /* rotate the tile dimensions to match the GTT view */
2926         pitch_tiles = intel_fb->rotated[plane].pitch / tile_height;
2927         swap(tile_width, tile_height);
2928
2929         /*
2930          * We only keep the x/y offsets, so push all of the
2931          * gtt offset into the x/y offsets.
2932          */
2933         intel_adjust_tile_offset(&x, &y,
2934                                  tile_width, tile_height,
2935                                  tile_size, pitch_tiles,
2936                                  gtt_offset_rotated * tile_size, 0);
2937
2938         /*
2939          * First pixel of the framebuffer from
2940          * the start of the rotated gtt mapping.
2941          */
2942         intel_fb->rotated[plane].x = x;
2943         intel_fb->rotated[plane].y = y;
2944
2945         return plane_info->width * plane_info->height;
2946 }
2947
2948 static int
2949 intel_fill_fb_info(struct drm_i915_private *dev_priv,
2950                    struct drm_framebuffer *fb)
2951 {
2952         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2953         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2954         u32 gtt_offset_rotated = 0;
2955         unsigned int max_size = 0;
2956         int i, num_planes = fb->format->num_planes;
2957         unsigned int tile_size = intel_tile_size(dev_priv);
2958
2959         for (i = 0; i < num_planes; i++) {
2960                 unsigned int width, height;
2961                 unsigned int cpp, size;
2962                 u32 offset;
2963                 int x, y;
2964                 int ret;
2965
2966                 cpp = fb->format->cpp[i];
2967                 intel_fb_plane_dims(&width, &height, fb, i);
2968
2969                 ret = intel_fb_offset_to_xy(&x, &y, fb, i);
2970                 if (ret) {
2971                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2972                                       i, fb->offsets[i]);
2973                         return ret;
2974                 }
2975
2976                 ret = intel_fb_check_ccs_xy(fb, i, x, y);
2977                 if (ret)
2978                         return ret;
2979
2980                 /*
2981                  * The fence (if used) is aligned to the start of the object
2982                  * so having the framebuffer wrap around across the edge of the
2983                  * fenced region doesn't really work. We have no API to configure
2984                  * the fence start offset within the object (nor could we probably
2985                  * on gen2/3). So it's just easier if we just require that the
2986                  * fb layout agrees with the fence layout. We already check that the
2987                  * fb stride matches the fence stride elsewhere.
2988                  */
2989                 if (i == 0 && i915_gem_object_is_tiled(obj) &&
2990                     (x + width) * cpp > fb->pitches[i]) {
2991                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2992                                       i, fb->offsets[i]);
2993                         return -EINVAL;
2994                 }
2995
2996                 /*
2997                  * First pixel of the framebuffer from
2998                  * the start of the normal gtt mapping.
2999                  */
3000                 intel_fb->normal[i].x = x;
3001                 intel_fb->normal[i].y = y;
3002
3003                 offset = intel_compute_aligned_offset(dev_priv, &x, &y, fb, i,
3004                                                       fb->pitches[i],
3005                                                       DRM_MODE_ROTATE_0,
3006                                                       tile_size);
3007                 offset /= tile_size;
3008
3009                 if (!is_surface_linear(fb, i)) {
3010                         struct intel_remapped_plane_info plane_info;
3011                         unsigned int tile_width, tile_height;
3012
3013                         intel_tile_dims(fb, i, &tile_width, &tile_height);
3014
3015                         plane_info.offset = offset;
3016                         plane_info.stride = DIV_ROUND_UP(fb->pitches[i],
3017                                                          tile_width * cpp);
3018                         plane_info.width = DIV_ROUND_UP(x + width, tile_width);
3019                         plane_info.height = DIV_ROUND_UP(y + height,
3020                                                          tile_height);
3021
3022                         /* how many tiles does this plane need */
3023                         size = plane_info.stride * plane_info.height;
3024                         /*
3025                          * If the plane isn't horizontally tile aligned,
3026                          * we need one more tile.
3027                          */
3028                         if (x != 0)
3029                                 size++;
3030
3031                         gtt_offset_rotated +=
3032                                 setup_fb_rotation(i, &plane_info,
3033                                                   gtt_offset_rotated,
3034                                                   x, y, width, height,
3035                                                   tile_size,
3036                                                   tile_width, tile_height,
3037                                                   fb);
3038                 } else {
3039                         size = DIV_ROUND_UP((y + height) * fb->pitches[i] +
3040                                             x * cpp, tile_size);
3041                 }
3042
3043                 /* how many tiles in total needed in the bo */
3044                 max_size = max(max_size, offset + size);
3045         }
3046
3047         if (mul_u32_u32(max_size, tile_size) > obj->base.size) {
3048                 DRM_DEBUG_KMS("fb too big for bo (need %llu bytes, have %zu bytes)\n",
3049                               mul_u32_u32(max_size, tile_size), obj->base.size);
3050                 return -EINVAL;
3051         }
3052
3053         return 0;
3054 }
3055
3056 static void
3057 intel_plane_remap_gtt(struct intel_plane_state *plane_state)
3058 {
3059         struct drm_i915_private *dev_priv =
3060                 to_i915(plane_state->uapi.plane->dev);
3061         struct drm_framebuffer *fb = plane_state->hw.fb;
3062         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
3063         struct intel_rotation_info *info = &plane_state->view.rotated;
3064         unsigned int rotation = plane_state->hw.rotation;
3065         int i, num_planes = fb->format->num_planes;
3066         unsigned int tile_size = intel_tile_size(dev_priv);
3067         unsigned int src_x, src_y;
3068         unsigned int src_w, src_h;
3069         u32 gtt_offset = 0;
3070
3071         memset(&plane_state->view, 0, sizeof(plane_state->view));
3072         plane_state->view.type = drm_rotation_90_or_270(rotation) ?
3073                 I915_GGTT_VIEW_ROTATED : I915_GGTT_VIEW_REMAPPED;
3074
3075         src_x = plane_state->uapi.src.x1 >> 16;
3076         src_y = plane_state->uapi.src.y1 >> 16;
3077         src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
3078         src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
3079
3080         WARN_ON(is_ccs_modifier(fb->modifier));
3081
3082         /* Make src coordinates relative to the viewport */
3083         drm_rect_translate(&plane_state->uapi.src,
3084                            -(src_x << 16), -(src_y << 16));
3085
3086         /* Rotate src coordinates to match rotated GTT view */
3087         if (drm_rotation_90_or_270(rotation))
3088                 drm_rect_rotate(&plane_state->uapi.src,
3089                                 src_w << 16, src_h << 16,
3090                                 DRM_MODE_ROTATE_270);
3091
3092         for (i = 0; i < num_planes; i++) {
3093                 unsigned int hsub = i ? fb->format->hsub : 1;
3094                 unsigned int vsub = i ? fb->format->vsub : 1;
3095                 unsigned int cpp = fb->format->cpp[i];
3096                 unsigned int tile_width, tile_height;
3097                 unsigned int width, height;
3098                 unsigned int pitch_tiles;
3099                 unsigned int x, y;
3100                 u32 offset;
3101
3102                 intel_tile_dims(fb, i, &tile_width, &tile_height);
3103
3104                 x = src_x / hsub;
3105                 y = src_y / vsub;
3106                 width = src_w / hsub;
3107                 height = src_h / vsub;
3108
3109                 /*
3110                  * First pixel of the src viewport from the
3111                  * start of the normal gtt mapping.
3112                  */
3113                 x += intel_fb->normal[i].x;
3114                 y += intel_fb->normal[i].y;
3115
3116                 offset = intel_compute_aligned_offset(dev_priv, &x, &y,
3117                                                       fb, i, fb->pitches[i],
3118                                                       DRM_MODE_ROTATE_0, tile_size);
3119                 offset /= tile_size;
3120
3121                 WARN_ON(i >= ARRAY_SIZE(info->plane));
3122                 info->plane[i].offset = offset;
3123                 info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i],
3124                                                      tile_width * cpp);
3125                 info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
3126                 info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
3127
3128                 if (drm_rotation_90_or_270(rotation)) {
3129                         struct drm_rect r;
3130
3131                         /* rotate the x/y offsets to match the GTT view */
3132                         drm_rect_init(&r, x, y, width, height);
3133                         drm_rect_rotate(&r,
3134                                         info->plane[i].width * tile_width,
3135                                         info->plane[i].height * tile_height,
3136                                         DRM_MODE_ROTATE_270);
3137                         x = r.x1;
3138                         y = r.y1;
3139
3140                         pitch_tiles = info->plane[i].height;
3141                         plane_state->color_plane[i].stride = pitch_tiles * tile_height;
3142
3143                         /* rotate the tile dimensions to match the GTT view */
3144                         swap(tile_width, tile_height);
3145                 } else {
3146                         pitch_tiles = info->plane[i].width;
3147                         plane_state->color_plane[i].stride = pitch_tiles * tile_width * cpp;
3148                 }
3149
3150                 /*
3151                  * We only keep the x/y offsets, so push all of the
3152                  * gtt offset into the x/y offsets.
3153                  */
3154                 intel_adjust_tile_offset(&x, &y,
3155                                          tile_width, tile_height,
3156                                          tile_size, pitch_tiles,
3157                                          gtt_offset * tile_size, 0);
3158
3159                 gtt_offset += info->plane[i].width * info->plane[i].height;
3160
3161                 plane_state->color_plane[i].offset = 0;
3162                 plane_state->color_plane[i].x = x;
3163                 plane_state->color_plane[i].y = y;
3164         }
3165 }
3166
3167 static int
3168 intel_plane_compute_gtt(struct intel_plane_state *plane_state)
3169 {
3170         const struct intel_framebuffer *fb =
3171                 to_intel_framebuffer(plane_state->hw.fb);
3172         unsigned int rotation = plane_state->hw.rotation;
3173         int i, num_planes;
3174
3175         if (!fb)
3176                 return 0;
3177
3178         num_planes = fb->base.format->num_planes;
3179
3180         if (intel_plane_needs_remap(plane_state)) {
3181                 intel_plane_remap_gtt(plane_state);
3182
3183                 /*
3184                  * Sometimes even remapping can't overcome
3185                  * the stride limitations :( Can happen with
3186                  * big plane sizes and suitably misaligned
3187                  * offsets.
3188                  */
3189                 return intel_plane_check_stride(plane_state);
3190         }
3191
3192         intel_fill_fb_ggtt_view(&plane_state->view, &fb->base, rotation);
3193
3194         for (i = 0; i < num_planes; i++) {
3195                 plane_state->color_plane[i].stride = intel_fb_pitch(&fb->base, i, rotation);
3196                 plane_state->color_plane[i].offset = 0;
3197
3198                 if (drm_rotation_90_or_270(rotation)) {
3199                         plane_state->color_plane[i].x = fb->rotated[i].x;
3200                         plane_state->color_plane[i].y = fb->rotated[i].y;
3201                 } else {
3202                         plane_state->color_plane[i].x = fb->normal[i].x;
3203                         plane_state->color_plane[i].y = fb->normal[i].y;
3204                 }
3205         }
3206
3207         /* Rotate src coordinates to match rotated GTT view */
3208         if (drm_rotation_90_or_270(rotation))
3209                 drm_rect_rotate(&plane_state->uapi.src,
3210                                 fb->base.width << 16, fb->base.height << 16,
3211                                 DRM_MODE_ROTATE_270);
3212
3213         return intel_plane_check_stride(plane_state);
3214 }
3215
3216 static int i9xx_format_to_fourcc(int format)
3217 {
3218         switch (format) {
3219         case DISPPLANE_8BPP:
3220                 return DRM_FORMAT_C8;
3221         case DISPPLANE_BGRA555:
3222                 return DRM_FORMAT_ARGB1555;
3223         case DISPPLANE_BGRX555:
3224                 return DRM_FORMAT_XRGB1555;
3225         case DISPPLANE_BGRX565:
3226                 return DRM_FORMAT_RGB565;
3227         default:
3228         case DISPPLANE_BGRX888:
3229                 return DRM_FORMAT_XRGB8888;
3230         case DISPPLANE_RGBX888:
3231                 return DRM_FORMAT_XBGR8888;
3232         case DISPPLANE_BGRA888:
3233                 return DRM_FORMAT_ARGB8888;
3234         case DISPPLANE_RGBA888:
3235                 return DRM_FORMAT_ABGR8888;
3236         case DISPPLANE_BGRX101010:
3237                 return DRM_FORMAT_XRGB2101010;
3238         case DISPPLANE_RGBX101010:
3239                 return DRM_FORMAT_XBGR2101010;
3240         case DISPPLANE_BGRA101010:
3241                 return DRM_FORMAT_ARGB2101010;
3242         case DISPPLANE_RGBA101010:
3243                 return DRM_FORMAT_ABGR2101010;
3244         case DISPPLANE_RGBX161616:
3245                 return DRM_FORMAT_XBGR16161616F;
3246         }
3247 }
3248
3249 int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
3250 {
3251         switch (format) {
3252         case PLANE_CTL_FORMAT_RGB_565:
3253                 return DRM_FORMAT_RGB565;
3254         case PLANE_CTL_FORMAT_NV12:
3255                 return DRM_FORMAT_NV12;
3256         case PLANE_CTL_FORMAT_P010:
3257                 return DRM_FORMAT_P010;
3258         case PLANE_CTL_FORMAT_P012:
3259                 return DRM_FORMAT_P012;
3260         case PLANE_CTL_FORMAT_P016:
3261                 return DRM_FORMAT_P016;
3262         case PLANE_CTL_FORMAT_Y210:
3263                 return DRM_FORMAT_Y210;
3264         case PLANE_CTL_FORMAT_Y212:
3265                 return DRM_FORMAT_Y212;
3266         case PLANE_CTL_FORMAT_Y216:
3267                 return DRM_FORMAT_Y216;
3268         case PLANE_CTL_FORMAT_Y410:
3269                 return DRM_FORMAT_XVYU2101010;
3270         case PLANE_CTL_FORMAT_Y412:
3271                 return DRM_FORMAT_XVYU12_16161616;
3272         case PLANE_CTL_FORMAT_Y416:
3273                 return DRM_FORMAT_XVYU16161616;
3274         default:
3275         case PLANE_CTL_FORMAT_XRGB_8888:
3276                 if (rgb_order) {
3277                         if (alpha)
3278                                 return DRM_FORMAT_ABGR8888;
3279                         else
3280                                 return DRM_FORMAT_XBGR8888;
3281                 } else {
3282                         if (alpha)
3283                                 return DRM_FORMAT_ARGB8888;
3284                         else
3285                                 return DRM_FORMAT_XRGB8888;
3286                 }
3287         case PLANE_CTL_FORMAT_XRGB_2101010:
3288                 if (rgb_order) {
3289                         if (alpha)
3290                                 return DRM_FORMAT_ABGR2101010;
3291                         else
3292                                 return DRM_FORMAT_XBGR2101010;
3293                 } else {
3294                         if (alpha)
3295                                 return DRM_FORMAT_ARGB2101010;
3296                         else
3297                                 return DRM_FORMAT_XRGB2101010;
3298                 }
3299         case PLANE_CTL_FORMAT_XRGB_16161616F:
3300                 if (rgb_order) {
3301                         if (alpha)
3302                                 return DRM_FORMAT_ABGR16161616F;
3303                         else
3304                                 return DRM_FORMAT_XBGR16161616F;
3305                 } else {
3306                         if (alpha)
3307                                 return DRM_FORMAT_ARGB16161616F;
3308                         else
3309                                 return DRM_FORMAT_XRGB16161616F;
3310                 }
3311         }
3312 }
3313
3314 static bool
3315 intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
3316                               struct intel_initial_plane_config *plane_config)
3317 {
3318         struct drm_device *dev = crtc->base.dev;
3319         struct drm_i915_private *dev_priv = to_i915(dev);
3320         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
3321         struct drm_framebuffer *fb = &plane_config->fb->base;
3322         u32 base_aligned = round_down(plane_config->base, PAGE_SIZE);
3323         u32 size_aligned = round_up(plane_config->base + plane_config->size,
3324                                     PAGE_SIZE);
3325         struct drm_i915_gem_object *obj;
3326         bool ret = false;
3327
3328         size_aligned -= base_aligned;
3329
3330         if (plane_config->size == 0)
3331                 return false;
3332
3333         /* If the FB is too big, just don't use it since fbdev is not very
3334          * important and we should probably use that space with FBC or other
3335          * features. */
3336         if (size_aligned * 2 > dev_priv->stolen_usable_size)
3337                 return false;
3338
3339         switch (fb->modifier) {
3340         case DRM_FORMAT_MOD_LINEAR:
3341         case I915_FORMAT_MOD_X_TILED:
3342         case I915_FORMAT_MOD_Y_TILED:
3343                 break;
3344         default:
3345                 DRM_DEBUG_DRIVER("Unsupported modifier for initial FB: 0x%llx\n",
3346                                  fb->modifier);
3347                 return false;
3348         }
3349
3350         obj = i915_gem_object_create_stolen_for_preallocated(dev_priv,
3351                                                              base_aligned,
3352                                                              base_aligned,
3353                                                              size_aligned);
3354         if (IS_ERR(obj))
3355                 return false;
3356
3357         switch (plane_config->tiling) {
3358         case I915_TILING_NONE:
3359                 break;
3360         case I915_TILING_X:
3361         case I915_TILING_Y:
3362                 obj->tiling_and_stride = fb->pitches[0] | plane_config->tiling;
3363                 break;
3364         default:
3365                 MISSING_CASE(plane_config->tiling);
3366                 goto out;
3367         }
3368
3369         mode_cmd.pixel_format = fb->format->format;
3370         mode_cmd.width = fb->width;
3371         mode_cmd.height = fb->height;
3372         mode_cmd.pitches[0] = fb->pitches[0];
3373         mode_cmd.modifier[0] = fb->modifier;
3374         mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
3375
3376         if (intel_framebuffer_init(to_intel_framebuffer(fb), obj, &mode_cmd)) {
3377                 DRM_DEBUG_KMS("intel fb init failed\n");
3378                 goto out;
3379         }
3380
3381
3382         DRM_DEBUG_KMS("initial plane fb obj %p\n", obj);
3383         ret = true;
3384 out:
3385         i915_gem_object_put(obj);
3386         return ret;
3387 }
3388
3389 static void
3390 intel_set_plane_visible(struct intel_crtc_state *crtc_state,
3391                         struct intel_plane_state *plane_state,
3392                         bool visible)
3393 {
3394         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
3395
3396         plane_state->uapi.visible = visible;
3397
3398         if (visible)
3399                 crtc_state->uapi.plane_mask |= drm_plane_mask(&plane->base);
3400         else
3401                 crtc_state->uapi.plane_mask &= ~drm_plane_mask(&plane->base);
3402 }
3403
3404 static void fixup_active_planes(struct intel_crtc_state *crtc_state)
3405 {
3406         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3407         struct drm_plane *plane;
3408
3409         /*
3410          * Active_planes aliases if multiple "primary" or cursor planes
3411          * have been used on the same (or wrong) pipe. plane_mask uses
3412          * unique ids, hence we can use that to reconstruct active_planes.
3413          */
3414         crtc_state->active_planes = 0;
3415
3416         drm_for_each_plane_mask(plane, &dev_priv->drm,
3417                                 crtc_state->uapi.plane_mask)
3418                 crtc_state->active_planes |= BIT(to_intel_plane(plane)->id);
3419 }
3420
3421 static void intel_plane_disable_noatomic(struct intel_crtc *crtc,
3422                                          struct intel_plane *plane)
3423 {
3424         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3425         struct intel_crtc_state *crtc_state =
3426                 to_intel_crtc_state(crtc->base.state);
3427         struct intel_plane_state *plane_state =
3428                 to_intel_plane_state(plane->base.state);
3429
3430         DRM_DEBUG_KMS("Disabling [PLANE:%d:%s] on [CRTC:%d:%s]\n",
3431                       plane->base.base.id, plane->base.name,
3432                       crtc->base.base.id, crtc->base.name);
3433
3434         intel_set_plane_visible(crtc_state, plane_state, false);
3435         fixup_active_planes(crtc_state);
3436         crtc_state->data_rate[plane->id] = 0;
3437         crtc_state->min_cdclk[plane->id] = 0;
3438
3439         if (plane->id == PLANE_PRIMARY)
3440                 hsw_disable_ips(crtc_state);
3441
3442         /*
3443          * Vblank time updates from the shadow to live plane control register
3444          * are blocked if the memory self-refresh mode is active at that
3445          * moment. So to make sure the plane gets truly disabled, disable
3446          * first the self-refresh mode. The self-refresh enable bit in turn
3447          * will be checked/applied by the HW only at the next frame start
3448          * event which is after the vblank start event, so we need to have a
3449          * wait-for-vblank between disabling the plane and the pipe.
3450          */
3451         if (HAS_GMCH(dev_priv) &&
3452             intel_set_memory_cxsr(dev_priv, false))
3453                 intel_wait_for_vblank(dev_priv, crtc->pipe);
3454
3455         /*
3456          * Gen2 reports pipe underruns whenever all planes are disabled.
3457          * So disable underrun reporting before all the planes get disabled.
3458          */
3459         if (IS_GEN(dev_priv, 2) && !crtc_state->active_planes)
3460                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
3461
3462         intel_disable_plane(plane, crtc_state);
3463 }
3464
3465 static struct intel_frontbuffer *
3466 to_intel_frontbuffer(struct drm_framebuffer *fb)
3467 {
3468         return fb ? to_intel_framebuffer(fb)->frontbuffer : NULL;
3469 }
3470
3471 static void
3472 intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
3473                              struct intel_initial_plane_config *plane_config)
3474 {
3475         struct drm_device *dev = intel_crtc->base.dev;
3476         struct drm_i915_private *dev_priv = to_i915(dev);
3477         struct drm_crtc *c;
3478         struct drm_plane *primary = intel_crtc->base.primary;
3479         struct drm_plane_state *plane_state = primary->state;
3480         struct intel_plane *intel_plane = to_intel_plane(primary);
3481         struct intel_plane_state *intel_state =
3482                 to_intel_plane_state(plane_state);
3483         struct drm_framebuffer *fb;
3484
3485         if (!plane_config->fb)
3486                 return;
3487
3488         if (intel_alloc_initial_plane_obj(intel_crtc, plane_config)) {
3489                 fb = &plane_config->fb->base;
3490                 goto valid_fb;
3491         }
3492
3493         kfree(plane_config->fb);
3494
3495         /*
3496          * Failed to alloc the obj, check to see if we should share
3497          * an fb with another CRTC instead
3498          */
3499         for_each_crtc(dev, c) {
3500                 struct intel_plane_state *state;
3501
3502                 if (c == &intel_crtc->base)
3503                         continue;
3504
3505                 if (!to_intel_crtc(c)->active)
3506                         continue;
3507
3508                 state = to_intel_plane_state(c->primary->state);
3509                 if (!state->vma)
3510                         continue;
3511
3512                 if (intel_plane_ggtt_offset(state) == plane_config->base) {
3513                         fb = state->hw.fb;
3514                         drm_framebuffer_get(fb);
3515                         goto valid_fb;
3516                 }
3517         }
3518
3519         /*
3520          * We've failed to reconstruct the BIOS FB.  Current display state
3521          * indicates that the primary plane is visible, but has a NULL FB,
3522          * which will lead to problems later if we don't fix it up.  The
3523          * simplest solution is to just disable the primary plane now and
3524          * pretend the BIOS never had it enabled.
3525          */
3526         intel_plane_disable_noatomic(intel_crtc, intel_plane);
3527
3528         return;
3529
3530 valid_fb:
3531         intel_state->hw.rotation = plane_config->rotation;
3532         intel_fill_fb_ggtt_view(&intel_state->view, fb,
3533                                 intel_state->hw.rotation);
3534         intel_state->color_plane[0].stride =
3535                 intel_fb_pitch(fb, 0, intel_state->hw.rotation);
3536
3537         intel_state->vma =
3538                 intel_pin_and_fence_fb_obj(fb,
3539                                            &intel_state->view,
3540                                            intel_plane_uses_fence(intel_state),
3541                                            &intel_state->flags);
3542         if (IS_ERR(intel_state->vma)) {
3543                 DRM_ERROR("failed to pin boot fb on pipe %d: %li\n",
3544                           intel_crtc->pipe, PTR_ERR(intel_state->vma));
3545
3546                 intel_state->vma = NULL;
3547                 drm_framebuffer_put(fb);
3548                 return;
3549         }
3550
3551         intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB);
3552
3553         plane_state->src_x = 0;
3554         plane_state->src_y = 0;
3555         plane_state->src_w = fb->width << 16;
3556         plane_state->src_h = fb->height << 16;
3557
3558         plane_state->crtc_x = 0;
3559         plane_state->crtc_y = 0;
3560         plane_state->crtc_w = fb->width;
3561         plane_state->crtc_h = fb->height;
3562
3563         intel_state->uapi.src = drm_plane_state_src(plane_state);
3564         intel_state->uapi.dst = drm_plane_state_dest(plane_state);
3565
3566         if (plane_config->tiling)
3567                 dev_priv->preserve_bios_swizzle = true;
3568
3569         plane_state->fb = fb;
3570         plane_state->crtc = &intel_crtc->base;
3571         intel_plane_copy_uapi_to_hw_state(intel_state, intel_state);
3572
3573         atomic_or(to_intel_plane(primary)->frontbuffer_bit,
3574                   &to_intel_frontbuffer(fb)->bits);
3575 }
3576
3577 static int skl_max_plane_width(const struct drm_framebuffer *fb,
3578                                int color_plane,
3579                                unsigned int rotation)
3580 {
3581         int cpp = fb->format->cpp[color_plane];
3582
3583         switch (fb->modifier) {
3584         case DRM_FORMAT_MOD_LINEAR:
3585         case I915_FORMAT_MOD_X_TILED:
3586                 /*
3587                  * Validated limit is 4k, but has 5k should
3588                  * work apart from the following features:
3589                  * - Ytile (already limited to 4k)
3590                  * - FP16 (already limited to 4k)
3591                  * - render compression (already limited to 4k)
3592                  * - KVMR sprite and cursor (don't care)
3593                  * - horizontal panning (TODO verify this)
3594                  * - pipe and plane scaling (TODO verify this)
3595                  */
3596                 if (cpp == 8)
3597                         return 4096;
3598                 else
3599                         return 5120;
3600         case I915_FORMAT_MOD_Y_TILED_CCS:
3601         case I915_FORMAT_MOD_Yf_TILED_CCS:
3602                 /* FIXME AUX plane? */
3603         case I915_FORMAT_MOD_Y_TILED:
3604         case I915_FORMAT_MOD_Yf_TILED:
3605                 if (cpp == 8)
3606                         return 2048;
3607                 else
3608                         return 4096;
3609         default:
3610                 MISSING_CASE(fb->modifier);
3611                 return 2048;
3612         }
3613 }
3614
3615 static int glk_max_plane_width(const struct drm_framebuffer *fb,
3616                                int color_plane,
3617                                unsigned int rotation)
3618 {
3619         int cpp = fb->format->cpp[color_plane];
3620
3621         switch (fb->modifier) {
3622         case DRM_FORMAT_MOD_LINEAR:
3623         case I915_FORMAT_MOD_X_TILED:
3624                 if (cpp == 8)
3625                         return 4096;
3626                 else
3627                         return 5120;
3628         case I915_FORMAT_MOD_Y_TILED_CCS:
3629         case I915_FORMAT_MOD_Yf_TILED_CCS:
3630                 /* FIXME AUX plane? */
3631         case I915_FORMAT_MOD_Y_TILED:
3632         case I915_FORMAT_MOD_Yf_TILED:
3633                 if (cpp == 8)
3634                         return 2048;
3635                 else
3636                         return 5120;
3637         default:
3638                 MISSING_CASE(fb->modifier);
3639                 return 2048;
3640         }
3641 }
3642
3643 static int icl_max_plane_width(const struct drm_framebuffer *fb,
3644                                int color_plane,
3645                                unsigned int rotation)
3646 {
3647         return 5120;
3648 }
3649
3650 static int skl_max_plane_height(void)
3651 {
3652         return 4096;
3653 }
3654
3655 static int icl_max_plane_height(void)
3656 {
3657         return 4320;
3658 }
3659
3660 static bool skl_check_main_ccs_coordinates(struct intel_plane_state *plane_state,
3661                                            int main_x, int main_y, u32 main_offset)
3662 {
3663         const struct drm_framebuffer *fb = plane_state->hw.fb;
3664         int ccs_plane = main_to_ccs_plane(fb, 0);
3665         int aux_x = plane_state->color_plane[ccs_plane].x;
3666         int aux_y = plane_state->color_plane[ccs_plane].y;
3667         u32 aux_offset = plane_state->color_plane[ccs_plane].offset;
3668         u32 alignment = intel_surf_alignment(fb, ccs_plane);
3669         int hsub;
3670         int vsub;
3671
3672         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, ccs_plane);
3673         while (aux_offset >= main_offset && aux_y <= main_y) {
3674                 int x, y;
3675
3676                 if (aux_x == main_x && aux_y == main_y)
3677                         break;
3678
3679                 if (aux_offset == 0)
3680                         break;
3681
3682                 x = aux_x / hsub;
3683                 y = aux_y / vsub;
3684                 aux_offset = intel_plane_adjust_aligned_offset(&x, &y,
3685                                                                plane_state,
3686                                                                ccs_plane,
3687                                                                aux_offset,
3688                                                                aux_offset -
3689                                                                 alignment);
3690                 aux_x = x * hsub + aux_x % hsub;
3691                 aux_y = y * vsub + aux_y % vsub;
3692         }
3693
3694         if (aux_x != main_x || aux_y != main_y)
3695                 return false;
3696
3697         plane_state->color_plane[ccs_plane].offset = aux_offset;
3698         plane_state->color_plane[ccs_plane].x = aux_x;
3699         plane_state->color_plane[ccs_plane].y = aux_y;
3700
3701         return true;
3702 }
3703
3704 static int skl_check_main_surface(struct intel_plane_state *plane_state)
3705 {
3706         struct drm_i915_private *dev_priv = to_i915(plane_state->uapi.plane->dev);
3707         const struct drm_framebuffer *fb = plane_state->hw.fb;
3708         unsigned int rotation = plane_state->hw.rotation;
3709         int x = plane_state->uapi.src.x1 >> 16;
3710         int y = plane_state->uapi.src.y1 >> 16;
3711         int w = drm_rect_width(&plane_state->uapi.src) >> 16;
3712         int h = drm_rect_height(&plane_state->uapi.src) >> 16;
3713         int max_width;
3714         int max_height;
3715         u32 alignment;
3716         u32 offset;
3717         int aux_plane = intel_main_to_aux_plane(fb, 0);
3718         u32 aux_offset = plane_state->color_plane[aux_plane].offset;
3719
3720         if (INTEL_GEN(dev_priv) >= 11)
3721                 max_width = icl_max_plane_width(fb, 0, rotation);
3722         else if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
3723                 max_width = glk_max_plane_width(fb, 0, rotation);
3724         else
3725                 max_width = skl_max_plane_width(fb, 0, rotation);
3726
3727         if (INTEL_GEN(dev_priv) >= 11)
3728                 max_height = icl_max_plane_height();
3729         else
3730                 max_height = skl_max_plane_height();
3731
3732         if (w > max_width || h > max_height) {
3733                 DRM_DEBUG_KMS("requested Y/RGB source size %dx%d too big (limit %dx%d)\n",
3734                               w, h, max_width, max_height);
3735                 return -EINVAL;
3736         }
3737
3738         intel_add_fb_offsets(&x, &y, plane_state, 0);
3739         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 0);
3740         alignment = intel_surf_alignment(fb, 0);
3741
3742         /*
3743          * AUX surface offset is specified as the distance from the
3744          * main surface offset, and it must be non-negative. Make
3745          * sure that is what we will get.
3746          */
3747         if (offset > aux_offset)
3748                 offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3749                                                            offset, aux_offset & ~(alignment - 1));
3750
3751         /*
3752          * When using an X-tiled surface, the plane blows up
3753          * if the x offset + width exceed the stride.
3754          *
3755          * TODO: linear and Y-tiled seem fine, Yf untested,
3756          */
3757         if (fb->modifier == I915_FORMAT_MOD_X_TILED) {
3758                 int cpp = fb->format->cpp[0];
3759
3760                 while ((x + w) * cpp > plane_state->color_plane[0].stride) {
3761                         if (offset == 0) {
3762                                 DRM_DEBUG_KMS("Unable to find suitable display surface offset due to X-tiling\n");
3763                                 return -EINVAL;
3764                         }
3765
3766                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3767                                                                    offset, offset - alignment);
3768                 }
3769         }
3770
3771         /*
3772          * CCS AUX surface doesn't have its own x/y offsets, we must make sure
3773          * they match with the main surface x/y offsets.
3774          */
3775         if (is_ccs_modifier(fb->modifier)) {
3776                 while (!skl_check_main_ccs_coordinates(plane_state, x, y, offset)) {
3777                         if (offset == 0)
3778                                 break;
3779
3780                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3781                                                                    offset, offset - alignment);
3782                 }
3783
3784                 if (x != plane_state->color_plane[aux_plane].x ||
3785                     y != plane_state->color_plane[aux_plane].y) {
3786                         DRM_DEBUG_KMS("Unable to find suitable display surface offset due to CCS\n");
3787                         return -EINVAL;
3788                 }
3789         }
3790
3791         plane_state->color_plane[0].offset = offset;
3792         plane_state->color_plane[0].x = x;
3793         plane_state->color_plane[0].y = y;
3794
3795         /*
3796          * Put the final coordinates back so that the src
3797          * coordinate checks will see the right values.
3798          */
3799         drm_rect_translate_to(&plane_state->uapi.src,
3800                               x << 16, y << 16);
3801
3802         return 0;
3803 }
3804
3805 static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
3806 {
3807         const struct drm_framebuffer *fb = plane_state->hw.fb;
3808         unsigned int rotation = plane_state->hw.rotation;
3809         int max_width = skl_max_plane_width(fb, 1, rotation);
3810         int max_height = 4096;
3811         int x = plane_state->uapi.src.x1 >> 17;
3812         int y = plane_state->uapi.src.y1 >> 17;
3813         int w = drm_rect_width(&plane_state->uapi.src) >> 17;
3814         int h = drm_rect_height(&plane_state->uapi.src) >> 17;
3815         u32 offset;
3816
3817         intel_add_fb_offsets(&x, &y, plane_state, 1);
3818         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3819
3820         /* FIXME not quite sure how/if these apply to the chroma plane */
3821         if (w > max_width || h > max_height) {
3822                 DRM_DEBUG_KMS("CbCr source size %dx%d too big (limit %dx%d)\n",
3823                               w, h, max_width, max_height);
3824                 return -EINVAL;
3825         }
3826
3827         plane_state->color_plane[1].offset = offset;
3828         plane_state->color_plane[1].x = x;
3829         plane_state->color_plane[1].y = y;
3830
3831         return 0;
3832 }
3833
3834 static int skl_check_ccs_aux_surface(struct intel_plane_state *plane_state)
3835 {
3836         const struct drm_framebuffer *fb = plane_state->hw.fb;
3837         int src_x = plane_state->uapi.src.x1 >> 16;
3838         int src_y = plane_state->uapi.src.y1 >> 16;
3839         int hsub;
3840         int vsub;
3841         int x;
3842         int y;
3843         u32 offset;
3844
3845         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, 1);
3846         x = src_x / hsub;
3847         y = src_y / vsub;
3848         intel_add_fb_offsets(&x, &y, plane_state, 1);
3849         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3850
3851         plane_state->color_plane[1].offset = offset;
3852         plane_state->color_plane[1].x = x * hsub + src_x % hsub;
3853         plane_state->color_plane[1].y = y * vsub + src_y % vsub;
3854
3855         return 0;
3856 }
3857
3858 int skl_check_plane_surface(struct intel_plane_state *plane_state)
3859 {
3860         const struct drm_framebuffer *fb = plane_state->hw.fb;
3861         int ret;
3862
3863         ret = intel_plane_compute_gtt(plane_state);
3864         if (ret)
3865                 return ret;
3866
3867         if (!plane_state->uapi.visible)
3868                 return 0;
3869
3870         /*
3871          * Handle the AUX surface first since
3872          * the main surface setup depends on it.
3873          */
3874         if (intel_format_info_is_yuv_semiplanar(fb->format,
3875                                                 fb->modifier)) {
3876                 ret = skl_check_nv12_aux_surface(plane_state);
3877                 if (ret)
3878                         return ret;
3879         } else if (is_ccs_modifier(fb->modifier)) {
3880                 ret = skl_check_ccs_aux_surface(plane_state);
3881                 if (ret)
3882                         return ret;
3883         } else {
3884                 plane_state->color_plane[1].offset = ~0xfff;
3885                 plane_state->color_plane[1].x = 0;
3886                 plane_state->color_plane[1].y = 0;
3887         }
3888
3889         ret = skl_check_main_surface(plane_state);
3890         if (ret)
3891                 return ret;
3892
3893         return 0;
3894 }
3895
3896 static void i9xx_plane_ratio(const struct intel_crtc_state *crtc_state,
3897                              const struct intel_plane_state *plane_state,
3898                              unsigned int *num, unsigned int *den)
3899 {
3900         const struct drm_framebuffer *fb = plane_state->hw.fb;
3901         unsigned int cpp = fb->format->cpp[0];
3902
3903         /*
3904          * g4x bspec says 64bpp pixel rate can't exceed 80%
3905          * of cdclk when the sprite plane is enabled on the
3906          * same pipe. ilk/snb bspec says 64bpp pixel rate is
3907          * never allowed to exceed 80% of cdclk. Let's just go
3908          * with the ilk/snb limit always.
3909          */
3910         if (cpp == 8) {
3911                 *num = 10;
3912                 *den = 8;
3913         } else {
3914                 *num = 1;
3915                 *den = 1;
3916         }
3917 }
3918
3919 static int i9xx_plane_min_cdclk(const struct intel_crtc_state *crtc_state,
3920                                 const struct intel_plane_state *plane_state)
3921 {
3922         unsigned int pixel_rate;
3923         unsigned int num, den;
3924
3925         /*
3926          * Note that crtc_state->pixel_rate accounts for both
3927          * horizontal and vertical panel fitter downscaling factors.
3928          * Pre-HSW bspec tells us to only consider the horizontal
3929          * downscaling factor here. We ignore that and just consider
3930          * both for simplicity.
3931          */
3932         pixel_rate = crtc_state->pixel_rate;
3933
3934         i9xx_plane_ratio(crtc_state, plane_state, &num, &den);
3935
3936         /* two pixels per clock with double wide pipe */
3937         if (crtc_state->double_wide)
3938                 den *= 2;
3939
3940         return DIV_ROUND_UP(pixel_rate * num, den);
3941 }
3942
3943 unsigned int
3944 i9xx_plane_max_stride(struct intel_plane *plane,
3945                       u32 pixel_format, u64 modifier,
3946                       unsigned int rotation)
3947 {
3948         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3949
3950         if (!HAS_GMCH(dev_priv)) {
3951                 return 32*1024;
3952         } else if (INTEL_GEN(dev_priv) >= 4) {
3953                 if (modifier == I915_FORMAT_MOD_X_TILED)
3954                         return 16*1024;
3955                 else
3956                         return 32*1024;
3957         } else if (INTEL_GEN(dev_priv) >= 3) {
3958                 if (modifier == I915_FORMAT_MOD_X_TILED)
3959                         return 8*1024;
3960                 else
3961                         return 16*1024;
3962         } else {
3963                 if (plane->i9xx_plane == PLANE_C)
3964                         return 4*1024;
3965                 else
3966                         return 8*1024;
3967         }
3968 }
3969
3970 static u32 i9xx_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
3971 {
3972         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3973         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3974         u32 dspcntr = 0;
3975
3976         if (crtc_state->gamma_enable)
3977                 dspcntr |= DISPPLANE_GAMMA_ENABLE;
3978
3979         if (crtc_state->csc_enable)
3980                 dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
3981
3982         if (INTEL_GEN(dev_priv) < 5)
3983                 dspcntr |= DISPPLANE_SEL_PIPE(crtc->pipe);
3984
3985         return dspcntr;
3986 }
3987
3988 static u32 i9xx_plane_ctl(const struct intel_crtc_state *crtc_state,
3989                           const struct intel_plane_state *plane_state)
3990 {
3991         struct drm_i915_private *dev_priv =
3992                 to_i915(plane_state->uapi.plane->dev);
3993         const struct drm_framebuffer *fb = plane_state->hw.fb;
3994         unsigned int rotation = plane_state->hw.rotation;
3995         u32 dspcntr;
3996
3997         dspcntr = DISPLAY_PLANE_ENABLE;
3998
3999         if (IS_G4X(dev_priv) || IS_GEN(dev_priv, 5) ||
4000             IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
4001                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
4002
4003         switch (fb->format->format) {
4004         case DRM_FORMAT_C8:
4005                 dspcntr |= DISPPLANE_8BPP;
4006                 break;
4007         case DRM_FORMAT_XRGB1555:
4008                 dspcntr |= DISPPLANE_BGRX555;
4009                 break;
4010         case DRM_FORMAT_ARGB1555:
4011                 dspcntr |= DISPPLANE_BGRA555;
4012                 break;
4013         case DRM_FORMAT_RGB565:
4014                 dspcntr |= DISPPLANE_BGRX565;
4015                 break;
4016         case DRM_FORMAT_XRGB8888:
4017                 dspcntr |= DISPPLANE_BGRX888;
4018                 break;
4019         case DRM_FORMAT_XBGR8888:
4020                 dspcntr |= DISPPLANE_RGBX888;
4021                 break;
4022         case DRM_FORMAT_ARGB8888:
4023                 dspcntr |= DISPPLANE_BGRA888;
4024                 break;
4025         case DRM_FORMAT_ABGR8888:
4026                 dspcntr |= DISPPLANE_RGBA888;
4027                 break;
4028         case DRM_FORMAT_XRGB2101010:
4029                 dspcntr |= DISPPLANE_BGRX101010;
4030                 break;
4031         case DRM_FORMAT_XBGR2101010:
4032                 dspcntr |= DISPPLANE_RGBX101010;
4033                 break;
4034         case DRM_FORMAT_ARGB2101010:
4035                 dspcntr |= DISPPLANE_BGRA101010;
4036                 break;
4037         case DRM_FORMAT_ABGR2101010:
4038                 dspcntr |= DISPPLANE_RGBA101010;
4039                 break;
4040         case DRM_FORMAT_XBGR16161616F:
4041                 dspcntr |= DISPPLANE_RGBX161616;
4042                 break;
4043         default:
4044                 MISSING_CASE(fb->format->format);
4045                 return 0;
4046         }
4047
4048         if (INTEL_GEN(dev_priv) >= 4 &&
4049             fb->modifier == I915_FORMAT_MOD_X_TILED)
4050                 dspcntr |= DISPPLANE_TILED;
4051
4052         if (rotation & DRM_MODE_ROTATE_180)
4053                 dspcntr |= DISPPLANE_ROTATE_180;
4054
4055         if (rotation & DRM_MODE_REFLECT_X)
4056                 dspcntr |= DISPPLANE_MIRROR;
4057
4058         return dspcntr;
4059 }
4060
4061 int i9xx_check_plane_surface(struct intel_plane_state *plane_state)
4062 {
4063         struct drm_i915_private *dev_priv =
4064                 to_i915(plane_state->uapi.plane->dev);
4065         const struct drm_framebuffer *fb = plane_state->hw.fb;
4066         int src_x, src_y, src_w;
4067         u32 offset;
4068         int ret;
4069
4070         ret = intel_plane_compute_gtt(plane_state);
4071         if (ret)
4072                 return ret;
4073
4074         if (!plane_state->uapi.visible)
4075                 return 0;
4076
4077         src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4078         src_x = plane_state->uapi.src.x1 >> 16;
4079         src_y = plane_state->uapi.src.y1 >> 16;
4080
4081         /* Undocumented hardware limit on i965/g4x/vlv/chv */
4082         if (HAS_GMCH(dev_priv) && fb->format->cpp[0] == 8 && src_w > 2048)
4083                 return -EINVAL;
4084
4085         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
4086
4087         if (INTEL_GEN(dev_priv) >= 4)
4088                 offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
4089                                                             plane_state, 0);
4090         else
4091                 offset = 0;
4092
4093         /*
4094          * Put the final coordinates back so that the src
4095          * coordinate checks will see the right values.
4096          */
4097         drm_rect_translate_to(&plane_state->uapi.src,
4098                               src_x << 16, src_y << 16);
4099
4100         /* HSW/BDW do this automagically in hardware */
4101         if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) {
4102                 unsigned int rotation = plane_state->hw.rotation;
4103                 int src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4104                 int src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
4105
4106                 if (rotation & DRM_MODE_ROTATE_180) {
4107                         src_x += src_w - 1;
4108                         src_y += src_h - 1;
4109                 } else if (rotation & DRM_MODE_REFLECT_X) {
4110                         src_x += src_w - 1;
4111                 }
4112         }
4113
4114         plane_state->color_plane[0].offset = offset;
4115         plane_state->color_plane[0].x = src_x;
4116         plane_state->color_plane[0].y = src_y;
4117
4118         return 0;
4119 }
4120
4121 static bool i9xx_plane_has_windowing(struct intel_plane *plane)
4122 {
4123         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4124         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4125
4126         if (IS_CHERRYVIEW(dev_priv))
4127                 return i9xx_plane == PLANE_B;
4128         else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
4129                 return false;
4130         else if (IS_GEN(dev_priv, 4))
4131                 return i9xx_plane == PLANE_C;
4132         else
4133                 return i9xx_plane == PLANE_B ||
4134                         i9xx_plane == PLANE_C;
4135 }
4136
4137 static int
4138 i9xx_plane_check(struct intel_crtc_state *crtc_state,
4139                  struct intel_plane_state *plane_state)
4140 {
4141         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4142         int ret;
4143
4144         ret = chv_plane_check_rotation(plane_state);
4145         if (ret)
4146                 return ret;
4147
4148         ret = drm_atomic_helper_check_plane_state(&plane_state->uapi,
4149                                                   &crtc_state->uapi,
4150                                                   DRM_PLANE_HELPER_NO_SCALING,
4151                                                   DRM_PLANE_HELPER_NO_SCALING,
4152                                                   i9xx_plane_has_windowing(plane),
4153                                                   true);
4154         if (ret)
4155                 return ret;
4156
4157         ret = i9xx_check_plane_surface(plane_state);
4158         if (ret)
4159                 return ret;
4160
4161         if (!plane_state->uapi.visible)
4162                 return 0;
4163
4164         ret = intel_plane_check_src_coordinates(plane_state);
4165         if (ret)
4166                 return ret;
4167
4168         plane_state->ctl = i9xx_plane_ctl(crtc_state, plane_state);
4169
4170         return 0;
4171 }
4172
4173 static void i9xx_update_plane(struct intel_plane *plane,
4174                               const struct intel_crtc_state *crtc_state,
4175                               const struct intel_plane_state *plane_state)
4176 {
4177         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4178         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4179         u32 linear_offset;
4180         int x = plane_state->color_plane[0].x;
4181         int y = plane_state->color_plane[0].y;
4182         int crtc_x = plane_state->uapi.dst.x1;
4183         int crtc_y = plane_state->uapi.dst.y1;
4184         int crtc_w = drm_rect_width(&plane_state->uapi.dst);
4185         int crtc_h = drm_rect_height(&plane_state->uapi.dst);
4186         unsigned long irqflags;
4187         u32 dspaddr_offset;
4188         u32 dspcntr;
4189
4190         dspcntr = plane_state->ctl | i9xx_plane_ctl_crtc(crtc_state);
4191
4192         linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
4193
4194         if (INTEL_GEN(dev_priv) >= 4)
4195                 dspaddr_offset = plane_state->color_plane[0].offset;
4196         else
4197                 dspaddr_offset = linear_offset;
4198
4199         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
4200
4201         I915_WRITE_FW(DSPSTRIDE(i9xx_plane), plane_state->color_plane[0].stride);
4202
4203         if (INTEL_GEN(dev_priv) < 4) {
4204                 /*
4205                  * PLANE_A doesn't actually have a full window
4206                  * generator but let's assume we still need to
4207                  * program whatever is there.
4208                  */
4209                 I915_WRITE_FW(DSPPOS(i9xx_plane), (crtc_y << 16) | crtc_x);
4210                 I915_WRITE_FW(DSPSIZE(i9xx_plane),
4211                               ((crtc_h - 1) << 16) | (crtc_w - 1));
4212         } else if (IS_CHERRYVIEW(dev_priv) && i9xx_plane == PLANE_B) {
4213                 I915_WRITE_FW(PRIMPOS(i9xx_plane), (crtc_y << 16) | crtc_x);
4214                 I915_WRITE_FW(PRIMSIZE(i9xx_plane),
4215                               ((crtc_h - 1) << 16) | (crtc_w - 1));
4216                 I915_WRITE_FW(PRIMCNSTALPHA(i9xx_plane), 0);
4217         }
4218
4219         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
4220                 I915_WRITE_FW(DSPOFFSET(i9xx_plane), (y << 16) | x);
4221         } else if (INTEL_GEN(dev_priv) >= 4) {
4222                 I915_WRITE_FW(DSPLINOFF(i9xx_plane), linear_offset);
4223                 I915_WRITE_FW(DSPTILEOFF(i9xx_plane), (y << 16) | x);
4224         }
4225
4226         /*
4227          * The control register self-arms if the plane was previously
4228          * disabled. Try to make the plane enable atomic by writing
4229          * the control register just before the surface register.
4230          */
4231         I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr);
4232         if (INTEL_GEN(dev_priv) >= 4)
4233                 I915_WRITE_FW(DSPSURF(i9xx_plane),
4234                               intel_plane_ggtt_offset(plane_state) +
4235                               dspaddr_offset);
4236         else
4237                 I915_WRITE_FW(DSPADDR(i9xx_plane),
4238                               intel_plane_ggtt_offset(plane_state) +
4239                               dspaddr_offset);
4240
4241         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
4242 }
4243
4244 static void i9xx_disable_plane(struct intel_plane *plane,
4245                                const struct intel_crtc_state *crtc_state)
4246 {
4247         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4248         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4249         unsigned long irqflags;
4250         u32 dspcntr;
4251
4252         /*
4253          * DSPCNTR pipe gamma enable on g4x+ and pipe csc
4254          * enable on ilk+ affect the pipe bottom color as
4255          * well, so we must configure them even if the plane
4256          * is disabled.
4257          *
4258          * On pre-g4x there is no way to gamma correct the
4259          * pipe bottom color but we'll keep on doing this
4260          * anyway so that the crtc state readout works correctly.
4261          */
4262         dspcntr = i9xx_plane_ctl_crtc(crtc_state);
4263
4264         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
4265
4266         I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr);
4267         if (INTEL_GEN(dev_priv) >= 4)
4268                 I915_WRITE_FW(DSPSURF(i9xx_plane), 0);
4269         else
4270                 I915_WRITE_FW(DSPADDR(i9xx_plane), 0);
4271
4272         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
4273 }
4274
4275 static bool i9xx_plane_get_hw_state(struct intel_plane *plane,
4276                                     enum pipe *pipe)
4277 {
4278         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4279         enum intel_display_power_domain power_domain;
4280         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4281         intel_wakeref_t wakeref;
4282         bool ret;
4283         u32 val;
4284
4285         /*
4286          * Not 100% correct for planes that can move between pipes,
4287          * but that's only the case for gen2-4 which don't have any
4288          * display power wells.
4289          */
4290         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
4291         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4292         if (!wakeref)
4293                 return false;
4294
4295         val = I915_READ(DSPCNTR(i9xx_plane));
4296
4297         ret = val & DISPLAY_PLANE_ENABLE;
4298
4299         if (INTEL_GEN(dev_priv) >= 5)
4300                 *pipe = plane->pipe;
4301         else
4302                 *pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
4303                         DISPPLANE_SEL_PIPE_SHIFT;
4304
4305         intel_display_power_put(dev_priv, power_domain, wakeref);
4306
4307         return ret;
4308 }
4309
4310 static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
4311 {
4312         struct drm_device *dev = intel_crtc->base.dev;
4313         struct drm_i915_private *dev_priv = to_i915(dev);
4314
4315         I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, id), 0);
4316         I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, id), 0);
4317         I915_WRITE(SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0);
4318 }
4319
4320 /*
4321  * This function detaches (aka. unbinds) unused scalers in hardware
4322  */
4323 static void skl_detach_scalers(const struct intel_crtc_state *crtc_state)
4324 {
4325         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->uapi.crtc);
4326         const struct intel_crtc_scaler_state *scaler_state =
4327                 &crtc_state->scaler_state;
4328         int i;
4329
4330         /* loop through and disable scalers that aren't in use */
4331         for (i = 0; i < intel_crtc->num_scalers; i++) {
4332                 if (!scaler_state->scalers[i].in_use)
4333                         skl_detach_scaler(intel_crtc, i);
4334         }
4335 }
4336
4337 static unsigned int skl_plane_stride_mult(const struct drm_framebuffer *fb,
4338                                           int color_plane, unsigned int rotation)
4339 {
4340         /*
4341          * The stride is either expressed as a multiple of 64 bytes chunks for
4342          * linear buffers or in number of tiles for tiled buffers.
4343          */
4344         if (is_surface_linear(fb, color_plane))
4345                 return 64;
4346         else if (drm_rotation_90_or_270(rotation))
4347                 return intel_tile_height(fb, color_plane);
4348         else
4349                 return intel_tile_width_bytes(fb, color_plane);
4350 }
4351
4352 u32 skl_plane_stride(const struct intel_plane_state *plane_state,
4353                      int color_plane)
4354 {
4355         const struct drm_framebuffer *fb = plane_state->hw.fb;
4356         unsigned int rotation = plane_state->hw.rotation;
4357         u32 stride = plane_state->color_plane[color_plane].stride;
4358
4359         if (color_plane >= fb->format->num_planes)
4360                 return 0;
4361
4362         return stride / skl_plane_stride_mult(fb, color_plane, rotation);
4363 }
4364
4365 static u32 skl_plane_ctl_format(u32 pixel_format)
4366 {
4367         switch (pixel_format) {
4368         case DRM_FORMAT_C8:
4369                 return PLANE_CTL_FORMAT_INDEXED;
4370         case DRM_FORMAT_RGB565:
4371                 return PLANE_CTL_FORMAT_RGB_565;
4372         case DRM_FORMAT_XBGR8888:
4373         case DRM_FORMAT_ABGR8888:
4374                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX;
4375         case DRM_FORMAT_XRGB8888:
4376         case DRM_FORMAT_ARGB8888:
4377                 return PLANE_CTL_FORMAT_XRGB_8888;
4378         case DRM_FORMAT_XBGR2101010:
4379         case DRM_FORMAT_ABGR2101010:
4380                 return PLANE_CTL_FORMAT_XRGB_2101010 | PLANE_CTL_ORDER_RGBX;
4381         case DRM_FORMAT_XRGB2101010:
4382         case DRM_FORMAT_ARGB2101010:
4383                 return PLANE_CTL_FORMAT_XRGB_2101010;
4384         case DRM_FORMAT_XBGR16161616F:
4385         case DRM_FORMAT_ABGR16161616F:
4386                 return PLANE_CTL_FORMAT_XRGB_16161616F | PLANE_CTL_ORDER_RGBX;
4387         case DRM_FORMAT_XRGB16161616F:
4388         case DRM_FORMAT_ARGB16161616F:
4389                 return PLANE_CTL_FORMAT_XRGB_16161616F;
4390         case DRM_FORMAT_YUYV:
4391                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV;
4392         case DRM_FORMAT_YVYU:
4393                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YVYU;
4394         case DRM_FORMAT_UYVY:
4395                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_UYVY;
4396         case DRM_FORMAT_VYUY:
4397                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY;
4398         case DRM_FORMAT_NV12:
4399                 return PLANE_CTL_FORMAT_NV12;
4400         case DRM_FORMAT_P010:
4401                 return PLANE_CTL_FORMAT_P010;
4402         case DRM_FORMAT_P012:
4403                 return PLANE_CTL_FORMAT_P012;
4404         case DRM_FORMAT_P016:
4405                 return PLANE_CTL_FORMAT_P016;
4406         case DRM_FORMAT_Y210:
4407                 return PLANE_CTL_FORMAT_Y210;
4408         case DRM_FORMAT_Y212:
4409                 return PLANE_CTL_FORMAT_Y212;
4410         case DRM_FORMAT_Y216:
4411                 return PLANE_CTL_FORMAT_Y216;
4412         case DRM_FORMAT_XVYU2101010:
4413                 return PLANE_CTL_FORMAT_Y410;
4414         case DRM_FORMAT_XVYU12_16161616:
4415                 return PLANE_CTL_FORMAT_Y412;
4416         case DRM_FORMAT_XVYU16161616:
4417                 return PLANE_CTL_FORMAT_Y416;
4418         default:
4419                 MISSING_CASE(pixel_format);
4420         }
4421
4422         return 0;
4423 }
4424
4425 static u32 skl_plane_ctl_alpha(const struct intel_plane_state *plane_state)
4426 {
4427         if (!plane_state->hw.fb->format->has_alpha)
4428                 return PLANE_CTL_ALPHA_DISABLE;
4429
4430         switch (plane_state->hw.pixel_blend_mode) {
4431         case DRM_MODE_BLEND_PIXEL_NONE:
4432                 return PLANE_CTL_ALPHA_DISABLE;
4433         case DRM_MODE_BLEND_PREMULTI:
4434                 return PLANE_CTL_ALPHA_SW_PREMULTIPLY;
4435         case DRM_MODE_BLEND_COVERAGE:
4436                 return PLANE_CTL_ALPHA_HW_PREMULTIPLY;
4437         default:
4438                 MISSING_CASE(plane_state->hw.pixel_blend_mode);
4439                 return PLANE_CTL_ALPHA_DISABLE;
4440         }
4441 }
4442
4443 static u32 glk_plane_color_ctl_alpha(const struct intel_plane_state *plane_state)
4444 {
4445         if (!plane_state->hw.fb->format->has_alpha)
4446                 return PLANE_COLOR_ALPHA_DISABLE;
4447
4448         switch (plane_state->hw.pixel_blend_mode) {
4449         case DRM_MODE_BLEND_PIXEL_NONE:
4450                 return PLANE_COLOR_ALPHA_DISABLE;
4451         case DRM_MODE_BLEND_PREMULTI:
4452                 return PLANE_COLOR_ALPHA_SW_PREMULTIPLY;
4453         case DRM_MODE_BLEND_COVERAGE:
4454                 return PLANE_COLOR_ALPHA_HW_PREMULTIPLY;
4455         default:
4456                 MISSING_CASE(plane_state->hw.pixel_blend_mode);
4457                 return PLANE_COLOR_ALPHA_DISABLE;
4458         }
4459 }
4460
4461 static u32 skl_plane_ctl_tiling(u64 fb_modifier)
4462 {
4463         switch (fb_modifier) {
4464         case DRM_FORMAT_MOD_LINEAR:
4465                 break;
4466         case I915_FORMAT_MOD_X_TILED:
4467                 return PLANE_CTL_TILED_X;
4468         case I915_FORMAT_MOD_Y_TILED:
4469                 return PLANE_CTL_TILED_Y;
4470         case I915_FORMAT_MOD_Y_TILED_CCS:
4471                 return PLANE_CTL_TILED_Y | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
4472         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
4473                 return PLANE_CTL_TILED_Y |
4474                        PLANE_CTL_RENDER_DECOMPRESSION_ENABLE |
4475                        PLANE_CTL_CLEAR_COLOR_DISABLE;
4476         case I915_FORMAT_MOD_Yf_TILED:
4477                 return PLANE_CTL_TILED_YF;
4478         case I915_FORMAT_MOD_Yf_TILED_CCS:
4479                 return PLANE_CTL_TILED_YF | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
4480         default:
4481                 MISSING_CASE(fb_modifier);
4482         }
4483
4484         return 0;
4485 }
4486
4487 static u32 skl_plane_ctl_rotate(unsigned int rotate)
4488 {
4489         switch (rotate) {
4490         case DRM_MODE_ROTATE_0:
4491                 break;
4492         /*
4493          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
4494          * while i915 HW rotation is clockwise, thats why this swapping.
4495          */
4496         case DRM_MODE_ROTATE_90:
4497                 return PLANE_CTL_ROTATE_270;
4498         case DRM_MODE_ROTATE_180:
4499                 return PLANE_CTL_ROTATE_180;
4500         case DRM_MODE_ROTATE_270:
4501                 return PLANE_CTL_ROTATE_90;
4502         default:
4503                 MISSING_CASE(rotate);
4504         }
4505
4506         return 0;
4507 }
4508
4509 static u32 cnl_plane_ctl_flip(unsigned int reflect)
4510 {
4511         switch (reflect) {
4512         case 0:
4513                 break;
4514         case DRM_MODE_REFLECT_X:
4515                 return PLANE_CTL_FLIP_HORIZONTAL;
4516         case DRM_MODE_REFLECT_Y:
4517         default:
4518                 MISSING_CASE(reflect);
4519         }
4520
4521         return 0;
4522 }
4523
4524 u32 skl_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
4525 {
4526         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4527         u32 plane_ctl = 0;
4528
4529         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
4530                 return plane_ctl;
4531
4532         if (crtc_state->gamma_enable)
4533                 plane_ctl |= PLANE_CTL_PIPE_GAMMA_ENABLE;
4534
4535         if (crtc_state->csc_enable)
4536                 plane_ctl |= PLANE_CTL_PIPE_CSC_ENABLE;
4537
4538         return plane_ctl;
4539 }
4540
4541 u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
4542                   const struct intel_plane_state *plane_state)
4543 {
4544         struct drm_i915_private *dev_priv =
4545                 to_i915(plane_state->uapi.plane->dev);
4546         const struct drm_framebuffer *fb = plane_state->hw.fb;
4547         unsigned int rotation = plane_state->hw.rotation;
4548         const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
4549         u32 plane_ctl;
4550
4551         plane_ctl = PLANE_CTL_ENABLE;
4552
4553         if (INTEL_GEN(dev_priv) < 10 && !IS_GEMINILAKE(dev_priv)) {
4554                 plane_ctl |= skl_plane_ctl_alpha(plane_state);
4555                 plane_ctl |= PLANE_CTL_PLANE_GAMMA_DISABLE;
4556
4557                 if (plane_state->hw.color_encoding == DRM_COLOR_YCBCR_BT709)
4558                         plane_ctl |= PLANE_CTL_YUV_TO_RGB_CSC_FORMAT_BT709;
4559
4560                 if (plane_state->hw.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
4561                         plane_ctl |= PLANE_CTL_YUV_RANGE_CORRECTION_DISABLE;
4562         }
4563
4564         plane_ctl |= skl_plane_ctl_format(fb->format->format);
4565         plane_ctl |= skl_plane_ctl_tiling(fb->modifier);
4566         plane_ctl |= skl_plane_ctl_rotate(rotation & DRM_MODE_ROTATE_MASK);
4567
4568         if (INTEL_GEN(dev_priv) >= 10)
4569                 plane_ctl |= cnl_plane_ctl_flip(rotation &
4570                                                 DRM_MODE_REFLECT_MASK);
4571
4572         if (key->flags & I915_SET_COLORKEY_DESTINATION)
4573                 plane_ctl |= PLANE_CTL_KEY_ENABLE_DESTINATION;
4574         else if (key->flags & I915_SET_COLORKEY_SOURCE)
4575                 plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE;
4576
4577         return plane_ctl;
4578 }
4579
4580 u32 glk_plane_color_ctl_crtc(const struct intel_crtc_state *crtc_state)
4581 {
4582         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4583         u32 plane_color_ctl = 0;
4584
4585         if (INTEL_GEN(dev_priv) >= 11)
4586                 return plane_color_ctl;
4587
4588         if (crtc_state->gamma_enable)
4589                 plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE;
4590
4591         if (crtc_state->csc_enable)
4592                 plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE;
4593
4594         return plane_color_ctl;
4595 }
4596
4597 u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
4598                         const struct intel_plane_state *plane_state)
4599 {
4600         struct drm_i915_private *dev_priv =
4601                 to_i915(plane_state->uapi.plane->dev);
4602         const struct drm_framebuffer *fb = plane_state->hw.fb;
4603         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4604         u32 plane_color_ctl = 0;
4605
4606         plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE;
4607         plane_color_ctl |= glk_plane_color_ctl_alpha(plane_state);
4608
4609         if (fb->format->is_yuv && !icl_is_hdr_plane(dev_priv, plane->id)) {
4610                 if (plane_state->hw.color_encoding == DRM_COLOR_YCBCR_BT709)
4611                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709;
4612                 else
4613                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV601_TO_RGB709;
4614
4615                 if (plane_state->hw.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
4616                         plane_color_ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE;
4617         } else if (fb->format->is_yuv) {
4618                 plane_color_ctl |= PLANE_COLOR_INPUT_CSC_ENABLE;
4619         }
4620
4621         return plane_color_ctl;
4622 }
4623
4624 static int
4625 __intel_display_resume(struct drm_device *dev,
4626                        struct drm_atomic_state *state,
4627                        struct drm_modeset_acquire_ctx *ctx)
4628 {
4629         struct drm_crtc_state *crtc_state;
4630         struct drm_crtc *crtc;
4631         int i, ret;
4632
4633         intel_modeset_setup_hw_state(dev, ctx);
4634         intel_vga_redisable(to_i915(dev));
4635
4636         if (!state)
4637                 return 0;
4638
4639         /*
4640          * We've duplicated the state, pointers to the old state are invalid.
4641          *
4642          * Don't attempt to use the old state until we commit the duplicated state.
4643          */
4644         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
4645                 /*
4646                  * Force recalculation even if we restore
4647                  * current state. With fast modeset this may not result
4648                  * in a modeset when the state is compatible.
4649                  */
4650                 crtc_state->mode_changed = true;
4651         }
4652
4653         /* ignore any reset values/BIOS leftovers in the WM registers */
4654         if (!HAS_GMCH(to_i915(dev)))
4655                 to_intel_atomic_state(state)->skip_intermediate_wm = true;
4656
4657         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
4658
4659         WARN_ON(ret == -EDEADLK);
4660         return ret;
4661 }
4662
4663 static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv)
4664 {
4665         return (INTEL_INFO(dev_priv)->gpu_reset_clobbers_display &&
4666                 intel_has_gpu_reset(&dev_priv->gt));
4667 }
4668
4669 void intel_prepare_reset(struct drm_i915_private *dev_priv)
4670 {
4671         struct drm_device *dev = &dev_priv->drm;
4672         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
4673         struct drm_atomic_state *state;
4674         int ret;
4675
4676         /* reset doesn't touch the display */
4677         if (!i915_modparams.force_reset_modeset_test &&
4678             !gpu_reset_clobbers_display(dev_priv))
4679                 return;
4680
4681         /* We have a modeset vs reset deadlock, defensively unbreak it. */
4682         set_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags);
4683         smp_mb__after_atomic();
4684         wake_up_bit(&dev_priv->gt.reset.flags, I915_RESET_MODESET);
4685
4686         if (atomic_read(&dev_priv->gpu_error.pending_fb_pin)) {
4687                 DRM_DEBUG_KMS("Modeset potentially stuck, unbreaking through wedging\n");
4688                 intel_gt_set_wedged(&dev_priv->gt);
4689         }
4690
4691         /*
4692          * Need mode_config.mutex so that we don't
4693          * trample ongoing ->detect() and whatnot.
4694          */
4695         mutex_lock(&dev->mode_config.mutex);
4696         drm_modeset_acquire_init(ctx, 0);
4697         while (1) {
4698                 ret = drm_modeset_lock_all_ctx(dev, ctx);
4699                 if (ret != -EDEADLK)
4700                         break;
4701
4702                 drm_modeset_backoff(ctx);
4703         }
4704         /*
4705          * Disabling the crtcs gracefully seems nicer. Also the
4706          * g33 docs say we should at least disable all the planes.
4707          */
4708         state = drm_atomic_helper_duplicate_state(dev, ctx);
4709         if (IS_ERR(state)) {
4710                 ret = PTR_ERR(state);
4711                 DRM_ERROR("Duplicating state failed with %i\n", ret);
4712                 return;
4713         }
4714
4715         ret = drm_atomic_helper_disable_all(dev, ctx);
4716         if (ret) {
4717                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
4718                 drm_atomic_state_put(state);
4719                 return;
4720         }
4721
4722         dev_priv->modeset_restore_state = state;
4723         state->acquire_ctx = ctx;
4724 }
4725
4726 void intel_finish_reset(struct drm_i915_private *dev_priv)
4727 {
4728         struct drm_device *dev = &dev_priv->drm;
4729         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
4730         struct drm_atomic_state *state;
4731         int ret;
4732
4733         /* reset doesn't touch the display */
4734         if (!test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags))
4735                 return;
4736
4737         state = fetch_and_zero(&dev_priv->modeset_restore_state);
4738         if (!state)
4739                 goto unlock;
4740
4741         /* reset doesn't touch the display */
4742         if (!gpu_reset_clobbers_display(dev_priv)) {
4743                 /* for testing only restore the display */
4744                 ret = __intel_display_resume(dev, state, ctx);
4745                 if (ret)
4746                         DRM_ERROR("Restoring old state failed with %i\n", ret);
4747         } else {
4748                 /*
4749                  * The display has been reset as well,
4750                  * so need a full re-initialization.
4751                  */
4752                 intel_pps_unlock_regs_wa(dev_priv);
4753                 intel_modeset_init_hw(dev_priv);
4754                 intel_init_clock_gating(dev_priv);
4755
4756                 spin_lock_irq(&dev_priv->irq_lock);
4757                 if (dev_priv->display.hpd_irq_setup)
4758                         dev_priv->display.hpd_irq_setup(dev_priv);
4759                 spin_unlock_irq(&dev_priv->irq_lock);
4760
4761                 ret = __intel_display_resume(dev, state, ctx);
4762                 if (ret)
4763                         DRM_ERROR("Restoring old state failed with %i\n", ret);
4764
4765                 intel_hpd_init(dev_priv);
4766         }
4767
4768         drm_atomic_state_put(state);
4769 unlock:
4770         drm_modeset_drop_locks(ctx);
4771         drm_modeset_acquire_fini(ctx);
4772         mutex_unlock(&dev->mode_config.mutex);
4773
4774         clear_bit_unlock(I915_RESET_MODESET, &dev_priv->gt.reset.flags);
4775 }
4776
4777 static void icl_set_pipe_chicken(struct intel_crtc *crtc)
4778 {
4779         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4780         enum pipe pipe = crtc->pipe;
4781         u32 tmp;
4782
4783         tmp = I915_READ(PIPE_CHICKEN(pipe));
4784
4785         /*
4786          * Display WA #1153: icl
4787          * enable hardware to bypass the alpha math
4788          * and rounding for per-pixel values 00 and 0xff
4789          */
4790         tmp |= PER_PIXEL_ALPHA_BYPASS_EN;
4791         /*
4792          * Display WA # 1605353570: icl
4793          * Set the pixel rounding bit to 1 for allowing
4794          * passthrough of Frame buffer pixels unmodified
4795          * across pipe
4796          */
4797         tmp |= PIXEL_ROUNDING_TRUNC_FB_PASSTHRU;
4798         I915_WRITE(PIPE_CHICKEN(pipe), tmp);
4799 }
4800
4801 static void icl_enable_trans_port_sync(const struct intel_crtc_state *crtc_state)
4802 {
4803         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4804         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4805         u32 trans_ddi_func_ctl2_val;
4806         u8 master_select;
4807
4808         /*
4809          * Configure the master select and enable Transcoder Port Sync for
4810          * Slave CRTCs transcoder.
4811          */
4812         if (crtc_state->master_transcoder == INVALID_TRANSCODER)
4813                 return;
4814
4815         if (crtc_state->master_transcoder == TRANSCODER_EDP)
4816                 master_select = 0;
4817         else
4818                 master_select = crtc_state->master_transcoder + 1;
4819
4820         /* Set the master select bits for Tranascoder Port Sync */
4821         trans_ddi_func_ctl2_val = (PORT_SYNC_MODE_MASTER_SELECT(master_select) &
4822                                    PORT_SYNC_MODE_MASTER_SELECT_MASK) <<
4823                 PORT_SYNC_MODE_MASTER_SELECT_SHIFT;
4824         /* Enable Transcoder Port Sync */
4825         trans_ddi_func_ctl2_val |= PORT_SYNC_MODE_ENABLE;
4826
4827         I915_WRITE(TRANS_DDI_FUNC_CTL2(crtc_state->cpu_transcoder),
4828                    trans_ddi_func_ctl2_val);
4829 }
4830
4831 static void intel_fdi_normal_train(struct intel_crtc *crtc)
4832 {
4833         struct drm_device *dev = crtc->base.dev;
4834         struct drm_i915_private *dev_priv = to_i915(dev);
4835         enum pipe pipe = crtc->pipe;
4836         i915_reg_t reg;
4837         u32 temp;
4838
4839         /* enable normal train */
4840         reg = FDI_TX_CTL(pipe);
4841         temp = I915_READ(reg);
4842         if (IS_IVYBRIDGE(dev_priv)) {
4843                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
4844                 temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
4845         } else {
4846                 temp &= ~FDI_LINK_TRAIN_NONE;
4847                 temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
4848         }
4849         I915_WRITE(reg, temp);
4850
4851         reg = FDI_RX_CTL(pipe);
4852         temp = I915_READ(reg);
4853         if (HAS_PCH_CPT(dev_priv)) {
4854                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4855                 temp |= FDI_LINK_TRAIN_NORMAL_CPT;
4856         } else {
4857                 temp &= ~FDI_LINK_TRAIN_NONE;
4858                 temp |= FDI_LINK_TRAIN_NONE;
4859         }
4860         I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
4861
4862         /* wait one idle pattern time */
4863         POSTING_READ(reg);
4864         udelay(1000);
4865
4866         /* IVB wants error correction enabled */
4867         if (IS_IVYBRIDGE(dev_priv))
4868                 I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
4869                            FDI_FE_ERRC_ENABLE);
4870 }
4871
4872 /* The FDI link training functions for ILK/Ibexpeak. */
4873 static void ilk_fdi_link_train(struct intel_crtc *crtc,
4874                                const struct intel_crtc_state *crtc_state)
4875 {
4876         struct drm_device *dev = crtc->base.dev;
4877         struct drm_i915_private *dev_priv = to_i915(dev);
4878         enum pipe pipe = crtc->pipe;
4879         i915_reg_t reg;
4880         u32 temp, tries;
4881
4882         /* FDI needs bits from pipe first */
4883         assert_pipe_enabled(dev_priv, crtc_state->cpu_transcoder);
4884
4885         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4886            for train result */
4887         reg = FDI_RX_IMR(pipe);
4888         temp = I915_READ(reg);
4889         temp &= ~FDI_RX_SYMBOL_LOCK;
4890         temp &= ~FDI_RX_BIT_LOCK;
4891         I915_WRITE(reg, temp);
4892         I915_READ(reg);
4893         udelay(150);
4894
4895         /* enable CPU FDI TX and PCH FDI RX */
4896         reg = FDI_TX_CTL(pipe);
4897         temp = I915_READ(reg);
4898         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4899         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4900         temp &= ~FDI_LINK_TRAIN_NONE;
4901         temp |= FDI_LINK_TRAIN_PATTERN_1;
4902         I915_WRITE(reg, temp | FDI_TX_ENABLE);
4903
4904         reg = FDI_RX_CTL(pipe);
4905         temp = I915_READ(reg);
4906         temp &= ~FDI_LINK_TRAIN_NONE;
4907         temp |= FDI_LINK_TRAIN_PATTERN_1;
4908         I915_WRITE(reg, temp | FDI_RX_ENABLE);
4909
4910         POSTING_READ(reg);
4911         udelay(150);
4912
4913         /* Ironlake workaround, enable clock pointer after FDI enable*/
4914         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
4915         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR |
4916                    FDI_RX_PHASE_SYNC_POINTER_EN);
4917
4918         reg = FDI_RX_IIR(pipe);
4919         for (tries = 0; tries < 5; tries++) {
4920                 temp = I915_READ(reg);
4921                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4922
4923                 if ((temp & FDI_RX_BIT_LOCK)) {
4924                         DRM_DEBUG_KMS("FDI train 1 done.\n");
4925                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4926                         break;
4927                 }
4928         }
4929         if (tries == 5)
4930                 DRM_ERROR("FDI train 1 fail!\n");
4931
4932         /* Train 2 */
4933         reg = FDI_TX_CTL(pipe);
4934         temp = I915_READ(reg);
4935         temp &= ~FDI_LINK_TRAIN_NONE;
4936         temp |= FDI_LINK_TRAIN_PATTERN_2;
4937         I915_WRITE(reg, temp);
4938
4939         reg = FDI_RX_CTL(pipe);
4940         temp = I915_READ(reg);
4941         temp &= ~FDI_LINK_TRAIN_NONE;
4942         temp |= FDI_LINK_TRAIN_PATTERN_2;
4943         I915_WRITE(reg, temp);
4944
4945         POSTING_READ(reg);
4946         udelay(150);
4947
4948         reg = FDI_RX_IIR(pipe);
4949         for (tries = 0; tries < 5; tries++) {
4950                 temp = I915_READ(reg);
4951                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4952
4953                 if (temp & FDI_RX_SYMBOL_LOCK) {
4954                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4955                         DRM_DEBUG_KMS("FDI train 2 done.\n");
4956                         break;
4957                 }
4958         }
4959         if (tries == 5)
4960                 DRM_ERROR("FDI train 2 fail!\n");
4961
4962         DRM_DEBUG_KMS("FDI train done\n");
4963
4964 }
4965
4966 static const int snb_b_fdi_train_param[] = {
4967         FDI_LINK_TRAIN_400MV_0DB_SNB_B,
4968         FDI_LINK_TRAIN_400MV_6DB_SNB_B,
4969         FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
4970         FDI_LINK_TRAIN_800MV_0DB_SNB_B,
4971 };
4972
4973 /* The FDI link training functions for SNB/Cougarpoint. */
4974 static void gen6_fdi_link_train(struct intel_crtc *crtc,
4975                                 const struct intel_crtc_state *crtc_state)
4976 {
4977         struct drm_device *dev = crtc->base.dev;
4978         struct drm_i915_private *dev_priv = to_i915(dev);
4979         enum pipe pipe = crtc->pipe;
4980         i915_reg_t reg;
4981         u32 temp, i, retry;
4982
4983         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4984            for train result */
4985         reg = FDI_RX_IMR(pipe);
4986         temp = I915_READ(reg);
4987         temp &= ~FDI_RX_SYMBOL_LOCK;
4988         temp &= ~FDI_RX_BIT_LOCK;
4989         I915_WRITE(reg, temp);
4990
4991         POSTING_READ(reg);
4992         udelay(150);
4993
4994         /* enable CPU FDI TX and PCH FDI RX */
4995         reg = FDI_TX_CTL(pipe);
4996         temp = I915_READ(reg);
4997         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4998         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4999         temp &= ~FDI_LINK_TRAIN_NONE;
5000         temp |= FDI_LINK_TRAIN_PATTERN_1;
5001         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5002         /* SNB-B */
5003         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
5004         I915_WRITE(reg, temp | FDI_TX_ENABLE);
5005
5006         I915_WRITE(FDI_RX_MISC(pipe),
5007                    FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
5008
5009         reg = FDI_RX_CTL(pipe);
5010         temp = I915_READ(reg);
5011         if (HAS_PCH_CPT(dev_priv)) {
5012                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5013                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
5014         } else {
5015                 temp &= ~FDI_LINK_TRAIN_NONE;
5016                 temp |= FDI_LINK_TRAIN_PATTERN_1;
5017         }
5018         I915_WRITE(reg, temp | FDI_RX_ENABLE);
5019
5020         POSTING_READ(reg);
5021         udelay(150);
5022
5023         for (i = 0; i < 4; i++) {
5024                 reg = FDI_TX_CTL(pipe);
5025                 temp = I915_READ(reg);
5026                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5027                 temp |= snb_b_fdi_train_param[i];
5028                 I915_WRITE(reg, temp);
5029
5030                 POSTING_READ(reg);
5031                 udelay(500);
5032
5033                 for (retry = 0; retry < 5; retry++) {
5034                         reg = FDI_RX_IIR(pipe);
5035                         temp = I915_READ(reg);
5036                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
5037                         if (temp & FDI_RX_BIT_LOCK) {
5038                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
5039                                 DRM_DEBUG_KMS("FDI train 1 done.\n");
5040                                 break;
5041                         }
5042                         udelay(50);
5043                 }
5044                 if (retry < 5)
5045                         break;
5046         }
5047         if (i == 4)
5048                 DRM_ERROR("FDI train 1 fail!\n");
5049
5050         /* Train 2 */
5051         reg = FDI_TX_CTL(pipe);
5052         temp = I915_READ(reg);
5053         temp &= ~FDI_LINK_TRAIN_NONE;
5054         temp |= FDI_LINK_TRAIN_PATTERN_2;
5055         if (IS_GEN(dev_priv, 6)) {
5056                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5057                 /* SNB-B */
5058                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
5059         }
5060         I915_WRITE(reg, temp);
5061
5062         reg = FDI_RX_CTL(pipe);
5063         temp = I915_READ(reg);
5064         if (HAS_PCH_CPT(dev_priv)) {
5065                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5066                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
5067         } else {
5068                 temp &= ~FDI_LINK_TRAIN_NONE;
5069                 temp |= FDI_LINK_TRAIN_PATTERN_2;
5070         }
5071         I915_WRITE(reg, temp);
5072
5073         POSTING_READ(reg);
5074         udelay(150);
5075
5076         for (i = 0; i < 4; i++) {
5077                 reg = FDI_TX_CTL(pipe);
5078                 temp = I915_READ(reg);
5079                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5080                 temp |= snb_b_fdi_train_param[i];
5081                 I915_WRITE(reg, temp);
5082
5083                 POSTING_READ(reg);
5084                 udelay(500);
5085
5086                 for (retry = 0; retry < 5; retry++) {
5087                         reg = FDI_RX_IIR(pipe);
5088                         temp = I915_READ(reg);
5089                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
5090                         if (temp & FDI_RX_SYMBOL_LOCK) {
5091                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
5092                                 DRM_DEBUG_KMS("FDI train 2 done.\n");
5093                                 break;
5094                         }
5095                         udelay(50);
5096                 }
5097                 if (retry < 5)
5098                         break;
5099         }
5100         if (i == 4)
5101                 DRM_ERROR("FDI train 2 fail!\n");
5102
5103         DRM_DEBUG_KMS("FDI train done.\n");
5104 }
5105
5106 /* Manual link training for Ivy Bridge A0 parts */
5107 static void ivb_manual_fdi_link_train(struct intel_crtc *crtc,
5108                                       const struct intel_crtc_state *crtc_state)
5109 {
5110         struct drm_device *dev = crtc->base.dev;
5111         struct drm_i915_private *dev_priv = to_i915(dev);
5112         enum pipe pipe = crtc->pipe;
5113         i915_reg_t reg;
5114         u32 temp, i, j;
5115
5116         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
5117            for train result */
5118         reg = FDI_RX_IMR(pipe);
5119         temp = I915_READ(reg);
5120         temp &= ~FDI_RX_SYMBOL_LOCK;
5121         temp &= ~FDI_RX_BIT_LOCK;
5122         I915_WRITE(reg, temp);
5123
5124         POSTING_READ(reg);
5125         udelay(150);
5126
5127         DRM_DEBUG_KMS("FDI_RX_IIR before link train 0x%x\n",
5128                       I915_READ(FDI_RX_IIR(pipe)));
5129
5130         /* Try each vswing and preemphasis setting twice before moving on */
5131         for (j = 0; j < ARRAY_SIZE(snb_b_fdi_train_param) * 2; j++) {
5132                 /* disable first in case we need to retry */
5133                 reg = FDI_TX_CTL(pipe);
5134                 temp = I915_READ(reg);
5135                 temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
5136                 temp &= ~FDI_TX_ENABLE;
5137                 I915_WRITE(reg, temp);
5138
5139                 reg = FDI_RX_CTL(pipe);
5140                 temp = I915_READ(reg);
5141                 temp &= ~FDI_LINK_TRAIN_AUTO;
5142                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5143                 temp &= ~FDI_RX_ENABLE;
5144                 I915_WRITE(reg, temp);
5145
5146                 /* enable CPU FDI TX and PCH FDI RX */
5147                 reg = FDI_TX_CTL(pipe);
5148                 temp = I915_READ(reg);
5149                 temp &= ~FDI_DP_PORT_WIDTH_MASK;
5150                 temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
5151                 temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
5152                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5153                 temp |= snb_b_fdi_train_param[j/2];
5154                 temp |= FDI_COMPOSITE_SYNC;
5155                 I915_WRITE(reg, temp | FDI_TX_ENABLE);
5156
5157                 I915_WRITE(FDI_RX_MISC(pipe),
5158                            FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
5159
5160                 reg = FDI_RX_CTL(pipe);
5161                 temp = I915_READ(reg);
5162                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
5163                 temp |= FDI_COMPOSITE_SYNC;
5164                 I915_WRITE(reg, temp | FDI_RX_ENABLE);
5165
5166                 POSTING_READ(reg);
5167                 udelay(1); /* should be 0.5us */
5168
5169                 for (i = 0; i < 4; i++) {
5170                         reg = FDI_RX_IIR(pipe);
5171                         temp = I915_READ(reg);
5172                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
5173
5174                         if (temp & FDI_RX_BIT_LOCK ||
5175                             (I915_READ(reg) & FDI_RX_BIT_LOCK)) {
5176                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
5177                                 DRM_DEBUG_KMS("FDI train 1 done, level %i.\n",
5178                                               i);
5179                                 break;
5180                         }
5181                         udelay(1); /* should be 0.5us */
5182                 }
5183                 if (i == 4) {
5184                         DRM_DEBUG_KMS("FDI train 1 fail on vswing %d\n", j / 2);
5185                         continue;
5186                 }
5187
5188                 /* Train 2 */
5189                 reg = FDI_TX_CTL(pipe);
5190                 temp = I915_READ(reg);
5191                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
5192                 temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
5193                 I915_WRITE(reg, temp);
5194
5195                 reg = FDI_RX_CTL(pipe);
5196                 temp = I915_READ(reg);
5197                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5198                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
5199                 I915_WRITE(reg, temp);
5200
5201                 POSTING_READ(reg);
5202                 udelay(2); /* should be 1.5us */
5203
5204                 for (i = 0; i < 4; i++) {
5205                         reg = FDI_RX_IIR(pipe);
5206                         temp = I915_READ(reg);
5207                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
5208
5209                         if (temp & FDI_RX_SYMBOL_LOCK ||
5210                             (I915_READ(reg) & FDI_RX_SYMBOL_LOCK)) {
5211                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
5212                                 DRM_DEBUG_KMS("FDI train 2 done, level %i.\n",
5213                                               i);
5214                                 goto train_done;
5215                         }
5216                         udelay(2); /* should be 1.5us */
5217                 }
5218                 if (i == 4)
5219                         DRM_DEBUG_KMS("FDI train 2 fail on vswing %d\n", j / 2);
5220         }
5221
5222 train_done:
5223         DRM_DEBUG_KMS("FDI train done.\n");
5224 }
5225
5226 static void ilk_fdi_pll_enable(const struct intel_crtc_state *crtc_state)
5227 {
5228         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->uapi.crtc);
5229         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
5230         enum pipe pipe = intel_crtc->pipe;
5231         i915_reg_t reg;
5232         u32 temp;
5233
5234         /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
5235         reg = FDI_RX_CTL(pipe);
5236         temp = I915_READ(reg);
5237         temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16));
5238         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
5239         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
5240         I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
5241
5242         POSTING_READ(reg);
5243         udelay(200);
5244
5245         /* Switch from Rawclk to PCDclk */
5246         temp = I915_READ(reg);
5247         I915_WRITE(reg, temp | FDI_PCDCLK);
5248
5249         POSTING_READ(reg);
5250         udelay(200);
5251
5252         /* Enable CPU FDI TX PLL, always on for Ironlake */
5253         reg = FDI_TX_CTL(pipe);
5254         temp = I915_READ(reg);
5255         if ((temp & FDI_TX_PLL_ENABLE) == 0) {
5256                 I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
5257
5258                 POSTING_READ(reg);
5259                 udelay(100);
5260         }
5261 }
5262
5263 static void ilk_fdi_pll_disable(struct intel_crtc *intel_crtc)
5264 {
5265         struct drm_device *dev = intel_crtc->base.dev;
5266         struct drm_i915_private *dev_priv = to_i915(dev);
5267         enum pipe pipe = intel_crtc->pipe;
5268         i915_reg_t reg;
5269         u32 temp;
5270
5271         /* Switch from PCDclk to Rawclk */
5272         reg = FDI_RX_CTL(pipe);
5273         temp = I915_READ(reg);
5274         I915_WRITE(reg, temp & ~FDI_PCDCLK);
5275
5276         /* Disable CPU FDI TX PLL */
5277         reg = FDI_TX_CTL(pipe);
5278         temp = I915_READ(reg);
5279         I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
5280
5281         POSTING_READ(reg);
5282         udelay(100);
5283
5284         reg = FDI_RX_CTL(pipe);
5285         temp = I915_READ(reg);
5286         I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
5287
5288         /* Wait for the clocks to turn off. */
5289         POSTING_READ(reg);
5290         udelay(100);
5291 }
5292
5293 static void ilk_fdi_disable(struct intel_crtc *crtc)
5294 {
5295         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5296         enum pipe pipe = crtc->pipe;
5297         i915_reg_t reg;
5298         u32 temp;
5299
5300         /* disable CPU FDI tx and PCH FDI rx */
5301         reg = FDI_TX_CTL(pipe);
5302         temp = I915_READ(reg);
5303         I915_WRITE(reg, temp & ~FDI_TX_ENABLE);
5304         POSTING_READ(reg);
5305
5306         reg = FDI_RX_CTL(pipe);
5307         temp = I915_READ(reg);
5308         temp &= ~(0x7 << 16);
5309         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
5310         I915_WRITE(reg, temp & ~FDI_RX_ENABLE);
5311
5312         POSTING_READ(reg);
5313         udelay(100);
5314
5315         /* Ironlake workaround, disable clock pointer after downing FDI */
5316         if (HAS_PCH_IBX(dev_priv))
5317                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
5318
5319         /* still set train pattern 1 */
5320         reg = FDI_TX_CTL(pipe);
5321         temp = I915_READ(reg);
5322         temp &= ~FDI_LINK_TRAIN_NONE;
5323         temp |= FDI_LINK_TRAIN_PATTERN_1;
5324         I915_WRITE(reg, temp);
5325
5326         reg = FDI_RX_CTL(pipe);
5327         temp = I915_READ(reg);
5328         if (HAS_PCH_CPT(dev_priv)) {
5329                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5330                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
5331         } else {
5332                 temp &= ~FDI_LINK_TRAIN_NONE;
5333                 temp |= FDI_LINK_TRAIN_PATTERN_1;
5334         }
5335         /* BPC in FDI rx is consistent with that in PIPECONF */
5336         temp &= ~(0x07 << 16);
5337         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
5338         I915_WRITE(reg, temp);
5339
5340         POSTING_READ(reg);
5341         udelay(100);
5342 }
5343
5344 bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv)
5345 {
5346         struct drm_crtc *crtc;
5347         bool cleanup_done;
5348
5349         drm_for_each_crtc(crtc, &dev_priv->drm) {
5350                 struct drm_crtc_commit *commit;
5351                 spin_lock(&crtc->commit_lock);
5352                 commit = list_first_entry_or_null(&crtc->commit_list,
5353                                                   struct drm_crtc_commit, commit_entry);
5354                 cleanup_done = commit ?
5355                         try_wait_for_completion(&commit->cleanup_done) : true;
5356                 spin_unlock(&crtc->commit_lock);
5357
5358                 if (cleanup_done)
5359                         continue;
5360
5361                 drm_crtc_wait_one_vblank(crtc);
5362
5363                 return true;
5364         }
5365
5366         return false;
5367 }
5368
5369 void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
5370 {
5371         u32 temp;
5372
5373         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
5374
5375         mutex_lock(&dev_priv->sb_lock);
5376
5377         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5378         temp |= SBI_SSCCTL_DISABLE;
5379         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
5380
5381         mutex_unlock(&dev_priv->sb_lock);
5382 }
5383
5384 /* Program iCLKIP clock to the desired frequency */
5385 static void lpt_program_iclkip(const struct intel_crtc_state *crtc_state)
5386 {
5387         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5388         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5389         int clock = crtc_state->hw.adjusted_mode.crtc_clock;
5390         u32 divsel, phaseinc, auxdiv, phasedir = 0;
5391         u32 temp;
5392
5393         lpt_disable_iclkip(dev_priv);
5394
5395         /* The iCLK virtual clock root frequency is in MHz,
5396          * but the adjusted_mode->crtc_clock in in KHz. To get the
5397          * divisors, it is necessary to divide one by another, so we
5398          * convert the virtual clock precision to KHz here for higher
5399          * precision.
5400          */
5401         for (auxdiv = 0; auxdiv < 2; auxdiv++) {
5402                 u32 iclk_virtual_root_freq = 172800 * 1000;
5403                 u32 iclk_pi_range = 64;
5404                 u32 desired_divisor;
5405
5406                 desired_divisor = DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
5407                                                     clock << auxdiv);
5408                 divsel = (desired_divisor / iclk_pi_range) - 2;
5409                 phaseinc = desired_divisor % iclk_pi_range;
5410
5411                 /*
5412                  * Near 20MHz is a corner case which is
5413                  * out of range for the 7-bit divisor
5414                  */
5415                 if (divsel <= 0x7f)
5416                         break;
5417         }
5418
5419         /* This should not happen with any sane values */
5420         WARN_ON(SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
5421                 ~SBI_SSCDIVINTPHASE_DIVSEL_MASK);
5422         WARN_ON(SBI_SSCDIVINTPHASE_DIR(phasedir) &
5423                 ~SBI_SSCDIVINTPHASE_INCVAL_MASK);
5424
5425         DRM_DEBUG_KMS("iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
5426                         clock,
5427                         auxdiv,
5428                         divsel,
5429                         phasedir,
5430                         phaseinc);
5431
5432         mutex_lock(&dev_priv->sb_lock);
5433
5434         /* Program SSCDIVINTPHASE6 */
5435         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
5436         temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
5437         temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
5438         temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
5439         temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
5440         temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
5441         temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
5442         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK);
5443
5444         /* Program SSCAUXDIV */
5445         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
5446         temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
5447         temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
5448         intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK);
5449
5450         /* Enable modulator and associated divider */
5451         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5452         temp &= ~SBI_SSCCTL_DISABLE;
5453         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
5454
5455         mutex_unlock(&dev_priv->sb_lock);
5456
5457         /* Wait for initialization time */
5458         udelay(24);
5459
5460         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE);
5461 }
5462
5463 int lpt_get_iclkip(struct drm_i915_private *dev_priv)
5464 {
5465         u32 divsel, phaseinc, auxdiv;
5466         u32 iclk_virtual_root_freq = 172800 * 1000;
5467         u32 iclk_pi_range = 64;
5468         u32 desired_divisor;
5469         u32 temp;
5470
5471         if ((I915_READ(PIXCLK_GATE) & PIXCLK_GATE_UNGATE) == 0)
5472                 return 0;
5473
5474         mutex_lock(&dev_priv->sb_lock);
5475
5476         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5477         if (temp & SBI_SSCCTL_DISABLE) {
5478                 mutex_unlock(&dev_priv->sb_lock);
5479                 return 0;
5480         }
5481
5482         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
5483         divsel = (temp & SBI_SSCDIVINTPHASE_DIVSEL_MASK) >>
5484                 SBI_SSCDIVINTPHASE_DIVSEL_SHIFT;
5485         phaseinc = (temp & SBI_SSCDIVINTPHASE_INCVAL_MASK) >>
5486                 SBI_SSCDIVINTPHASE_INCVAL_SHIFT;
5487
5488         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
5489         auxdiv = (temp & SBI_SSCAUXDIV_FINALDIV2SEL_MASK) >>
5490                 SBI_SSCAUXDIV_FINALDIV2SEL_SHIFT;
5491
5492         mutex_unlock(&dev_priv->sb_lock);
5493
5494         desired_divisor = (divsel + 2) * iclk_pi_range + phaseinc;
5495
5496         return DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
5497                                  desired_divisor << auxdiv);
5498 }
5499
5500 static void ilk_pch_transcoder_set_timings(const struct intel_crtc_state *crtc_state,
5501                                            enum pipe pch_transcoder)
5502 {
5503         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5504         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5505         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
5506
5507         I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder),
5508                    I915_READ(HTOTAL(cpu_transcoder)));
5509         I915_WRITE(PCH_TRANS_HBLANK(pch_transcoder),
5510                    I915_READ(HBLANK(cpu_transcoder)));
5511         I915_WRITE(PCH_TRANS_HSYNC(pch_transcoder),
5512                    I915_READ(HSYNC(cpu_transcoder)));
5513
5514         I915_WRITE(PCH_TRANS_VTOTAL(pch_transcoder),
5515                    I915_READ(VTOTAL(cpu_transcoder)));
5516         I915_WRITE(PCH_TRANS_VBLANK(pch_transcoder),
5517                    I915_READ(VBLANK(cpu_transcoder)));
5518         I915_WRITE(PCH_TRANS_VSYNC(pch_transcoder),
5519                    I915_READ(VSYNC(cpu_transcoder)));
5520         I915_WRITE(PCH_TRANS_VSYNCSHIFT(pch_transcoder),
5521                    I915_READ(VSYNCSHIFT(cpu_transcoder)));
5522 }
5523
5524 static void cpt_set_fdi_bc_bifurcation(struct drm_i915_private *dev_priv, bool enable)
5525 {
5526         u32 temp;
5527
5528         temp = I915_READ(SOUTH_CHICKEN1);
5529         if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable)
5530                 return;
5531
5532         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_B)) & FDI_RX_ENABLE);
5533         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_C)) & FDI_RX_ENABLE);
5534
5535         temp &= ~FDI_BC_BIFURCATION_SELECT;
5536         if (enable)
5537                 temp |= FDI_BC_BIFURCATION_SELECT;
5538
5539         DRM_DEBUG_KMS("%sabling fdi C rx\n", enable ? "en" : "dis");
5540         I915_WRITE(SOUTH_CHICKEN1, temp);
5541         POSTING_READ(SOUTH_CHICKEN1);
5542 }
5543
5544 static void ivb_update_fdi_bc_bifurcation(const struct intel_crtc_state *crtc_state)
5545 {
5546         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5547         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5548
5549         switch (crtc->pipe) {
5550         case PIPE_A:
5551                 break;
5552         case PIPE_B:
5553                 if (crtc_state->fdi_lanes > 2)
5554                         cpt_set_fdi_bc_bifurcation(dev_priv, false);
5555                 else
5556                         cpt_set_fdi_bc_bifurcation(dev_priv, true);
5557
5558                 break;
5559         case PIPE_C:
5560                 cpt_set_fdi_bc_bifurcation(dev_priv, true);
5561
5562                 break;
5563         default:
5564                 BUG();
5565         }
5566 }
5567
5568 /*
5569  * Finds the encoder associated with the given CRTC. This can only be
5570  * used when we know that the CRTC isn't feeding multiple encoders!
5571  */
5572 static struct intel_encoder *
5573 intel_get_crtc_new_encoder(const struct intel_atomic_state *state,
5574                            const struct intel_crtc_state *crtc_state)
5575 {
5576         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5577         const struct drm_connector_state *connector_state;
5578         const struct drm_connector *connector;
5579         struct intel_encoder *encoder = NULL;
5580         int num_encoders = 0;
5581         int i;
5582
5583         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
5584                 if (connector_state->crtc != &crtc->base)
5585                         continue;
5586
5587                 encoder = to_intel_encoder(connector_state->best_encoder);
5588                 num_encoders++;
5589         }
5590
5591         WARN(num_encoders != 1, "%d encoders for pipe %c\n",
5592              num_encoders, pipe_name(crtc->pipe));
5593
5594         return encoder;
5595 }
5596
5597 /*
5598  * Enable PCH resources required for PCH ports:
5599  *   - PCH PLLs
5600  *   - FDI training & RX/TX
5601  *   - update transcoder timings
5602  *   - DP transcoding bits
5603  *   - transcoder
5604  */
5605 static void ilk_pch_enable(const struct intel_atomic_state *state,
5606                            const struct intel_crtc_state *crtc_state)
5607 {
5608         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5609         struct drm_device *dev = crtc->base.dev;
5610         struct drm_i915_private *dev_priv = to_i915(dev);
5611         enum pipe pipe = crtc->pipe;
5612         u32 temp;
5613
5614         assert_pch_transcoder_disabled(dev_priv, pipe);
5615
5616         if (IS_IVYBRIDGE(dev_priv))
5617                 ivb_update_fdi_bc_bifurcation(crtc_state);
5618
5619         /* Write the TU size bits before fdi link training, so that error
5620          * detection works. */
5621         I915_WRITE(FDI_RX_TUSIZE1(pipe),
5622                    I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
5623
5624         /* For PCH output, training FDI link */
5625         dev_priv->display.fdi_link_train(crtc, crtc_state);
5626
5627         /* We need to program the right clock selection before writing the pixel
5628          * mutliplier into the DPLL. */
5629         if (HAS_PCH_CPT(dev_priv)) {
5630                 u32 sel;
5631
5632                 temp = I915_READ(PCH_DPLL_SEL);
5633                 temp |= TRANS_DPLL_ENABLE(pipe);
5634                 sel = TRANS_DPLLB_SEL(pipe);
5635                 if (crtc_state->shared_dpll ==
5636                     intel_get_shared_dpll_by_id(dev_priv, DPLL_ID_PCH_PLL_B))
5637                         temp |= sel;
5638                 else
5639                         temp &= ~sel;
5640                 I915_WRITE(PCH_DPLL_SEL, temp);
5641         }
5642
5643         /* XXX: pch pll's can be enabled any time before we enable the PCH
5644          * transcoder, and we actually should do this to not upset any PCH
5645          * transcoder that already use the clock when we share it.
5646          *
5647          * Note that enable_shared_dpll tries to do the right thing, but
5648          * get_shared_dpll unconditionally resets the pll - we need that to have
5649          * the right LVDS enable sequence. */
5650         intel_enable_shared_dpll(crtc_state);
5651
5652         /* set transcoder timing, panel must allow it */
5653         assert_panel_unlocked(dev_priv, pipe);
5654         ilk_pch_transcoder_set_timings(crtc_state, pipe);
5655
5656         intel_fdi_normal_train(crtc);
5657
5658         /* For PCH DP, enable TRANS_DP_CTL */
5659         if (HAS_PCH_CPT(dev_priv) &&
5660             intel_crtc_has_dp_encoder(crtc_state)) {
5661                 const struct drm_display_mode *adjusted_mode =
5662                         &crtc_state->hw.adjusted_mode;
5663                 u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
5664                 i915_reg_t reg = TRANS_DP_CTL(pipe);
5665                 enum port port;
5666
5667                 temp = I915_READ(reg);
5668                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
5669                           TRANS_DP_SYNC_MASK |
5670                           TRANS_DP_BPC_MASK);
5671                 temp |= TRANS_DP_OUTPUT_ENABLE;
5672                 temp |= bpc << 9; /* same format but at 11:9 */
5673
5674                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
5675                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
5676                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
5677                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
5678
5679                 port = intel_get_crtc_new_encoder(state, crtc_state)->port;
5680                 WARN_ON(port < PORT_B || port > PORT_D);
5681                 temp |= TRANS_DP_PORT_SEL(port);
5682
5683                 I915_WRITE(reg, temp);
5684         }
5685
5686         ilk_enable_pch_transcoder(crtc_state);
5687 }
5688
5689 static void lpt_pch_enable(const struct intel_atomic_state *state,
5690                            const struct intel_crtc_state *crtc_state)
5691 {
5692         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5693         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5694         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
5695
5696         assert_pch_transcoder_disabled(dev_priv, PIPE_A);
5697
5698         lpt_program_iclkip(crtc_state);
5699
5700         /* Set transcoder timing. */
5701         ilk_pch_transcoder_set_timings(crtc_state, PIPE_A);
5702
5703         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
5704 }
5705
5706 static void cpt_verify_modeset(struct drm_i915_private *dev_priv,
5707                                enum pipe pipe)
5708 {
5709         i915_reg_t dslreg = PIPEDSL(pipe);
5710         u32 temp;
5711
5712         temp = I915_READ(dslreg);
5713         udelay(500);
5714         if (wait_for(I915_READ(dslreg) != temp, 5)) {
5715                 if (wait_for(I915_READ(dslreg) != temp, 5))
5716                         DRM_ERROR("mode set failed: pipe %c stuck\n", pipe_name(pipe));
5717         }
5718 }
5719
5720 /*
5721  * The hardware phase 0.0 refers to the center of the pixel.
5722  * We want to start from the top/left edge which is phase
5723  * -0.5. That matches how the hardware calculates the scaling
5724  * factors (from top-left of the first pixel to bottom-right
5725  * of the last pixel, as opposed to the pixel centers).
5726  *
5727  * For 4:2:0 subsampled chroma planes we obviously have to
5728  * adjust that so that the chroma sample position lands in
5729  * the right spot.
5730  *
5731  * Note that for packed YCbCr 4:2:2 formats there is no way to
5732  * control chroma siting. The hardware simply replicates the
5733  * chroma samples for both of the luma samples, and thus we don't
5734  * actually get the expected MPEG2 chroma siting convention :(
5735  * The same behaviour is observed on pre-SKL platforms as well.
5736  *
5737  * Theory behind the formula (note that we ignore sub-pixel
5738  * source coordinates):
5739  * s = source sample position
5740  * d = destination sample position
5741  *
5742  * Downscaling 4:1:
5743  * -0.5
5744  * | 0.0
5745  * | |     1.5 (initial phase)
5746  * | |     |
5747  * v v     v
5748  * | s | s | s | s |
5749  * |       d       |
5750  *
5751  * Upscaling 1:4:
5752  * -0.5
5753  * | -0.375 (initial phase)
5754  * | |     0.0
5755  * | |     |
5756  * v v     v
5757  * |       s       |
5758  * | d | d | d | d |
5759  */
5760 u16 skl_scaler_calc_phase(int sub, int scale, bool chroma_cosited)
5761 {
5762         int phase = -0x8000;
5763         u16 trip = 0;
5764
5765         if (chroma_cosited)
5766                 phase += (sub - 1) * 0x8000 / sub;
5767
5768         phase += scale / (2 * sub);
5769
5770         /*
5771          * Hardware initial phase limited to [-0.5:1.5].
5772          * Since the max hardware scale factor is 3.0, we
5773          * should never actually excdeed 1.0 here.
5774          */
5775         WARN_ON(phase < -0x8000 || phase > 0x18000);
5776
5777         if (phase < 0)
5778                 phase = 0x10000 + phase;
5779         else
5780                 trip = PS_PHASE_TRIP;
5781
5782         return ((phase >> 2) & PS_PHASE_MASK) | trip;
5783 }
5784
5785 #define SKL_MIN_SRC_W 8
5786 #define SKL_MAX_SRC_W 4096
5787 #define SKL_MIN_SRC_H 8
5788 #define SKL_MAX_SRC_H 4096
5789 #define SKL_MIN_DST_W 8
5790 #define SKL_MAX_DST_W 4096
5791 #define SKL_MIN_DST_H 8
5792 #define SKL_MAX_DST_H 4096
5793 #define ICL_MAX_SRC_W 5120
5794 #define ICL_MAX_SRC_H 4096
5795 #define ICL_MAX_DST_W 5120
5796 #define ICL_MAX_DST_H 4096
5797 #define SKL_MIN_YUV_420_SRC_W 16
5798 #define SKL_MIN_YUV_420_SRC_H 16
5799
5800 static int
5801 skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
5802                   unsigned int scaler_user, int *scaler_id,
5803                   int src_w, int src_h, int dst_w, int dst_h,
5804                   const struct drm_format_info *format,
5805                   u64 modifier, bool need_scaler)
5806 {
5807         struct intel_crtc_scaler_state *scaler_state =
5808                 &crtc_state->scaler_state;
5809         struct intel_crtc *intel_crtc =
5810                 to_intel_crtc(crtc_state->uapi.crtc);
5811         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
5812         const struct drm_display_mode *adjusted_mode =
5813                 &crtc_state->hw.adjusted_mode;
5814
5815         /*
5816          * Src coordinates are already rotated by 270 degrees for
5817          * the 90/270 degree plane rotation cases (to match the
5818          * GTT mapping), hence no need to account for rotation here.
5819          */
5820         if (src_w != dst_w || src_h != dst_h)
5821                 need_scaler = true;
5822
5823         /*
5824          * Scaling/fitting not supported in IF-ID mode in GEN9+
5825          * TODO: Interlace fetch mode doesn't support YUV420 planar formats.
5826          * Once NV12 is enabled, handle it here while allocating scaler
5827          * for NV12.
5828          */
5829         if (INTEL_GEN(dev_priv) >= 9 && crtc_state->hw.enable &&
5830             need_scaler && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
5831                 DRM_DEBUG_KMS("Pipe/Plane scaling not supported with IF-ID mode\n");
5832                 return -EINVAL;
5833         }
5834
5835         /*
5836          * if plane is being disabled or scaler is no more required or force detach
5837          *  - free scaler binded to this plane/crtc
5838          *  - in order to do this, update crtc->scaler_usage
5839          *
5840          * Here scaler state in crtc_state is set free so that
5841          * scaler can be assigned to other user. Actual register
5842          * update to free the scaler is done in plane/panel-fit programming.
5843          * For this purpose crtc/plane_state->scaler_id isn't reset here.
5844          */
5845         if (force_detach || !need_scaler) {
5846                 if (*scaler_id >= 0) {
5847                         scaler_state->scaler_users &= ~(1 << scaler_user);
5848                         scaler_state->scalers[*scaler_id].in_use = 0;
5849
5850                         DRM_DEBUG_KMS("scaler_user index %u.%u: "
5851                                 "Staged freeing scaler id %d scaler_users = 0x%x\n",
5852                                 intel_crtc->pipe, scaler_user, *scaler_id,
5853                                 scaler_state->scaler_users);
5854                         *scaler_id = -1;
5855                 }
5856                 return 0;
5857         }
5858
5859         if (format && intel_format_info_is_yuv_semiplanar(format, modifier) &&
5860             (src_h < SKL_MIN_YUV_420_SRC_H || src_w < SKL_MIN_YUV_420_SRC_W)) {
5861                 DRM_DEBUG_KMS("Planar YUV: src dimensions not met\n");
5862                 return -EINVAL;
5863         }
5864
5865         /* range checks */
5866         if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H ||
5867             dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
5868             (INTEL_GEN(dev_priv) >= 11 &&
5869              (src_w > ICL_MAX_SRC_W || src_h > ICL_MAX_SRC_H ||
5870               dst_w > ICL_MAX_DST_W || dst_h > ICL_MAX_DST_H)) ||
5871             (INTEL_GEN(dev_priv) < 11 &&
5872              (src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
5873               dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H))) {
5874                 DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u "
5875                         "size is out of scaler range\n",
5876                         intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h);
5877                 return -EINVAL;
5878         }
5879
5880         /* mark this plane as a scaler user in crtc_state */
5881         scaler_state->scaler_users |= (1 << scaler_user);
5882         DRM_DEBUG_KMS("scaler_user index %u.%u: "
5883                 "staged scaling request for %ux%u->%ux%u scaler_users = 0x%x\n",
5884                 intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h,
5885                 scaler_state->scaler_users);
5886
5887         return 0;
5888 }
5889
5890 /**
5891  * skl_update_scaler_crtc - Stages update to scaler state for a given crtc.
5892  *
5893  * @state: crtc's scaler state
5894  *
5895  * Return
5896  *     0 - scaler_usage updated successfully
5897  *    error - requested scaling cannot be supported or other error condition
5898  */
5899 int skl_update_scaler_crtc(struct intel_crtc_state *state)
5900 {
5901         const struct drm_display_mode *adjusted_mode = &state->hw.adjusted_mode;
5902         bool need_scaler = false;
5903
5904         if (state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
5905                 need_scaler = true;
5906
5907         return skl_update_scaler(state, !state->hw.active, SKL_CRTC_INDEX,
5908                                  &state->scaler_state.scaler_id,
5909                                  state->pipe_src_w, state->pipe_src_h,
5910                                  adjusted_mode->crtc_hdisplay,
5911                                  adjusted_mode->crtc_vdisplay, NULL, 0,
5912                                  need_scaler);
5913 }
5914
5915 /**
5916  * skl_update_scaler_plane - Stages update to scaler state for a given plane.
5917  * @crtc_state: crtc's scaler state
5918  * @plane_state: atomic plane state to update
5919  *
5920  * Return
5921  *     0 - scaler_usage updated successfully
5922  *    error - requested scaling cannot be supported or other error condition
5923  */
5924 static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
5925                                    struct intel_plane_state *plane_state)
5926 {
5927         struct intel_plane *intel_plane =
5928                 to_intel_plane(plane_state->uapi.plane);
5929         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
5930         struct drm_framebuffer *fb = plane_state->hw.fb;
5931         int ret;
5932         bool force_detach = !fb || !plane_state->uapi.visible;
5933         bool need_scaler = false;
5934
5935         /* Pre-gen11 and SDR planes always need a scaler for planar formats. */
5936         if (!icl_is_hdr_plane(dev_priv, intel_plane->id) &&
5937             fb && intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
5938                 need_scaler = true;
5939
5940         ret = skl_update_scaler(crtc_state, force_detach,
5941                                 drm_plane_index(&intel_plane->base),
5942                                 &plane_state->scaler_id,
5943                                 drm_rect_width(&plane_state->uapi.src) >> 16,
5944                                 drm_rect_height(&plane_state->uapi.src) >> 16,
5945                                 drm_rect_width(&plane_state->uapi.dst),
5946                                 drm_rect_height(&plane_state->uapi.dst),
5947                                 fb ? fb->format : NULL,
5948                                 fb ? fb->modifier : 0,
5949                                 need_scaler);
5950
5951         if (ret || plane_state->scaler_id < 0)
5952                 return ret;
5953
5954         /* check colorkey */
5955         if (plane_state->ckey.flags) {
5956                 DRM_DEBUG_KMS("[PLANE:%d:%s] scaling with color key not allowed",
5957                               intel_plane->base.base.id,
5958                               intel_plane->base.name);
5959                 return -EINVAL;
5960         }
5961
5962         /* Check src format */
5963         switch (fb->format->format) {
5964         case DRM_FORMAT_RGB565:
5965         case DRM_FORMAT_XBGR8888:
5966         case DRM_FORMAT_XRGB8888:
5967         case DRM_FORMAT_ABGR8888:
5968         case DRM_FORMAT_ARGB8888:
5969         case DRM_FORMAT_XRGB2101010:
5970         case DRM_FORMAT_XBGR2101010:
5971         case DRM_FORMAT_ARGB2101010:
5972         case DRM_FORMAT_ABGR2101010:
5973         case DRM_FORMAT_YUYV:
5974         case DRM_FORMAT_YVYU:
5975         case DRM_FORMAT_UYVY:
5976         case DRM_FORMAT_VYUY:
5977         case DRM_FORMAT_NV12:
5978         case DRM_FORMAT_P010:
5979         case DRM_FORMAT_P012:
5980         case DRM_FORMAT_P016:
5981         case DRM_FORMAT_Y210:
5982         case DRM_FORMAT_Y212:
5983         case DRM_FORMAT_Y216:
5984         case DRM_FORMAT_XVYU2101010:
5985         case DRM_FORMAT_XVYU12_16161616:
5986         case DRM_FORMAT_XVYU16161616:
5987                 break;
5988         case DRM_FORMAT_XBGR16161616F:
5989         case DRM_FORMAT_ABGR16161616F:
5990         case DRM_FORMAT_XRGB16161616F:
5991         case DRM_FORMAT_ARGB16161616F:
5992                 if (INTEL_GEN(dev_priv) >= 11)
5993                         break;
5994                 /* fall through */
5995         default:
5996                 DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n",
5997                               intel_plane->base.base.id, intel_plane->base.name,
5998                               fb->base.id, fb->format->format);
5999                 return -EINVAL;
6000         }
6001
6002         return 0;
6003 }
6004
6005 void skl_scaler_disable(const struct intel_crtc_state *old_crtc_state)
6006 {
6007         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
6008         int i;
6009
6010         for (i = 0; i < crtc->num_scalers; i++)
6011                 skl_detach_scaler(crtc, i);
6012 }
6013
6014 static void skl_pfit_enable(const struct intel_crtc_state *crtc_state)
6015 {
6016         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6017         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6018         enum pipe pipe = crtc->pipe;
6019         const struct intel_crtc_scaler_state *scaler_state =
6020                 &crtc_state->scaler_state;
6021
6022         if (crtc_state->pch_pfit.enabled) {
6023                 u16 uv_rgb_hphase, uv_rgb_vphase;
6024                 int pfit_w, pfit_h, hscale, vscale;
6025                 int id;
6026
6027                 if (WARN_ON(crtc_state->scaler_state.scaler_id < 0))
6028                         return;
6029
6030                 pfit_w = (crtc_state->pch_pfit.size >> 16) & 0xFFFF;
6031                 pfit_h = crtc_state->pch_pfit.size & 0xFFFF;
6032
6033                 hscale = (crtc_state->pipe_src_w << 16) / pfit_w;
6034                 vscale = (crtc_state->pipe_src_h << 16) / pfit_h;
6035
6036                 uv_rgb_hphase = skl_scaler_calc_phase(1, hscale, false);
6037                 uv_rgb_vphase = skl_scaler_calc_phase(1, vscale, false);
6038
6039                 id = scaler_state->scaler_id;
6040                 I915_WRITE(SKL_PS_CTRL(pipe, id), PS_SCALER_EN |
6041                         PS_FILTER_MEDIUM | scaler_state->scalers[id].mode);
6042                 I915_WRITE_FW(SKL_PS_VPHASE(pipe, id),
6043                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_vphase));
6044                 I915_WRITE_FW(SKL_PS_HPHASE(pipe, id),
6045                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_hphase));
6046                 I915_WRITE(SKL_PS_WIN_POS(pipe, id), crtc_state->pch_pfit.pos);
6047                 I915_WRITE(SKL_PS_WIN_SZ(pipe, id), crtc_state->pch_pfit.size);
6048         }
6049 }
6050
6051 static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state)
6052 {
6053         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6054         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6055         enum pipe pipe = crtc->pipe;
6056
6057         if (crtc_state->pch_pfit.enabled) {
6058                 /* Force use of hard-coded filter coefficients
6059                  * as some pre-programmed values are broken,
6060                  * e.g. x201.
6061                  */
6062                 if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv))
6063                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
6064                                                  PF_PIPE_SEL_IVB(pipe));
6065                 else
6066                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
6067                 I915_WRITE(PF_WIN_POS(pipe), crtc_state->pch_pfit.pos);
6068                 I915_WRITE(PF_WIN_SZ(pipe), crtc_state->pch_pfit.size);
6069         }
6070 }
6071
6072 void hsw_enable_ips(const struct intel_crtc_state *crtc_state)
6073 {
6074         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6075         struct drm_device *dev = crtc->base.dev;
6076         struct drm_i915_private *dev_priv = to_i915(dev);
6077
6078         if (!crtc_state->ips_enabled)
6079                 return;
6080
6081         /*
6082          * We can only enable IPS after we enable a plane and wait for a vblank
6083          * This function is called from post_plane_update, which is run after
6084          * a vblank wait.
6085          */
6086         WARN_ON(!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)));
6087
6088         if (IS_BROADWELL(dev_priv)) {
6089                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL,
6090                                                 IPS_ENABLE | IPS_PCODE_CONTROL));
6091                 /* Quoting Art Runyan: "its not safe to expect any particular
6092                  * value in IPS_CTL bit 31 after enabling IPS through the
6093                  * mailbox." Moreover, the mailbox may return a bogus state,
6094                  * so we need to just enable it and continue on.
6095                  */
6096         } else {
6097                 I915_WRITE(IPS_CTL, IPS_ENABLE);
6098                 /* The bit only becomes 1 in the next vblank, so this wait here
6099                  * is essentially intel_wait_for_vblank. If we don't have this
6100                  * and don't wait for vblanks until the end of crtc_enable, then
6101                  * the HW state readout code will complain that the expected
6102                  * IPS_CTL value is not the one we read. */
6103                 if (intel_de_wait_for_set(dev_priv, IPS_CTL, IPS_ENABLE, 50))
6104                         DRM_ERROR("Timed out waiting for IPS enable\n");
6105         }
6106 }
6107
6108 void hsw_disable_ips(const struct intel_crtc_state *crtc_state)
6109 {
6110         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6111         struct drm_device *dev = crtc->base.dev;
6112         struct drm_i915_private *dev_priv = to_i915(dev);
6113
6114         if (!crtc_state->ips_enabled)
6115                 return;
6116
6117         if (IS_BROADWELL(dev_priv)) {
6118                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
6119                 /*
6120                  * Wait for PCODE to finish disabling IPS. The BSpec specified
6121                  * 42ms timeout value leads to occasional timeouts so use 100ms
6122                  * instead.
6123                  */
6124                 if (intel_de_wait_for_clear(dev_priv, IPS_CTL, IPS_ENABLE, 100))
6125                         DRM_ERROR("Timed out waiting for IPS disable\n");
6126         } else {
6127                 I915_WRITE(IPS_CTL, 0);
6128                 POSTING_READ(IPS_CTL);
6129         }
6130
6131         /* We need to wait for a vblank before we can disable the plane. */
6132         intel_wait_for_vblank(dev_priv, crtc->pipe);
6133 }
6134
6135 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
6136 {
6137         if (intel_crtc->overlay)
6138                 (void) intel_overlay_switch_off(intel_crtc->overlay);
6139
6140         /* Let userspace switch the overlay on again. In most cases userspace
6141          * has to recompute where to put it anyway.
6142          */
6143 }
6144
6145 static bool hsw_pre_update_disable_ips(const struct intel_crtc_state *old_crtc_state,
6146                                        const struct intel_crtc_state *new_crtc_state)
6147 {
6148         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6149         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6150
6151         if (!old_crtc_state->ips_enabled)
6152                 return false;
6153
6154         if (needs_modeset(new_crtc_state))
6155                 return true;
6156
6157         /*
6158          * Workaround : Do not read or write the pipe palette/gamma data while
6159          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
6160          *
6161          * Disable IPS before we program the LUT.
6162          */
6163         if (IS_HASWELL(dev_priv) &&
6164             (new_crtc_state->uapi.color_mgmt_changed ||
6165              new_crtc_state->update_pipe) &&
6166             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
6167                 return true;
6168
6169         return !new_crtc_state->ips_enabled;
6170 }
6171
6172 static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_state,
6173                                        const struct intel_crtc_state *new_crtc_state)
6174 {
6175         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6176         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6177
6178         if (!new_crtc_state->ips_enabled)
6179                 return false;
6180
6181         if (needs_modeset(new_crtc_state))
6182                 return true;
6183
6184         /*
6185          * Workaround : Do not read or write the pipe palette/gamma data while
6186          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
6187          *
6188          * Re-enable IPS after the LUT has been programmed.
6189          */
6190         if (IS_HASWELL(dev_priv) &&
6191             (new_crtc_state->uapi.color_mgmt_changed ||
6192              new_crtc_state->update_pipe) &&
6193             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
6194                 return true;
6195
6196         /*
6197          * We can't read out IPS on broadwell, assume the worst and
6198          * forcibly enable IPS on the first fastset.
6199          */
6200         if (new_crtc_state->update_pipe &&
6201             old_crtc_state->hw.adjusted_mode.private_flags & I915_MODE_FLAG_INHERITED)
6202                 return true;
6203
6204         return !old_crtc_state->ips_enabled;
6205 }
6206
6207 static bool needs_nv12_wa(const struct intel_crtc_state *crtc_state)
6208 {
6209         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
6210
6211         if (!crtc_state->nv12_planes)
6212                 return false;
6213
6214         /* WA Display #0827: Gen9:all */
6215         if (IS_GEN(dev_priv, 9) && !IS_GEMINILAKE(dev_priv))
6216                 return true;
6217
6218         return false;
6219 }
6220
6221 static bool needs_scalerclk_wa(const struct intel_crtc_state *crtc_state)
6222 {
6223         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
6224
6225         /* Wa_2006604312:icl */
6226         if (crtc_state->scaler_state.scaler_users > 0 && IS_ICELAKE(dev_priv))
6227                 return true;
6228
6229         return false;
6230 }
6231
6232 static bool planes_enabling(const struct intel_crtc_state *old_crtc_state,
6233                             const struct intel_crtc_state *new_crtc_state)
6234 {
6235         return (!old_crtc_state->active_planes || needs_modeset(new_crtc_state)) &&
6236                 new_crtc_state->active_planes;
6237 }
6238
6239 static bool planes_disabling(const struct intel_crtc_state *old_crtc_state,
6240                              const struct intel_crtc_state *new_crtc_state)
6241 {
6242         return old_crtc_state->active_planes &&
6243                 (!new_crtc_state->active_planes || needs_modeset(new_crtc_state));
6244 }
6245
6246 static void intel_post_plane_update(struct intel_atomic_state *state,
6247                                     struct intel_crtc *crtc)
6248 {
6249         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6250         struct intel_plane *primary = to_intel_plane(crtc->base.primary);
6251         const struct intel_crtc_state *old_crtc_state =
6252                 intel_atomic_get_old_crtc_state(state, crtc);
6253         const struct intel_crtc_state *new_crtc_state =
6254                 intel_atomic_get_new_crtc_state(state, crtc);
6255         const struct intel_plane_state *new_primary_state =
6256                 intel_atomic_get_new_plane_state(state, primary);
6257         enum pipe pipe = crtc->pipe;
6258
6259         intel_frontbuffer_flip(dev_priv, new_crtc_state->fb_bits);
6260
6261         if (new_crtc_state->update_wm_post && new_crtc_state->hw.active)
6262                 intel_update_watermarks(crtc);
6263
6264         if (hsw_post_update_enable_ips(old_crtc_state, new_crtc_state))
6265                 hsw_enable_ips(new_crtc_state);
6266
6267         if (new_primary_state)
6268                 intel_fbc_post_update(crtc);
6269
6270         if (needs_nv12_wa(old_crtc_state) &&
6271             !needs_nv12_wa(new_crtc_state))
6272                 skl_wa_827(dev_priv, pipe, false);
6273
6274         if (needs_scalerclk_wa(old_crtc_state) &&
6275             !needs_scalerclk_wa(new_crtc_state))
6276                 icl_wa_scalerclkgating(dev_priv, pipe, false);
6277 }
6278
6279 static void intel_pre_plane_update(struct intel_atomic_state *state,
6280                                    struct intel_crtc *crtc)
6281 {
6282         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6283         struct intel_plane *primary = to_intel_plane(crtc->base.primary);
6284         const struct intel_crtc_state *old_crtc_state =
6285                 intel_atomic_get_old_crtc_state(state, crtc);
6286         const struct intel_crtc_state *new_crtc_state =
6287                 intel_atomic_get_new_crtc_state(state, crtc);
6288         const struct intel_plane_state *new_primary_state =
6289                 intel_atomic_get_new_plane_state(state, primary);
6290         enum pipe pipe = crtc->pipe;
6291
6292         if (hsw_pre_update_disable_ips(old_crtc_state, new_crtc_state))
6293                 hsw_disable_ips(old_crtc_state);
6294
6295         if (new_primary_state &&
6296             intel_fbc_pre_update(crtc, new_crtc_state, new_primary_state))
6297                 intel_wait_for_vblank(dev_priv, pipe);
6298
6299         /* Display WA 827 */
6300         if (!needs_nv12_wa(old_crtc_state) &&
6301             needs_nv12_wa(new_crtc_state))
6302                 skl_wa_827(dev_priv, pipe, true);
6303
6304         /* Wa_2006604312:icl */
6305         if (!needs_scalerclk_wa(old_crtc_state) &&
6306             needs_scalerclk_wa(new_crtc_state))
6307                 icl_wa_scalerclkgating(dev_priv, pipe, true);
6308
6309         /*
6310          * Vblank time updates from the shadow to live plane control register
6311          * are blocked if the memory self-refresh mode is active at that
6312          * moment. So to make sure the plane gets truly disabled, disable
6313          * first the self-refresh mode. The self-refresh enable bit in turn
6314          * will be checked/applied by the HW only at the next frame start
6315          * event which is after the vblank start event, so we need to have a
6316          * wait-for-vblank between disabling the plane and the pipe.
6317          */
6318         if (HAS_GMCH(dev_priv) && old_crtc_state->hw.active &&
6319             new_crtc_state->disable_cxsr && intel_set_memory_cxsr(dev_priv, false))
6320                 intel_wait_for_vblank(dev_priv, pipe);
6321
6322         /*
6323          * IVB workaround: must disable low power watermarks for at least
6324          * one frame before enabling scaling.  LP watermarks can be re-enabled
6325          * when scaling is disabled.
6326          *
6327          * WaCxSRDisabledForSpriteScaling:ivb
6328          */
6329         if (old_crtc_state->hw.active &&
6330             new_crtc_state->disable_lp_wm && ilk_disable_lp_wm(dev_priv))
6331                 intel_wait_for_vblank(dev_priv, pipe);
6332
6333         /*
6334          * If we're doing a modeset we don't need to do any
6335          * pre-vblank watermark programming here.
6336          */
6337         if (!needs_modeset(new_crtc_state)) {
6338                 /*
6339                  * For platforms that support atomic watermarks, program the
6340                  * 'intermediate' watermarks immediately.  On pre-gen9 platforms, these
6341                  * will be the intermediate values that are safe for both pre- and
6342                  * post- vblank; when vblank happens, the 'active' values will be set
6343                  * to the final 'target' values and we'll do this again to get the
6344                  * optimal watermarks.  For gen9+ platforms, the values we program here
6345                  * will be the final target values which will get automatically latched
6346                  * at vblank time; no further programming will be necessary.
6347                  *
6348                  * If a platform hasn't been transitioned to atomic watermarks yet,
6349                  * we'll continue to update watermarks the old way, if flags tell
6350                  * us to.
6351                  */
6352                 if (dev_priv->display.initial_watermarks)
6353                         dev_priv->display.initial_watermarks(state, crtc);
6354                 else if (new_crtc_state->update_wm_pre)
6355                         intel_update_watermarks(crtc);
6356         }
6357
6358         /*
6359          * Gen2 reports pipe underruns whenever all planes are disabled.
6360          * So disable underrun reporting before all the planes get disabled.
6361          *
6362          * We do this after .initial_watermarks() so that we have a
6363          * chance of catching underruns with the intermediate watermarks
6364          * vs. the old plane configuration.
6365          */
6366         if (IS_GEN(dev_priv, 2) && planes_disabling(old_crtc_state, new_crtc_state))
6367                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6368 }
6369
6370 static void intel_crtc_disable_planes(struct intel_atomic_state *state,
6371                                       struct intel_crtc *crtc)
6372 {
6373         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6374         const struct intel_crtc_state *new_crtc_state =
6375                 intel_atomic_get_new_crtc_state(state, crtc);
6376         unsigned int update_mask = new_crtc_state->update_planes;
6377         const struct intel_plane_state *old_plane_state;
6378         struct intel_plane *plane;
6379         unsigned fb_bits = 0;
6380         int i;
6381
6382         intel_crtc_dpms_overlay_disable(crtc);
6383
6384         for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
6385                 if (crtc->pipe != plane->pipe ||
6386                     !(update_mask & BIT(plane->id)))
6387                         continue;
6388
6389                 intel_disable_plane(plane, new_crtc_state);
6390
6391                 if (old_plane_state->uapi.visible)
6392                         fb_bits |= plane->frontbuffer_bit;
6393         }
6394
6395         intel_frontbuffer_flip(dev_priv, fb_bits);
6396 }
6397
6398 /*
6399  * intel_connector_primary_encoder - get the primary encoder for a connector
6400  * @connector: connector for which to return the encoder
6401  *
6402  * Returns the primary encoder for a connector. There is a 1:1 mapping from
6403  * all connectors to their encoder, except for DP-MST connectors which have
6404  * both a virtual and a primary encoder. These DP-MST primary encoders can be
6405  * pointed to by as many DP-MST connectors as there are pipes.
6406  */
6407 static struct intel_encoder *
6408 intel_connector_primary_encoder(struct intel_connector *connector)
6409 {
6410         struct intel_encoder *encoder;
6411
6412         if (connector->mst_port)
6413                 return &dp_to_dig_port(connector->mst_port)->base;
6414
6415         encoder = intel_attached_encoder(&connector->base);
6416         WARN_ON(!encoder);
6417
6418         return encoder;
6419 }
6420
6421 static void intel_encoders_update_prepare(struct intel_atomic_state *state)
6422 {
6423         struct drm_connector_state *new_conn_state;
6424         struct drm_connector *connector;
6425         int i;
6426
6427         for_each_new_connector_in_state(&state->base, connector, new_conn_state,
6428                                         i) {
6429                 struct intel_connector *intel_connector;
6430                 struct intel_encoder *encoder;
6431                 struct intel_crtc *crtc;
6432
6433                 if (!intel_connector_needs_modeset(state, connector))
6434                         continue;
6435
6436                 intel_connector = to_intel_connector(connector);
6437                 encoder = intel_connector_primary_encoder(intel_connector);
6438                 if (!encoder->update_prepare)
6439                         continue;
6440
6441                 crtc = new_conn_state->crtc ?
6442                         to_intel_crtc(new_conn_state->crtc) : NULL;
6443                 encoder->update_prepare(state, encoder, crtc);
6444         }
6445 }
6446
6447 static void intel_encoders_update_complete(struct intel_atomic_state *state)
6448 {
6449         struct drm_connector_state *new_conn_state;
6450         struct drm_connector *connector;
6451         int i;
6452
6453         for_each_new_connector_in_state(&state->base, connector, new_conn_state,
6454                                         i) {
6455                 struct intel_connector *intel_connector;
6456                 struct intel_encoder *encoder;
6457                 struct intel_crtc *crtc;
6458
6459                 if (!intel_connector_needs_modeset(state, connector))
6460                         continue;
6461
6462                 intel_connector = to_intel_connector(connector);
6463                 encoder = intel_connector_primary_encoder(intel_connector);
6464                 if (!encoder->update_complete)
6465                         continue;
6466
6467                 crtc = new_conn_state->crtc ?
6468                         to_intel_crtc(new_conn_state->crtc) : NULL;
6469                 encoder->update_complete(state, encoder, crtc);
6470         }
6471 }
6472
6473 static void intel_encoders_pre_pll_enable(struct intel_atomic_state *state,
6474                                           struct intel_crtc *crtc)
6475 {
6476         const struct intel_crtc_state *crtc_state =
6477                 intel_atomic_get_new_crtc_state(state, crtc);
6478         const struct drm_connector_state *conn_state;
6479         struct drm_connector *conn;
6480         int i;
6481
6482         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6483                 struct intel_encoder *encoder =
6484                         to_intel_encoder(conn_state->best_encoder);
6485
6486                 if (conn_state->crtc != &crtc->base)
6487                         continue;
6488
6489                 if (encoder->pre_pll_enable)
6490                         encoder->pre_pll_enable(encoder, crtc_state, conn_state);
6491         }
6492 }
6493
6494 static void intel_encoders_pre_enable(struct intel_atomic_state *state,
6495                                       struct intel_crtc *crtc)
6496 {
6497         const struct intel_crtc_state *crtc_state =
6498                 intel_atomic_get_new_crtc_state(state, crtc);
6499         const struct drm_connector_state *conn_state;
6500         struct drm_connector *conn;
6501         int i;
6502
6503         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6504                 struct intel_encoder *encoder =
6505                         to_intel_encoder(conn_state->best_encoder);
6506
6507                 if (conn_state->crtc != &crtc->base)
6508                         continue;
6509
6510                 if (encoder->pre_enable)
6511                         encoder->pre_enable(encoder, crtc_state, conn_state);
6512         }
6513 }
6514
6515 static void intel_encoders_enable(struct intel_atomic_state *state,
6516                                   struct intel_crtc *crtc)
6517 {
6518         const struct intel_crtc_state *crtc_state =
6519                 intel_atomic_get_new_crtc_state(state, crtc);
6520         const struct drm_connector_state *conn_state;
6521         struct drm_connector *conn;
6522         int i;
6523
6524         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6525                 struct intel_encoder *encoder =
6526                         to_intel_encoder(conn_state->best_encoder);
6527
6528                 if (conn_state->crtc != &crtc->base)
6529                         continue;
6530
6531                 if (encoder->enable)
6532                         encoder->enable(encoder, crtc_state, conn_state);
6533                 intel_opregion_notify_encoder(encoder, true);
6534         }
6535 }
6536
6537 static void intel_encoders_disable(struct intel_atomic_state *state,
6538                                    struct intel_crtc *crtc)
6539 {
6540         const struct intel_crtc_state *old_crtc_state =
6541                 intel_atomic_get_old_crtc_state(state, crtc);
6542         const struct drm_connector_state *old_conn_state;
6543         struct drm_connector *conn;
6544         int i;
6545
6546         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6547                 struct intel_encoder *encoder =
6548                         to_intel_encoder(old_conn_state->best_encoder);
6549
6550                 if (old_conn_state->crtc != &crtc->base)
6551                         continue;
6552
6553                 intel_opregion_notify_encoder(encoder, false);
6554                 if (encoder->disable)
6555                         encoder->disable(encoder, old_crtc_state, old_conn_state);
6556         }
6557 }
6558
6559 static void intel_encoders_post_disable(struct intel_atomic_state *state,
6560                                         struct intel_crtc *crtc)
6561 {
6562         const struct intel_crtc_state *old_crtc_state =
6563                 intel_atomic_get_old_crtc_state(state, crtc);
6564         const struct drm_connector_state *old_conn_state;
6565         struct drm_connector *conn;
6566         int i;
6567
6568         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6569                 struct intel_encoder *encoder =
6570                         to_intel_encoder(old_conn_state->best_encoder);
6571
6572                 if (old_conn_state->crtc != &crtc->base)
6573                         continue;
6574
6575                 if (encoder->post_disable)
6576                         encoder->post_disable(encoder, old_crtc_state, old_conn_state);
6577         }
6578 }
6579
6580 static void intel_encoders_post_pll_disable(struct intel_atomic_state *state,
6581                                             struct intel_crtc *crtc)
6582 {
6583         const struct intel_crtc_state *old_crtc_state =
6584                 intel_atomic_get_old_crtc_state(state, crtc);
6585         const struct drm_connector_state *old_conn_state;
6586         struct drm_connector *conn;
6587         int i;
6588
6589         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6590                 struct intel_encoder *encoder =
6591                         to_intel_encoder(old_conn_state->best_encoder);
6592
6593                 if (old_conn_state->crtc != &crtc->base)
6594                         continue;
6595
6596                 if (encoder->post_pll_disable)
6597                         encoder->post_pll_disable(encoder, old_crtc_state, old_conn_state);
6598         }
6599 }
6600
6601 static void intel_encoders_update_pipe(struct intel_atomic_state *state,
6602                                        struct intel_crtc *crtc)
6603 {
6604         const struct intel_crtc_state *crtc_state =
6605                 intel_atomic_get_new_crtc_state(state, crtc);
6606         const struct drm_connector_state *conn_state;
6607         struct drm_connector *conn;
6608         int i;
6609
6610         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6611                 struct intel_encoder *encoder =
6612                         to_intel_encoder(conn_state->best_encoder);
6613
6614                 if (conn_state->crtc != &crtc->base)
6615                         continue;
6616
6617                 if (encoder->update_pipe)
6618                         encoder->update_pipe(encoder, crtc_state, conn_state);
6619         }
6620 }
6621
6622 static void intel_disable_primary_plane(const struct intel_crtc_state *crtc_state)
6623 {
6624         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6625         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
6626
6627         plane->disable_plane(plane, crtc_state);
6628 }
6629
6630 static void ilk_crtc_enable(struct intel_atomic_state *state,
6631                             struct intel_crtc *crtc)
6632 {
6633         const struct intel_crtc_state *new_crtc_state =
6634                 intel_atomic_get_new_crtc_state(state, crtc);
6635         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6636         enum pipe pipe = crtc->pipe;
6637
6638         if (WARN_ON(crtc->active))
6639                 return;
6640
6641         /*
6642          * Sometimes spurious CPU pipe underruns happen during FDI
6643          * training, at least with VGA+HDMI cloning. Suppress them.
6644          *
6645          * On ILK we get an occasional spurious CPU pipe underruns
6646          * between eDP port A enable and vdd enable. Also PCH port
6647          * enable seems to result in the occasional CPU pipe underrun.
6648          *
6649          * Spurious PCH underruns also occur during PCH enabling.
6650          */
6651         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6652         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
6653
6654         if (new_crtc_state->has_pch_encoder)
6655                 intel_prepare_shared_dpll(new_crtc_state);
6656
6657         if (intel_crtc_has_dp_encoder(new_crtc_state))
6658                 intel_dp_set_m_n(new_crtc_state, M1_N1);
6659
6660         intel_set_pipe_timings(new_crtc_state);
6661         intel_set_pipe_src_size(new_crtc_state);
6662
6663         if (new_crtc_state->has_pch_encoder)
6664                 intel_cpu_transcoder_set_m_n(new_crtc_state,
6665                                              &new_crtc_state->fdi_m_n, NULL);
6666
6667         ilk_set_pipeconf(new_crtc_state);
6668
6669         crtc->active = true;
6670
6671         intel_encoders_pre_enable(state, crtc);
6672
6673         if (new_crtc_state->has_pch_encoder) {
6674                 /* Note: FDI PLL enabling _must_ be done before we enable the
6675                  * cpu pipes, hence this is separate from all the other fdi/pch
6676                  * enabling. */
6677                 ilk_fdi_pll_enable(new_crtc_state);
6678         } else {
6679                 assert_fdi_tx_disabled(dev_priv, pipe);
6680                 assert_fdi_rx_disabled(dev_priv, pipe);
6681         }
6682
6683         ilk_pfit_enable(new_crtc_state);
6684
6685         /*
6686          * On ILK+ LUT must be loaded before the pipe is running but with
6687          * clocks enabled
6688          */
6689         intel_color_load_luts(new_crtc_state);
6690         intel_color_commit(new_crtc_state);
6691         /* update DSPCNTR to configure gamma for pipe bottom color */
6692         intel_disable_primary_plane(new_crtc_state);
6693
6694         if (dev_priv->display.initial_watermarks)
6695                 dev_priv->display.initial_watermarks(state, crtc);
6696         intel_enable_pipe(new_crtc_state);
6697
6698         if (new_crtc_state->has_pch_encoder)
6699                 ilk_pch_enable(state, new_crtc_state);
6700
6701         intel_crtc_vblank_on(new_crtc_state);
6702
6703         intel_encoders_enable(state, crtc);
6704
6705         if (HAS_PCH_CPT(dev_priv))
6706                 cpt_verify_modeset(dev_priv, pipe);
6707
6708         /*
6709          * Must wait for vblank to avoid spurious PCH FIFO underruns.
6710          * And a second vblank wait is needed at least on ILK with
6711          * some interlaced HDMI modes. Let's do the double wait always
6712          * in case there are more corner cases we don't know about.
6713          */
6714         if (new_crtc_state->has_pch_encoder) {
6715                 intel_wait_for_vblank(dev_priv, pipe);
6716                 intel_wait_for_vblank(dev_priv, pipe);
6717         }
6718         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6719         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
6720 }
6721
6722 /* IPS only exists on ULT machines and is tied to pipe A. */
6723 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
6724 {
6725         return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A;
6726 }
6727
6728 static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv,
6729                                             enum pipe pipe, bool apply)
6730 {
6731         u32 val = I915_READ(CLKGATE_DIS_PSL(pipe));
6732         u32 mask = DPF_GATING_DIS | DPF_RAM_GATING_DIS | DPFR_GATING_DIS;
6733
6734         if (apply)
6735                 val |= mask;
6736         else
6737                 val &= ~mask;
6738
6739         I915_WRITE(CLKGATE_DIS_PSL(pipe), val);
6740 }
6741
6742 static void icl_pipe_mbus_enable(struct intel_crtc *crtc)
6743 {
6744         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6745         enum pipe pipe = crtc->pipe;
6746         u32 val;
6747
6748         val = MBUS_DBOX_A_CREDIT(2);
6749
6750         if (INTEL_GEN(dev_priv) >= 12) {
6751                 val |= MBUS_DBOX_BW_CREDIT(2);
6752                 val |= MBUS_DBOX_B_CREDIT(12);
6753         } else {
6754                 val |= MBUS_DBOX_BW_CREDIT(1);
6755                 val |= MBUS_DBOX_B_CREDIT(8);
6756         }
6757
6758         I915_WRITE(PIPE_MBUS_DBOX_CTL(pipe), val);
6759 }
6760
6761 static void hsw_set_frame_start_delay(const struct intel_crtc_state *crtc_state)
6762 {
6763         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6764         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6765         i915_reg_t reg = CHICKEN_TRANS(crtc_state->cpu_transcoder);
6766         u32 val;
6767
6768         val = I915_READ(reg);
6769         val &= ~HSW_FRAME_START_DELAY_MASK;
6770         val |= HSW_FRAME_START_DELAY(0);
6771         I915_WRITE(reg, val);
6772 }
6773
6774 static void hsw_crtc_enable(struct intel_atomic_state *state,
6775                             struct intel_crtc *crtc)
6776 {
6777         const struct intel_crtc_state *new_crtc_state =
6778                 intel_atomic_get_new_crtc_state(state, crtc);
6779         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6780         enum pipe pipe = crtc->pipe, hsw_workaround_pipe;
6781         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
6782         bool psl_clkgate_wa;
6783
6784         if (WARN_ON(crtc->active))
6785                 return;
6786
6787         intel_encoders_pre_pll_enable(state, crtc);
6788
6789         if (new_crtc_state->shared_dpll)
6790                 intel_enable_shared_dpll(new_crtc_state);
6791
6792         intel_encoders_pre_enable(state, crtc);
6793
6794         if (intel_crtc_has_dp_encoder(new_crtc_state))
6795                 intel_dp_set_m_n(new_crtc_state, M1_N1);
6796
6797         if (!transcoder_is_dsi(cpu_transcoder))
6798                 intel_set_pipe_timings(new_crtc_state);
6799
6800         if (INTEL_GEN(dev_priv) >= 11)
6801                 icl_enable_trans_port_sync(new_crtc_state);
6802
6803         intel_set_pipe_src_size(new_crtc_state);
6804
6805         if (cpu_transcoder != TRANSCODER_EDP &&
6806             !transcoder_is_dsi(cpu_transcoder))
6807                 I915_WRITE(PIPE_MULT(cpu_transcoder),
6808                            new_crtc_state->pixel_multiplier - 1);
6809
6810         if (new_crtc_state->has_pch_encoder)
6811                 intel_cpu_transcoder_set_m_n(new_crtc_state,
6812                                              &new_crtc_state->fdi_m_n, NULL);
6813
6814         if (!transcoder_is_dsi(cpu_transcoder)) {
6815                 hsw_set_frame_start_delay(new_crtc_state);
6816                 hsw_set_pipeconf(new_crtc_state);
6817         }
6818
6819         if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
6820                 bdw_set_pipemisc(new_crtc_state);
6821
6822         crtc->active = true;
6823
6824         /* Display WA #1180: WaDisableScalarClockGating: glk, cnl */
6825         psl_clkgate_wa = (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) &&
6826                 new_crtc_state->pch_pfit.enabled;
6827         if (psl_clkgate_wa)
6828                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, true);
6829
6830         if (INTEL_GEN(dev_priv) >= 9)
6831                 skl_pfit_enable(new_crtc_state);
6832         else
6833                 ilk_pfit_enable(new_crtc_state);
6834
6835         /*
6836          * On ILK+ LUT must be loaded before the pipe is running but with
6837          * clocks enabled
6838          */
6839         intel_color_load_luts(new_crtc_state);
6840         intel_color_commit(new_crtc_state);
6841         /* update DSPCNTR to configure gamma/csc for pipe bottom color */
6842         if (INTEL_GEN(dev_priv) < 9)
6843                 intel_disable_primary_plane(new_crtc_state);
6844
6845         if (INTEL_GEN(dev_priv) >= 11)
6846                 icl_set_pipe_chicken(crtc);
6847
6848         if (!transcoder_is_dsi(cpu_transcoder))
6849                 intel_ddi_enable_transcoder_func(new_crtc_state);
6850
6851         if (dev_priv->display.initial_watermarks)
6852                 dev_priv->display.initial_watermarks(state, crtc);
6853
6854         if (INTEL_GEN(dev_priv) >= 11)
6855                 icl_pipe_mbus_enable(crtc);
6856
6857         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
6858         if (!transcoder_is_dsi(cpu_transcoder))
6859                 intel_enable_pipe(new_crtc_state);
6860
6861         if (new_crtc_state->has_pch_encoder)
6862                 lpt_pch_enable(state, new_crtc_state);
6863
6864         intel_crtc_vblank_on(new_crtc_state);
6865
6866         intel_encoders_enable(state, crtc);
6867
6868         if (psl_clkgate_wa) {
6869                 intel_wait_for_vblank(dev_priv, pipe);
6870                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, false);
6871         }
6872
6873         /* If we change the relative order between pipe/planes enabling, we need
6874          * to change the workaround. */
6875         hsw_workaround_pipe = new_crtc_state->hsw_workaround_pipe;
6876         if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
6877                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
6878                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
6879         }
6880 }
6881
6882 void ilk_pfit_disable(const struct intel_crtc_state *old_crtc_state)
6883 {
6884         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
6885         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6886         enum pipe pipe = crtc->pipe;
6887
6888         /* To avoid upsetting the power well on haswell only disable the pfit if
6889          * it's in use. The hw state code will make sure we get this right. */
6890         if (old_crtc_state->pch_pfit.enabled) {
6891                 I915_WRITE(PF_CTL(pipe), 0);
6892                 I915_WRITE(PF_WIN_POS(pipe), 0);
6893                 I915_WRITE(PF_WIN_SZ(pipe), 0);
6894         }
6895 }
6896
6897 static void ilk_crtc_disable(struct intel_atomic_state *state,
6898                              struct intel_crtc *crtc)
6899 {
6900         const struct intel_crtc_state *old_crtc_state =
6901                 intel_atomic_get_old_crtc_state(state, crtc);
6902         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6903         enum pipe pipe = crtc->pipe;
6904
6905         /*
6906          * Sometimes spurious CPU pipe underruns happen when the
6907          * pipe is already disabled, but FDI RX/TX is still enabled.
6908          * Happens at least with VGA+HDMI cloning. Suppress them.
6909          */
6910         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6911         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
6912
6913         intel_encoders_disable(state, crtc);
6914
6915         intel_crtc_vblank_off(old_crtc_state);
6916
6917         intel_disable_pipe(old_crtc_state);
6918
6919         ilk_pfit_disable(old_crtc_state);
6920
6921         if (old_crtc_state->has_pch_encoder)
6922                 ilk_fdi_disable(crtc);
6923
6924         intel_encoders_post_disable(state, crtc);
6925
6926         if (old_crtc_state->has_pch_encoder) {
6927                 ilk_disable_pch_transcoder(dev_priv, pipe);
6928
6929                 if (HAS_PCH_CPT(dev_priv)) {
6930                         i915_reg_t reg;
6931                         u32 temp;
6932
6933                         /* disable TRANS_DP_CTL */
6934                         reg = TRANS_DP_CTL(pipe);
6935                         temp = I915_READ(reg);
6936                         temp &= ~(TRANS_DP_OUTPUT_ENABLE |
6937                                   TRANS_DP_PORT_SEL_MASK);
6938                         temp |= TRANS_DP_PORT_SEL_NONE;
6939                         I915_WRITE(reg, temp);
6940
6941                         /* disable DPLL_SEL */
6942                         temp = I915_READ(PCH_DPLL_SEL);
6943                         temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
6944                         I915_WRITE(PCH_DPLL_SEL, temp);
6945                 }
6946
6947                 ilk_fdi_pll_disable(crtc);
6948         }
6949
6950         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6951         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
6952 }
6953
6954 static void hsw_crtc_disable(struct intel_atomic_state *state,
6955                              struct intel_crtc *crtc)
6956 {
6957         /*
6958          * FIXME collapse everything to one hook.
6959          * Need care with mst->ddi interactions.
6960          */
6961         intel_encoders_disable(state, crtc);
6962         intel_encoders_post_disable(state, crtc);
6963 }
6964
6965 static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state)
6966 {
6967         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6968         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6969
6970         if (!crtc_state->gmch_pfit.control)
6971                 return;
6972
6973         /*
6974          * The panel fitter should only be adjusted whilst the pipe is disabled,
6975          * according to register description and PRM.
6976          */
6977         WARN_ON(I915_READ(PFIT_CONTROL) & PFIT_ENABLE);
6978         assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder);
6979
6980         I915_WRITE(PFIT_PGM_RATIOS, crtc_state->gmch_pfit.pgm_ratios);
6981         I915_WRITE(PFIT_CONTROL, crtc_state->gmch_pfit.control);
6982
6983         /* Border color in case we don't scale up to the full screen. Black by
6984          * default, change to something else for debugging. */
6985         I915_WRITE(BCLRPAT(crtc->pipe), 0);
6986 }
6987
6988 bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy)
6989 {
6990         if (phy == PHY_NONE)
6991                 return false;
6992
6993         if (IS_ELKHARTLAKE(dev_priv))
6994                 return phy <= PHY_C;
6995
6996         if (INTEL_GEN(dev_priv) >= 11)
6997                 return phy <= PHY_B;
6998
6999         return false;
7000 }
7001
7002 bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy)
7003 {
7004         if (INTEL_GEN(dev_priv) >= 12)
7005                 return phy >= PHY_D && phy <= PHY_I;
7006
7007         if (INTEL_GEN(dev_priv) >= 11 && !IS_ELKHARTLAKE(dev_priv))
7008                 return phy >= PHY_C && phy <= PHY_F;
7009
7010         return false;
7011 }
7012
7013 enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port)
7014 {
7015         if (IS_ELKHARTLAKE(i915) && port == PORT_D)
7016                 return PHY_A;
7017
7018         return (enum phy)port;
7019 }
7020
7021 enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port)
7022 {
7023         if (!intel_phy_is_tc(dev_priv, intel_port_to_phy(dev_priv, port)))
7024                 return PORT_TC_NONE;
7025
7026         if (INTEL_GEN(dev_priv) >= 12)
7027                 return port - PORT_D;
7028
7029         return port - PORT_C;
7030 }
7031
7032 enum intel_display_power_domain intel_port_to_power_domain(enum port port)
7033 {
7034         switch (port) {
7035         case PORT_A:
7036                 return POWER_DOMAIN_PORT_DDI_A_LANES;
7037         case PORT_B:
7038                 return POWER_DOMAIN_PORT_DDI_B_LANES;
7039         case PORT_C:
7040                 return POWER_DOMAIN_PORT_DDI_C_LANES;
7041         case PORT_D:
7042                 return POWER_DOMAIN_PORT_DDI_D_LANES;
7043         case PORT_E:
7044                 return POWER_DOMAIN_PORT_DDI_E_LANES;
7045         case PORT_F:
7046                 return POWER_DOMAIN_PORT_DDI_F_LANES;
7047         case PORT_G:
7048                 return POWER_DOMAIN_PORT_DDI_G_LANES;
7049         default:
7050                 MISSING_CASE(port);
7051                 return POWER_DOMAIN_PORT_OTHER;
7052         }
7053 }
7054
7055 enum intel_display_power_domain
7056 intel_aux_power_domain(struct intel_digital_port *dig_port)
7057 {
7058         struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
7059         enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port);
7060
7061         if (intel_phy_is_tc(dev_priv, phy) &&
7062             dig_port->tc_mode == TC_PORT_TBT_ALT) {
7063                 switch (dig_port->aux_ch) {
7064                 case AUX_CH_C:
7065                         return POWER_DOMAIN_AUX_C_TBT;
7066                 case AUX_CH_D:
7067                         return POWER_DOMAIN_AUX_D_TBT;
7068                 case AUX_CH_E:
7069                         return POWER_DOMAIN_AUX_E_TBT;
7070                 case AUX_CH_F:
7071                         return POWER_DOMAIN_AUX_F_TBT;
7072                 case AUX_CH_G:
7073                         return POWER_DOMAIN_AUX_G_TBT;
7074                 default:
7075                         MISSING_CASE(dig_port->aux_ch);
7076                         return POWER_DOMAIN_AUX_C_TBT;
7077                 }
7078         }
7079
7080         switch (dig_port->aux_ch) {
7081         case AUX_CH_A:
7082                 return POWER_DOMAIN_AUX_A;
7083         case AUX_CH_B:
7084                 return POWER_DOMAIN_AUX_B;
7085         case AUX_CH_C:
7086                 return POWER_DOMAIN_AUX_C;
7087         case AUX_CH_D:
7088                 return POWER_DOMAIN_AUX_D;
7089         case AUX_CH_E:
7090                 return POWER_DOMAIN_AUX_E;
7091         case AUX_CH_F:
7092                 return POWER_DOMAIN_AUX_F;
7093         case AUX_CH_G:
7094                 return POWER_DOMAIN_AUX_G;
7095         default:
7096                 MISSING_CASE(dig_port->aux_ch);
7097                 return POWER_DOMAIN_AUX_A;
7098         }
7099 }
7100
7101 static u64 get_crtc_power_domains(struct intel_crtc_state *crtc_state)
7102 {
7103         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7104         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7105         struct drm_encoder *encoder;
7106         enum pipe pipe = crtc->pipe;
7107         u64 mask;
7108         enum transcoder transcoder = crtc_state->cpu_transcoder;
7109
7110         if (!crtc_state->hw.active)
7111                 return 0;
7112
7113         mask = BIT_ULL(POWER_DOMAIN_PIPE(pipe));
7114         mask |= BIT_ULL(POWER_DOMAIN_TRANSCODER(transcoder));
7115         if (crtc_state->pch_pfit.enabled ||
7116             crtc_state->pch_pfit.force_thru)
7117                 mask |= BIT_ULL(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
7118
7119         drm_for_each_encoder_mask(encoder, &dev_priv->drm,
7120                                   crtc_state->uapi.encoder_mask) {
7121                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
7122
7123                 mask |= BIT_ULL(intel_encoder->power_domain);
7124         }
7125
7126         if (HAS_DDI(dev_priv) && crtc_state->has_audio)
7127                 mask |= BIT_ULL(POWER_DOMAIN_AUDIO);
7128
7129         if (crtc_state->shared_dpll)
7130                 mask |= BIT_ULL(POWER_DOMAIN_DISPLAY_CORE);
7131
7132         return mask;
7133 }
7134
7135 static u64
7136 modeset_get_crtc_power_domains(struct intel_crtc_state *crtc_state)
7137 {
7138         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7139         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7140         enum intel_display_power_domain domain;
7141         u64 domains, new_domains, old_domains;
7142
7143         old_domains = crtc->enabled_power_domains;
7144         crtc->enabled_power_domains = new_domains =
7145                 get_crtc_power_domains(crtc_state);
7146
7147         domains = new_domains & ~old_domains;
7148
7149         for_each_power_domain(domain, domains)
7150                 intel_display_power_get(dev_priv, domain);
7151
7152         return old_domains & ~new_domains;
7153 }
7154
7155 static void modeset_put_power_domains(struct drm_i915_private *dev_priv,
7156                                       u64 domains)
7157 {
7158         enum intel_display_power_domain domain;
7159
7160         for_each_power_domain(domain, domains)
7161                 intel_display_power_put_unchecked(dev_priv, domain);
7162 }
7163
7164 static void valleyview_crtc_enable(struct intel_atomic_state *state,
7165                                    struct intel_crtc *crtc)
7166 {
7167         const struct intel_crtc_state *new_crtc_state =
7168                 intel_atomic_get_new_crtc_state(state, crtc);
7169         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7170         enum pipe pipe = crtc->pipe;
7171
7172         if (WARN_ON(crtc->active))
7173                 return;
7174
7175         if (intel_crtc_has_dp_encoder(new_crtc_state))
7176                 intel_dp_set_m_n(new_crtc_state, M1_N1);
7177
7178         intel_set_pipe_timings(new_crtc_state);
7179         intel_set_pipe_src_size(new_crtc_state);
7180
7181         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
7182                 I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY);
7183                 I915_WRITE(CHV_CANVAS(pipe), 0);
7184         }
7185
7186         i9xx_set_pipeconf(new_crtc_state);
7187
7188         crtc->active = true;
7189
7190         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
7191
7192         intel_encoders_pre_pll_enable(state, crtc);
7193
7194         if (IS_CHERRYVIEW(dev_priv)) {
7195                 chv_prepare_pll(crtc, new_crtc_state);
7196                 chv_enable_pll(crtc, new_crtc_state);
7197         } else {
7198                 vlv_prepare_pll(crtc, new_crtc_state);
7199                 vlv_enable_pll(crtc, new_crtc_state);
7200         }
7201
7202         intel_encoders_pre_enable(state, crtc);
7203
7204         i9xx_pfit_enable(new_crtc_state);
7205
7206         intel_color_load_luts(new_crtc_state);
7207         intel_color_commit(new_crtc_state);
7208         /* update DSPCNTR to configure gamma for pipe bottom color */
7209         intel_disable_primary_plane(new_crtc_state);
7210
7211         dev_priv->display.initial_watermarks(state, crtc);
7212         intel_enable_pipe(new_crtc_state);
7213
7214         intel_crtc_vblank_on(new_crtc_state);
7215
7216         intel_encoders_enable(state, crtc);
7217 }
7218
7219 static void i9xx_set_pll_dividers(const struct intel_crtc_state *crtc_state)
7220 {
7221         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7222         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7223
7224         I915_WRITE(FP0(crtc->pipe), crtc_state->dpll_hw_state.fp0);
7225         I915_WRITE(FP1(crtc->pipe), crtc_state->dpll_hw_state.fp1);
7226 }
7227
7228 static void i9xx_crtc_enable(struct intel_atomic_state *state,
7229                              struct intel_crtc *crtc)
7230 {
7231         const struct intel_crtc_state *new_crtc_state =
7232                 intel_atomic_get_new_crtc_state(state, crtc);
7233         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7234         enum pipe pipe = crtc->pipe;
7235
7236         if (WARN_ON(crtc->active))
7237                 return;
7238
7239         i9xx_set_pll_dividers(new_crtc_state);
7240
7241         if (intel_crtc_has_dp_encoder(new_crtc_state))
7242                 intel_dp_set_m_n(new_crtc_state, M1_N1);
7243
7244         intel_set_pipe_timings(new_crtc_state);
7245         intel_set_pipe_src_size(new_crtc_state);
7246
7247         i9xx_set_pipeconf(new_crtc_state);
7248
7249         crtc->active = true;
7250
7251         if (!IS_GEN(dev_priv, 2))
7252                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
7253
7254         intel_encoders_pre_enable(state, crtc);
7255
7256         i9xx_enable_pll(crtc, new_crtc_state);
7257
7258         i9xx_pfit_enable(new_crtc_state);
7259
7260         intel_color_load_luts(new_crtc_state);
7261         intel_color_commit(new_crtc_state);
7262         /* update DSPCNTR to configure gamma for pipe bottom color */
7263         intel_disable_primary_plane(new_crtc_state);
7264
7265         if (dev_priv->display.initial_watermarks)
7266                 dev_priv->display.initial_watermarks(state, crtc);
7267         else
7268                 intel_update_watermarks(crtc);
7269         intel_enable_pipe(new_crtc_state);
7270
7271         intel_crtc_vblank_on(new_crtc_state);
7272
7273         intel_encoders_enable(state, crtc);
7274 }
7275
7276 static void i9xx_pfit_disable(const struct intel_crtc_state *old_crtc_state)
7277 {
7278         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
7279         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7280
7281         if (!old_crtc_state->gmch_pfit.control)
7282                 return;
7283
7284         assert_pipe_disabled(dev_priv, old_crtc_state->cpu_transcoder);
7285
7286         DRM_DEBUG_KMS("disabling pfit, current: 0x%08x\n",
7287                       I915_READ(PFIT_CONTROL));
7288         I915_WRITE(PFIT_CONTROL, 0);
7289 }
7290
7291 static void i9xx_crtc_disable(struct intel_atomic_state *state,
7292                               struct intel_crtc *crtc)
7293 {
7294         struct intel_crtc_state *old_crtc_state =
7295                 intel_atomic_get_old_crtc_state(state, crtc);
7296         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7297         enum pipe pipe = crtc->pipe;
7298
7299         /*
7300          * On gen2 planes are double buffered but the pipe isn't, so we must
7301          * wait for planes to fully turn off before disabling the pipe.
7302          */
7303         if (IS_GEN(dev_priv, 2))
7304                 intel_wait_for_vblank(dev_priv, pipe);
7305
7306         intel_encoders_disable(state, crtc);
7307
7308         intel_crtc_vblank_off(old_crtc_state);
7309
7310         intel_disable_pipe(old_crtc_state);
7311
7312         i9xx_pfit_disable(old_crtc_state);
7313
7314         intel_encoders_post_disable(state, crtc);
7315
7316         if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DSI)) {
7317                 if (IS_CHERRYVIEW(dev_priv))
7318                         chv_disable_pll(dev_priv, pipe);
7319                 else if (IS_VALLEYVIEW(dev_priv))
7320                         vlv_disable_pll(dev_priv, pipe);
7321                 else
7322                         i9xx_disable_pll(old_crtc_state);
7323         }
7324
7325         intel_encoders_post_pll_disable(state, crtc);
7326
7327         if (!IS_GEN(dev_priv, 2))
7328                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
7329
7330         if (!dev_priv->display.initial_watermarks)
7331                 intel_update_watermarks(crtc);
7332
7333         /* clock the pipe down to 640x480@60 to potentially save power */
7334         if (IS_I830(dev_priv))
7335                 i830_enable_pipe(dev_priv, pipe);
7336 }
7337
7338 static void intel_crtc_disable_noatomic(struct intel_crtc *crtc,
7339                                         struct drm_modeset_acquire_ctx *ctx)
7340 {
7341         struct intel_encoder *encoder;
7342         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7343         struct intel_bw_state *bw_state =
7344                 to_intel_bw_state(dev_priv->bw_obj.state);
7345         struct intel_crtc_state *crtc_state =
7346                 to_intel_crtc_state(crtc->base.state);
7347         enum intel_display_power_domain domain;
7348         struct intel_plane *plane;
7349         struct drm_atomic_state *state;
7350         struct intel_crtc_state *temp_crtc_state;
7351         enum pipe pipe = crtc->pipe;
7352         u64 domains;
7353         int ret;
7354
7355         if (!crtc_state->hw.active)
7356                 return;
7357
7358         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
7359                 const struct intel_plane_state *plane_state =
7360                         to_intel_plane_state(plane->base.state);
7361
7362                 if (plane_state->uapi.visible)
7363                         intel_plane_disable_noatomic(crtc, plane);
7364         }
7365
7366         state = drm_atomic_state_alloc(&dev_priv->drm);
7367         if (!state) {
7368                 DRM_DEBUG_KMS("failed to disable [CRTC:%d:%s], out of memory",
7369                               crtc->base.base.id, crtc->base.name);
7370                 return;
7371         }
7372
7373         state->acquire_ctx = ctx;
7374
7375         /* Everything's already locked, -EDEADLK can't happen. */
7376         temp_crtc_state = intel_atomic_get_crtc_state(state, crtc);
7377         ret = drm_atomic_add_affected_connectors(state, &crtc->base);
7378
7379         WARN_ON(IS_ERR(temp_crtc_state) || ret);
7380
7381         dev_priv->display.crtc_disable(to_intel_atomic_state(state), crtc);
7382
7383         drm_atomic_state_put(state);
7384
7385         DRM_DEBUG_KMS("[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
7386                       crtc->base.base.id, crtc->base.name);
7387
7388         crtc->active = false;
7389         crtc->base.enabled = false;
7390
7391         WARN_ON(drm_atomic_set_mode_for_crtc(&crtc_state->uapi, NULL) < 0);
7392         crtc_state->uapi.active = false;
7393         crtc_state->uapi.connector_mask = 0;
7394         crtc_state->uapi.encoder_mask = 0;
7395         intel_crtc_free_hw_state(crtc_state);
7396         memset(&crtc_state->hw, 0, sizeof(crtc_state->hw));
7397
7398         for_each_encoder_on_crtc(&dev_priv->drm, &crtc->base, encoder)
7399                 encoder->base.crtc = NULL;
7400
7401         intel_fbc_disable(crtc);
7402         intel_update_watermarks(crtc);
7403         intel_disable_shared_dpll(crtc_state);
7404
7405         domains = crtc->enabled_power_domains;
7406         for_each_power_domain(domain, domains)
7407                 intel_display_power_put_unchecked(dev_priv, domain);
7408         crtc->enabled_power_domains = 0;
7409
7410         dev_priv->active_pipes &= ~BIT(pipe);
7411         dev_priv->min_cdclk[pipe] = 0;
7412         dev_priv->min_voltage_level[pipe] = 0;
7413
7414         bw_state->data_rate[pipe] = 0;
7415         bw_state->num_active_planes[pipe] = 0;
7416 }
7417
7418 /*
7419  * turn all crtc's off, but do not adjust state
7420  * This has to be paired with a call to intel_modeset_setup_hw_state.
7421  */
7422 int intel_display_suspend(struct drm_device *dev)
7423 {
7424         struct drm_i915_private *dev_priv = to_i915(dev);
7425         struct drm_atomic_state *state;
7426         int ret;
7427
7428         state = drm_atomic_helper_suspend(dev);
7429         ret = PTR_ERR_OR_ZERO(state);
7430         if (ret)
7431                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
7432         else
7433                 dev_priv->modeset_restore_state = state;
7434         return ret;
7435 }
7436
7437 void intel_encoder_destroy(struct drm_encoder *encoder)
7438 {
7439         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
7440
7441         drm_encoder_cleanup(encoder);
7442         kfree(intel_encoder);
7443 }
7444
7445 /* Cross check the actual hw state with our own modeset state tracking (and it's
7446  * internal consistency). */
7447 static void intel_connector_verify_state(struct intel_crtc_state *crtc_state,
7448                                          struct drm_connector_state *conn_state)
7449 {
7450         struct intel_connector *connector = to_intel_connector(conn_state->connector);
7451
7452         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
7453                       connector->base.base.id,
7454                       connector->base.name);
7455
7456         if (connector->get_hw_state(connector)) {
7457                 struct intel_encoder *encoder = connector->encoder;
7458
7459                 I915_STATE_WARN(!crtc_state,
7460                          "connector enabled without attached crtc\n");
7461
7462                 if (!crtc_state)
7463                         return;
7464
7465                 I915_STATE_WARN(!crtc_state->hw.active,
7466                                 "connector is active, but attached crtc isn't\n");
7467
7468                 if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST)
7469                         return;
7470
7471                 I915_STATE_WARN(conn_state->best_encoder != &encoder->base,
7472                         "atomic encoder doesn't match attached encoder\n");
7473
7474                 I915_STATE_WARN(conn_state->crtc != encoder->base.crtc,
7475                         "attached encoder crtc differs from connector crtc\n");
7476         } else {
7477                 I915_STATE_WARN(crtc_state && crtc_state->hw.active,
7478                                 "attached crtc is active, but connector isn't\n");
7479                 I915_STATE_WARN(!crtc_state && conn_state->best_encoder,
7480                         "best encoder set without crtc!\n");
7481         }
7482 }
7483
7484 static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state)
7485 {
7486         if (crtc_state->hw.enable && crtc_state->has_pch_encoder)
7487                 return crtc_state->fdi_lanes;
7488
7489         return 0;
7490 }
7491
7492 static int ilk_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
7493                                struct intel_crtc_state *pipe_config)
7494 {
7495         struct drm_i915_private *dev_priv = to_i915(dev);
7496         struct drm_atomic_state *state = pipe_config->uapi.state;
7497         struct intel_crtc *other_crtc;
7498         struct intel_crtc_state *other_crtc_state;
7499
7500         DRM_DEBUG_KMS("checking fdi config on pipe %c, lanes %i\n",
7501                       pipe_name(pipe), pipe_config->fdi_lanes);
7502         if (pipe_config->fdi_lanes > 4) {
7503                 DRM_DEBUG_KMS("invalid fdi lane config on pipe %c: %i lanes\n",
7504                               pipe_name(pipe), pipe_config->fdi_lanes);
7505                 return -EINVAL;
7506         }
7507
7508         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
7509                 if (pipe_config->fdi_lanes > 2) {
7510                         DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n",
7511                                       pipe_config->fdi_lanes);
7512                         return -EINVAL;
7513                 } else {
7514                         return 0;
7515                 }
7516         }
7517
7518         if (INTEL_NUM_PIPES(dev_priv) == 2)
7519                 return 0;
7520
7521         /* Ivybridge 3 pipe is really complicated */
7522         switch (pipe) {
7523         case PIPE_A:
7524                 return 0;
7525         case PIPE_B:
7526                 if (pipe_config->fdi_lanes <= 2)
7527                         return 0;
7528
7529                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_C);
7530                 other_crtc_state =
7531                         intel_atomic_get_crtc_state(state, other_crtc);
7532                 if (IS_ERR(other_crtc_state))
7533                         return PTR_ERR(other_crtc_state);
7534
7535                 if (pipe_required_fdi_lanes(other_crtc_state) > 0) {
7536                         DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %c: %i lanes\n",
7537                                       pipe_name(pipe), pipe_config->fdi_lanes);
7538                         return -EINVAL;
7539                 }
7540                 return 0;
7541         case PIPE_C:
7542                 if (pipe_config->fdi_lanes > 2) {
7543                         DRM_DEBUG_KMS("only 2 lanes on pipe %c: required %i lanes\n",
7544                                       pipe_name(pipe), pipe_config->fdi_lanes);
7545                         return -EINVAL;
7546                 }
7547
7548                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_B);
7549                 other_crtc_state =
7550                         intel_atomic_get_crtc_state(state, other_crtc);
7551                 if (IS_ERR(other_crtc_state))
7552                         return PTR_ERR(other_crtc_state);
7553
7554                 if (pipe_required_fdi_lanes(other_crtc_state) > 2) {
7555                         DRM_DEBUG_KMS("fdi link B uses too many lanes to enable link C\n");
7556                         return -EINVAL;
7557                 }
7558                 return 0;
7559         default:
7560                 BUG();
7561         }
7562 }
7563
7564 #define RETRY 1
7565 static int ilk_fdi_compute_config(struct intel_crtc *intel_crtc,
7566                                   struct intel_crtc_state *pipe_config)
7567 {
7568         struct drm_device *dev = intel_crtc->base.dev;
7569         const struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
7570         int lane, link_bw, fdi_dotclock, ret;
7571         bool needs_recompute = false;
7572
7573 retry:
7574         /* FDI is a binary signal running at ~2.7GHz, encoding
7575          * each output octet as 10 bits. The actual frequency
7576          * is stored as a divider into a 100MHz clock, and the
7577          * mode pixel clock is stored in units of 1KHz.
7578          * Hence the bw of each lane in terms of the mode signal
7579          * is:
7580          */
7581         link_bw = intel_fdi_link_freq(to_i915(dev), pipe_config);
7582
7583         fdi_dotclock = adjusted_mode->crtc_clock;
7584
7585         lane = ilk_get_lanes_required(fdi_dotclock, link_bw,
7586                                       pipe_config->pipe_bpp);
7587
7588         pipe_config->fdi_lanes = lane;
7589
7590         intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock,
7591                                link_bw, &pipe_config->fdi_m_n, false, false);
7592
7593         ret = ilk_check_fdi_lanes(dev, intel_crtc->pipe, pipe_config);
7594         if (ret == -EDEADLK)
7595                 return ret;
7596
7597         if (ret == -EINVAL && pipe_config->pipe_bpp > 6*3) {
7598                 pipe_config->pipe_bpp -= 2*3;
7599                 DRM_DEBUG_KMS("fdi link bw constraint, reducing pipe bpp to %i\n",
7600                               pipe_config->pipe_bpp);
7601                 needs_recompute = true;
7602                 pipe_config->bw_constrained = true;
7603
7604                 goto retry;
7605         }
7606
7607         if (needs_recompute)
7608                 return RETRY;
7609
7610         return ret;
7611 }
7612
7613 bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state)
7614 {
7615         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7616         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7617
7618         /* IPS only exists on ULT machines and is tied to pipe A. */
7619         if (!hsw_crtc_supports_ips(crtc))
7620                 return false;
7621
7622         if (!i915_modparams.enable_ips)
7623                 return false;
7624
7625         if (crtc_state->pipe_bpp > 24)
7626                 return false;
7627
7628         /*
7629          * We compare against max which means we must take
7630          * the increased cdclk requirement into account when
7631          * calculating the new cdclk.
7632          *
7633          * Should measure whether using a lower cdclk w/o IPS
7634          */
7635         if (IS_BROADWELL(dev_priv) &&
7636             crtc_state->pixel_rate > dev_priv->max_cdclk_freq * 95 / 100)
7637                 return false;
7638
7639         return true;
7640 }
7641
7642 static bool hsw_compute_ips_config(struct intel_crtc_state *crtc_state)
7643 {
7644         struct drm_i915_private *dev_priv =
7645                 to_i915(crtc_state->uapi.crtc->dev);
7646         struct intel_atomic_state *intel_state =
7647                 to_intel_atomic_state(crtc_state->uapi.state);
7648
7649         if (!hsw_crtc_state_ips_capable(crtc_state))
7650                 return false;
7651
7652         /*
7653          * When IPS gets enabled, the pipe CRC changes. Since IPS gets
7654          * enabled and disabled dynamically based on package C states,
7655          * user space can't make reliable use of the CRCs, so let's just
7656          * completely disable it.
7657          */
7658         if (crtc_state->crc_enabled)
7659                 return false;
7660
7661         /* IPS should be fine as long as at least one plane is enabled. */
7662         if (!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)))
7663                 return false;
7664
7665         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
7666         if (IS_BROADWELL(dev_priv) &&
7667             crtc_state->pixel_rate > intel_state->cdclk.logical.cdclk * 95 / 100)
7668                 return false;
7669
7670         return true;
7671 }
7672
7673 static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc)
7674 {
7675         const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7676
7677         /* GDG double wide on either pipe, otherwise pipe A only */
7678         return INTEL_GEN(dev_priv) < 4 &&
7679                 (crtc->pipe == PIPE_A || IS_I915G(dev_priv));
7680 }
7681
7682 static u32 ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config)
7683 {
7684         u32 pixel_rate;
7685
7686         pixel_rate = pipe_config->hw.adjusted_mode.crtc_clock;
7687
7688         /*
7689          * We only use IF-ID interlacing. If we ever use
7690          * PF-ID we'll need to adjust the pixel_rate here.
7691          */
7692
7693         if (pipe_config->pch_pfit.enabled) {
7694                 u64 pipe_w, pipe_h, pfit_w, pfit_h;
7695                 u32 pfit_size = pipe_config->pch_pfit.size;
7696
7697                 pipe_w = pipe_config->pipe_src_w;
7698                 pipe_h = pipe_config->pipe_src_h;
7699
7700                 pfit_w = (pfit_size >> 16) & 0xFFFF;
7701                 pfit_h = pfit_size & 0xFFFF;
7702                 if (pipe_w < pfit_w)
7703                         pipe_w = pfit_w;
7704                 if (pipe_h < pfit_h)
7705                         pipe_h = pfit_h;
7706
7707                 if (WARN_ON(!pfit_w || !pfit_h))
7708                         return pixel_rate;
7709
7710                 pixel_rate = div_u64(mul_u32_u32(pixel_rate, pipe_w * pipe_h),
7711                                      pfit_w * pfit_h);
7712         }
7713
7714         return pixel_rate;
7715 }
7716
7717 static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state)
7718 {
7719         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
7720
7721         if (HAS_GMCH(dev_priv))
7722                 /* FIXME calculate proper pipe pixel rate for GMCH pfit */
7723                 crtc_state->pixel_rate =
7724                         crtc_state->hw.adjusted_mode.crtc_clock;
7725         else
7726                 crtc_state->pixel_rate =
7727                         ilk_pipe_pixel_rate(crtc_state);
7728 }
7729
7730 static int intel_crtc_compute_config(struct intel_crtc *crtc,
7731                                      struct intel_crtc_state *pipe_config)
7732 {
7733         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7734         const struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
7735         int clock_limit = dev_priv->max_dotclk_freq;
7736
7737         if (INTEL_GEN(dev_priv) < 4) {
7738                 clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
7739
7740                 /*
7741                  * Enable double wide mode when the dot clock
7742                  * is > 90% of the (display) core speed.
7743                  */
7744                 if (intel_crtc_supports_double_wide(crtc) &&
7745                     adjusted_mode->crtc_clock > clock_limit) {
7746                         clock_limit = dev_priv->max_dotclk_freq;
7747                         pipe_config->double_wide = true;
7748                 }
7749         }
7750
7751         if (adjusted_mode->crtc_clock > clock_limit) {
7752                 DRM_DEBUG_KMS("requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
7753                               adjusted_mode->crtc_clock, clock_limit,
7754                               yesno(pipe_config->double_wide));
7755                 return -EINVAL;
7756         }
7757
7758         if ((pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
7759              pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) &&
7760              pipe_config->hw.ctm) {
7761                 /*
7762                  * There is only one pipe CSC unit per pipe, and we need that
7763                  * for output conversion from RGB->YCBCR. So if CTM is already
7764                  * applied we can't support YCBCR420 output.
7765                  */
7766                 DRM_DEBUG_KMS("YCBCR420 and CTM together are not possible\n");
7767                 return -EINVAL;
7768         }
7769
7770         /*
7771          * Pipe horizontal size must be even in:
7772          * - DVO ganged mode
7773          * - LVDS dual channel mode
7774          * - Double wide pipe
7775          */
7776         if (pipe_config->pipe_src_w & 1) {
7777                 if (pipe_config->double_wide) {
7778                         DRM_DEBUG_KMS("Odd pipe source width not supported with double wide pipe\n");
7779                         return -EINVAL;
7780                 }
7781
7782                 if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_LVDS) &&
7783                     intel_is_dual_link_lvds(dev_priv)) {
7784                         DRM_DEBUG_KMS("Odd pipe source width not supported with dual link LVDS\n");
7785                         return -EINVAL;
7786                 }
7787         }
7788
7789         /* Cantiga+ cannot handle modes with a hsync front porch of 0.
7790          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
7791          */
7792         if ((INTEL_GEN(dev_priv) > 4 || IS_G4X(dev_priv)) &&
7793                 adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay)
7794                 return -EINVAL;
7795
7796         intel_crtc_compute_pixel_rate(pipe_config);
7797
7798         if (pipe_config->has_pch_encoder)
7799                 return ilk_fdi_compute_config(crtc, pipe_config);
7800
7801         return 0;
7802 }
7803
7804 static void
7805 intel_reduce_m_n_ratio(u32 *num, u32 *den)
7806 {
7807         while (*num > DATA_LINK_M_N_MASK ||
7808                *den > DATA_LINK_M_N_MASK) {
7809                 *num >>= 1;
7810                 *den >>= 1;
7811         }
7812 }
7813
7814 static void compute_m_n(unsigned int m, unsigned int n,
7815                         u32 *ret_m, u32 *ret_n,
7816                         bool constant_n)
7817 {
7818         /*
7819          * Several DP dongles in particular seem to be fussy about
7820          * too large link M/N values. Give N value as 0x8000 that
7821          * should be acceptable by specific devices. 0x8000 is the
7822          * specified fixed N value for asynchronous clock mode,
7823          * which the devices expect also in synchronous clock mode.
7824          */
7825         if (constant_n)
7826                 *ret_n = 0x8000;
7827         else
7828                 *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
7829
7830         *ret_m = div_u64(mul_u32_u32(m, *ret_n), n);
7831         intel_reduce_m_n_ratio(ret_m, ret_n);
7832 }
7833
7834 void
7835 intel_link_compute_m_n(u16 bits_per_pixel, int nlanes,
7836                        int pixel_clock, int link_clock,
7837                        struct intel_link_m_n *m_n,
7838                        bool constant_n, bool fec_enable)
7839 {
7840         u32 data_clock = bits_per_pixel * pixel_clock;
7841
7842         if (fec_enable)
7843                 data_clock = intel_dp_mode_to_fec_clock(data_clock);
7844
7845         m_n->tu = 64;
7846         compute_m_n(data_clock,
7847                     link_clock * nlanes * 8,
7848                     &m_n->gmch_m, &m_n->gmch_n,
7849                     constant_n);
7850
7851         compute_m_n(pixel_clock, link_clock,
7852                     &m_n->link_m, &m_n->link_n,
7853                     constant_n);
7854 }
7855
7856 static void intel_panel_sanitize_ssc(struct drm_i915_private *dev_priv)
7857 {
7858         /*
7859          * There may be no VBT; and if the BIOS enabled SSC we can
7860          * just keep using it to avoid unnecessary flicker.  Whereas if the
7861          * BIOS isn't using it, don't assume it will work even if the VBT
7862          * indicates as much.
7863          */
7864         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
7865                 bool bios_lvds_use_ssc = I915_READ(PCH_DREF_CONTROL) &
7866                         DREF_SSC1_ENABLE;
7867
7868                 if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) {
7869                         DRM_DEBUG_KMS("SSC %s by BIOS, overriding VBT which says %s\n",
7870                                       enableddisabled(bios_lvds_use_ssc),
7871                                       enableddisabled(dev_priv->vbt.lvds_use_ssc));
7872                         dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc;
7873                 }
7874         }
7875 }
7876
7877 static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
7878 {
7879         if (i915_modparams.panel_use_ssc >= 0)
7880                 return i915_modparams.panel_use_ssc != 0;
7881         return dev_priv->vbt.lvds_use_ssc
7882                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
7883 }
7884
7885 static u32 pnv_dpll_compute_fp(struct dpll *dpll)
7886 {
7887         return (1 << dpll->n) << 16 | dpll->m2;
7888 }
7889
7890 static u32 i9xx_dpll_compute_fp(struct dpll *dpll)
7891 {
7892         return dpll->n << 16 | dpll->m1 << 8 | dpll->m2;
7893 }
7894
7895 static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
7896                                      struct intel_crtc_state *crtc_state,
7897                                      struct dpll *reduced_clock)
7898 {
7899         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7900         u32 fp, fp2 = 0;
7901
7902         if (IS_PINEVIEW(dev_priv)) {
7903                 fp = pnv_dpll_compute_fp(&crtc_state->dpll);
7904                 if (reduced_clock)
7905                         fp2 = pnv_dpll_compute_fp(reduced_clock);
7906         } else {
7907                 fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
7908                 if (reduced_clock)
7909                         fp2 = i9xx_dpll_compute_fp(reduced_clock);
7910         }
7911
7912         crtc_state->dpll_hw_state.fp0 = fp;
7913
7914         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7915             reduced_clock) {
7916                 crtc_state->dpll_hw_state.fp1 = fp2;
7917         } else {
7918                 crtc_state->dpll_hw_state.fp1 = fp;
7919         }
7920 }
7921
7922 static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe
7923                 pipe)
7924 {
7925         u32 reg_val;
7926
7927         /*
7928          * PLLB opamp always calibrates to max value of 0x3f, force enable it
7929          * and set it to a reasonable value instead.
7930          */
7931         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7932         reg_val &= 0xffffff00;
7933         reg_val |= 0x00000030;
7934         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7935
7936         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7937         reg_val &= 0x00ffffff;
7938         reg_val |= 0x8c000000;
7939         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7940
7941         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7942         reg_val &= 0xffffff00;
7943         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7944
7945         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7946         reg_val &= 0x00ffffff;
7947         reg_val |= 0xb0000000;
7948         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7949 }
7950
7951 static void intel_pch_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
7952                                          const struct intel_link_m_n *m_n)
7953 {
7954         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7955         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7956         enum pipe pipe = crtc->pipe;
7957
7958         I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
7959         I915_WRITE(PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
7960         I915_WRITE(PCH_TRANS_LINK_M1(pipe), m_n->link_m);
7961         I915_WRITE(PCH_TRANS_LINK_N1(pipe), m_n->link_n);
7962 }
7963
7964 static bool transcoder_has_m2_n2(struct drm_i915_private *dev_priv,
7965                                  enum transcoder transcoder)
7966 {
7967         if (IS_HASWELL(dev_priv))
7968                 return transcoder == TRANSCODER_EDP;
7969
7970         /*
7971          * Strictly speaking some registers are available before
7972          * gen7, but we only support DRRS on gen7+
7973          */
7974         return IS_GEN(dev_priv, 7) || IS_CHERRYVIEW(dev_priv);
7975 }
7976
7977 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
7978                                          const struct intel_link_m_n *m_n,
7979                                          const struct intel_link_m_n *m2_n2)
7980 {
7981         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7982         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7983         enum pipe pipe = crtc->pipe;
7984         enum transcoder transcoder = crtc_state->cpu_transcoder;
7985
7986         if (INTEL_GEN(dev_priv) >= 5) {
7987                 I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m);
7988                 I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
7989                 I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
7990                 I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n);
7991                 /*
7992                  *  M2_N2 registers are set only if DRRS is supported
7993                  * (to make sure the registers are not unnecessarily accessed).
7994                  */
7995                 if (m2_n2 && crtc_state->has_drrs &&
7996                     transcoder_has_m2_n2(dev_priv, transcoder)) {
7997                         I915_WRITE(PIPE_DATA_M2(transcoder),
7998                                         TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
7999                         I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n);
8000                         I915_WRITE(PIPE_LINK_M2(transcoder), m2_n2->link_m);
8001                         I915_WRITE(PIPE_LINK_N2(transcoder), m2_n2->link_n);
8002                 }
8003         } else {
8004                 I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
8005                 I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
8006                 I915_WRITE(PIPE_LINK_M_G4X(pipe), m_n->link_m);
8007                 I915_WRITE(PIPE_LINK_N_G4X(pipe), m_n->link_n);
8008         }
8009 }
8010
8011 void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, enum link_m_n_set m_n)
8012 {
8013         const struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL;
8014
8015         if (m_n == M1_N1) {
8016                 dp_m_n = &crtc_state->dp_m_n;
8017                 dp_m2_n2 = &crtc_state->dp_m2_n2;
8018         } else if (m_n == M2_N2) {
8019
8020                 /*
8021                  * M2_N2 registers are not supported. Hence m2_n2 divider value
8022                  * needs to be programmed into M1_N1.
8023                  */
8024                 dp_m_n = &crtc_state->dp_m2_n2;
8025         } else {
8026                 DRM_ERROR("Unsupported divider value\n");
8027                 return;
8028         }
8029
8030         if (crtc_state->has_pch_encoder)
8031                 intel_pch_transcoder_set_m_n(crtc_state, &crtc_state->dp_m_n);
8032         else
8033                 intel_cpu_transcoder_set_m_n(crtc_state, dp_m_n, dp_m2_n2);
8034 }
8035
8036 static void vlv_compute_dpll(struct intel_crtc *crtc,
8037                              struct intel_crtc_state *pipe_config)
8038 {
8039         pipe_config->dpll_hw_state.dpll = DPLL_INTEGRATED_REF_CLK_VLV |
8040                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
8041         if (crtc->pipe != PIPE_A)
8042                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
8043
8044         /* DPLL not used with DSI, but still need the rest set up */
8045         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
8046                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE |
8047                         DPLL_EXT_BUFFER_ENABLE_VLV;
8048
8049         pipe_config->dpll_hw_state.dpll_md =
8050                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
8051 }
8052
8053 static void chv_compute_dpll(struct intel_crtc *crtc,
8054                              struct intel_crtc_state *pipe_config)
8055 {
8056         pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV |
8057                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
8058         if (crtc->pipe != PIPE_A)
8059                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
8060
8061         /* DPLL not used with DSI, but still need the rest set up */
8062         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
8063                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE;
8064
8065         pipe_config->dpll_hw_state.dpll_md =
8066                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
8067 }
8068
8069 static void vlv_prepare_pll(struct intel_crtc *crtc,
8070                             const struct intel_crtc_state *pipe_config)
8071 {
8072         struct drm_device *dev = crtc->base.dev;
8073         struct drm_i915_private *dev_priv = to_i915(dev);
8074         enum pipe pipe = crtc->pipe;
8075         u32 mdiv;
8076         u32 bestn, bestm1, bestm2, bestp1, bestp2;
8077         u32 coreclk, reg_val;
8078
8079         /* Enable Refclk */
8080         I915_WRITE(DPLL(pipe),
8081                    pipe_config->dpll_hw_state.dpll &
8082                    ~(DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV));
8083
8084         /* No need to actually set up the DPLL with DSI */
8085         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8086                 return;
8087
8088         vlv_dpio_get(dev_priv);
8089
8090         bestn = pipe_config->dpll.n;
8091         bestm1 = pipe_config->dpll.m1;
8092         bestm2 = pipe_config->dpll.m2;
8093         bestp1 = pipe_config->dpll.p1;
8094         bestp2 = pipe_config->dpll.p2;
8095
8096         /* See eDP HDMI DPIO driver vbios notes doc */
8097
8098         /* PLL B needs special handling */
8099         if (pipe == PIPE_B)
8100                 vlv_pllb_recal_opamp(dev_priv, pipe);
8101
8102         /* Set up Tx target for periodic Rcomp update */
8103         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9_BCAST, 0x0100000f);
8104
8105         /* Disable target IRef on PLL */
8106         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW8(pipe));
8107         reg_val &= 0x00ffffff;
8108         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW8(pipe), reg_val);
8109
8110         /* Disable fast lock */
8111         vlv_dpio_write(dev_priv, pipe, VLV_CMN_DW0, 0x610);
8112
8113         /* Set idtafcrecal before PLL is enabled */
8114         mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
8115         mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT));
8116         mdiv |= ((bestn << DPIO_N_SHIFT));
8117         mdiv |= (1 << DPIO_K_SHIFT);
8118
8119         /*
8120          * Post divider depends on pixel clock rate, DAC vs digital (and LVDS,
8121          * but we don't support that).
8122          * Note: don't use the DAC post divider as it seems unstable.
8123          */
8124         mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
8125         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
8126
8127         mdiv |= DPIO_ENABLE_CALIBRATION;
8128         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
8129
8130         /* Set HBR and RBR LPF coefficients */
8131         if (pipe_config->port_clock == 162000 ||
8132             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_ANALOG) ||
8133             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI))
8134                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
8135                                  0x009f0003);
8136         else
8137                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
8138                                  0x00d0000f);
8139
8140         if (intel_crtc_has_dp_encoder(pipe_config)) {
8141                 /* Use SSC source */
8142                 if (pipe == PIPE_A)
8143                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
8144                                          0x0df40000);
8145                 else
8146                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
8147                                          0x0df70000);
8148         } else { /* HDMI or VGA */
8149                 /* Use bend source */
8150                 if (pipe == PIPE_A)
8151                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
8152                                          0x0df70000);
8153                 else
8154                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
8155                                          0x0df40000);
8156         }
8157
8158         coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe));
8159         coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
8160         if (intel_crtc_has_dp_encoder(pipe_config))
8161                 coreclk |= 0x01000000;
8162         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk);
8163
8164         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000);
8165
8166         vlv_dpio_put(dev_priv);
8167 }
8168
8169 static void chv_prepare_pll(struct intel_crtc *crtc,
8170                             const struct intel_crtc_state *pipe_config)
8171 {
8172         struct drm_device *dev = crtc->base.dev;
8173         struct drm_i915_private *dev_priv = to_i915(dev);
8174         enum pipe pipe = crtc->pipe;
8175         enum dpio_channel port = vlv_pipe_to_channel(pipe);
8176         u32 loopfilter, tribuf_calcntr;
8177         u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
8178         u32 dpio_val;
8179         int vco;
8180
8181         /* Enable Refclk and SSC */
8182         I915_WRITE(DPLL(pipe),
8183                    pipe_config->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE);
8184
8185         /* No need to actually set up the DPLL with DSI */
8186         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8187                 return;
8188
8189         bestn = pipe_config->dpll.n;
8190         bestm2_frac = pipe_config->dpll.m2 & 0x3fffff;
8191         bestm1 = pipe_config->dpll.m1;
8192         bestm2 = pipe_config->dpll.m2 >> 22;
8193         bestp1 = pipe_config->dpll.p1;
8194         bestp2 = pipe_config->dpll.p2;
8195         vco = pipe_config->dpll.vco;
8196         dpio_val = 0;
8197         loopfilter = 0;
8198
8199         vlv_dpio_get(dev_priv);
8200
8201         /* p1 and p2 divider */
8202         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port),
8203                         5 << DPIO_CHV_S1_DIV_SHIFT |
8204                         bestp1 << DPIO_CHV_P1_DIV_SHIFT |
8205                         bestp2 << DPIO_CHV_P2_DIV_SHIFT |
8206                         1 << DPIO_CHV_K_DIV_SHIFT);
8207
8208         /* Feedback post-divider - m2 */
8209         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2);
8210
8211         /* Feedback refclk divider - n and m1 */
8212         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port),
8213                         DPIO_CHV_M1_DIV_BY_2 |
8214                         1 << DPIO_CHV_N_DIV_SHIFT);
8215
8216         /* M2 fraction division */
8217         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
8218
8219         /* M2 fraction division enable */
8220         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
8221         dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
8222         dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT);
8223         if (bestm2_frac)
8224                 dpio_val |= DPIO_CHV_FRAC_DIV_EN;
8225         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), dpio_val);
8226
8227         /* Program digital lock detect threshold */
8228         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW9(port));
8229         dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
8230                                         DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
8231         dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT);
8232         if (!bestm2_frac)
8233                 dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
8234         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW9(port), dpio_val);
8235
8236         /* Loop filter */
8237         if (vco == 5400000) {
8238                 loopfilter |= (0x3 << DPIO_CHV_PROP_COEFF_SHIFT);
8239                 loopfilter |= (0x8 << DPIO_CHV_INT_COEFF_SHIFT);
8240                 loopfilter |= (0x1 << DPIO_CHV_GAIN_CTRL_SHIFT);
8241                 tribuf_calcntr = 0x9;
8242         } else if (vco <= 6200000) {
8243                 loopfilter |= (0x5 << DPIO_CHV_PROP_COEFF_SHIFT);
8244                 loopfilter |= (0xB << DPIO_CHV_INT_COEFF_SHIFT);
8245                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
8246                 tribuf_calcntr = 0x9;
8247         } else if (vco <= 6480000) {
8248                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
8249                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
8250                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
8251                 tribuf_calcntr = 0x8;
8252         } else {
8253                 /* Not supported. Apply the same limits as in the max case */
8254                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
8255                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
8256                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
8257                 tribuf_calcntr = 0;
8258         }
8259         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter);
8260
8261         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW8(port));
8262         dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
8263         dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT);
8264         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW8(port), dpio_val);
8265
8266         /* AFC Recal */
8267         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port),
8268                         vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) |
8269                         DPIO_AFC_RECAL);
8270
8271         vlv_dpio_put(dev_priv);
8272 }
8273
8274 /**
8275  * vlv_force_pll_on - forcibly enable just the PLL
8276  * @dev_priv: i915 private structure
8277  * @pipe: pipe PLL to enable
8278  * @dpll: PLL configuration
8279  *
8280  * Enable the PLL for @pipe using the supplied @dpll config. To be used
8281  * in cases where we need the PLL enabled even when @pipe is not going to
8282  * be enabled.
8283  */
8284 int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe,
8285                      const struct dpll *dpll)
8286 {
8287         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
8288         struct intel_crtc_state *pipe_config;
8289
8290         pipe_config = intel_crtc_state_alloc(crtc);
8291         if (!pipe_config)
8292                 return -ENOMEM;
8293
8294         pipe_config->cpu_transcoder = (enum transcoder)pipe;
8295         pipe_config->pixel_multiplier = 1;
8296         pipe_config->dpll = *dpll;
8297
8298         if (IS_CHERRYVIEW(dev_priv)) {
8299                 chv_compute_dpll(crtc, pipe_config);
8300                 chv_prepare_pll(crtc, pipe_config);
8301                 chv_enable_pll(crtc, pipe_config);
8302         } else {
8303                 vlv_compute_dpll(crtc, pipe_config);
8304                 vlv_prepare_pll(crtc, pipe_config);
8305                 vlv_enable_pll(crtc, pipe_config);
8306         }
8307
8308         kfree(pipe_config);
8309
8310         return 0;
8311 }
8312
8313 /**
8314  * vlv_force_pll_off - forcibly disable just the PLL
8315  * @dev_priv: i915 private structure
8316  * @pipe: pipe PLL to disable
8317  *
8318  * Disable the PLL for @pipe. To be used in cases where we need
8319  * the PLL enabled even when @pipe is not going to be enabled.
8320  */
8321 void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe)
8322 {
8323         if (IS_CHERRYVIEW(dev_priv))
8324                 chv_disable_pll(dev_priv, pipe);
8325         else
8326                 vlv_disable_pll(dev_priv, pipe);
8327 }
8328
8329 static void i9xx_compute_dpll(struct intel_crtc *crtc,
8330                               struct intel_crtc_state *crtc_state,
8331                               struct dpll *reduced_clock)
8332 {
8333         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8334         u32 dpll;
8335         struct dpll *clock = &crtc_state->dpll;
8336
8337         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
8338
8339         dpll = DPLL_VGA_MODE_DIS;
8340
8341         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
8342                 dpll |= DPLLB_MODE_LVDS;
8343         else
8344                 dpll |= DPLLB_MODE_DAC_SERIAL;
8345
8346         if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
8347             IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
8348                 dpll |= (crtc_state->pixel_multiplier - 1)
8349                         << SDVO_MULTIPLIER_SHIFT_HIRES;
8350         }
8351
8352         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
8353             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
8354                 dpll |= DPLL_SDVO_HIGH_SPEED;
8355
8356         if (intel_crtc_has_dp_encoder(crtc_state))
8357                 dpll |= DPLL_SDVO_HIGH_SPEED;
8358
8359         /* compute bitmask from p1 value */
8360         if (IS_PINEVIEW(dev_priv))
8361                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
8362         else {
8363                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8364                 if (IS_G4X(dev_priv) && reduced_clock)
8365                         dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
8366         }
8367         switch (clock->p2) {
8368         case 5:
8369                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
8370                 break;
8371         case 7:
8372                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
8373                 break;
8374         case 10:
8375                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
8376                 break;
8377         case 14:
8378                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
8379                 break;
8380         }
8381         if (INTEL_GEN(dev_priv) >= 4)
8382                 dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
8383
8384         if (crtc_state->sdvo_tv_clock)
8385                 dpll |= PLL_REF_INPUT_TVCLKINBC;
8386         else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8387                  intel_panel_use_ssc(dev_priv))
8388                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8389         else
8390                 dpll |= PLL_REF_INPUT_DREFCLK;
8391
8392         dpll |= DPLL_VCO_ENABLE;
8393         crtc_state->dpll_hw_state.dpll = dpll;
8394
8395         if (INTEL_GEN(dev_priv) >= 4) {
8396                 u32 dpll_md = (crtc_state->pixel_multiplier - 1)
8397                         << DPLL_MD_UDI_MULTIPLIER_SHIFT;
8398                 crtc_state->dpll_hw_state.dpll_md = dpll_md;
8399         }
8400 }
8401
8402 static void i8xx_compute_dpll(struct intel_crtc *crtc,
8403                               struct intel_crtc_state *crtc_state,
8404                               struct dpll *reduced_clock)
8405 {
8406         struct drm_device *dev = crtc->base.dev;
8407         struct drm_i915_private *dev_priv = to_i915(dev);
8408         u32 dpll;
8409         struct dpll *clock = &crtc_state->dpll;
8410
8411         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
8412
8413         dpll = DPLL_VGA_MODE_DIS;
8414
8415         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8416                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8417         } else {
8418                 if (clock->p1 == 2)
8419                         dpll |= PLL_P1_DIVIDE_BY_TWO;
8420                 else
8421                         dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8422                 if (clock->p2 == 4)
8423                         dpll |= PLL_P2_DIVIDE_BY_4;
8424         }
8425
8426         /*
8427          * Bspec:
8428          * "[Almador Errata}: For the correct operation of the muxed DVO pins
8429          *  (GDEVSELB/I2Cdata, GIRDBY/I2CClk) and (GFRAMEB/DVI_Data,
8430          *  GTRDYB/DVI_Clk): Bit 31 (DPLL VCO Enable) and Bit 30 (2X Clock
8431          *  Enable) must be set to “1” in both the DPLL A Control Register
8432          *  (06014h-06017h) and DPLL B Control Register (06018h-0601Bh)."
8433          *
8434          * For simplicity We simply keep both bits always enabled in
8435          * both DPLLS. The spec says we should disable the DVO 2X clock
8436          * when not needed, but this seems to work fine in practice.
8437          */
8438         if (IS_I830(dev_priv) ||
8439             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO))
8440                 dpll |= DPLL_DVO_2X_MODE;
8441
8442         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8443             intel_panel_use_ssc(dev_priv))
8444                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8445         else
8446                 dpll |= PLL_REF_INPUT_DREFCLK;
8447
8448         dpll |= DPLL_VCO_ENABLE;
8449         crtc_state->dpll_hw_state.dpll = dpll;
8450 }
8451
8452 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state)
8453 {
8454         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8455         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8456         enum pipe pipe = crtc->pipe;
8457         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
8458         const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
8459         u32 crtc_vtotal, crtc_vblank_end;
8460         int vsyncshift = 0;
8461
8462         /* We need to be careful not to changed the adjusted mode, for otherwise
8463          * the hw state checker will get angry at the mismatch. */
8464         crtc_vtotal = adjusted_mode->crtc_vtotal;
8465         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
8466
8467         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
8468                 /* the chip adds 2 halflines automatically */
8469                 crtc_vtotal -= 1;
8470                 crtc_vblank_end -= 1;
8471
8472                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
8473                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
8474                 else
8475                         vsyncshift = adjusted_mode->crtc_hsync_start -
8476                                 adjusted_mode->crtc_htotal / 2;
8477                 if (vsyncshift < 0)
8478                         vsyncshift += adjusted_mode->crtc_htotal;
8479         }
8480
8481         if (INTEL_GEN(dev_priv) > 3)
8482                 I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift);
8483
8484         I915_WRITE(HTOTAL(cpu_transcoder),
8485                    (adjusted_mode->crtc_hdisplay - 1) |
8486                    ((adjusted_mode->crtc_htotal - 1) << 16));
8487         I915_WRITE(HBLANK(cpu_transcoder),
8488                    (adjusted_mode->crtc_hblank_start - 1) |
8489                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
8490         I915_WRITE(HSYNC(cpu_transcoder),
8491                    (adjusted_mode->crtc_hsync_start - 1) |
8492                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
8493
8494         I915_WRITE(VTOTAL(cpu_transcoder),
8495                    (adjusted_mode->crtc_vdisplay - 1) |
8496                    ((crtc_vtotal - 1) << 16));
8497         I915_WRITE(VBLANK(cpu_transcoder),
8498                    (adjusted_mode->crtc_vblank_start - 1) |
8499                    ((crtc_vblank_end - 1) << 16));
8500         I915_WRITE(VSYNC(cpu_transcoder),
8501                    (adjusted_mode->crtc_vsync_start - 1) |
8502                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
8503
8504         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
8505          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
8506          * documented on the DDI_FUNC_CTL register description, EDP Input Select
8507          * bits. */
8508         if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP &&
8509             (pipe == PIPE_B || pipe == PIPE_C))
8510                 I915_WRITE(VTOTAL(pipe), I915_READ(VTOTAL(cpu_transcoder)));
8511
8512 }
8513
8514 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state)
8515 {
8516         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8517         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8518         enum pipe pipe = crtc->pipe;
8519
8520         /* pipesrc controls the size that is scaled from, which should
8521          * always be the user's requested size.
8522          */
8523         I915_WRITE(PIPESRC(pipe),
8524                    ((crtc_state->pipe_src_w - 1) << 16) |
8525                    (crtc_state->pipe_src_h - 1));
8526 }
8527
8528 static bool intel_pipe_is_interlaced(const struct intel_crtc_state *crtc_state)
8529 {
8530         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
8531         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
8532
8533         if (IS_GEN(dev_priv, 2))
8534                 return false;
8535
8536         if (INTEL_GEN(dev_priv) >= 9 ||
8537             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
8538                 return I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK_HSW;
8539         else
8540                 return I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK;
8541 }
8542
8543 static void intel_get_pipe_timings(struct intel_crtc *crtc,
8544                                    struct intel_crtc_state *pipe_config)
8545 {
8546         struct drm_device *dev = crtc->base.dev;
8547         struct drm_i915_private *dev_priv = to_i915(dev);
8548         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
8549         u32 tmp;
8550
8551         tmp = I915_READ(HTOTAL(cpu_transcoder));
8552         pipe_config->hw.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
8553         pipe_config->hw.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
8554
8555         if (!transcoder_is_dsi(cpu_transcoder)) {
8556                 tmp = I915_READ(HBLANK(cpu_transcoder));
8557                 pipe_config->hw.adjusted_mode.crtc_hblank_start =
8558                                                         (tmp & 0xffff) + 1;
8559                 pipe_config->hw.adjusted_mode.crtc_hblank_end =
8560                                                 ((tmp >> 16) & 0xffff) + 1;
8561         }
8562         tmp = I915_READ(HSYNC(cpu_transcoder));
8563         pipe_config->hw.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
8564         pipe_config->hw.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
8565
8566         tmp = I915_READ(VTOTAL(cpu_transcoder));
8567         pipe_config->hw.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
8568         pipe_config->hw.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
8569
8570         if (!transcoder_is_dsi(cpu_transcoder)) {
8571                 tmp = I915_READ(VBLANK(cpu_transcoder));
8572                 pipe_config->hw.adjusted_mode.crtc_vblank_start =
8573                                                         (tmp & 0xffff) + 1;
8574                 pipe_config->hw.adjusted_mode.crtc_vblank_end =
8575                                                 ((tmp >> 16) & 0xffff) + 1;
8576         }
8577         tmp = I915_READ(VSYNC(cpu_transcoder));
8578         pipe_config->hw.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
8579         pipe_config->hw.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
8580
8581         if (intel_pipe_is_interlaced(pipe_config)) {
8582                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
8583                 pipe_config->hw.adjusted_mode.crtc_vtotal += 1;
8584                 pipe_config->hw.adjusted_mode.crtc_vblank_end += 1;
8585         }
8586 }
8587
8588 static void intel_get_pipe_src_size(struct intel_crtc *crtc,
8589                                     struct intel_crtc_state *pipe_config)
8590 {
8591         struct drm_device *dev = crtc->base.dev;
8592         struct drm_i915_private *dev_priv = to_i915(dev);
8593         u32 tmp;
8594
8595         tmp = I915_READ(PIPESRC(crtc->pipe));
8596         pipe_config->pipe_src_h = (tmp & 0xffff) + 1;
8597         pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1;
8598
8599         pipe_config->hw.mode.vdisplay = pipe_config->pipe_src_h;
8600         pipe_config->hw.mode.hdisplay = pipe_config->pipe_src_w;
8601 }
8602
8603 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
8604                                  struct intel_crtc_state *pipe_config)
8605 {
8606         mode->hdisplay = pipe_config->hw.adjusted_mode.crtc_hdisplay;
8607         mode->htotal = pipe_config->hw.adjusted_mode.crtc_htotal;
8608         mode->hsync_start = pipe_config->hw.adjusted_mode.crtc_hsync_start;
8609         mode->hsync_end = pipe_config->hw.adjusted_mode.crtc_hsync_end;
8610
8611         mode->vdisplay = pipe_config->hw.adjusted_mode.crtc_vdisplay;
8612         mode->vtotal = pipe_config->hw.adjusted_mode.crtc_vtotal;
8613         mode->vsync_start = pipe_config->hw.adjusted_mode.crtc_vsync_start;
8614         mode->vsync_end = pipe_config->hw.adjusted_mode.crtc_vsync_end;
8615
8616         mode->flags = pipe_config->hw.adjusted_mode.flags;
8617         mode->type = DRM_MODE_TYPE_DRIVER;
8618
8619         mode->clock = pipe_config->hw.adjusted_mode.crtc_clock;
8620
8621         mode->hsync = drm_mode_hsync(mode);
8622         mode->vrefresh = drm_mode_vrefresh(mode);
8623         drm_mode_set_name(mode);
8624 }
8625
8626 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
8627 {
8628         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8629         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8630         u32 pipeconf;
8631
8632         pipeconf = 0;
8633
8634         /* we keep both pipes enabled on 830 */
8635         if (IS_I830(dev_priv))
8636                 pipeconf |= I915_READ(PIPECONF(crtc->pipe)) & PIPECONF_ENABLE;
8637
8638         if (crtc_state->double_wide)
8639                 pipeconf |= PIPECONF_DOUBLE_WIDE;
8640
8641         /* only g4x and later have fancy bpc/dither controls */
8642         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
8643             IS_CHERRYVIEW(dev_priv)) {
8644                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
8645                 if (crtc_state->dither && crtc_state->pipe_bpp != 30)
8646                         pipeconf |= PIPECONF_DITHER_EN |
8647                                     PIPECONF_DITHER_TYPE_SP;
8648
8649                 switch (crtc_state->pipe_bpp) {
8650                 case 18:
8651                         pipeconf |= PIPECONF_6BPC;
8652                         break;
8653                 case 24:
8654                         pipeconf |= PIPECONF_8BPC;
8655                         break;
8656                 case 30:
8657                         pipeconf |= PIPECONF_10BPC;
8658                         break;
8659                 default:
8660                         /* Case prevented by intel_choose_pipe_bpp_dither. */
8661                         BUG();
8662                 }
8663         }
8664
8665         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
8666                 if (INTEL_GEN(dev_priv) < 4 ||
8667                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
8668                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
8669                 else
8670                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
8671         } else {
8672                 pipeconf |= PIPECONF_PROGRESSIVE;
8673         }
8674
8675         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
8676              crtc_state->limited_color_range)
8677                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
8678
8679         pipeconf |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
8680
8681         pipeconf |= PIPECONF_FRAME_START_DELAY(0);
8682
8683         I915_WRITE(PIPECONF(crtc->pipe), pipeconf);
8684         POSTING_READ(PIPECONF(crtc->pipe));
8685 }
8686
8687 static int i8xx_crtc_compute_clock(struct intel_crtc *crtc,
8688                                    struct intel_crtc_state *crtc_state)
8689 {
8690         struct drm_device *dev = crtc->base.dev;
8691         struct drm_i915_private *dev_priv = to_i915(dev);
8692         const struct intel_limit *limit;
8693         int refclk = 48000;
8694
8695         memset(&crtc_state->dpll_hw_state, 0,
8696                sizeof(crtc_state->dpll_hw_state));
8697
8698         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8699                 if (intel_panel_use_ssc(dev_priv)) {
8700                         refclk = dev_priv->vbt.lvds_ssc_freq;
8701                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8702                 }
8703
8704                 limit = &intel_limits_i8xx_lvds;
8705         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO)) {
8706                 limit = &intel_limits_i8xx_dvo;
8707         } else {
8708                 limit = &intel_limits_i8xx_dac;
8709         }
8710
8711         if (!crtc_state->clock_set &&
8712             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8713                                  refclk, NULL, &crtc_state->dpll)) {
8714                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8715                 return -EINVAL;
8716         }
8717
8718         i8xx_compute_dpll(crtc, crtc_state, NULL);
8719
8720         return 0;
8721 }
8722
8723 static int g4x_crtc_compute_clock(struct intel_crtc *crtc,
8724                                   struct intel_crtc_state *crtc_state)
8725 {
8726         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8727         const struct intel_limit *limit;
8728         int refclk = 96000;
8729
8730         memset(&crtc_state->dpll_hw_state, 0,
8731                sizeof(crtc_state->dpll_hw_state));
8732
8733         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8734                 if (intel_panel_use_ssc(dev_priv)) {
8735                         refclk = dev_priv->vbt.lvds_ssc_freq;
8736                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8737                 }
8738
8739                 if (intel_is_dual_link_lvds(dev_priv))
8740                         limit = &intel_limits_g4x_dual_channel_lvds;
8741                 else
8742                         limit = &intel_limits_g4x_single_channel_lvds;
8743         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) ||
8744                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
8745                 limit = &intel_limits_g4x_hdmi;
8746         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) {
8747                 limit = &intel_limits_g4x_sdvo;
8748         } else {
8749                 /* The option is for other outputs */
8750                 limit = &intel_limits_i9xx_sdvo;
8751         }
8752
8753         if (!crtc_state->clock_set &&
8754             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8755                                 refclk, NULL, &crtc_state->dpll)) {
8756                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8757                 return -EINVAL;
8758         }
8759
8760         i9xx_compute_dpll(crtc, crtc_state, NULL);
8761
8762         return 0;
8763 }
8764
8765 static int pnv_crtc_compute_clock(struct intel_crtc *crtc,
8766                                   struct intel_crtc_state *crtc_state)
8767 {
8768         struct drm_device *dev = crtc->base.dev;
8769         struct drm_i915_private *dev_priv = to_i915(dev);
8770         const struct intel_limit *limit;
8771         int refclk = 96000;
8772
8773         memset(&crtc_state->dpll_hw_state, 0,
8774                sizeof(crtc_state->dpll_hw_state));
8775
8776         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8777                 if (intel_panel_use_ssc(dev_priv)) {
8778                         refclk = dev_priv->vbt.lvds_ssc_freq;
8779                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8780                 }
8781
8782                 limit = &pnv_limits_lvds;
8783         } else {
8784                 limit = &pnv_limits_sdvo;
8785         }
8786
8787         if (!crtc_state->clock_set &&
8788             !pnv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8789                                 refclk, NULL, &crtc_state->dpll)) {
8790                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8791                 return -EINVAL;
8792         }
8793
8794         i9xx_compute_dpll(crtc, crtc_state, NULL);
8795
8796         return 0;
8797 }
8798
8799 static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
8800                                    struct intel_crtc_state *crtc_state)
8801 {
8802         struct drm_device *dev = crtc->base.dev;
8803         struct drm_i915_private *dev_priv = to_i915(dev);
8804         const struct intel_limit *limit;
8805         int refclk = 96000;
8806
8807         memset(&crtc_state->dpll_hw_state, 0,
8808                sizeof(crtc_state->dpll_hw_state));
8809
8810         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8811                 if (intel_panel_use_ssc(dev_priv)) {
8812                         refclk = dev_priv->vbt.lvds_ssc_freq;
8813                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8814                 }
8815
8816                 limit = &intel_limits_i9xx_lvds;
8817         } else {
8818                 limit = &intel_limits_i9xx_sdvo;
8819         }
8820
8821         if (!crtc_state->clock_set &&
8822             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8823                                  refclk, NULL, &crtc_state->dpll)) {
8824                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8825                 return -EINVAL;
8826         }
8827
8828         i9xx_compute_dpll(crtc, crtc_state, NULL);
8829
8830         return 0;
8831 }
8832
8833 static int chv_crtc_compute_clock(struct intel_crtc *crtc,
8834                                   struct intel_crtc_state *crtc_state)
8835 {
8836         int refclk = 100000;
8837         const struct intel_limit *limit = &intel_limits_chv;
8838
8839         memset(&crtc_state->dpll_hw_state, 0,
8840                sizeof(crtc_state->dpll_hw_state));
8841
8842         if (!crtc_state->clock_set &&
8843             !chv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8844                                 refclk, NULL, &crtc_state->dpll)) {
8845                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8846                 return -EINVAL;
8847         }
8848
8849         chv_compute_dpll(crtc, crtc_state);
8850
8851         return 0;
8852 }
8853
8854 static int vlv_crtc_compute_clock(struct intel_crtc *crtc,
8855                                   struct intel_crtc_state *crtc_state)
8856 {
8857         int refclk = 100000;
8858         const struct intel_limit *limit = &intel_limits_vlv;
8859
8860         memset(&crtc_state->dpll_hw_state, 0,
8861                sizeof(crtc_state->dpll_hw_state));
8862
8863         if (!crtc_state->clock_set &&
8864             !vlv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8865                                 refclk, NULL, &crtc_state->dpll)) {
8866                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8867                 return -EINVAL;
8868         }
8869
8870         vlv_compute_dpll(crtc, crtc_state);
8871
8872         return 0;
8873 }
8874
8875 static bool i9xx_has_pfit(struct drm_i915_private *dev_priv)
8876 {
8877         if (IS_I830(dev_priv))
8878                 return false;
8879
8880         return INTEL_GEN(dev_priv) >= 4 ||
8881                 IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
8882 }
8883
8884 static void i9xx_get_pfit_config(struct intel_crtc *crtc,
8885                                  struct intel_crtc_state *pipe_config)
8886 {
8887         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8888         u32 tmp;
8889
8890         if (!i9xx_has_pfit(dev_priv))
8891                 return;
8892
8893         tmp = I915_READ(PFIT_CONTROL);
8894         if (!(tmp & PFIT_ENABLE))
8895                 return;
8896
8897         /* Check whether the pfit is attached to our pipe. */
8898         if (INTEL_GEN(dev_priv) < 4) {
8899                 if (crtc->pipe != PIPE_B)
8900                         return;
8901         } else {
8902                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
8903                         return;
8904         }
8905
8906         pipe_config->gmch_pfit.control = tmp;
8907         pipe_config->gmch_pfit.pgm_ratios = I915_READ(PFIT_PGM_RATIOS);
8908 }
8909
8910 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
8911                                struct intel_crtc_state *pipe_config)
8912 {
8913         struct drm_device *dev = crtc->base.dev;
8914         struct drm_i915_private *dev_priv = to_i915(dev);
8915         enum pipe pipe = crtc->pipe;
8916         struct dpll clock;
8917         u32 mdiv;
8918         int refclk = 100000;
8919
8920         /* In case of DSI, DPLL will not be used */
8921         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8922                 return;
8923
8924         vlv_dpio_get(dev_priv);
8925         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
8926         vlv_dpio_put(dev_priv);
8927
8928         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
8929         clock.m2 = mdiv & DPIO_M2DIV_MASK;
8930         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
8931         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
8932         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
8933
8934         pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
8935 }
8936
8937 static void
8938 i9xx_get_initial_plane_config(struct intel_crtc *crtc,
8939                               struct intel_initial_plane_config *plane_config)
8940 {
8941         struct drm_device *dev = crtc->base.dev;
8942         struct drm_i915_private *dev_priv = to_i915(dev);
8943         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
8944         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
8945         enum pipe pipe;
8946         u32 val, base, offset;
8947         int fourcc, pixel_format;
8948         unsigned int aligned_height;
8949         struct drm_framebuffer *fb;
8950         struct intel_framebuffer *intel_fb;
8951
8952         if (!plane->get_hw_state(plane, &pipe))
8953                 return;
8954
8955         WARN_ON(pipe != crtc->pipe);
8956
8957         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
8958         if (!intel_fb) {
8959                 DRM_DEBUG_KMS("failed to alloc fb\n");
8960                 return;
8961         }
8962
8963         fb = &intel_fb->base;
8964
8965         fb->dev = dev;
8966
8967         val = I915_READ(DSPCNTR(i9xx_plane));
8968
8969         if (INTEL_GEN(dev_priv) >= 4) {
8970                 if (val & DISPPLANE_TILED) {
8971                         plane_config->tiling = I915_TILING_X;
8972                         fb->modifier = I915_FORMAT_MOD_X_TILED;
8973                 }
8974
8975                 if (val & DISPPLANE_ROTATE_180)
8976                         plane_config->rotation = DRM_MODE_ROTATE_180;
8977         }
8978
8979         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B &&
8980             val & DISPPLANE_MIRROR)
8981                 plane_config->rotation |= DRM_MODE_REFLECT_X;
8982
8983         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
8984         fourcc = i9xx_format_to_fourcc(pixel_format);
8985         fb->format = drm_format_info(fourcc);
8986
8987         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
8988                 offset = I915_READ(DSPOFFSET(i9xx_plane));
8989                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
8990         } else if (INTEL_GEN(dev_priv) >= 4) {
8991                 if (plane_config->tiling)
8992                         offset = I915_READ(DSPTILEOFF(i9xx_plane));
8993                 else
8994                         offset = I915_READ(DSPLINOFF(i9xx_plane));
8995                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
8996         } else {
8997                 base = I915_READ(DSPADDR(i9xx_plane));
8998         }
8999         plane_config->base = base;
9000
9001         val = I915_READ(PIPESRC(pipe));
9002         fb->width = ((val >> 16) & 0xfff) + 1;
9003         fb->height = ((val >> 0) & 0xfff) + 1;
9004
9005         val = I915_READ(DSPSTRIDE(i9xx_plane));
9006         fb->pitches[0] = val & 0xffffffc0;
9007
9008         aligned_height = intel_fb_align_height(fb, 0, fb->height);
9009
9010         plane_config->size = fb->pitches[0] * aligned_height;
9011
9012         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
9013                       crtc->base.name, plane->base.name, fb->width, fb->height,
9014                       fb->format->cpp[0] * 8, base, fb->pitches[0],
9015                       plane_config->size);
9016
9017         plane_config->fb = intel_fb;
9018 }
9019
9020 static void chv_crtc_clock_get(struct intel_crtc *crtc,
9021                                struct intel_crtc_state *pipe_config)
9022 {
9023         struct drm_device *dev = crtc->base.dev;
9024         struct drm_i915_private *dev_priv = to_i915(dev);
9025         enum pipe pipe = crtc->pipe;
9026         enum dpio_channel port = vlv_pipe_to_channel(pipe);
9027         struct dpll clock;
9028         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
9029         int refclk = 100000;
9030
9031         /* In case of DSI, DPLL will not be used */
9032         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
9033                 return;
9034
9035         vlv_dpio_get(dev_priv);
9036         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
9037         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
9038         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
9039         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
9040         pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
9041         vlv_dpio_put(dev_priv);
9042
9043         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
9044         clock.m2 = (pll_dw0 & 0xff) << 22;
9045         if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
9046                 clock.m2 |= pll_dw2 & 0x3fffff;
9047         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
9048         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
9049         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
9050
9051         pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
9052 }
9053
9054 static enum intel_output_format
9055 bdw_get_pipemisc_output_format(struct intel_crtc *crtc)
9056 {
9057         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9058         u32 tmp;
9059
9060         tmp = I915_READ(PIPEMISC(crtc->pipe));
9061
9062         if (tmp & PIPEMISC_YUV420_ENABLE) {
9063                 /* We support 4:2:0 in full blend mode only */
9064                 WARN_ON((tmp & PIPEMISC_YUV420_MODE_FULL_BLEND) == 0);
9065
9066                 return INTEL_OUTPUT_FORMAT_YCBCR420;
9067         } else if (tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV) {
9068                 return INTEL_OUTPUT_FORMAT_YCBCR444;
9069         } else {
9070                 return INTEL_OUTPUT_FORMAT_RGB;
9071         }
9072 }
9073
9074 static void i9xx_get_pipe_color_config(struct intel_crtc_state *crtc_state)
9075 {
9076         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9077         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
9078         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9079         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
9080         u32 tmp;
9081
9082         tmp = I915_READ(DSPCNTR(i9xx_plane));
9083
9084         if (tmp & DISPPLANE_GAMMA_ENABLE)
9085                 crtc_state->gamma_enable = true;
9086
9087         if (!HAS_GMCH(dev_priv) &&
9088             tmp & DISPPLANE_PIPE_CSC_ENABLE)
9089                 crtc_state->csc_enable = true;
9090 }
9091
9092 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
9093                                  struct intel_crtc_state *pipe_config)
9094 {
9095         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9096         enum intel_display_power_domain power_domain;
9097         intel_wakeref_t wakeref;
9098         u32 tmp;
9099         bool ret;
9100
9101         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
9102         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
9103         if (!wakeref)
9104                 return false;
9105
9106         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
9107         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9108         pipe_config->shared_dpll = NULL;
9109         pipe_config->master_transcoder = INVALID_TRANSCODER;
9110
9111         ret = false;
9112
9113         tmp = I915_READ(PIPECONF(crtc->pipe));
9114         if (!(tmp & PIPECONF_ENABLE))
9115                 goto out;
9116
9117         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
9118             IS_CHERRYVIEW(dev_priv)) {
9119                 switch (tmp & PIPECONF_BPC_MASK) {
9120                 case PIPECONF_6BPC:
9121                         pipe_config->pipe_bpp = 18;
9122                         break;
9123                 case PIPECONF_8BPC:
9124                         pipe_config->pipe_bpp = 24;
9125                         break;
9126                 case PIPECONF_10BPC:
9127                         pipe_config->pipe_bpp = 30;
9128                         break;
9129                 default:
9130                         break;
9131                 }
9132         }
9133
9134         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
9135             (tmp & PIPECONF_COLOR_RANGE_SELECT))
9136                 pipe_config->limited_color_range = true;
9137
9138         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_I9XX) >>
9139                 PIPECONF_GAMMA_MODE_SHIFT;
9140
9141         if (IS_CHERRYVIEW(dev_priv))
9142                 pipe_config->cgm_mode = I915_READ(CGM_PIPE_MODE(crtc->pipe));
9143
9144         i9xx_get_pipe_color_config(pipe_config);
9145         intel_color_get_config(pipe_config);
9146
9147         if (INTEL_GEN(dev_priv) < 4)
9148                 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
9149
9150         intel_get_pipe_timings(crtc, pipe_config);
9151         intel_get_pipe_src_size(crtc, pipe_config);
9152
9153         i9xx_get_pfit_config(crtc, pipe_config);
9154
9155         if (INTEL_GEN(dev_priv) >= 4) {
9156                 /* No way to read it out on pipes B and C */
9157                 if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
9158                         tmp = dev_priv->chv_dpll_md[crtc->pipe];
9159                 else
9160                         tmp = I915_READ(DPLL_MD(crtc->pipe));
9161                 pipe_config->pixel_multiplier =
9162                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
9163                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
9164                 pipe_config->dpll_hw_state.dpll_md = tmp;
9165         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
9166                    IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
9167                 tmp = I915_READ(DPLL(crtc->pipe));
9168                 pipe_config->pixel_multiplier =
9169                         ((tmp & SDVO_MULTIPLIER_MASK)
9170                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
9171         } else {
9172                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
9173                  * port and will be fixed up in the encoder->get_config
9174                  * function. */
9175                 pipe_config->pixel_multiplier = 1;
9176         }
9177         pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
9178         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
9179                 pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
9180                 pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe));
9181         } else {
9182                 /* Mask out read-only status bits. */
9183                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
9184                                                      DPLL_PORTC_READY_MASK |
9185                                                      DPLL_PORTB_READY_MASK);
9186         }
9187
9188         if (IS_CHERRYVIEW(dev_priv))
9189                 chv_crtc_clock_get(crtc, pipe_config);
9190         else if (IS_VALLEYVIEW(dev_priv))
9191                 vlv_crtc_clock_get(crtc, pipe_config);
9192         else
9193                 i9xx_crtc_clock_get(crtc, pipe_config);
9194
9195         /*
9196          * Normally the dotclock is filled in by the encoder .get_config()
9197          * but in case the pipe is enabled w/o any ports we need a sane
9198          * default.
9199          */
9200         pipe_config->hw.adjusted_mode.crtc_clock =
9201                 pipe_config->port_clock / pipe_config->pixel_multiplier;
9202
9203         ret = true;
9204
9205 out:
9206         intel_display_power_put(dev_priv, power_domain, wakeref);
9207
9208         return ret;
9209 }
9210
9211 static void ilk_init_pch_refclk(struct drm_i915_private *dev_priv)
9212 {
9213         struct intel_encoder *encoder;
9214         int i;
9215         u32 val, final;
9216         bool has_lvds = false;
9217         bool has_cpu_edp = false;
9218         bool has_panel = false;
9219         bool has_ck505 = false;
9220         bool can_ssc = false;
9221         bool using_ssc_source = false;
9222
9223         /* We need to take the global config into account */
9224         for_each_intel_encoder(&dev_priv->drm, encoder) {
9225                 switch (encoder->type) {
9226                 case INTEL_OUTPUT_LVDS:
9227                         has_panel = true;
9228                         has_lvds = true;
9229                         break;
9230                 case INTEL_OUTPUT_EDP:
9231                         has_panel = true;
9232                         if (encoder->port == PORT_A)
9233                                 has_cpu_edp = true;
9234                         break;
9235                 default:
9236                         break;
9237                 }
9238         }
9239
9240         if (HAS_PCH_IBX(dev_priv)) {
9241                 has_ck505 = dev_priv->vbt.display_clock_mode;
9242                 can_ssc = has_ck505;
9243         } else {
9244                 has_ck505 = false;
9245                 can_ssc = true;
9246         }
9247
9248         /* Check if any DPLLs are using the SSC source */
9249         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
9250                 u32 temp = I915_READ(PCH_DPLL(i));
9251
9252                 if (!(temp & DPLL_VCO_ENABLE))
9253                         continue;
9254
9255                 if ((temp & PLL_REF_INPUT_MASK) ==
9256                     PLLB_REF_INPUT_SPREADSPECTRUMIN) {
9257                         using_ssc_source = true;
9258                         break;
9259                 }
9260         }
9261
9262         DRM_DEBUG_KMS("has_panel %d has_lvds %d has_ck505 %d using_ssc_source %d\n",
9263                       has_panel, has_lvds, has_ck505, using_ssc_source);
9264
9265         /* Ironlake: try to setup display ref clock before DPLL
9266          * enabling. This is only under driver's control after
9267          * PCH B stepping, previous chipset stepping should be
9268          * ignoring this setting.
9269          */
9270         val = I915_READ(PCH_DREF_CONTROL);
9271
9272         /* As we must carefully and slowly disable/enable each source in turn,
9273          * compute the final state we want first and check if we need to
9274          * make any changes at all.
9275          */
9276         final = val;
9277         final &= ~DREF_NONSPREAD_SOURCE_MASK;
9278         if (has_ck505)
9279                 final |= DREF_NONSPREAD_CK505_ENABLE;
9280         else
9281                 final |= DREF_NONSPREAD_SOURCE_ENABLE;
9282
9283         final &= ~DREF_SSC_SOURCE_MASK;
9284         final &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9285         final &= ~DREF_SSC1_ENABLE;
9286
9287         if (has_panel) {
9288                 final |= DREF_SSC_SOURCE_ENABLE;
9289
9290                 if (intel_panel_use_ssc(dev_priv) && can_ssc)
9291                         final |= DREF_SSC1_ENABLE;
9292
9293                 if (has_cpu_edp) {
9294                         if (intel_panel_use_ssc(dev_priv) && can_ssc)
9295                                 final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
9296                         else
9297                                 final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
9298                 } else
9299                         final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9300         } else if (using_ssc_source) {
9301                 final |= DREF_SSC_SOURCE_ENABLE;
9302                 final |= DREF_SSC1_ENABLE;
9303         }
9304
9305         if (final == val)
9306                 return;
9307
9308         /* Always enable nonspread source */
9309         val &= ~DREF_NONSPREAD_SOURCE_MASK;
9310
9311         if (has_ck505)
9312                 val |= DREF_NONSPREAD_CK505_ENABLE;
9313         else
9314                 val |= DREF_NONSPREAD_SOURCE_ENABLE;
9315
9316         if (has_panel) {
9317                 val &= ~DREF_SSC_SOURCE_MASK;
9318                 val |= DREF_SSC_SOURCE_ENABLE;
9319
9320                 /* SSC must be turned on before enabling the CPU output  */
9321                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
9322                         DRM_DEBUG_KMS("Using SSC on panel\n");
9323                         val |= DREF_SSC1_ENABLE;
9324                 } else
9325                         val &= ~DREF_SSC1_ENABLE;
9326
9327                 /* Get SSC going before enabling the outputs */
9328                 I915_WRITE(PCH_DREF_CONTROL, val);
9329                 POSTING_READ(PCH_DREF_CONTROL);
9330                 udelay(200);
9331
9332                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9333
9334                 /* Enable CPU source on CPU attached eDP */
9335                 if (has_cpu_edp) {
9336                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
9337                                 DRM_DEBUG_KMS("Using SSC on eDP\n");
9338                                 val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
9339                         } else
9340                                 val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
9341                 } else
9342                         val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9343
9344                 I915_WRITE(PCH_DREF_CONTROL, val);
9345                 POSTING_READ(PCH_DREF_CONTROL);
9346                 udelay(200);
9347         } else {
9348                 DRM_DEBUG_KMS("Disabling CPU source output\n");
9349
9350                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9351
9352                 /* Turn off CPU output */
9353                 val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9354
9355                 I915_WRITE(PCH_DREF_CONTROL, val);
9356                 POSTING_READ(PCH_DREF_CONTROL);
9357                 udelay(200);
9358
9359                 if (!using_ssc_source) {
9360                         DRM_DEBUG_KMS("Disabling SSC source\n");
9361
9362                         /* Turn off the SSC source */
9363                         val &= ~DREF_SSC_SOURCE_MASK;
9364                         val |= DREF_SSC_SOURCE_DISABLE;
9365
9366                         /* Turn off SSC1 */
9367                         val &= ~DREF_SSC1_ENABLE;
9368
9369                         I915_WRITE(PCH_DREF_CONTROL, val);
9370                         POSTING_READ(PCH_DREF_CONTROL);
9371                         udelay(200);
9372                 }
9373         }
9374
9375         BUG_ON(val != final);
9376 }
9377
9378 static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv)
9379 {
9380         u32 tmp;
9381
9382         tmp = I915_READ(SOUTH_CHICKEN2);
9383         tmp |= FDI_MPHY_IOSFSB_RESET_CTL;
9384         I915_WRITE(SOUTH_CHICKEN2, tmp);
9385
9386         if (wait_for_us(I915_READ(SOUTH_CHICKEN2) &
9387                         FDI_MPHY_IOSFSB_RESET_STATUS, 100))
9388                 DRM_ERROR("FDI mPHY reset assert timeout\n");
9389
9390         tmp = I915_READ(SOUTH_CHICKEN2);
9391         tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL;
9392         I915_WRITE(SOUTH_CHICKEN2, tmp);
9393
9394         if (wait_for_us((I915_READ(SOUTH_CHICKEN2) &
9395                          FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100))
9396                 DRM_ERROR("FDI mPHY reset de-assert timeout\n");
9397 }
9398
9399 /* WaMPhyProgramming:hsw */
9400 static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv)
9401 {
9402         u32 tmp;
9403
9404         tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY);
9405         tmp &= ~(0xFF << 24);
9406         tmp |= (0x12 << 24);
9407         intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY);
9408
9409         tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY);
9410         tmp |= (1 << 11);
9411         intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY);
9412
9413         tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY);
9414         tmp |= (1 << 11);
9415         intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY);
9416
9417         tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY);
9418         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
9419         intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY);
9420
9421         tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY);
9422         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
9423         intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY);
9424
9425         tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY);
9426         tmp &= ~(7 << 13);
9427         tmp |= (5 << 13);
9428         intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY);
9429
9430         tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY);
9431         tmp &= ~(7 << 13);
9432         tmp |= (5 << 13);
9433         intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY);
9434
9435         tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY);
9436         tmp &= ~0xFF;
9437         tmp |= 0x1C;
9438         intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY);
9439
9440         tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY);
9441         tmp &= ~0xFF;
9442         tmp |= 0x1C;
9443         intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY);
9444
9445         tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY);
9446         tmp &= ~(0xFF << 16);
9447         tmp |= (0x1C << 16);
9448         intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY);
9449
9450         tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY);
9451         tmp &= ~(0xFF << 16);
9452         tmp |= (0x1C << 16);
9453         intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY);
9454
9455         tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY);
9456         tmp |= (1 << 27);
9457         intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY);
9458
9459         tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY);
9460         tmp |= (1 << 27);
9461         intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY);
9462
9463         tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY);
9464         tmp &= ~(0xF << 28);
9465         tmp |= (4 << 28);
9466         intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY);
9467
9468         tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY);
9469         tmp &= ~(0xF << 28);
9470         tmp |= (4 << 28);
9471         intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY);
9472 }
9473
9474 /* Implements 3 different sequences from BSpec chapter "Display iCLK
9475  * Programming" based on the parameters passed:
9476  * - Sequence to enable CLKOUT_DP
9477  * - Sequence to enable CLKOUT_DP without spread
9478  * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O
9479  */
9480 static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv,
9481                                  bool with_spread, bool with_fdi)
9482 {
9483         u32 reg, tmp;
9484
9485         if (WARN(with_fdi && !with_spread, "FDI requires downspread\n"))
9486                 with_spread = true;
9487         if (WARN(HAS_PCH_LPT_LP(dev_priv) &&
9488             with_fdi, "LP PCH doesn't have FDI\n"))
9489                 with_fdi = false;
9490
9491         mutex_lock(&dev_priv->sb_lock);
9492
9493         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9494         tmp &= ~SBI_SSCCTL_DISABLE;
9495         tmp |= SBI_SSCCTL_PATHALT;
9496         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9497
9498         udelay(24);
9499
9500         if (with_spread) {
9501                 tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9502                 tmp &= ~SBI_SSCCTL_PATHALT;
9503                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9504
9505                 if (with_fdi) {
9506                         lpt_reset_fdi_mphy(dev_priv);
9507                         lpt_program_fdi_mphy(dev_priv);
9508                 }
9509         }
9510
9511         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
9512         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
9513         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
9514         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
9515
9516         mutex_unlock(&dev_priv->sb_lock);
9517 }
9518
9519 /* Sequence to disable CLKOUT_DP */
9520 void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv)
9521 {
9522         u32 reg, tmp;
9523
9524         mutex_lock(&dev_priv->sb_lock);
9525
9526         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
9527         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
9528         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
9529         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
9530
9531         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9532         if (!(tmp & SBI_SSCCTL_DISABLE)) {
9533                 if (!(tmp & SBI_SSCCTL_PATHALT)) {
9534                         tmp |= SBI_SSCCTL_PATHALT;
9535                         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9536                         udelay(32);
9537                 }
9538                 tmp |= SBI_SSCCTL_DISABLE;
9539                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9540         }
9541
9542         mutex_unlock(&dev_priv->sb_lock);
9543 }
9544
9545 #define BEND_IDX(steps) ((50 + (steps)) / 5)
9546
9547 static const u16 sscdivintphase[] = {
9548         [BEND_IDX( 50)] = 0x3B23,
9549         [BEND_IDX( 45)] = 0x3B23,
9550         [BEND_IDX( 40)] = 0x3C23,
9551         [BEND_IDX( 35)] = 0x3C23,
9552         [BEND_IDX( 30)] = 0x3D23,
9553         [BEND_IDX( 25)] = 0x3D23,
9554         [BEND_IDX( 20)] = 0x3E23,
9555         [BEND_IDX( 15)] = 0x3E23,
9556         [BEND_IDX( 10)] = 0x3F23,
9557         [BEND_IDX(  5)] = 0x3F23,
9558         [BEND_IDX(  0)] = 0x0025,
9559         [BEND_IDX( -5)] = 0x0025,
9560         [BEND_IDX(-10)] = 0x0125,
9561         [BEND_IDX(-15)] = 0x0125,
9562         [BEND_IDX(-20)] = 0x0225,
9563         [BEND_IDX(-25)] = 0x0225,
9564         [BEND_IDX(-30)] = 0x0325,
9565         [BEND_IDX(-35)] = 0x0325,
9566         [BEND_IDX(-40)] = 0x0425,
9567         [BEND_IDX(-45)] = 0x0425,
9568         [BEND_IDX(-50)] = 0x0525,
9569 };
9570
9571 /*
9572  * Bend CLKOUT_DP
9573  * steps -50 to 50 inclusive, in steps of 5
9574  * < 0 slow down the clock, > 0 speed up the clock, 0 == no bend (135MHz)
9575  * change in clock period = -(steps / 10) * 5.787 ps
9576  */
9577 static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps)
9578 {
9579         u32 tmp;
9580         int idx = BEND_IDX(steps);
9581
9582         if (WARN_ON(steps % 5 != 0))
9583                 return;
9584
9585         if (WARN_ON(idx >= ARRAY_SIZE(sscdivintphase)))
9586                 return;
9587
9588         mutex_lock(&dev_priv->sb_lock);
9589
9590         if (steps % 10 != 0)
9591                 tmp = 0xAAAAAAAB;
9592         else
9593                 tmp = 0x00000000;
9594         intel_sbi_write(dev_priv, SBI_SSCDITHPHASE, tmp, SBI_ICLK);
9595
9596         tmp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE, SBI_ICLK);
9597         tmp &= 0xffff0000;
9598         tmp |= sscdivintphase[idx];
9599         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE, tmp, SBI_ICLK);
9600
9601         mutex_unlock(&dev_priv->sb_lock);
9602 }
9603
9604 #undef BEND_IDX
9605
9606 static bool spll_uses_pch_ssc(struct drm_i915_private *dev_priv)
9607 {
9608         u32 fuse_strap = I915_READ(FUSE_STRAP);
9609         u32 ctl = I915_READ(SPLL_CTL);
9610
9611         if ((ctl & SPLL_PLL_ENABLE) == 0)
9612                 return false;
9613
9614         if ((ctl & SPLL_REF_MASK) == SPLL_REF_MUXED_SSC &&
9615             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
9616                 return true;
9617
9618         if (IS_BROADWELL(dev_priv) &&
9619             (ctl & SPLL_REF_MASK) == SPLL_REF_PCH_SSC_BDW)
9620                 return true;
9621
9622         return false;
9623 }
9624
9625 static bool wrpll_uses_pch_ssc(struct drm_i915_private *dev_priv,
9626                                enum intel_dpll_id id)
9627 {
9628         u32 fuse_strap = I915_READ(FUSE_STRAP);
9629         u32 ctl = I915_READ(WRPLL_CTL(id));
9630
9631         if ((ctl & WRPLL_PLL_ENABLE) == 0)
9632                 return false;
9633
9634         if ((ctl & WRPLL_REF_MASK) == WRPLL_REF_PCH_SSC)
9635                 return true;
9636
9637         if ((IS_BROADWELL(dev_priv) || IS_HSW_ULT(dev_priv)) &&
9638             (ctl & WRPLL_REF_MASK) == WRPLL_REF_MUXED_SSC_BDW &&
9639             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
9640                 return true;
9641
9642         return false;
9643 }
9644
9645 static void lpt_init_pch_refclk(struct drm_i915_private *dev_priv)
9646 {
9647         struct intel_encoder *encoder;
9648         bool has_fdi = false;
9649
9650         for_each_intel_encoder(&dev_priv->drm, encoder) {
9651                 switch (encoder->type) {
9652                 case INTEL_OUTPUT_ANALOG:
9653                         has_fdi = true;
9654                         break;
9655                 default:
9656                         break;
9657                 }
9658         }
9659
9660         /*
9661          * The BIOS may have decided to use the PCH SSC
9662          * reference so we must not disable it until the
9663          * relevant PLLs have stopped relying on it. We'll
9664          * just leave the PCH SSC reference enabled in case
9665          * any active PLL is using it. It will get disabled
9666          * after runtime suspend if we don't have FDI.
9667          *
9668          * TODO: Move the whole reference clock handling
9669          * to the modeset sequence proper so that we can
9670          * actually enable/disable/reconfigure these things
9671          * safely. To do that we need to introduce a real
9672          * clock hierarchy. That would also allow us to do
9673          * clock bending finally.
9674          */
9675         dev_priv->pch_ssc_use = 0;
9676
9677         if (spll_uses_pch_ssc(dev_priv)) {
9678                 DRM_DEBUG_KMS("SPLL using PCH SSC\n");
9679                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_SPLL);
9680         }
9681
9682         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL1)) {
9683                 DRM_DEBUG_KMS("WRPLL1 using PCH SSC\n");
9684                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_WRPLL1);
9685         }
9686
9687         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL2)) {
9688                 DRM_DEBUG_KMS("WRPLL2 using PCH SSC\n");
9689                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_WRPLL2);
9690         }
9691
9692         if (dev_priv->pch_ssc_use)
9693                 return;
9694
9695         if (has_fdi) {
9696                 lpt_bend_clkout_dp(dev_priv, 0);
9697                 lpt_enable_clkout_dp(dev_priv, true, true);
9698         } else {
9699                 lpt_disable_clkout_dp(dev_priv);
9700         }
9701 }
9702
9703 /*
9704  * Initialize reference clocks when the driver loads
9705  */
9706 void intel_init_pch_refclk(struct drm_i915_private *dev_priv)
9707 {
9708         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
9709                 ilk_init_pch_refclk(dev_priv);
9710         else if (HAS_PCH_LPT(dev_priv))
9711                 lpt_init_pch_refclk(dev_priv);
9712 }
9713
9714 static void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state)
9715 {
9716         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9717         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9718         enum pipe pipe = crtc->pipe;
9719         u32 val;
9720
9721         val = 0;
9722
9723         switch (crtc_state->pipe_bpp) {
9724         case 18:
9725                 val |= PIPECONF_6BPC;
9726                 break;
9727         case 24:
9728                 val |= PIPECONF_8BPC;
9729                 break;
9730         case 30:
9731                 val |= PIPECONF_10BPC;
9732                 break;
9733         case 36:
9734                 val |= PIPECONF_12BPC;
9735                 break;
9736         default:
9737                 /* Case prevented by intel_choose_pipe_bpp_dither. */
9738                 BUG();
9739         }
9740
9741         if (crtc_state->dither)
9742                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
9743
9744         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
9745                 val |= PIPECONF_INTERLACED_ILK;
9746         else
9747                 val |= PIPECONF_PROGRESSIVE;
9748
9749         /*
9750          * This would end up with an odd purple hue over
9751          * the entire display. Make sure we don't do it.
9752          */
9753         WARN_ON(crtc_state->limited_color_range &&
9754                 crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB);
9755
9756         if (crtc_state->limited_color_range)
9757                 val |= PIPECONF_COLOR_RANGE_SELECT;
9758
9759         if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
9760                 val |= PIPECONF_OUTPUT_COLORSPACE_YUV709;
9761
9762         val |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
9763
9764         val |= PIPECONF_FRAME_START_DELAY(0);
9765
9766         I915_WRITE(PIPECONF(pipe), val);
9767         POSTING_READ(PIPECONF(pipe));
9768 }
9769
9770 static void hsw_set_pipeconf(const struct intel_crtc_state *crtc_state)
9771 {
9772         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9773         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9774         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
9775         u32 val = 0;
9776
9777         if (IS_HASWELL(dev_priv) && crtc_state->dither)
9778                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
9779
9780         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
9781                 val |= PIPECONF_INTERLACED_ILK;
9782         else
9783                 val |= PIPECONF_PROGRESSIVE;
9784
9785         if (IS_HASWELL(dev_priv) &&
9786             crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
9787                 val |= PIPECONF_OUTPUT_COLORSPACE_YUV_HSW;
9788
9789         I915_WRITE(PIPECONF(cpu_transcoder), val);
9790         POSTING_READ(PIPECONF(cpu_transcoder));
9791 }
9792
9793 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state)
9794 {
9795         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9796         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9797         u32 val = 0;
9798
9799         switch (crtc_state->pipe_bpp) {
9800         case 18:
9801                 val |= PIPEMISC_DITHER_6_BPC;
9802                 break;
9803         case 24:
9804                 val |= PIPEMISC_DITHER_8_BPC;
9805                 break;
9806         case 30:
9807                 val |= PIPEMISC_DITHER_10_BPC;
9808                 break;
9809         case 36:
9810                 val |= PIPEMISC_DITHER_12_BPC;
9811                 break;
9812         default:
9813                 MISSING_CASE(crtc_state->pipe_bpp);
9814                 break;
9815         }
9816
9817         if (crtc_state->dither)
9818                 val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
9819
9820         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
9821             crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444)
9822                 val |= PIPEMISC_OUTPUT_COLORSPACE_YUV;
9823
9824         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
9825                 val |= PIPEMISC_YUV420_ENABLE |
9826                         PIPEMISC_YUV420_MODE_FULL_BLEND;
9827
9828         if (INTEL_GEN(dev_priv) >= 11 &&
9829             (crtc_state->active_planes & ~(icl_hdr_plane_mask() |
9830                                            BIT(PLANE_CURSOR))) == 0)
9831                 val |= PIPEMISC_HDR_MODE_PRECISION;
9832
9833         I915_WRITE(PIPEMISC(crtc->pipe), val);
9834 }
9835
9836 int bdw_get_pipemisc_bpp(struct intel_crtc *crtc)
9837 {
9838         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9839         u32 tmp;
9840
9841         tmp = I915_READ(PIPEMISC(crtc->pipe));
9842
9843         switch (tmp & PIPEMISC_DITHER_BPC_MASK) {
9844         case PIPEMISC_DITHER_6_BPC:
9845                 return 18;
9846         case PIPEMISC_DITHER_8_BPC:
9847                 return 24;
9848         case PIPEMISC_DITHER_10_BPC:
9849                 return 30;
9850         case PIPEMISC_DITHER_12_BPC:
9851                 return 36;
9852         default:
9853                 MISSING_CASE(tmp);
9854                 return 0;
9855         }
9856 }
9857
9858 int ilk_get_lanes_required(int target_clock, int link_bw, int bpp)
9859 {
9860         /*
9861          * Account for spread spectrum to avoid
9862          * oversubscribing the link. Max center spread
9863          * is 2.5%; use 5% for safety's sake.
9864          */
9865         u32 bps = target_clock * bpp * 21 / 20;
9866         return DIV_ROUND_UP(bps, link_bw * 8);
9867 }
9868
9869 static bool ilk_needs_fb_cb_tune(struct dpll *dpll, int factor)
9870 {
9871         return i9xx_dpll_compute_m(dpll) < factor * dpll->n;
9872 }
9873
9874 static void ilk_compute_dpll(struct intel_crtc *crtc,
9875                              struct intel_crtc_state *crtc_state,
9876                              struct dpll *reduced_clock)
9877 {
9878         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9879         u32 dpll, fp, fp2;
9880         int factor;
9881
9882         /* Enable autotuning of the PLL clock (if permissible) */
9883         factor = 21;
9884         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
9885                 if ((intel_panel_use_ssc(dev_priv) &&
9886                      dev_priv->vbt.lvds_ssc_freq == 100000) ||
9887                     (HAS_PCH_IBX(dev_priv) &&
9888                      intel_is_dual_link_lvds(dev_priv)))
9889                         factor = 25;
9890         } else if (crtc_state->sdvo_tv_clock) {
9891                 factor = 20;
9892         }
9893
9894         fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
9895
9896         if (ilk_needs_fb_cb_tune(&crtc_state->dpll, factor))
9897                 fp |= FP_CB_TUNE;
9898
9899         if (reduced_clock) {
9900                 fp2 = i9xx_dpll_compute_fp(reduced_clock);
9901
9902                 if (reduced_clock->m < factor * reduced_clock->n)
9903                         fp2 |= FP_CB_TUNE;
9904         } else {
9905                 fp2 = fp;
9906         }
9907
9908         dpll = 0;
9909
9910         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
9911                 dpll |= DPLLB_MODE_LVDS;
9912         else
9913                 dpll |= DPLLB_MODE_DAC_SERIAL;
9914
9915         dpll |= (crtc_state->pixel_multiplier - 1)
9916                 << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
9917
9918         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
9919             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
9920                 dpll |= DPLL_SDVO_HIGH_SPEED;
9921
9922         if (intel_crtc_has_dp_encoder(crtc_state))
9923                 dpll |= DPLL_SDVO_HIGH_SPEED;
9924
9925         /*
9926          * The high speed IO clock is only really required for
9927          * SDVO/HDMI/DP, but we also enable it for CRT to make it
9928          * possible to share the DPLL between CRT and HDMI. Enabling
9929          * the clock needlessly does no real harm, except use up a
9930          * bit of power potentially.
9931          *
9932          * We'll limit this to IVB with 3 pipes, since it has only two
9933          * DPLLs and so DPLL sharing is the only way to get three pipes
9934          * driving PCH ports at the same time. On SNB we could do this,
9935          * and potentially avoid enabling the second DPLL, but it's not
9936          * clear if it''s a win or loss power wise. No point in doing
9937          * this on ILK at all since it has a fixed DPLL<->pipe mapping.
9938          */
9939         if (INTEL_NUM_PIPES(dev_priv) == 3 &&
9940             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG))
9941                 dpll |= DPLL_SDVO_HIGH_SPEED;
9942
9943         /* compute bitmask from p1 value */
9944         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
9945         /* also FPA1 */
9946         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
9947
9948         switch (crtc_state->dpll.p2) {
9949         case 5:
9950                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
9951                 break;
9952         case 7:
9953                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
9954                 break;
9955         case 10:
9956                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
9957                 break;
9958         case 14:
9959                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
9960                 break;
9961         }
9962
9963         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
9964             intel_panel_use_ssc(dev_priv))
9965                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
9966         else
9967                 dpll |= PLL_REF_INPUT_DREFCLK;
9968
9969         dpll |= DPLL_VCO_ENABLE;
9970
9971         crtc_state->dpll_hw_state.dpll = dpll;
9972         crtc_state->dpll_hw_state.fp0 = fp;
9973         crtc_state->dpll_hw_state.fp1 = fp2;
9974 }
9975
9976 static int ilk_crtc_compute_clock(struct intel_crtc *crtc,
9977                                   struct intel_crtc_state *crtc_state)
9978 {
9979         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9980         struct intel_atomic_state *state =
9981                 to_intel_atomic_state(crtc_state->uapi.state);
9982         const struct intel_limit *limit;
9983         int refclk = 120000;
9984
9985         memset(&crtc_state->dpll_hw_state, 0,
9986                sizeof(crtc_state->dpll_hw_state));
9987
9988         /* CPU eDP is the only output that doesn't need a PCH PLL of its own. */
9989         if (!crtc_state->has_pch_encoder)
9990                 return 0;
9991
9992         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
9993                 if (intel_panel_use_ssc(dev_priv)) {
9994                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n",
9995                                       dev_priv->vbt.lvds_ssc_freq);
9996                         refclk = dev_priv->vbt.lvds_ssc_freq;
9997                 }
9998
9999                 if (intel_is_dual_link_lvds(dev_priv)) {
10000                         if (refclk == 100000)
10001                                 limit = &ilk_limits_dual_lvds_100m;
10002                         else
10003                                 limit = &ilk_limits_dual_lvds;
10004                 } else {
10005                         if (refclk == 100000)
10006                                 limit = &ilk_limits_single_lvds_100m;
10007                         else
10008                                 limit = &ilk_limits_single_lvds;
10009                 }
10010         } else {
10011                 limit = &ilk_limits_dac;
10012         }
10013
10014         if (!crtc_state->clock_set &&
10015             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
10016                                 refclk, NULL, &crtc_state->dpll)) {
10017                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
10018                 return -EINVAL;
10019         }
10020
10021         ilk_compute_dpll(crtc, crtc_state, NULL);
10022
10023         if (!intel_reserve_shared_dplls(state, crtc, NULL)) {
10024                 DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
10025                               pipe_name(crtc->pipe));
10026                 return -EINVAL;
10027         }
10028
10029         return 0;
10030 }
10031
10032 static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
10033                                          struct intel_link_m_n *m_n)
10034 {
10035         struct drm_device *dev = crtc->base.dev;
10036         struct drm_i915_private *dev_priv = to_i915(dev);
10037         enum pipe pipe = crtc->pipe;
10038
10039         m_n->link_m = I915_READ(PCH_TRANS_LINK_M1(pipe));
10040         m_n->link_n = I915_READ(PCH_TRANS_LINK_N1(pipe));
10041         m_n->gmch_m = I915_READ(PCH_TRANS_DATA_M1(pipe))
10042                 & ~TU_SIZE_MASK;
10043         m_n->gmch_n = I915_READ(PCH_TRANS_DATA_N1(pipe));
10044         m_n->tu = ((I915_READ(PCH_TRANS_DATA_M1(pipe))
10045                     & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
10046 }
10047
10048 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
10049                                          enum transcoder transcoder,
10050                                          struct intel_link_m_n *m_n,
10051                                          struct intel_link_m_n *m2_n2)
10052 {
10053         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10054         enum pipe pipe = crtc->pipe;
10055
10056         if (INTEL_GEN(dev_priv) >= 5) {
10057                 m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder));
10058                 m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder));
10059                 m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder))
10060                         & ~TU_SIZE_MASK;
10061                 m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder));
10062                 m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder))
10063                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
10064
10065                 if (m2_n2 && transcoder_has_m2_n2(dev_priv, transcoder)) {
10066                         m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder));
10067                         m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder));
10068                         m2_n2->gmch_m = I915_READ(PIPE_DATA_M2(transcoder))
10069                                         & ~TU_SIZE_MASK;
10070                         m2_n2->gmch_n = I915_READ(PIPE_DATA_N2(transcoder));
10071                         m2_n2->tu = ((I915_READ(PIPE_DATA_M2(transcoder))
10072                                         & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
10073                 }
10074         } else {
10075                 m_n->link_m = I915_READ(PIPE_LINK_M_G4X(pipe));
10076                 m_n->link_n = I915_READ(PIPE_LINK_N_G4X(pipe));
10077                 m_n->gmch_m = I915_READ(PIPE_DATA_M_G4X(pipe))
10078                         & ~TU_SIZE_MASK;
10079                 m_n->gmch_n = I915_READ(PIPE_DATA_N_G4X(pipe));
10080                 m_n->tu = ((I915_READ(PIPE_DATA_M_G4X(pipe))
10081                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
10082         }
10083 }
10084
10085 void intel_dp_get_m_n(struct intel_crtc *crtc,
10086                       struct intel_crtc_state *pipe_config)
10087 {
10088         if (pipe_config->has_pch_encoder)
10089                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
10090         else
10091                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
10092                                              &pipe_config->dp_m_n,
10093                                              &pipe_config->dp_m2_n2);
10094 }
10095
10096 static void ilk_get_fdi_m_n_config(struct intel_crtc *crtc,
10097                                    struct intel_crtc_state *pipe_config)
10098 {
10099         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
10100                                      &pipe_config->fdi_m_n, NULL);
10101 }
10102
10103 static void skl_get_pfit_config(struct intel_crtc *crtc,
10104                                 struct intel_crtc_state *pipe_config)
10105 {
10106         struct drm_device *dev = crtc->base.dev;
10107         struct drm_i915_private *dev_priv = to_i915(dev);
10108         struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state;
10109         u32 ps_ctrl = 0;
10110         int id = -1;
10111         int i;
10112
10113         /* find scaler attached to this pipe */
10114         for (i = 0; i < crtc->num_scalers; i++) {
10115                 ps_ctrl = I915_READ(SKL_PS_CTRL(crtc->pipe, i));
10116                 if (ps_ctrl & PS_SCALER_EN && !(ps_ctrl & PS_PLANE_SEL_MASK)) {
10117                         id = i;
10118                         pipe_config->pch_pfit.enabled = true;
10119                         pipe_config->pch_pfit.pos = I915_READ(SKL_PS_WIN_POS(crtc->pipe, i));
10120                         pipe_config->pch_pfit.size = I915_READ(SKL_PS_WIN_SZ(crtc->pipe, i));
10121                         scaler_state->scalers[i].in_use = true;
10122                         break;
10123                 }
10124         }
10125
10126         scaler_state->scaler_id = id;
10127         if (id >= 0) {
10128                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
10129         } else {
10130                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
10131         }
10132 }
10133
10134 static void
10135 skl_get_initial_plane_config(struct intel_crtc *crtc,
10136                              struct intel_initial_plane_config *plane_config)
10137 {
10138         struct drm_device *dev = crtc->base.dev;
10139         struct drm_i915_private *dev_priv = to_i915(dev);
10140         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
10141         enum plane_id plane_id = plane->id;
10142         enum pipe pipe;
10143         u32 val, base, offset, stride_mult, tiling, alpha;
10144         int fourcc, pixel_format;
10145         unsigned int aligned_height;
10146         struct drm_framebuffer *fb;
10147         struct intel_framebuffer *intel_fb;
10148
10149         if (!plane->get_hw_state(plane, &pipe))
10150                 return;
10151
10152         WARN_ON(pipe != crtc->pipe);
10153
10154         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
10155         if (!intel_fb) {
10156                 DRM_DEBUG_KMS("failed to alloc fb\n");
10157                 return;
10158         }
10159
10160         fb = &intel_fb->base;
10161
10162         fb->dev = dev;
10163
10164         val = I915_READ(PLANE_CTL(pipe, plane_id));
10165
10166         if (INTEL_GEN(dev_priv) >= 11)
10167                 pixel_format = val & ICL_PLANE_CTL_FORMAT_MASK;
10168         else
10169                 pixel_format = val & PLANE_CTL_FORMAT_MASK;
10170
10171         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
10172                 alpha = I915_READ(PLANE_COLOR_CTL(pipe, plane_id));
10173                 alpha &= PLANE_COLOR_ALPHA_MASK;
10174         } else {
10175                 alpha = val & PLANE_CTL_ALPHA_MASK;
10176         }
10177
10178         fourcc = skl_format_to_fourcc(pixel_format,
10179                                       val & PLANE_CTL_ORDER_RGBX, alpha);
10180         fb->format = drm_format_info(fourcc);
10181
10182         tiling = val & PLANE_CTL_TILED_MASK;
10183         switch (tiling) {
10184         case PLANE_CTL_TILED_LINEAR:
10185                 fb->modifier = DRM_FORMAT_MOD_LINEAR;
10186                 break;
10187         case PLANE_CTL_TILED_X:
10188                 plane_config->tiling = I915_TILING_X;
10189                 fb->modifier = I915_FORMAT_MOD_X_TILED;
10190                 break;
10191         case PLANE_CTL_TILED_Y:
10192                 plane_config->tiling = I915_TILING_Y;
10193                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
10194                         fb->modifier = INTEL_GEN(dev_priv) >= 12 ?
10195                                 I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS :
10196                                 I915_FORMAT_MOD_Y_TILED_CCS;
10197                 else
10198                         fb->modifier = I915_FORMAT_MOD_Y_TILED;
10199                 break;
10200         case PLANE_CTL_TILED_YF:
10201                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
10202                         fb->modifier = I915_FORMAT_MOD_Yf_TILED_CCS;
10203                 else
10204                         fb->modifier = I915_FORMAT_MOD_Yf_TILED;
10205                 break;
10206         default:
10207                 MISSING_CASE(tiling);
10208                 goto error;
10209         }
10210
10211         /*
10212          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
10213          * while i915 HW rotation is clockwise, thats why this swapping.
10214          */
10215         switch (val & PLANE_CTL_ROTATE_MASK) {
10216         case PLANE_CTL_ROTATE_0:
10217                 plane_config->rotation = DRM_MODE_ROTATE_0;
10218                 break;
10219         case PLANE_CTL_ROTATE_90:
10220                 plane_config->rotation = DRM_MODE_ROTATE_270;
10221                 break;
10222         case PLANE_CTL_ROTATE_180:
10223                 plane_config->rotation = DRM_MODE_ROTATE_180;
10224                 break;
10225         case PLANE_CTL_ROTATE_270:
10226                 plane_config->rotation = DRM_MODE_ROTATE_90;
10227                 break;
10228         }
10229
10230         if (INTEL_GEN(dev_priv) >= 10 &&
10231             val & PLANE_CTL_FLIP_HORIZONTAL)
10232                 plane_config->rotation |= DRM_MODE_REFLECT_X;
10233
10234         base = I915_READ(PLANE_SURF(pipe, plane_id)) & 0xfffff000;
10235         plane_config->base = base;
10236
10237         offset = I915_READ(PLANE_OFFSET(pipe, plane_id));
10238
10239         val = I915_READ(PLANE_SIZE(pipe, plane_id));
10240         fb->height = ((val >> 16) & 0xffff) + 1;
10241         fb->width = ((val >> 0) & 0xffff) + 1;
10242
10243         val = I915_READ(PLANE_STRIDE(pipe, plane_id));
10244         stride_mult = skl_plane_stride_mult(fb, 0, DRM_MODE_ROTATE_0);
10245         fb->pitches[0] = (val & 0x3ff) * stride_mult;
10246
10247         aligned_height = intel_fb_align_height(fb, 0, fb->height);
10248
10249         plane_config->size = fb->pitches[0] * aligned_height;
10250
10251         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
10252                       crtc->base.name, plane->base.name, fb->width, fb->height,
10253                       fb->format->cpp[0] * 8, base, fb->pitches[0],
10254                       plane_config->size);
10255
10256         plane_config->fb = intel_fb;
10257         return;
10258
10259 error:
10260         kfree(intel_fb);
10261 }
10262
10263 static void ilk_get_pfit_config(struct intel_crtc *crtc,
10264                                 struct intel_crtc_state *pipe_config)
10265 {
10266         struct drm_device *dev = crtc->base.dev;
10267         struct drm_i915_private *dev_priv = to_i915(dev);
10268         u32 tmp;
10269
10270         tmp = I915_READ(PF_CTL(crtc->pipe));
10271
10272         if (tmp & PF_ENABLE) {
10273                 pipe_config->pch_pfit.enabled = true;
10274                 pipe_config->pch_pfit.pos = I915_READ(PF_WIN_POS(crtc->pipe));
10275                 pipe_config->pch_pfit.size = I915_READ(PF_WIN_SZ(crtc->pipe));
10276
10277                 /* We currently do not free assignements of panel fitters on
10278                  * ivb/hsw (since we don't use the higher upscaling modes which
10279                  * differentiates them) so just WARN about this case for now. */
10280                 if (IS_GEN(dev_priv, 7)) {
10281                         WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) !=
10282                                 PF_PIPE_SEL_IVB(crtc->pipe));
10283                 }
10284         }
10285 }
10286
10287 static bool ilk_get_pipe_config(struct intel_crtc *crtc,
10288                                 struct intel_crtc_state *pipe_config)
10289 {
10290         struct drm_device *dev = crtc->base.dev;
10291         struct drm_i915_private *dev_priv = to_i915(dev);
10292         enum intel_display_power_domain power_domain;
10293         intel_wakeref_t wakeref;
10294         u32 tmp;
10295         bool ret;
10296
10297         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
10298         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
10299         if (!wakeref)
10300                 return false;
10301
10302         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
10303         pipe_config->shared_dpll = NULL;
10304         pipe_config->master_transcoder = INVALID_TRANSCODER;
10305
10306         ret = false;
10307         tmp = I915_READ(PIPECONF(crtc->pipe));
10308         if (!(tmp & PIPECONF_ENABLE))
10309                 goto out;
10310
10311         switch (tmp & PIPECONF_BPC_MASK) {
10312         case PIPECONF_6BPC:
10313                 pipe_config->pipe_bpp = 18;
10314                 break;
10315         case PIPECONF_8BPC:
10316                 pipe_config->pipe_bpp = 24;
10317                 break;
10318         case PIPECONF_10BPC:
10319                 pipe_config->pipe_bpp = 30;
10320                 break;
10321         case PIPECONF_12BPC:
10322                 pipe_config->pipe_bpp = 36;
10323                 break;
10324         default:
10325                 break;
10326         }
10327
10328         if (tmp & PIPECONF_COLOR_RANGE_SELECT)
10329                 pipe_config->limited_color_range = true;
10330
10331         switch (tmp & PIPECONF_OUTPUT_COLORSPACE_MASK) {
10332         case PIPECONF_OUTPUT_COLORSPACE_YUV601:
10333         case PIPECONF_OUTPUT_COLORSPACE_YUV709:
10334                 pipe_config->output_format = INTEL_OUTPUT_FORMAT_YCBCR444;
10335                 break;
10336         default:
10337                 pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
10338                 break;
10339         }
10340
10341         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_ILK) >>
10342                 PIPECONF_GAMMA_MODE_SHIFT;
10343
10344         pipe_config->csc_mode = I915_READ(PIPE_CSC_MODE(crtc->pipe));
10345
10346         i9xx_get_pipe_color_config(pipe_config);
10347         intel_color_get_config(pipe_config);
10348
10349         if (I915_READ(PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
10350                 struct intel_shared_dpll *pll;
10351                 enum intel_dpll_id pll_id;
10352
10353                 pipe_config->has_pch_encoder = true;
10354
10355                 tmp = I915_READ(FDI_RX_CTL(crtc->pipe));
10356                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
10357                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
10358
10359                 ilk_get_fdi_m_n_config(crtc, pipe_config);
10360
10361                 if (HAS_PCH_IBX(dev_priv)) {
10362                         /*
10363                          * The pipe->pch transcoder and pch transcoder->pll
10364                          * mapping is fixed.
10365                          */
10366                         pll_id = (enum intel_dpll_id) crtc->pipe;
10367                 } else {
10368                         tmp = I915_READ(PCH_DPLL_SEL);
10369                         if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
10370                                 pll_id = DPLL_ID_PCH_PLL_B;
10371                         else
10372                                 pll_id= DPLL_ID_PCH_PLL_A;
10373                 }
10374
10375                 pipe_config->shared_dpll =
10376                         intel_get_shared_dpll_by_id(dev_priv, pll_id);
10377                 pll = pipe_config->shared_dpll;
10378
10379                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
10380                                                 &pipe_config->dpll_hw_state));
10381
10382                 tmp = pipe_config->dpll_hw_state.dpll;
10383                 pipe_config->pixel_multiplier =
10384                         ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
10385                          >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
10386
10387                 ilk_pch_clock_get(crtc, pipe_config);
10388         } else {
10389                 pipe_config->pixel_multiplier = 1;
10390         }
10391
10392         intel_get_pipe_timings(crtc, pipe_config);
10393         intel_get_pipe_src_size(crtc, pipe_config);
10394
10395         ilk_get_pfit_config(crtc, pipe_config);
10396
10397         ret = true;
10398
10399 out:
10400         intel_display_power_put(dev_priv, power_domain, wakeref);
10401
10402         return ret;
10403 }
10404
10405 static int hsw_crtc_compute_clock(struct intel_crtc *crtc,
10406                                   struct intel_crtc_state *crtc_state)
10407 {
10408         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10409         struct intel_atomic_state *state =
10410                 to_intel_atomic_state(crtc_state->uapi.state);
10411
10412         if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) ||
10413             INTEL_GEN(dev_priv) >= 11) {
10414                 struct intel_encoder *encoder =
10415                         intel_get_crtc_new_encoder(state, crtc_state);
10416
10417                 if (!intel_reserve_shared_dplls(state, crtc, encoder)) {
10418                         DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
10419                                       pipe_name(crtc->pipe));
10420                         return -EINVAL;
10421                 }
10422         }
10423
10424         return 0;
10425 }
10426
10427 static void cnl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port,
10428                             struct intel_crtc_state *pipe_config)
10429 {
10430         enum intel_dpll_id id;
10431         u32 temp;
10432
10433         temp = I915_READ(DPCLKA_CFGCR0) & DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
10434         id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port);
10435
10436         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL2))
10437                 return;
10438
10439         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10440 }
10441
10442 static void icl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port,
10443                             struct intel_crtc_state *pipe_config)
10444 {
10445         enum phy phy = intel_port_to_phy(dev_priv, port);
10446         enum icl_port_dpll_id port_dpll_id;
10447         enum intel_dpll_id id;
10448         u32 temp;
10449
10450         if (intel_phy_is_combo(dev_priv, phy)) {
10451                 temp = I915_READ(ICL_DPCLKA_CFGCR0) &
10452                         ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy);
10453                 id = temp >> ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy);
10454                 port_dpll_id = ICL_PORT_DPLL_DEFAULT;
10455         } else if (intel_phy_is_tc(dev_priv, phy)) {
10456                 u32 clk_sel = I915_READ(DDI_CLK_SEL(port)) & DDI_CLK_SEL_MASK;
10457
10458                 if (clk_sel == DDI_CLK_SEL_MG) {
10459                         id = icl_tc_port_to_pll_id(intel_port_to_tc(dev_priv,
10460                                                                     port));
10461                         port_dpll_id = ICL_PORT_DPLL_MG_PHY;
10462                 } else {
10463                         WARN_ON(clk_sel < DDI_CLK_SEL_TBT_162);
10464                         id = DPLL_ID_ICL_TBTPLL;
10465                         port_dpll_id = ICL_PORT_DPLL_DEFAULT;
10466                 }
10467         } else {
10468                 WARN(1, "Invalid port %x\n", port);
10469                 return;
10470         }
10471
10472         pipe_config->icl_port_dplls[port_dpll_id].pll =
10473                 intel_get_shared_dpll_by_id(dev_priv, id);
10474
10475         icl_set_active_port_dpll(pipe_config, port_dpll_id);
10476 }
10477
10478 static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv,
10479                                 enum port port,
10480                                 struct intel_crtc_state *pipe_config)
10481 {
10482         enum intel_dpll_id id;
10483
10484         switch (port) {
10485         case PORT_A:
10486                 id = DPLL_ID_SKL_DPLL0;
10487                 break;
10488         case PORT_B:
10489                 id = DPLL_ID_SKL_DPLL1;
10490                 break;
10491         case PORT_C:
10492                 id = DPLL_ID_SKL_DPLL2;
10493                 break;
10494         default:
10495                 DRM_ERROR("Incorrect port type\n");
10496                 return;
10497         }
10498
10499         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10500 }
10501
10502 static void skl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port,
10503                             struct intel_crtc_state *pipe_config)
10504 {
10505         enum intel_dpll_id id;
10506         u32 temp;
10507
10508         temp = I915_READ(DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port);
10509         id = temp >> (port * 3 + 1);
10510
10511         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL3))
10512                 return;
10513
10514         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10515 }
10516
10517 static void hsw_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port,
10518                             struct intel_crtc_state *pipe_config)
10519 {
10520         enum intel_dpll_id id;
10521         u32 ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
10522
10523         switch (ddi_pll_sel) {
10524         case PORT_CLK_SEL_WRPLL1:
10525                 id = DPLL_ID_WRPLL1;
10526                 break;
10527         case PORT_CLK_SEL_WRPLL2:
10528                 id = DPLL_ID_WRPLL2;
10529                 break;
10530         case PORT_CLK_SEL_SPLL:
10531                 id = DPLL_ID_SPLL;
10532                 break;
10533         case PORT_CLK_SEL_LCPLL_810:
10534                 id = DPLL_ID_LCPLL_810;
10535                 break;
10536         case PORT_CLK_SEL_LCPLL_1350:
10537                 id = DPLL_ID_LCPLL_1350;
10538                 break;
10539         case PORT_CLK_SEL_LCPLL_2700:
10540                 id = DPLL_ID_LCPLL_2700;
10541                 break;
10542         default:
10543                 MISSING_CASE(ddi_pll_sel);
10544                 /* fall through */
10545         case PORT_CLK_SEL_NONE:
10546                 return;
10547         }
10548
10549         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10550 }
10551
10552 static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
10553                                      struct intel_crtc_state *pipe_config,
10554                                      u64 *power_domain_mask,
10555                                      intel_wakeref_t *wakerefs)
10556 {
10557         struct drm_device *dev = crtc->base.dev;
10558         struct drm_i915_private *dev_priv = to_i915(dev);
10559         enum intel_display_power_domain power_domain;
10560         unsigned long panel_transcoder_mask = 0;
10561         unsigned long enabled_panel_transcoders = 0;
10562         enum transcoder panel_transcoder;
10563         intel_wakeref_t wf;
10564         u32 tmp;
10565
10566         if (INTEL_GEN(dev_priv) >= 11)
10567                 panel_transcoder_mask |=
10568                         BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1);
10569
10570         if (HAS_TRANSCODER_EDP(dev_priv))
10571                 panel_transcoder_mask |= BIT(TRANSCODER_EDP);
10572
10573         /*
10574          * The pipe->transcoder mapping is fixed with the exception of the eDP
10575          * and DSI transcoders handled below.
10576          */
10577         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
10578
10579         /*
10580          * XXX: Do intel_display_power_get_if_enabled before reading this (for
10581          * consistency and less surprising code; it's in always on power).
10582          */
10583         for_each_set_bit(panel_transcoder,
10584                          &panel_transcoder_mask,
10585                          ARRAY_SIZE(INTEL_INFO(dev_priv)->trans_offsets)) {
10586                 bool force_thru = false;
10587                 enum pipe trans_pipe;
10588
10589                 tmp = I915_READ(TRANS_DDI_FUNC_CTL(panel_transcoder));
10590                 if (!(tmp & TRANS_DDI_FUNC_ENABLE))
10591                         continue;
10592
10593                 /*
10594                  * Log all enabled ones, only use the first one.
10595                  *
10596                  * FIXME: This won't work for two separate DSI displays.
10597                  */
10598                 enabled_panel_transcoders |= BIT(panel_transcoder);
10599                 if (enabled_panel_transcoders != BIT(panel_transcoder))
10600                         continue;
10601
10602                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
10603                 default:
10604                         WARN(1, "unknown pipe linked to transcoder %s\n",
10605                              transcoder_name(panel_transcoder));
10606                         /* fall through */
10607                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
10608                         force_thru = true;
10609                         /* fall through */
10610                 case TRANS_DDI_EDP_INPUT_A_ON:
10611                         trans_pipe = PIPE_A;
10612                         break;
10613                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
10614                         trans_pipe = PIPE_B;
10615                         break;
10616                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
10617                         trans_pipe = PIPE_C;
10618                         break;
10619                 case TRANS_DDI_EDP_INPUT_D_ONOFF:
10620                         trans_pipe = PIPE_D;
10621                         break;
10622                 }
10623
10624                 if (trans_pipe == crtc->pipe) {
10625                         pipe_config->cpu_transcoder = panel_transcoder;
10626                         pipe_config->pch_pfit.force_thru = force_thru;
10627                 }
10628         }
10629
10630         /*
10631          * Valid combos: none, eDP, DSI0, DSI1, DSI0+DSI1
10632          */
10633         WARN_ON((enabled_panel_transcoders & BIT(TRANSCODER_EDP)) &&
10634                 enabled_panel_transcoders != BIT(TRANSCODER_EDP));
10635
10636         power_domain = POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder);
10637         WARN_ON(*power_domain_mask & BIT_ULL(power_domain));
10638
10639         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10640         if (!wf)
10641                 return false;
10642
10643         wakerefs[power_domain] = wf;
10644         *power_domain_mask |= BIT_ULL(power_domain);
10645
10646         tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
10647
10648         return tmp & PIPECONF_ENABLE;
10649 }
10650
10651 static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
10652                                          struct intel_crtc_state *pipe_config,
10653                                          u64 *power_domain_mask,
10654                                          intel_wakeref_t *wakerefs)
10655 {
10656         struct drm_device *dev = crtc->base.dev;
10657         struct drm_i915_private *dev_priv = to_i915(dev);
10658         enum intel_display_power_domain power_domain;
10659         enum transcoder cpu_transcoder;
10660         intel_wakeref_t wf;
10661         enum port port;
10662         u32 tmp;
10663
10664         for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) {
10665                 if (port == PORT_A)
10666                         cpu_transcoder = TRANSCODER_DSI_A;
10667                 else
10668                         cpu_transcoder = TRANSCODER_DSI_C;
10669
10670                 power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
10671                 WARN_ON(*power_domain_mask & BIT_ULL(power_domain));
10672
10673                 wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10674                 if (!wf)
10675                         continue;
10676
10677                 wakerefs[power_domain] = wf;
10678                 *power_domain_mask |= BIT_ULL(power_domain);
10679
10680                 /*
10681                  * The PLL needs to be enabled with a valid divider
10682                  * configuration, otherwise accessing DSI registers will hang
10683                  * the machine. See BSpec North Display Engine
10684                  * registers/MIPI[BXT]. We can break out here early, since we
10685                  * need the same DSI PLL to be enabled for both DSI ports.
10686                  */
10687                 if (!bxt_dsi_pll_is_enabled(dev_priv))
10688                         break;
10689
10690                 /* XXX: this works for video mode only */
10691                 tmp = I915_READ(BXT_MIPI_PORT_CTRL(port));
10692                 if (!(tmp & DPI_ENABLE))
10693                         continue;
10694
10695                 tmp = I915_READ(MIPI_CTRL(port));
10696                 if ((tmp & BXT_PIPE_SELECT_MASK) != BXT_PIPE_SELECT(crtc->pipe))
10697                         continue;
10698
10699                 pipe_config->cpu_transcoder = cpu_transcoder;
10700                 break;
10701         }
10702
10703         return transcoder_is_dsi(pipe_config->cpu_transcoder);
10704 }
10705
10706 static void hsw_get_ddi_port_state(struct intel_crtc *crtc,
10707                                    struct intel_crtc_state *pipe_config)
10708 {
10709         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10710         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
10711         struct intel_shared_dpll *pll;
10712         enum port port;
10713         u32 tmp;
10714
10715         if (transcoder_is_dsi(cpu_transcoder)) {
10716                 port = (cpu_transcoder == TRANSCODER_DSI_A) ?
10717                                                 PORT_A : PORT_B;
10718         } else {
10719                 tmp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
10720                 if (INTEL_GEN(dev_priv) >= 12)
10721                         port = TGL_TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp);
10722                 else
10723                         port = TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp);
10724         }
10725
10726         if (INTEL_GEN(dev_priv) >= 11)
10727                 icl_get_ddi_pll(dev_priv, port, pipe_config);
10728         else if (IS_CANNONLAKE(dev_priv))
10729                 cnl_get_ddi_pll(dev_priv, port, pipe_config);
10730         else if (IS_GEN9_BC(dev_priv))
10731                 skl_get_ddi_pll(dev_priv, port, pipe_config);
10732         else if (IS_GEN9_LP(dev_priv))
10733                 bxt_get_ddi_pll(dev_priv, port, pipe_config);
10734         else
10735                 hsw_get_ddi_pll(dev_priv, port, pipe_config);
10736
10737         pll = pipe_config->shared_dpll;
10738         if (pll) {
10739                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
10740                                                 &pipe_config->dpll_hw_state));
10741         }
10742
10743         /*
10744          * Haswell has only FDI/PCH transcoder A. It is which is connected to
10745          * DDI E. So just check whether this pipe is wired to DDI E and whether
10746          * the PCH transcoder is on.
10747          */
10748         if (INTEL_GEN(dev_priv) < 9 &&
10749             (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) {
10750                 pipe_config->has_pch_encoder = true;
10751
10752                 tmp = I915_READ(FDI_RX_CTL(PIPE_A));
10753                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
10754                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
10755
10756                 ilk_get_fdi_m_n_config(crtc, pipe_config);
10757         }
10758 }
10759
10760 static enum transcoder transcoder_master_readout(struct drm_i915_private *dev_priv,
10761                                                  enum transcoder cpu_transcoder)
10762 {
10763         u32 trans_port_sync, master_select;
10764
10765         trans_port_sync = I915_READ(TRANS_DDI_FUNC_CTL2(cpu_transcoder));
10766
10767         if ((trans_port_sync & PORT_SYNC_MODE_ENABLE) == 0)
10768                 return INVALID_TRANSCODER;
10769
10770         master_select = trans_port_sync &
10771                         PORT_SYNC_MODE_MASTER_SELECT_MASK;
10772         if (master_select == 0)
10773                 return TRANSCODER_EDP;
10774         else
10775                 return master_select - 1;
10776 }
10777
10778 static void icl_get_trans_port_sync_config(struct intel_crtc_state *crtc_state)
10779 {
10780         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
10781         u32 transcoders;
10782         enum transcoder cpu_transcoder;
10783
10784         crtc_state->master_transcoder = transcoder_master_readout(dev_priv,
10785                                                                   crtc_state->cpu_transcoder);
10786
10787         transcoders = BIT(TRANSCODER_A) |
10788                 BIT(TRANSCODER_B) |
10789                 BIT(TRANSCODER_C) |
10790                 BIT(TRANSCODER_D);
10791         for_each_cpu_transcoder_masked(dev_priv, cpu_transcoder, transcoders) {
10792                 enum intel_display_power_domain power_domain;
10793                 intel_wakeref_t trans_wakeref;
10794
10795                 power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
10796                 trans_wakeref = intel_display_power_get_if_enabled(dev_priv,
10797                                                                    power_domain);
10798
10799                 if (!trans_wakeref)
10800                         continue;
10801
10802                 if (transcoder_master_readout(dev_priv, cpu_transcoder) ==
10803                     crtc_state->cpu_transcoder)
10804                         crtc_state->sync_mode_slaves_mask |= BIT(cpu_transcoder);
10805
10806                 intel_display_power_put(dev_priv, power_domain, trans_wakeref);
10807         }
10808
10809         WARN_ON(crtc_state->master_transcoder != INVALID_TRANSCODER &&
10810                 crtc_state->sync_mode_slaves_mask);
10811 }
10812
10813 static bool hsw_get_pipe_config(struct intel_crtc *crtc,
10814                                 struct intel_crtc_state *pipe_config)
10815 {
10816         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10817         intel_wakeref_t wakerefs[POWER_DOMAIN_NUM], wf;
10818         enum intel_display_power_domain power_domain;
10819         u64 power_domain_mask;
10820         bool active;
10821
10822         pipe_config->master_transcoder = INVALID_TRANSCODER;
10823
10824         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
10825         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10826         if (!wf)
10827                 return false;
10828
10829         wakerefs[power_domain] = wf;
10830         power_domain_mask = BIT_ULL(power_domain);
10831
10832         pipe_config->shared_dpll = NULL;
10833
10834         active = hsw_get_transcoder_state(crtc, pipe_config,
10835                                           &power_domain_mask, wakerefs);
10836
10837         if (IS_GEN9_LP(dev_priv) &&
10838             bxt_get_dsi_transcoder_state(crtc, pipe_config,
10839                                          &power_domain_mask, wakerefs)) {
10840                 WARN_ON(active);
10841                 active = true;
10842         }
10843
10844         if (!active)
10845                 goto out;
10846
10847         if (!transcoder_is_dsi(pipe_config->cpu_transcoder) ||
10848             INTEL_GEN(dev_priv) >= 11) {
10849                 hsw_get_ddi_port_state(crtc, pipe_config);
10850                 intel_get_pipe_timings(crtc, pipe_config);
10851         }
10852
10853         intel_get_pipe_src_size(crtc, pipe_config);
10854
10855         if (IS_HASWELL(dev_priv)) {
10856                 u32 tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
10857
10858                 if (tmp & PIPECONF_OUTPUT_COLORSPACE_YUV_HSW)
10859                         pipe_config->output_format = INTEL_OUTPUT_FORMAT_YCBCR444;
10860                 else
10861                         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
10862         } else {
10863                 pipe_config->output_format =
10864                         bdw_get_pipemisc_output_format(crtc);
10865
10866                 /*
10867                  * Currently there is no interface defined to
10868                  * check user preference between RGB/YCBCR444
10869                  * or YCBCR420. So the only possible case for
10870                  * YCBCR444 usage is driving YCBCR420 output
10871                  * with LSPCON, when pipe is configured for
10872                  * YCBCR444 output and LSPCON takes care of
10873                  * downsampling it.
10874                  */
10875                 pipe_config->lspcon_downsampling =
10876                         pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR444;
10877         }
10878
10879         pipe_config->gamma_mode = I915_READ(GAMMA_MODE(crtc->pipe));
10880
10881         pipe_config->csc_mode = I915_READ(PIPE_CSC_MODE(crtc->pipe));
10882
10883         if (INTEL_GEN(dev_priv) >= 9) {
10884                 u32 tmp = I915_READ(SKL_BOTTOM_COLOR(crtc->pipe));
10885
10886                 if (tmp & SKL_BOTTOM_COLOR_GAMMA_ENABLE)
10887                         pipe_config->gamma_enable = true;
10888
10889                 if (tmp & SKL_BOTTOM_COLOR_CSC_ENABLE)
10890                         pipe_config->csc_enable = true;
10891         } else {
10892                 i9xx_get_pipe_color_config(pipe_config);
10893         }
10894
10895         intel_color_get_config(pipe_config);
10896
10897         power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
10898         WARN_ON(power_domain_mask & BIT_ULL(power_domain));
10899
10900         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10901         if (wf) {
10902                 wakerefs[power_domain] = wf;
10903                 power_domain_mask |= BIT_ULL(power_domain);
10904
10905                 if (INTEL_GEN(dev_priv) >= 9)
10906                         skl_get_pfit_config(crtc, pipe_config);
10907                 else
10908                         ilk_get_pfit_config(crtc, pipe_config);
10909         }
10910
10911         if (hsw_crtc_supports_ips(crtc)) {
10912                 if (IS_HASWELL(dev_priv))
10913                         pipe_config->ips_enabled = I915_READ(IPS_CTL) & IPS_ENABLE;
10914                 else {
10915                         /*
10916                          * We cannot readout IPS state on broadwell, set to
10917                          * true so we can set it to a defined state on first
10918                          * commit.
10919                          */
10920                         pipe_config->ips_enabled = true;
10921                 }
10922         }
10923
10924         if (pipe_config->cpu_transcoder != TRANSCODER_EDP &&
10925             !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
10926                 pipe_config->pixel_multiplier =
10927                         I915_READ(PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
10928         } else {
10929                 pipe_config->pixel_multiplier = 1;
10930         }
10931
10932         if (INTEL_GEN(dev_priv) >= 11 &&
10933             !transcoder_is_dsi(pipe_config->cpu_transcoder))
10934                 icl_get_trans_port_sync_config(pipe_config);
10935
10936 out:
10937         for_each_power_domain(power_domain, power_domain_mask)
10938                 intel_display_power_put(dev_priv,
10939                                         power_domain, wakerefs[power_domain]);
10940
10941         return active;
10942 }
10943
10944 static u32 intel_cursor_base(const struct intel_plane_state *plane_state)
10945 {
10946         struct drm_i915_private *dev_priv =
10947                 to_i915(plane_state->uapi.plane->dev);
10948         const struct drm_framebuffer *fb = plane_state->hw.fb;
10949         const struct drm_i915_gem_object *obj = intel_fb_obj(fb);
10950         u32 base;
10951
10952         if (INTEL_INFO(dev_priv)->display.cursor_needs_physical)
10953                 base = obj->phys_handle->busaddr;
10954         else
10955                 base = intel_plane_ggtt_offset(plane_state);
10956
10957         return base + plane_state->color_plane[0].offset;
10958 }
10959
10960 static u32 intel_cursor_position(const struct intel_plane_state *plane_state)
10961 {
10962         int x = plane_state->uapi.dst.x1;
10963         int y = plane_state->uapi.dst.y1;
10964         u32 pos = 0;
10965
10966         if (x < 0) {
10967                 pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
10968                 x = -x;
10969         }
10970         pos |= x << CURSOR_X_SHIFT;
10971
10972         if (y < 0) {
10973                 pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
10974                 y = -y;
10975         }
10976         pos |= y << CURSOR_Y_SHIFT;
10977
10978         return pos;
10979 }
10980
10981 static bool intel_cursor_size_ok(const struct intel_plane_state *plane_state)
10982 {
10983         const struct drm_mode_config *config =
10984                 &plane_state->uapi.plane->dev->mode_config;
10985         int width = drm_rect_width(&plane_state->uapi.dst);
10986         int height = drm_rect_height(&plane_state->uapi.dst);
10987
10988         return width > 0 && width <= config->cursor_width &&
10989                 height > 0 && height <= config->cursor_height;
10990 }
10991
10992 static int intel_cursor_check_surface(struct intel_plane_state *plane_state)
10993 {
10994         struct drm_i915_private *dev_priv =
10995                 to_i915(plane_state->uapi.plane->dev);
10996         unsigned int rotation = plane_state->hw.rotation;
10997         int src_x, src_y;
10998         u32 offset;
10999         int ret;
11000
11001         ret = intel_plane_compute_gtt(plane_state);
11002         if (ret)
11003                 return ret;
11004
11005         if (!plane_state->uapi.visible)
11006                 return 0;
11007
11008         src_x = plane_state->uapi.src.x1 >> 16;
11009         src_y = plane_state->uapi.src.y1 >> 16;
11010
11011         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
11012         offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
11013                                                     plane_state, 0);
11014
11015         if (src_x != 0 || src_y != 0) {
11016                 DRM_DEBUG_KMS("Arbitrary cursor panning not supported\n");
11017                 return -EINVAL;
11018         }
11019
11020         /*
11021          * Put the final coordinates back so that the src
11022          * coordinate checks will see the right values.
11023          */
11024         drm_rect_translate_to(&plane_state->uapi.src,
11025                               src_x << 16, src_y << 16);
11026
11027         /* ILK+ do this automagically in hardware */
11028         if (HAS_GMCH(dev_priv) && rotation & DRM_MODE_ROTATE_180) {
11029                 const struct drm_framebuffer *fb = plane_state->hw.fb;
11030                 int src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
11031                 int src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
11032
11033                 offset += (src_h * src_w - 1) * fb->format->cpp[0];
11034         }
11035
11036         plane_state->color_plane[0].offset = offset;
11037         plane_state->color_plane[0].x = src_x;
11038         plane_state->color_plane[0].y = src_y;
11039
11040         return 0;
11041 }
11042
11043 static int intel_check_cursor(struct intel_crtc_state *crtc_state,
11044                               struct intel_plane_state *plane_state)
11045 {
11046         const struct drm_framebuffer *fb = plane_state->hw.fb;
11047         int ret;
11048
11049         if (fb && fb->modifier != DRM_FORMAT_MOD_LINEAR) {
11050                 DRM_DEBUG_KMS("cursor cannot be tiled\n");
11051                 return -EINVAL;
11052         }
11053
11054         ret = drm_atomic_helper_check_plane_state(&plane_state->uapi,
11055                                                   &crtc_state->uapi,
11056                                                   DRM_PLANE_HELPER_NO_SCALING,
11057                                                   DRM_PLANE_HELPER_NO_SCALING,
11058                                                   true, true);
11059         if (ret)
11060                 return ret;
11061
11062         /* Use the unclipped src/dst rectangles, which we program to hw */
11063         plane_state->uapi.src = drm_plane_state_src(&plane_state->uapi);
11064         plane_state->uapi.dst = drm_plane_state_dest(&plane_state->uapi);
11065
11066         ret = intel_cursor_check_surface(plane_state);
11067         if (ret)
11068                 return ret;
11069
11070         if (!plane_state->uapi.visible)
11071                 return 0;
11072
11073         ret = intel_plane_check_src_coordinates(plane_state);
11074         if (ret)
11075                 return ret;
11076
11077         return 0;
11078 }
11079
11080 static unsigned int
11081 i845_cursor_max_stride(struct intel_plane *plane,
11082                        u32 pixel_format, u64 modifier,
11083                        unsigned int rotation)
11084 {
11085         return 2048;
11086 }
11087
11088 static u32 i845_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state)
11089 {
11090         u32 cntl = 0;
11091
11092         if (crtc_state->gamma_enable)
11093                 cntl |= CURSOR_GAMMA_ENABLE;
11094
11095         return cntl;
11096 }
11097
11098 static u32 i845_cursor_ctl(const struct intel_crtc_state *crtc_state,
11099                            const struct intel_plane_state *plane_state)
11100 {
11101         return CURSOR_ENABLE |
11102                 CURSOR_FORMAT_ARGB |
11103                 CURSOR_STRIDE(plane_state->color_plane[0].stride);
11104 }
11105
11106 static bool i845_cursor_size_ok(const struct intel_plane_state *plane_state)
11107 {
11108         int width = drm_rect_width(&plane_state->uapi.dst);
11109
11110         /*
11111          * 845g/865g are only limited by the width of their cursors,
11112          * the height is arbitrary up to the precision of the register.
11113          */
11114         return intel_cursor_size_ok(plane_state) && IS_ALIGNED(width, 64);
11115 }
11116
11117 static int i845_check_cursor(struct intel_crtc_state *crtc_state,
11118                              struct intel_plane_state *plane_state)
11119 {
11120         const struct drm_framebuffer *fb = plane_state->hw.fb;
11121         int ret;
11122
11123         ret = intel_check_cursor(crtc_state, plane_state);
11124         if (ret)
11125                 return ret;
11126
11127         /* if we want to turn off the cursor ignore width and height */
11128         if (!fb)
11129                 return 0;
11130
11131         /* Check for which cursor types we support */
11132         if (!i845_cursor_size_ok(plane_state)) {
11133                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
11134                           drm_rect_width(&plane_state->uapi.dst),
11135                           drm_rect_height(&plane_state->uapi.dst));
11136                 return -EINVAL;
11137         }
11138
11139         WARN_ON(plane_state->uapi.visible &&
11140                 plane_state->color_plane[0].stride != fb->pitches[0]);
11141
11142         switch (fb->pitches[0]) {
11143         case 256:
11144         case 512:
11145         case 1024:
11146         case 2048:
11147                 break;
11148         default:
11149                 DRM_DEBUG_KMS("Invalid cursor stride (%u)\n",
11150                               fb->pitches[0]);
11151                 return -EINVAL;
11152         }
11153
11154         plane_state->ctl = i845_cursor_ctl(crtc_state, plane_state);
11155
11156         return 0;
11157 }
11158
11159 static void i845_update_cursor(struct intel_plane *plane,
11160                                const struct intel_crtc_state *crtc_state,
11161                                const struct intel_plane_state *plane_state)
11162 {
11163         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11164         u32 cntl = 0, base = 0, pos = 0, size = 0;
11165         unsigned long irqflags;
11166
11167         if (plane_state && plane_state->uapi.visible) {
11168                 unsigned int width = drm_rect_width(&plane_state->uapi.dst);
11169                 unsigned int height = drm_rect_height(&plane_state->uapi.dst);
11170
11171                 cntl = plane_state->ctl |
11172                         i845_cursor_ctl_crtc(crtc_state);
11173
11174                 size = (height << 12) | width;
11175
11176                 base = intel_cursor_base(plane_state);
11177                 pos = intel_cursor_position(plane_state);
11178         }
11179
11180         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
11181
11182         /* On these chipsets we can only modify the base/size/stride
11183          * whilst the cursor is disabled.
11184          */
11185         if (plane->cursor.base != base ||
11186             plane->cursor.size != size ||
11187             plane->cursor.cntl != cntl) {
11188                 I915_WRITE_FW(CURCNTR(PIPE_A), 0);
11189                 I915_WRITE_FW(CURBASE(PIPE_A), base);
11190                 I915_WRITE_FW(CURSIZE, size);
11191                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
11192                 I915_WRITE_FW(CURCNTR(PIPE_A), cntl);
11193
11194                 plane->cursor.base = base;
11195                 plane->cursor.size = size;
11196                 plane->cursor.cntl = cntl;
11197         } else {
11198                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
11199         }
11200
11201         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
11202 }
11203
11204 static void i845_disable_cursor(struct intel_plane *plane,
11205                                 const struct intel_crtc_state *crtc_state)
11206 {
11207         i845_update_cursor(plane, crtc_state, NULL);
11208 }
11209
11210 static bool i845_cursor_get_hw_state(struct intel_plane *plane,
11211                                      enum pipe *pipe)
11212 {
11213         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11214         enum intel_display_power_domain power_domain;
11215         intel_wakeref_t wakeref;
11216         bool ret;
11217
11218         power_domain = POWER_DOMAIN_PIPE(PIPE_A);
11219         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
11220         if (!wakeref)
11221                 return false;
11222
11223         ret = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
11224
11225         *pipe = PIPE_A;
11226
11227         intel_display_power_put(dev_priv, power_domain, wakeref);
11228
11229         return ret;
11230 }
11231
11232 static unsigned int
11233 i9xx_cursor_max_stride(struct intel_plane *plane,
11234                        u32 pixel_format, u64 modifier,
11235                        unsigned int rotation)
11236 {
11237         return plane->base.dev->mode_config.cursor_width * 4;
11238 }
11239
11240 static u32 i9xx_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state)
11241 {
11242         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
11243         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11244         u32 cntl = 0;
11245
11246         if (INTEL_GEN(dev_priv) >= 11)
11247                 return cntl;
11248
11249         if (crtc_state->gamma_enable)
11250                 cntl = MCURSOR_GAMMA_ENABLE;
11251
11252         if (crtc_state->csc_enable)
11253                 cntl |= MCURSOR_PIPE_CSC_ENABLE;
11254
11255         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
11256                 cntl |= MCURSOR_PIPE_SELECT(crtc->pipe);
11257
11258         return cntl;
11259 }
11260
11261 static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state,
11262                            const struct intel_plane_state *plane_state)
11263 {
11264         struct drm_i915_private *dev_priv =
11265                 to_i915(plane_state->uapi.plane->dev);
11266         u32 cntl = 0;
11267
11268         if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
11269                 cntl |= MCURSOR_TRICKLE_FEED_DISABLE;
11270
11271         switch (drm_rect_width(&plane_state->uapi.dst)) {
11272         case 64:
11273                 cntl |= MCURSOR_MODE_64_ARGB_AX;
11274                 break;
11275         case 128:
11276                 cntl |= MCURSOR_MODE_128_ARGB_AX;
11277                 break;
11278         case 256:
11279                 cntl |= MCURSOR_MODE_256_ARGB_AX;
11280                 break;
11281         default:
11282                 MISSING_CASE(drm_rect_width(&plane_state->uapi.dst));
11283                 return 0;
11284         }
11285
11286         if (plane_state->hw.rotation & DRM_MODE_ROTATE_180)
11287                 cntl |= MCURSOR_ROTATE_180;
11288
11289         return cntl;
11290 }
11291
11292 static bool i9xx_cursor_size_ok(const struct intel_plane_state *plane_state)
11293 {
11294         struct drm_i915_private *dev_priv =
11295                 to_i915(plane_state->uapi.plane->dev);
11296         int width = drm_rect_width(&plane_state->uapi.dst);
11297         int height = drm_rect_height(&plane_state->uapi.dst);
11298
11299         if (!intel_cursor_size_ok(plane_state))
11300                 return false;
11301
11302         /* Cursor width is limited to a few power-of-two sizes */
11303         switch (width) {
11304         case 256:
11305         case 128:
11306         case 64:
11307                 break;
11308         default:
11309                 return false;
11310         }
11311
11312         /*
11313          * IVB+ have CUR_FBC_CTL which allows an arbitrary cursor
11314          * height from 8 lines up to the cursor width, when the
11315          * cursor is not rotated. Everything else requires square
11316          * cursors.
11317          */
11318         if (HAS_CUR_FBC(dev_priv) &&
11319             plane_state->hw.rotation & DRM_MODE_ROTATE_0) {
11320                 if (height < 8 || height > width)
11321                         return false;
11322         } else {
11323                 if (height != width)
11324                         return false;
11325         }
11326
11327         return true;
11328 }
11329
11330 static int i9xx_check_cursor(struct intel_crtc_state *crtc_state,
11331                              struct intel_plane_state *plane_state)
11332 {
11333         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
11334         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11335         const struct drm_framebuffer *fb = plane_state->hw.fb;
11336         enum pipe pipe = plane->pipe;
11337         int ret;
11338
11339         ret = intel_check_cursor(crtc_state, plane_state);
11340         if (ret)
11341                 return ret;
11342
11343         /* if we want to turn off the cursor ignore width and height */
11344         if (!fb)
11345                 return 0;
11346
11347         /* Check for which cursor types we support */
11348         if (!i9xx_cursor_size_ok(plane_state)) {
11349                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
11350                           drm_rect_width(&plane_state->uapi.dst),
11351                           drm_rect_height(&plane_state->uapi.dst));
11352                 return -EINVAL;
11353         }
11354
11355         WARN_ON(plane_state->uapi.visible &&
11356                 plane_state->color_plane[0].stride != fb->pitches[0]);
11357
11358         if (fb->pitches[0] !=
11359             drm_rect_width(&plane_state->uapi.dst) * fb->format->cpp[0]) {
11360                 DRM_DEBUG_KMS("Invalid cursor stride (%u) (cursor width %d)\n",
11361                               fb->pitches[0],
11362                               drm_rect_width(&plane_state->uapi.dst));
11363                 return -EINVAL;
11364         }
11365
11366         /*
11367          * There's something wrong with the cursor on CHV pipe C.
11368          * If it straddles the left edge of the screen then
11369          * moving it away from the edge or disabling it often
11370          * results in a pipe underrun, and often that can lead to
11371          * dead pipe (constant underrun reported, and it scans
11372          * out just a solid color). To recover from that, the
11373          * display power well must be turned off and on again.
11374          * Refuse the put the cursor into that compromised position.
11375          */
11376         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_C &&
11377             plane_state->uapi.visible && plane_state->uapi.dst.x1 < 0) {
11378                 DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
11379                 return -EINVAL;
11380         }
11381
11382         plane_state->ctl = i9xx_cursor_ctl(crtc_state, plane_state);
11383
11384         return 0;
11385 }
11386
11387 static void i9xx_update_cursor(struct intel_plane *plane,
11388                                const struct intel_crtc_state *crtc_state,
11389                                const struct intel_plane_state *plane_state)
11390 {
11391         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11392         enum pipe pipe = plane->pipe;
11393         u32 cntl = 0, base = 0, pos = 0, fbc_ctl = 0;
11394         unsigned long irqflags;
11395
11396         if (plane_state && plane_state->uapi.visible) {
11397                 unsigned width = drm_rect_width(&plane_state->uapi.dst);
11398                 unsigned height = drm_rect_height(&plane_state->uapi.dst);
11399
11400                 cntl = plane_state->ctl |
11401                         i9xx_cursor_ctl_crtc(crtc_state);
11402
11403                 if (width != height)
11404                         fbc_ctl = CUR_FBC_CTL_EN | (height - 1);
11405
11406                 base = intel_cursor_base(plane_state);
11407                 pos = intel_cursor_position(plane_state);
11408         }
11409
11410         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
11411
11412         /*
11413          * On some platforms writing CURCNTR first will also
11414          * cause CURPOS to be armed by the CURBASE write.
11415          * Without the CURCNTR write the CURPOS write would
11416          * arm itself. Thus we always update CURCNTR before
11417          * CURPOS.
11418          *
11419          * On other platforms CURPOS always requires the
11420          * CURBASE write to arm the update. Additonally
11421          * a write to any of the cursor register will cancel
11422          * an already armed cursor update. Thus leaving out
11423          * the CURBASE write after CURPOS could lead to a
11424          * cursor that doesn't appear to move, or even change
11425          * shape. Thus we always write CURBASE.
11426          *
11427          * The other registers are armed by by the CURBASE write
11428          * except when the plane is getting enabled at which time
11429          * the CURCNTR write arms the update.
11430          */
11431
11432         if (INTEL_GEN(dev_priv) >= 9)
11433                 skl_write_cursor_wm(plane, crtc_state);
11434
11435         if (plane->cursor.base != base ||
11436             plane->cursor.size != fbc_ctl ||
11437             plane->cursor.cntl != cntl) {
11438                 if (HAS_CUR_FBC(dev_priv))
11439                         I915_WRITE_FW(CUR_FBC_CTL(pipe), fbc_ctl);
11440                 I915_WRITE_FW(CURCNTR(pipe), cntl);
11441                 I915_WRITE_FW(CURPOS(pipe), pos);
11442                 I915_WRITE_FW(CURBASE(pipe), base);
11443
11444                 plane->cursor.base = base;
11445                 plane->cursor.size = fbc_ctl;
11446                 plane->cursor.cntl = cntl;
11447         } else {
11448                 I915_WRITE_FW(CURPOS(pipe), pos);
11449                 I915_WRITE_FW(CURBASE(pipe), base);
11450         }
11451
11452         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
11453 }
11454
11455 static void i9xx_disable_cursor(struct intel_plane *plane,
11456                                 const struct intel_crtc_state *crtc_state)
11457 {
11458         i9xx_update_cursor(plane, crtc_state, NULL);
11459 }
11460
11461 static bool i9xx_cursor_get_hw_state(struct intel_plane *plane,
11462                                      enum pipe *pipe)
11463 {
11464         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11465         enum intel_display_power_domain power_domain;
11466         intel_wakeref_t wakeref;
11467         bool ret;
11468         u32 val;
11469
11470         /*
11471          * Not 100% correct for planes that can move between pipes,
11472          * but that's only the case for gen2-3 which don't have any
11473          * display power wells.
11474          */
11475         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
11476         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
11477         if (!wakeref)
11478                 return false;
11479
11480         val = I915_READ(CURCNTR(plane->pipe));
11481
11482         ret = val & MCURSOR_MODE;
11483
11484         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
11485                 *pipe = plane->pipe;
11486         else
11487                 *pipe = (val & MCURSOR_PIPE_SELECT_MASK) >>
11488                         MCURSOR_PIPE_SELECT_SHIFT;
11489
11490         intel_display_power_put(dev_priv, power_domain, wakeref);
11491
11492         return ret;
11493 }
11494
11495 /* VESA 640x480x72Hz mode to set on the pipe */
11496 static const struct drm_display_mode load_detect_mode = {
11497         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
11498                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
11499 };
11500
11501 struct drm_framebuffer *
11502 intel_framebuffer_create(struct drm_i915_gem_object *obj,
11503                          struct drm_mode_fb_cmd2 *mode_cmd)
11504 {
11505         struct intel_framebuffer *intel_fb;
11506         int ret;
11507
11508         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
11509         if (!intel_fb)
11510                 return ERR_PTR(-ENOMEM);
11511
11512         ret = intel_framebuffer_init(intel_fb, obj, mode_cmd);
11513         if (ret)
11514                 goto err;
11515
11516         return &intel_fb->base;
11517
11518 err:
11519         kfree(intel_fb);
11520         return ERR_PTR(ret);
11521 }
11522
11523 static int intel_modeset_disable_planes(struct drm_atomic_state *state,
11524                                         struct drm_crtc *crtc)
11525 {
11526         struct drm_plane *plane;
11527         struct drm_plane_state *plane_state;
11528         int ret, i;
11529
11530         ret = drm_atomic_add_affected_planes(state, crtc);
11531         if (ret)
11532                 return ret;
11533
11534         for_each_new_plane_in_state(state, plane, plane_state, i) {
11535                 if (plane_state->crtc != crtc)
11536                         continue;
11537
11538                 ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
11539                 if (ret)
11540                         return ret;
11541
11542                 drm_atomic_set_fb_for_plane(plane_state, NULL);
11543         }
11544
11545         return 0;
11546 }
11547
11548 int intel_get_load_detect_pipe(struct drm_connector *connector,
11549                                struct intel_load_detect_pipe *old,
11550                                struct drm_modeset_acquire_ctx *ctx)
11551 {
11552         struct intel_crtc *intel_crtc;
11553         struct intel_encoder *intel_encoder =
11554                 intel_attached_encoder(connector);
11555         struct drm_crtc *possible_crtc;
11556         struct drm_encoder *encoder = &intel_encoder->base;
11557         struct drm_crtc *crtc = NULL;
11558         struct drm_device *dev = encoder->dev;
11559         struct drm_i915_private *dev_priv = to_i915(dev);
11560         struct drm_mode_config *config = &dev->mode_config;
11561         struct drm_atomic_state *state = NULL, *restore_state = NULL;
11562         struct drm_connector_state *connector_state;
11563         struct intel_crtc_state *crtc_state;
11564         int ret, i = -1;
11565
11566         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
11567                       connector->base.id, connector->name,
11568                       encoder->base.id, encoder->name);
11569
11570         old->restore_state = NULL;
11571
11572         WARN_ON(!drm_modeset_is_locked(&config->connection_mutex));
11573
11574         /*
11575          * Algorithm gets a little messy:
11576          *
11577          *   - if the connector already has an assigned crtc, use it (but make
11578          *     sure it's on first)
11579          *
11580          *   - try to find the first unused crtc that can drive this connector,
11581          *     and use that if we find one
11582          */
11583
11584         /* See if we already have a CRTC for this connector */
11585         if (connector->state->crtc) {
11586                 crtc = connector->state->crtc;
11587
11588                 ret = drm_modeset_lock(&crtc->mutex, ctx);
11589                 if (ret)
11590                         goto fail;
11591
11592                 /* Make sure the crtc and connector are running */
11593                 goto found;
11594         }
11595
11596         /* Find an unused one (if possible) */
11597         for_each_crtc(dev, possible_crtc) {
11598                 i++;
11599                 if (!(encoder->possible_crtcs & (1 << i)))
11600                         continue;
11601
11602                 ret = drm_modeset_lock(&possible_crtc->mutex, ctx);
11603                 if (ret)
11604                         goto fail;
11605
11606                 if (possible_crtc->state->enable) {
11607                         drm_modeset_unlock(&possible_crtc->mutex);
11608                         continue;
11609                 }
11610
11611                 crtc = possible_crtc;
11612                 break;
11613         }
11614
11615         /*
11616          * If we didn't find an unused CRTC, don't use any.
11617          */
11618         if (!crtc) {
11619                 DRM_DEBUG_KMS("no pipe available for load-detect\n");
11620                 ret = -ENODEV;
11621                 goto fail;
11622         }
11623
11624 found:
11625         intel_crtc = to_intel_crtc(crtc);
11626
11627         state = drm_atomic_state_alloc(dev);
11628         restore_state = drm_atomic_state_alloc(dev);
11629         if (!state || !restore_state) {
11630                 ret = -ENOMEM;
11631                 goto fail;
11632         }
11633
11634         state->acquire_ctx = ctx;
11635         restore_state->acquire_ctx = ctx;
11636
11637         connector_state = drm_atomic_get_connector_state(state, connector);
11638         if (IS_ERR(connector_state)) {
11639                 ret = PTR_ERR(connector_state);
11640                 goto fail;
11641         }
11642
11643         ret = drm_atomic_set_crtc_for_connector(connector_state, crtc);
11644         if (ret)
11645                 goto fail;
11646
11647         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
11648         if (IS_ERR(crtc_state)) {
11649                 ret = PTR_ERR(crtc_state);
11650                 goto fail;
11651         }
11652
11653         crtc_state->uapi.active = true;
11654
11655         ret = drm_atomic_set_mode_for_crtc(&crtc_state->uapi,
11656                                            &load_detect_mode);
11657         if (ret)
11658                 goto fail;
11659
11660         ret = intel_modeset_disable_planes(state, crtc);
11661         if (ret)
11662                 goto fail;
11663
11664         ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector));
11665         if (!ret)
11666                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, crtc));
11667         if (!ret)
11668                 ret = drm_atomic_add_affected_planes(restore_state, crtc);
11669         if (ret) {
11670                 DRM_DEBUG_KMS("Failed to create a copy of old state to restore: %i\n", ret);
11671                 goto fail;
11672         }
11673
11674         ret = drm_atomic_commit(state);
11675         if (ret) {
11676                 DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
11677                 goto fail;
11678         }
11679
11680         old->restore_state = restore_state;
11681         drm_atomic_state_put(state);
11682
11683         /* let the connector get through one full cycle before testing */
11684         intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
11685         return true;
11686
11687 fail:
11688         if (state) {
11689                 drm_atomic_state_put(state);
11690                 state = NULL;
11691         }
11692         if (restore_state) {
11693                 drm_atomic_state_put(restore_state);
11694                 restore_state = NULL;
11695         }
11696
11697         if (ret == -EDEADLK)
11698                 return ret;
11699
11700         return false;
11701 }
11702
11703 void intel_release_load_detect_pipe(struct drm_connector *connector,
11704                                     struct intel_load_detect_pipe *old,
11705                                     struct drm_modeset_acquire_ctx *ctx)
11706 {
11707         struct intel_encoder *intel_encoder =
11708                 intel_attached_encoder(connector);
11709         struct drm_encoder *encoder = &intel_encoder->base;
11710         struct drm_atomic_state *state = old->restore_state;
11711         int ret;
11712
11713         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
11714                       connector->base.id, connector->name,
11715                       encoder->base.id, encoder->name);
11716
11717         if (!state)
11718                 return;
11719
11720         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
11721         if (ret)
11722                 DRM_DEBUG_KMS("Couldn't release load detect pipe: %i\n", ret);
11723         drm_atomic_state_put(state);
11724 }
11725
11726 static int i9xx_pll_refclk(struct drm_device *dev,
11727                            const struct intel_crtc_state *pipe_config)
11728 {
11729         struct drm_i915_private *dev_priv = to_i915(dev);
11730         u32 dpll = pipe_config->dpll_hw_state.dpll;
11731
11732         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
11733                 return dev_priv->vbt.lvds_ssc_freq;
11734         else if (HAS_PCH_SPLIT(dev_priv))
11735                 return 120000;
11736         else if (!IS_GEN(dev_priv, 2))
11737                 return 96000;
11738         else
11739                 return 48000;
11740 }
11741
11742 /* Returns the clock of the currently programmed mode of the given pipe. */
11743 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
11744                                 struct intel_crtc_state *pipe_config)
11745 {
11746         struct drm_device *dev = crtc->base.dev;
11747         struct drm_i915_private *dev_priv = to_i915(dev);
11748         enum pipe pipe = crtc->pipe;
11749         u32 dpll = pipe_config->dpll_hw_state.dpll;
11750         u32 fp;
11751         struct dpll clock;
11752         int port_clock;
11753         int refclk = i9xx_pll_refclk(dev, pipe_config);
11754
11755         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
11756                 fp = pipe_config->dpll_hw_state.fp0;
11757         else
11758                 fp = pipe_config->dpll_hw_state.fp1;
11759
11760         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
11761         if (IS_PINEVIEW(dev_priv)) {
11762                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
11763                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
11764         } else {
11765                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
11766                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
11767         }
11768
11769         if (!IS_GEN(dev_priv, 2)) {
11770                 if (IS_PINEVIEW(dev_priv))
11771                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
11772                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
11773                 else
11774                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
11775                                DPLL_FPA01_P1_POST_DIV_SHIFT);
11776
11777                 switch (dpll & DPLL_MODE_MASK) {
11778                 case DPLLB_MODE_DAC_SERIAL:
11779                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
11780                                 5 : 10;
11781                         break;
11782                 case DPLLB_MODE_LVDS:
11783                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
11784                                 7 : 14;
11785                         break;
11786                 default:
11787                         DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed "
11788                                   "mode\n", (int)(dpll & DPLL_MODE_MASK));
11789                         return;
11790                 }
11791
11792                 if (IS_PINEVIEW(dev_priv))
11793                         port_clock = pnv_calc_dpll_params(refclk, &clock);
11794                 else
11795                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
11796         } else {
11797                 u32 lvds = IS_I830(dev_priv) ? 0 : I915_READ(LVDS);
11798                 bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN);
11799
11800                 if (is_lvds) {
11801                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
11802                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
11803
11804                         if (lvds & LVDS_CLKB_POWER_UP)
11805                                 clock.p2 = 7;
11806                         else
11807                                 clock.p2 = 14;
11808                 } else {
11809                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
11810                                 clock.p1 = 2;
11811                         else {
11812                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
11813                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
11814                         }
11815                         if (dpll & PLL_P2_DIVIDE_BY_4)
11816                                 clock.p2 = 4;
11817                         else
11818                                 clock.p2 = 2;
11819                 }
11820
11821                 port_clock = i9xx_calc_dpll_params(refclk, &clock);
11822         }
11823
11824         /*
11825          * This value includes pixel_multiplier. We will use
11826          * port_clock to compute adjusted_mode.crtc_clock in the
11827          * encoder's get_config() function.
11828          */
11829         pipe_config->port_clock = port_clock;
11830 }
11831
11832 int intel_dotclock_calculate(int link_freq,
11833                              const struct intel_link_m_n *m_n)
11834 {
11835         /*
11836          * The calculation for the data clock is:
11837          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
11838          * But we want to avoid losing precison if possible, so:
11839          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
11840          *
11841          * and the link clock is simpler:
11842          * link_clock = (m * link_clock) / n
11843          */
11844
11845         if (!m_n->link_n)
11846                 return 0;
11847
11848         return div_u64(mul_u32_u32(m_n->link_m, link_freq), m_n->link_n);
11849 }
11850
11851 static void ilk_pch_clock_get(struct intel_crtc *crtc,
11852                               struct intel_crtc_state *pipe_config)
11853 {
11854         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11855
11856         /* read out port_clock from the DPLL */
11857         i9xx_crtc_clock_get(crtc, pipe_config);
11858
11859         /*
11860          * In case there is an active pipe without active ports,
11861          * we may need some idea for the dotclock anyway.
11862          * Calculate one based on the FDI configuration.
11863          */
11864         pipe_config->hw.adjusted_mode.crtc_clock =
11865                 intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
11866                                          &pipe_config->fdi_m_n);
11867 }
11868
11869 static void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
11870                                    struct intel_crtc *crtc)
11871 {
11872         memset(crtc_state, 0, sizeof(*crtc_state));
11873
11874         __drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
11875
11876         crtc_state->cpu_transcoder = INVALID_TRANSCODER;
11877         crtc_state->master_transcoder = INVALID_TRANSCODER;
11878         crtc_state->hsw_workaround_pipe = INVALID_PIPE;
11879         crtc_state->output_format = INTEL_OUTPUT_FORMAT_INVALID;
11880         crtc_state->scaler_state.scaler_id = -1;
11881         crtc_state->mst_master_transcoder = INVALID_TRANSCODER;
11882 }
11883
11884 static struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
11885 {
11886         struct intel_crtc_state *crtc_state;
11887
11888         crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
11889
11890         if (crtc_state)
11891                 intel_crtc_state_reset(crtc_state, crtc);
11892
11893         return crtc_state;
11894 }
11895
11896 /* Returns the currently programmed mode of the given encoder. */
11897 struct drm_display_mode *
11898 intel_encoder_current_mode(struct intel_encoder *encoder)
11899 {
11900         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
11901         struct intel_crtc_state *crtc_state;
11902         struct drm_display_mode *mode;
11903         struct intel_crtc *crtc;
11904         enum pipe pipe;
11905
11906         if (!encoder->get_hw_state(encoder, &pipe))
11907                 return NULL;
11908
11909         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
11910
11911         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
11912         if (!mode)
11913                 return NULL;
11914
11915         crtc_state = intel_crtc_state_alloc(crtc);
11916         if (!crtc_state) {
11917                 kfree(mode);
11918                 return NULL;
11919         }
11920
11921         if (!dev_priv->display.get_pipe_config(crtc, crtc_state)) {
11922                 kfree(crtc_state);
11923                 kfree(mode);
11924                 return NULL;
11925         }
11926
11927         encoder->get_config(encoder, crtc_state);
11928
11929         intel_mode_from_pipe_config(mode, crtc_state);
11930
11931         kfree(crtc_state);
11932
11933         return mode;
11934 }
11935
11936 static void intel_crtc_destroy(struct drm_crtc *crtc)
11937 {
11938         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11939
11940         drm_crtc_cleanup(crtc);
11941         kfree(intel_crtc);
11942 }
11943
11944 /**
11945  * intel_wm_need_update - Check whether watermarks need updating
11946  * @cur: current plane state
11947  * @new: new plane state
11948  *
11949  * Check current plane state versus the new one to determine whether
11950  * watermarks need to be recalculated.
11951  *
11952  * Returns true or false.
11953  */
11954 static bool intel_wm_need_update(const struct intel_plane_state *cur,
11955                                  struct intel_plane_state *new)
11956 {
11957         /* Update watermarks on tiling or size changes. */
11958         if (new->uapi.visible != cur->uapi.visible)
11959                 return true;
11960
11961         if (!cur->hw.fb || !new->hw.fb)
11962                 return false;
11963
11964         if (cur->hw.fb->modifier != new->hw.fb->modifier ||
11965             cur->hw.rotation != new->hw.rotation ||
11966             drm_rect_width(&new->uapi.src) != drm_rect_width(&cur->uapi.src) ||
11967             drm_rect_height(&new->uapi.src) != drm_rect_height(&cur->uapi.src) ||
11968             drm_rect_width(&new->uapi.dst) != drm_rect_width(&cur->uapi.dst) ||
11969             drm_rect_height(&new->uapi.dst) != drm_rect_height(&cur->uapi.dst))
11970                 return true;
11971
11972         return false;
11973 }
11974
11975 static bool needs_scaling(const struct intel_plane_state *state)
11976 {
11977         int src_w = drm_rect_width(&state->uapi.src) >> 16;
11978         int src_h = drm_rect_height(&state->uapi.src) >> 16;
11979         int dst_w = drm_rect_width(&state->uapi.dst);
11980         int dst_h = drm_rect_height(&state->uapi.dst);
11981
11982         return (src_w != dst_w || src_h != dst_h);
11983 }
11984
11985 int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state,
11986                                     struct intel_crtc_state *crtc_state,
11987                                     const struct intel_plane_state *old_plane_state,
11988                                     struct intel_plane_state *plane_state)
11989 {
11990         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
11991         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
11992         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11993         bool mode_changed = needs_modeset(crtc_state);
11994         bool was_crtc_enabled = old_crtc_state->hw.active;
11995         bool is_crtc_enabled = crtc_state->hw.active;
11996         bool turn_off, turn_on, visible, was_visible;
11997         int ret;
11998
11999         if (INTEL_GEN(dev_priv) >= 9 && plane->id != PLANE_CURSOR) {
12000                 ret = skl_update_scaler_plane(crtc_state, plane_state);
12001                 if (ret)
12002                         return ret;
12003         }
12004
12005         was_visible = old_plane_state->uapi.visible;
12006         visible = plane_state->uapi.visible;
12007
12008         if (!was_crtc_enabled && WARN_ON(was_visible))
12009                 was_visible = false;
12010
12011         /*
12012          * Visibility is calculated as if the crtc was on, but
12013          * after scaler setup everything depends on it being off
12014          * when the crtc isn't active.
12015          *
12016          * FIXME this is wrong for watermarks. Watermarks should also
12017          * be computed as if the pipe would be active. Perhaps move
12018          * per-plane wm computation to the .check_plane() hook, and
12019          * only combine the results from all planes in the current place?
12020          */
12021         if (!is_crtc_enabled) {
12022                 plane_state->uapi.visible = visible = false;
12023                 crtc_state->active_planes &= ~BIT(plane->id);
12024                 crtc_state->data_rate[plane->id] = 0;
12025                 crtc_state->min_cdclk[plane->id] = 0;
12026         }
12027
12028         if (!was_visible && !visible)
12029                 return 0;
12030
12031         turn_off = was_visible && (!visible || mode_changed);
12032         turn_on = visible && (!was_visible || mode_changed);
12033
12034         DRM_DEBUG_ATOMIC("[CRTC:%d:%s] with [PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n",
12035                          crtc->base.base.id, crtc->base.name,
12036                          plane->base.base.id, plane->base.name,
12037                          was_visible, visible,
12038                          turn_off, turn_on, mode_changed);
12039
12040         if (turn_on) {
12041                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
12042                         crtc_state->update_wm_pre = true;
12043
12044                 /* must disable cxsr around plane enable/disable */
12045                 if (plane->id != PLANE_CURSOR)
12046                         crtc_state->disable_cxsr = true;
12047         } else if (turn_off) {
12048                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
12049                         crtc_state->update_wm_post = true;
12050
12051                 /* must disable cxsr around plane enable/disable */
12052                 if (plane->id != PLANE_CURSOR)
12053                         crtc_state->disable_cxsr = true;
12054         } else if (intel_wm_need_update(old_plane_state, plane_state)) {
12055                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) {
12056                         /* FIXME bollocks */
12057                         crtc_state->update_wm_pre = true;
12058                         crtc_state->update_wm_post = true;
12059                 }
12060         }
12061
12062         if (visible || was_visible)
12063                 crtc_state->fb_bits |= plane->frontbuffer_bit;
12064
12065         /*
12066          * ILK/SNB DVSACNTR/Sprite Enable
12067          * IVB SPR_CTL/Sprite Enable
12068          * "When in Self Refresh Big FIFO mode, a write to enable the
12069          *  plane will be internally buffered and delayed while Big FIFO
12070          *  mode is exiting."
12071          *
12072          * Which means that enabling the sprite can take an extra frame
12073          * when we start in big FIFO mode (LP1+). Thus we need to drop
12074          * down to LP0 and wait for vblank in order to make sure the
12075          * sprite gets enabled on the next vblank after the register write.
12076          * Doing otherwise would risk enabling the sprite one frame after
12077          * we've already signalled flip completion. We can resume LP1+
12078          * once the sprite has been enabled.
12079          *
12080          *
12081          * WaCxSRDisabledForSpriteScaling:ivb
12082          * IVB SPR_SCALE/Scaling Enable
12083          * "Low Power watermarks must be disabled for at least one
12084          *  frame before enabling sprite scaling, and kept disabled
12085          *  until sprite scaling is disabled."
12086          *
12087          * ILK/SNB DVSASCALE/Scaling Enable
12088          * "When in Self Refresh Big FIFO mode, scaling enable will be
12089          *  masked off while Big FIFO mode is exiting."
12090          *
12091          * Despite the w/a only being listed for IVB we assume that
12092          * the ILK/SNB note has similar ramifications, hence we apply
12093          * the w/a on all three platforms.
12094          *
12095          * With experimental results seems this is needed also for primary
12096          * plane, not only sprite plane.
12097          */
12098         if (plane->id != PLANE_CURSOR &&
12099             (IS_GEN_RANGE(dev_priv, 5, 6) ||
12100              IS_IVYBRIDGE(dev_priv)) &&
12101             (turn_on || (!needs_scaling(old_plane_state) &&
12102                          needs_scaling(plane_state))))
12103                 crtc_state->disable_lp_wm = true;
12104
12105         return 0;
12106 }
12107
12108 static bool encoders_cloneable(const struct intel_encoder *a,
12109                                const struct intel_encoder *b)
12110 {
12111         /* masks could be asymmetric, so check both ways */
12112         return a == b || (a->cloneable & (1 << b->type) &&
12113                           b->cloneable & (1 << a->type));
12114 }
12115
12116 static bool check_single_encoder_cloning(struct drm_atomic_state *state,
12117                                          struct intel_crtc *crtc,
12118                                          struct intel_encoder *encoder)
12119 {
12120         struct intel_encoder *source_encoder;
12121         struct drm_connector *connector;
12122         struct drm_connector_state *connector_state;
12123         int i;
12124
12125         for_each_new_connector_in_state(state, connector, connector_state, i) {
12126                 if (connector_state->crtc != &crtc->base)
12127                         continue;
12128
12129                 source_encoder =
12130                         to_intel_encoder(connector_state->best_encoder);
12131                 if (!encoders_cloneable(encoder, source_encoder))
12132                         return false;
12133         }
12134
12135         return true;
12136 }
12137
12138 static int icl_add_linked_planes(struct intel_atomic_state *state)
12139 {
12140         struct intel_plane *plane, *linked;
12141         struct intel_plane_state *plane_state, *linked_plane_state;
12142         int i;
12143
12144         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12145                 linked = plane_state->planar_linked_plane;
12146
12147                 if (!linked)
12148                         continue;
12149
12150                 linked_plane_state = intel_atomic_get_plane_state(state, linked);
12151                 if (IS_ERR(linked_plane_state))
12152                         return PTR_ERR(linked_plane_state);
12153
12154                 WARN_ON(linked_plane_state->planar_linked_plane != plane);
12155                 WARN_ON(linked_plane_state->planar_slave == plane_state->planar_slave);
12156         }
12157
12158         return 0;
12159 }
12160
12161 static int icl_check_nv12_planes(struct intel_crtc_state *crtc_state)
12162 {
12163         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
12164         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12165         struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->uapi.state);
12166         struct intel_plane *plane, *linked;
12167         struct intel_plane_state *plane_state;
12168         int i;
12169
12170         if (INTEL_GEN(dev_priv) < 11)
12171                 return 0;
12172
12173         /*
12174          * Destroy all old plane links and make the slave plane invisible
12175          * in the crtc_state->active_planes mask.
12176          */
12177         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12178                 if (plane->pipe != crtc->pipe || !plane_state->planar_linked_plane)
12179                         continue;
12180
12181                 plane_state->planar_linked_plane = NULL;
12182                 if (plane_state->planar_slave && !plane_state->uapi.visible) {
12183                         crtc_state->active_planes &= ~BIT(plane->id);
12184                         crtc_state->update_planes |= BIT(plane->id);
12185                 }
12186
12187                 plane_state->planar_slave = false;
12188         }
12189
12190         if (!crtc_state->nv12_planes)
12191                 return 0;
12192
12193         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12194                 struct intel_plane_state *linked_state = NULL;
12195
12196                 if (plane->pipe != crtc->pipe ||
12197                     !(crtc_state->nv12_planes & BIT(plane->id)))
12198                         continue;
12199
12200                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, linked) {
12201                         if (!icl_is_nv12_y_plane(linked->id))
12202                                 continue;
12203
12204                         if (crtc_state->active_planes & BIT(linked->id))
12205                                 continue;
12206
12207                         linked_state = intel_atomic_get_plane_state(state, linked);
12208                         if (IS_ERR(linked_state))
12209                                 return PTR_ERR(linked_state);
12210
12211                         break;
12212                 }
12213
12214                 if (!linked_state) {
12215                         DRM_DEBUG_KMS("Need %d free Y planes for planar YUV\n",
12216                                       hweight8(crtc_state->nv12_planes));
12217
12218                         return -EINVAL;
12219                 }
12220
12221                 plane_state->planar_linked_plane = linked;
12222
12223                 linked_state->planar_slave = true;
12224                 linked_state->planar_linked_plane = plane;
12225                 crtc_state->active_planes |= BIT(linked->id);
12226                 crtc_state->update_planes |= BIT(linked->id);
12227                 DRM_DEBUG_KMS("Using %s as Y plane for %s\n", linked->base.name, plane->base.name);
12228
12229                 /* Copy parameters to slave plane */
12230                 linked_state->ctl = plane_state->ctl | PLANE_CTL_YUV420_Y_PLANE;
12231                 linked_state->color_ctl = plane_state->color_ctl;
12232                 memcpy(linked_state->color_plane, plane_state->color_plane,
12233                        sizeof(linked_state->color_plane));
12234
12235                 intel_plane_copy_uapi_to_hw_state(linked_state, plane_state);
12236                 linked_state->uapi.src = plane_state->uapi.src;
12237                 linked_state->uapi.dst = plane_state->uapi.dst;
12238
12239                 if (icl_is_hdr_plane(dev_priv, plane->id)) {
12240                         if (linked->id == PLANE_SPRITE5)
12241                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_7;
12242                         else if (linked->id == PLANE_SPRITE4)
12243                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_6;
12244                         else
12245                                 MISSING_CASE(linked->id);
12246                 }
12247         }
12248
12249         return 0;
12250 }
12251
12252 static bool c8_planes_changed(const struct intel_crtc_state *new_crtc_state)
12253 {
12254         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
12255         struct intel_atomic_state *state =
12256                 to_intel_atomic_state(new_crtc_state->uapi.state);
12257         const struct intel_crtc_state *old_crtc_state =
12258                 intel_atomic_get_old_crtc_state(state, crtc);
12259
12260         return !old_crtc_state->c8_planes != !new_crtc_state->c8_planes;
12261 }
12262
12263 static int icl_add_sync_mode_crtcs(struct intel_crtc_state *crtc_state)
12264 {
12265         struct drm_crtc *crtc = crtc_state->uapi.crtc;
12266         struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->uapi.state);
12267         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
12268         struct drm_connector *master_connector, *connector;
12269         struct drm_connector_state *connector_state;
12270         struct drm_connector_list_iter conn_iter;
12271         struct drm_crtc *master_crtc = NULL;
12272         struct drm_crtc_state *master_crtc_state;
12273         struct intel_crtc_state *master_pipe_config;
12274         int i, tile_group_id;
12275
12276         if (INTEL_GEN(dev_priv) < 11)
12277                 return 0;
12278
12279         /*
12280          * In case of tiled displays there could be one or more slaves but there is
12281          * only one master. Lets make the CRTC used by the connector corresponding
12282          * to the last horizonal and last vertical tile a master/genlock CRTC.
12283          * All the other CRTCs corresponding to other tiles of the same Tile group
12284          * are the slave CRTCs and hold a pointer to their genlock CRTC.
12285          */
12286         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
12287                 if (connector_state->crtc != crtc)
12288                         continue;
12289                 if (!connector->has_tile)
12290                         continue;
12291                 if (crtc_state->hw.mode.hdisplay != connector->tile_h_size ||
12292                     crtc_state->hw.mode.vdisplay != connector->tile_v_size)
12293                         return 0;
12294                 if (connector->tile_h_loc == connector->num_h_tile - 1 &&
12295                     connector->tile_v_loc == connector->num_v_tile - 1)
12296                         continue;
12297                 crtc_state->sync_mode_slaves_mask = 0;
12298                 tile_group_id = connector->tile_group->id;
12299                 drm_connector_list_iter_begin(&dev_priv->drm, &conn_iter);
12300                 drm_for_each_connector_iter(master_connector, &conn_iter) {
12301                         struct drm_connector_state *master_conn_state = NULL;
12302
12303                         if (!master_connector->has_tile)
12304                                 continue;
12305                         if (master_connector->tile_h_loc != master_connector->num_h_tile - 1 ||
12306                             master_connector->tile_v_loc != master_connector->num_v_tile - 1)
12307                                 continue;
12308                         if (master_connector->tile_group->id != tile_group_id)
12309                                 continue;
12310
12311                         master_conn_state = drm_atomic_get_connector_state(&state->base,
12312                                                                            master_connector);
12313                         if (IS_ERR(master_conn_state)) {
12314                                 drm_connector_list_iter_end(&conn_iter);
12315                                 return PTR_ERR(master_conn_state);
12316                         }
12317                         if (master_conn_state->crtc) {
12318                                 master_crtc = master_conn_state->crtc;
12319                                 break;
12320                         }
12321                 }
12322                 drm_connector_list_iter_end(&conn_iter);
12323
12324                 if (!master_crtc) {
12325                         DRM_DEBUG_KMS("Could not find Master CRTC for Slave CRTC %d\n",
12326                                       connector_state->crtc->base.id);
12327                         return -EINVAL;
12328                 }
12329
12330                 master_crtc_state = drm_atomic_get_crtc_state(&state->base,
12331                                                               master_crtc);
12332                 if (IS_ERR(master_crtc_state))
12333                         return PTR_ERR(master_crtc_state);
12334
12335                 master_pipe_config = to_intel_crtc_state(master_crtc_state);
12336                 crtc_state->master_transcoder = master_pipe_config->cpu_transcoder;
12337                 master_pipe_config->sync_mode_slaves_mask |=
12338                         BIT(crtc_state->cpu_transcoder);
12339                 DRM_DEBUG_KMS("Master Transcoder = %s added for Slave CRTC = %d, slave transcoder bitmask = %d\n",
12340                               transcoder_name(crtc_state->master_transcoder),
12341                               crtc_state->uapi.crtc->base.id,
12342                               master_pipe_config->sync_mode_slaves_mask);
12343         }
12344
12345         return 0;
12346 }
12347
12348 static int intel_crtc_atomic_check(struct intel_atomic_state *state,
12349                                    struct intel_crtc *crtc)
12350 {
12351         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12352         struct intel_crtc_state *crtc_state =
12353                 intel_atomic_get_new_crtc_state(state, crtc);
12354         bool mode_changed = needs_modeset(crtc_state);
12355         int ret;
12356
12357         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv) &&
12358             mode_changed && !crtc_state->hw.active)
12359                 crtc_state->update_wm_post = true;
12360
12361         if (mode_changed && crtc_state->hw.enable &&
12362             dev_priv->display.crtc_compute_clock &&
12363             !WARN_ON(crtc_state->shared_dpll)) {
12364                 ret = dev_priv->display.crtc_compute_clock(crtc, crtc_state);
12365                 if (ret)
12366                         return ret;
12367         }
12368
12369         /*
12370          * May need to update pipe gamma enable bits
12371          * when C8 planes are getting enabled/disabled.
12372          */
12373         if (c8_planes_changed(crtc_state))
12374                 crtc_state->uapi.color_mgmt_changed = true;
12375
12376         if (mode_changed || crtc_state->update_pipe ||
12377             crtc_state->uapi.color_mgmt_changed) {
12378                 ret = intel_color_check(crtc_state);
12379                 if (ret)
12380                         return ret;
12381         }
12382
12383         ret = 0;
12384         if (dev_priv->display.compute_pipe_wm) {
12385                 ret = dev_priv->display.compute_pipe_wm(crtc_state);
12386                 if (ret) {
12387                         DRM_DEBUG_KMS("Target pipe watermarks are invalid\n");
12388                         return ret;
12389                 }
12390         }
12391
12392         if (dev_priv->display.compute_intermediate_wm) {
12393                 if (WARN_ON(!dev_priv->display.compute_pipe_wm))
12394                         return 0;
12395
12396                 /*
12397                  * Calculate 'intermediate' watermarks that satisfy both the
12398                  * old state and the new state.  We can program these
12399                  * immediately.
12400                  */
12401                 ret = dev_priv->display.compute_intermediate_wm(crtc_state);
12402                 if (ret) {
12403                         DRM_DEBUG_KMS("No valid intermediate pipe watermarks are possible\n");
12404                         return ret;
12405                 }
12406         }
12407
12408         if (INTEL_GEN(dev_priv) >= 9) {
12409                 if (mode_changed || crtc_state->update_pipe)
12410                         ret = skl_update_scaler_crtc(crtc_state);
12411                 if (!ret)
12412                         ret = intel_atomic_setup_scalers(dev_priv, crtc,
12413                                                          crtc_state);
12414         }
12415
12416         if (HAS_IPS(dev_priv))
12417                 crtc_state->ips_enabled = hsw_compute_ips_config(crtc_state);
12418
12419         return ret;
12420 }
12421
12422 static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
12423 {
12424         struct intel_connector *connector;
12425         struct drm_connector_list_iter conn_iter;
12426
12427         drm_connector_list_iter_begin(dev, &conn_iter);
12428         for_each_intel_connector_iter(connector, &conn_iter) {
12429                 if (connector->base.state->crtc)
12430                         drm_connector_put(&connector->base);
12431
12432                 if (connector->base.encoder) {
12433                         connector->base.state->best_encoder =
12434                                 connector->base.encoder;
12435                         connector->base.state->crtc =
12436                                 connector->base.encoder->crtc;
12437
12438                         drm_connector_get(&connector->base);
12439                 } else {
12440                         connector->base.state->best_encoder = NULL;
12441                         connector->base.state->crtc = NULL;
12442                 }
12443         }
12444         drm_connector_list_iter_end(&conn_iter);
12445 }
12446
12447 static int
12448 compute_sink_pipe_bpp(const struct drm_connector_state *conn_state,
12449                       struct intel_crtc_state *pipe_config)
12450 {
12451         struct drm_connector *connector = conn_state->connector;
12452         const struct drm_display_info *info = &connector->display_info;
12453         int bpp;
12454
12455         switch (conn_state->max_bpc) {
12456         case 6 ... 7:
12457                 bpp = 6 * 3;
12458                 break;
12459         case 8 ... 9:
12460                 bpp = 8 * 3;
12461                 break;
12462         case 10 ... 11:
12463                 bpp = 10 * 3;
12464                 break;
12465         case 12:
12466                 bpp = 12 * 3;
12467                 break;
12468         default:
12469                 return -EINVAL;
12470         }
12471
12472         if (bpp < pipe_config->pipe_bpp) {
12473                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Limiting display bpp to %d instead of "
12474                               "EDID bpp %d, requested bpp %d, max platform bpp %d\n",
12475                               connector->base.id, connector->name,
12476                               bpp, 3 * info->bpc, 3 * conn_state->max_requested_bpc,
12477                               pipe_config->pipe_bpp);
12478
12479                 pipe_config->pipe_bpp = bpp;
12480         }
12481
12482         return 0;
12483 }
12484
12485 static int
12486 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
12487                           struct intel_crtc_state *pipe_config)
12488 {
12489         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12490         struct drm_atomic_state *state = pipe_config->uapi.state;
12491         struct drm_connector *connector;
12492         struct drm_connector_state *connector_state;
12493         int bpp, i;
12494
12495         if ((IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
12496             IS_CHERRYVIEW(dev_priv)))
12497                 bpp = 10*3;
12498         else if (INTEL_GEN(dev_priv) >= 5)
12499                 bpp = 12*3;
12500         else
12501                 bpp = 8*3;
12502
12503         pipe_config->pipe_bpp = bpp;
12504
12505         /* Clamp display bpp to connector max bpp */
12506         for_each_new_connector_in_state(state, connector, connector_state, i) {
12507                 int ret;
12508
12509                 if (connector_state->crtc != &crtc->base)
12510                         continue;
12511
12512                 ret = compute_sink_pipe_bpp(connector_state, pipe_config);
12513                 if (ret)
12514                         return ret;
12515         }
12516
12517         return 0;
12518 }
12519
12520 static void intel_dump_crtc_timings(const struct drm_display_mode *mode)
12521 {
12522         DRM_DEBUG_KMS("crtc timings: %d %d %d %d %d %d %d %d %d, "
12523                       "type: 0x%x flags: 0x%x\n",
12524                       mode->crtc_clock,
12525                       mode->crtc_hdisplay, mode->crtc_hsync_start,
12526                       mode->crtc_hsync_end, mode->crtc_htotal,
12527                       mode->crtc_vdisplay, mode->crtc_vsync_start,
12528                       mode->crtc_vsync_end, mode->crtc_vtotal,
12529                       mode->type, mode->flags);
12530 }
12531
12532 static inline void
12533 intel_dump_m_n_config(const struct intel_crtc_state *pipe_config,
12534                       const char *id, unsigned int lane_count,
12535                       const struct intel_link_m_n *m_n)
12536 {
12537         DRM_DEBUG_KMS("%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
12538                       id, lane_count,
12539                       m_n->gmch_m, m_n->gmch_n,
12540                       m_n->link_m, m_n->link_n, m_n->tu);
12541 }
12542
12543 static void
12544 intel_dump_infoframe(struct drm_i915_private *dev_priv,
12545                      const union hdmi_infoframe *frame)
12546 {
12547         if ((drm_debug & DRM_UT_KMS) == 0)
12548                 return;
12549
12550         hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, frame);
12551 }
12552
12553 #define OUTPUT_TYPE(x) [INTEL_OUTPUT_ ## x] = #x
12554
12555 static const char * const output_type_str[] = {
12556         OUTPUT_TYPE(UNUSED),
12557         OUTPUT_TYPE(ANALOG),
12558         OUTPUT_TYPE(DVO),
12559         OUTPUT_TYPE(SDVO),
12560         OUTPUT_TYPE(LVDS),
12561         OUTPUT_TYPE(TVOUT),
12562         OUTPUT_TYPE(HDMI),
12563         OUTPUT_TYPE(DP),
12564         OUTPUT_TYPE(EDP),
12565         OUTPUT_TYPE(DSI),
12566         OUTPUT_TYPE(DDI),
12567         OUTPUT_TYPE(DP_MST),
12568 };
12569
12570 #undef OUTPUT_TYPE
12571
12572 static void snprintf_output_types(char *buf, size_t len,
12573                                   unsigned int output_types)
12574 {
12575         char *str = buf;
12576         int i;
12577
12578         str[0] = '\0';
12579
12580         for (i = 0; i < ARRAY_SIZE(output_type_str); i++) {
12581                 int r;
12582
12583                 if ((output_types & BIT(i)) == 0)
12584                         continue;
12585
12586                 r = snprintf(str, len, "%s%s",
12587                              str != buf ? "," : "", output_type_str[i]);
12588                 if (r >= len)
12589                         break;
12590                 str += r;
12591                 len -= r;
12592
12593                 output_types &= ~BIT(i);
12594         }
12595
12596         WARN_ON_ONCE(output_types != 0);
12597 }
12598
12599 static const char * const output_format_str[] = {
12600         [INTEL_OUTPUT_FORMAT_INVALID] = "Invalid",
12601         [INTEL_OUTPUT_FORMAT_RGB] = "RGB",
12602         [INTEL_OUTPUT_FORMAT_YCBCR420] = "YCBCR4:2:0",
12603         [INTEL_OUTPUT_FORMAT_YCBCR444] = "YCBCR4:4:4",
12604 };
12605
12606 static const char *output_formats(enum intel_output_format format)
12607 {
12608         if (format >= ARRAY_SIZE(output_format_str))
12609                 format = INTEL_OUTPUT_FORMAT_INVALID;
12610         return output_format_str[format];
12611 }
12612
12613 static void intel_dump_plane_state(const struct intel_plane_state *plane_state)
12614 {
12615         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
12616         const struct drm_framebuffer *fb = plane_state->hw.fb;
12617         struct drm_format_name_buf format_name;
12618
12619         if (!fb) {
12620                 DRM_DEBUG_KMS("[PLANE:%d:%s] fb: [NOFB], visible: %s\n",
12621                               plane->base.base.id, plane->base.name,
12622                               yesno(plane_state->uapi.visible));
12623                 return;
12624         }
12625
12626         DRM_DEBUG_KMS("[PLANE:%d:%s] fb: [FB:%d] %ux%u format = %s, visible: %s\n",
12627                       plane->base.base.id, plane->base.name,
12628                       fb->base.id, fb->width, fb->height,
12629                       drm_get_format_name(fb->format->format, &format_name),
12630                       yesno(plane_state->uapi.visible));
12631         DRM_DEBUG_KMS("\trotation: 0x%x, scaler: %d\n",
12632                       plane_state->hw.rotation, plane_state->scaler_id);
12633         if (plane_state->uapi.visible)
12634                 DRM_DEBUG_KMS("\tsrc: " DRM_RECT_FP_FMT " dst: " DRM_RECT_FMT "\n",
12635                               DRM_RECT_FP_ARG(&plane_state->uapi.src),
12636                               DRM_RECT_ARG(&plane_state->uapi.dst));
12637 }
12638
12639 static void intel_dump_pipe_config(const struct intel_crtc_state *pipe_config,
12640                                    struct intel_atomic_state *state,
12641                                    const char *context)
12642 {
12643         struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
12644         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12645         const struct intel_plane_state *plane_state;
12646         struct intel_plane *plane;
12647         char buf[64];
12648         int i;
12649
12650         DRM_DEBUG_KMS("[CRTC:%d:%s] enable: %s %s\n",
12651                       crtc->base.base.id, crtc->base.name,
12652                       yesno(pipe_config->hw.enable), context);
12653
12654         if (!pipe_config->hw.enable)
12655                 goto dump_planes;
12656
12657         snprintf_output_types(buf, sizeof(buf), pipe_config->output_types);
12658         DRM_DEBUG_KMS("active: %s, output_types: %s (0x%x), output format: %s\n",
12659                       yesno(pipe_config->hw.active),
12660                       buf, pipe_config->output_types,
12661                       output_formats(pipe_config->output_format));
12662
12663         DRM_DEBUG_KMS("cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
12664                       transcoder_name(pipe_config->cpu_transcoder),
12665                       pipe_config->pipe_bpp, pipe_config->dither);
12666
12667         if (pipe_config->has_pch_encoder)
12668                 intel_dump_m_n_config(pipe_config, "fdi",
12669                                       pipe_config->fdi_lanes,
12670                                       &pipe_config->fdi_m_n);
12671
12672         if (intel_crtc_has_dp_encoder(pipe_config)) {
12673                 intel_dump_m_n_config(pipe_config, "dp m_n",
12674                                 pipe_config->lane_count, &pipe_config->dp_m_n);
12675                 if (pipe_config->has_drrs)
12676                         intel_dump_m_n_config(pipe_config, "dp m2_n2",
12677                                               pipe_config->lane_count,
12678                                               &pipe_config->dp_m2_n2);
12679         }
12680
12681         DRM_DEBUG_KMS("audio: %i, infoframes: %i, infoframes enabled: 0x%x\n",
12682                       pipe_config->has_audio, pipe_config->has_infoframe,
12683                       pipe_config->infoframes.enable);
12684
12685         if (pipe_config->infoframes.enable &
12686             intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GENERAL_CONTROL))
12687                 DRM_DEBUG_KMS("GCP: 0x%x\n", pipe_config->infoframes.gcp);
12688         if (pipe_config->infoframes.enable &
12689             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_AVI))
12690                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.avi);
12691         if (pipe_config->infoframes.enable &
12692             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_SPD))
12693                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.spd);
12694         if (pipe_config->infoframes.enable &
12695             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_VENDOR))
12696                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.hdmi);
12697
12698         DRM_DEBUG_KMS("requested mode:\n");
12699         drm_mode_debug_printmodeline(&pipe_config->hw.mode);
12700         DRM_DEBUG_KMS("adjusted mode:\n");
12701         drm_mode_debug_printmodeline(&pipe_config->hw.adjusted_mode);
12702         intel_dump_crtc_timings(&pipe_config->hw.adjusted_mode);
12703         DRM_DEBUG_KMS("port clock: %d, pipe src size: %dx%d, pixel rate %d\n",
12704                       pipe_config->port_clock,
12705                       pipe_config->pipe_src_w, pipe_config->pipe_src_h,
12706                       pipe_config->pixel_rate);
12707
12708         if (INTEL_GEN(dev_priv) >= 9)
12709                 DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
12710                               crtc->num_scalers,
12711                               pipe_config->scaler_state.scaler_users,
12712                               pipe_config->scaler_state.scaler_id);
12713
12714         if (HAS_GMCH(dev_priv))
12715                 DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
12716                               pipe_config->gmch_pfit.control,
12717                               pipe_config->gmch_pfit.pgm_ratios,
12718                               pipe_config->gmch_pfit.lvds_border_bits);
12719         else
12720                 DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s, force thru: %s\n",
12721                               pipe_config->pch_pfit.pos,
12722                               pipe_config->pch_pfit.size,
12723                               enableddisabled(pipe_config->pch_pfit.enabled),
12724                               yesno(pipe_config->pch_pfit.force_thru));
12725
12726         DRM_DEBUG_KMS("ips: %i, double wide: %i\n",
12727                       pipe_config->ips_enabled, pipe_config->double_wide);
12728
12729         intel_dpll_dump_hw_state(dev_priv, &pipe_config->dpll_hw_state);
12730
12731         if (IS_CHERRYVIEW(dev_priv))
12732                 DRM_DEBUG_KMS("cgm_mode: 0x%x gamma_mode: 0x%x gamma_enable: %d csc_enable: %d\n",
12733                               pipe_config->cgm_mode, pipe_config->gamma_mode,
12734                               pipe_config->gamma_enable, pipe_config->csc_enable);
12735         else
12736                 DRM_DEBUG_KMS("csc_mode: 0x%x gamma_mode: 0x%x gamma_enable: %d csc_enable: %d\n",
12737                               pipe_config->csc_mode, pipe_config->gamma_mode,
12738                               pipe_config->gamma_enable, pipe_config->csc_enable);
12739
12740         DRM_DEBUG_KMS("MST master transcoder: %s\n",
12741                       transcoder_name(pipe_config->mst_master_transcoder));
12742
12743 dump_planes:
12744         if (!state)
12745                 return;
12746
12747         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12748                 if (plane->pipe == crtc->pipe)
12749                         intel_dump_plane_state(plane_state);
12750         }
12751 }
12752
12753 static bool check_digital_port_conflicts(struct intel_atomic_state *state)
12754 {
12755         struct drm_device *dev = state->base.dev;
12756         struct drm_connector *connector;
12757         struct drm_connector_list_iter conn_iter;
12758         unsigned int used_ports = 0;
12759         unsigned int used_mst_ports = 0;
12760         bool ret = true;
12761
12762         /*
12763          * We're going to peek into connector->state,
12764          * hence connection_mutex must be held.
12765          */
12766         drm_modeset_lock_assert_held(&dev->mode_config.connection_mutex);
12767
12768         /*
12769          * Walk the connector list instead of the encoder
12770          * list to detect the problem on ddi platforms
12771          * where there's just one encoder per digital port.
12772          */
12773         drm_connector_list_iter_begin(dev, &conn_iter);
12774         drm_for_each_connector_iter(connector, &conn_iter) {
12775                 struct drm_connector_state *connector_state;
12776                 struct intel_encoder *encoder;
12777
12778                 connector_state =
12779                         drm_atomic_get_new_connector_state(&state->base,
12780                                                            connector);
12781                 if (!connector_state)
12782                         connector_state = connector->state;
12783
12784                 if (!connector_state->best_encoder)
12785                         continue;
12786
12787                 encoder = to_intel_encoder(connector_state->best_encoder);
12788
12789                 WARN_ON(!connector_state->crtc);
12790
12791                 switch (encoder->type) {
12792                         unsigned int port_mask;
12793                 case INTEL_OUTPUT_DDI:
12794                         if (WARN_ON(!HAS_DDI(to_i915(dev))))
12795                                 break;
12796                         /* else, fall through */
12797                 case INTEL_OUTPUT_DP:
12798                 case INTEL_OUTPUT_HDMI:
12799                 case INTEL_OUTPUT_EDP:
12800                         port_mask = 1 << encoder->port;
12801
12802                         /* the same port mustn't appear more than once */
12803                         if (used_ports & port_mask)
12804                                 ret = false;
12805
12806                         used_ports |= port_mask;
12807                         break;
12808                 case INTEL_OUTPUT_DP_MST:
12809                         used_mst_ports |=
12810                                 1 << encoder->port;
12811                         break;
12812                 default:
12813                         break;
12814                 }
12815         }
12816         drm_connector_list_iter_end(&conn_iter);
12817
12818         /* can't mix MST and SST/HDMI on the same port */
12819         if (used_ports & used_mst_ports)
12820                 return false;
12821
12822         return ret;
12823 }
12824
12825 static void
12826 intel_crtc_copy_uapi_to_hw_state_nomodeset(struct intel_crtc_state *crtc_state)
12827 {
12828         intel_crtc_copy_color_blobs(crtc_state);
12829 }
12830
12831 static void
12832 intel_crtc_copy_uapi_to_hw_state(struct intel_crtc_state *crtc_state)
12833 {
12834         crtc_state->hw.enable = crtc_state->uapi.enable;
12835         crtc_state->hw.active = crtc_state->uapi.active;
12836         crtc_state->hw.mode = crtc_state->uapi.mode;
12837         crtc_state->hw.adjusted_mode = crtc_state->uapi.adjusted_mode;
12838         intel_crtc_copy_uapi_to_hw_state_nomodeset(crtc_state);
12839 }
12840
12841 static void intel_crtc_copy_hw_to_uapi_state(struct intel_crtc_state *crtc_state)
12842 {
12843         crtc_state->uapi.enable = crtc_state->hw.enable;
12844         crtc_state->uapi.active = crtc_state->hw.active;
12845         WARN_ON(drm_atomic_set_mode_for_crtc(&crtc_state->uapi, &crtc_state->hw.mode) < 0);
12846
12847         crtc_state->uapi.adjusted_mode = crtc_state->hw.adjusted_mode;
12848
12849         /* copy color blobs to uapi */
12850         drm_property_replace_blob(&crtc_state->uapi.degamma_lut,
12851                                   crtc_state->hw.degamma_lut);
12852         drm_property_replace_blob(&crtc_state->uapi.gamma_lut,
12853                                   crtc_state->hw.gamma_lut);
12854         drm_property_replace_blob(&crtc_state->uapi.ctm,
12855                                   crtc_state->hw.ctm);
12856 }
12857
12858 static int
12859 intel_crtc_prepare_cleared_state(struct intel_crtc_state *crtc_state)
12860 {
12861         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
12862         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12863         struct intel_crtc_state *saved_state;
12864
12865         saved_state = intel_crtc_state_alloc(crtc);
12866         if (!saved_state)
12867                 return -ENOMEM;
12868
12869         /* free the old crtc_state->hw members */
12870         intel_crtc_free_hw_state(crtc_state);
12871
12872         /* FIXME: before the switch to atomic started, a new pipe_config was
12873          * kzalloc'd. Code that depends on any field being zero should be
12874          * fixed, so that the crtc_state can be safely duplicated. For now,
12875          * only fields that are know to not cause problems are preserved. */
12876
12877         saved_state->uapi = crtc_state->uapi;
12878         saved_state->scaler_state = crtc_state->scaler_state;
12879         saved_state->shared_dpll = crtc_state->shared_dpll;
12880         saved_state->dpll_hw_state = crtc_state->dpll_hw_state;
12881         memcpy(saved_state->icl_port_dplls, crtc_state->icl_port_dplls,
12882                sizeof(saved_state->icl_port_dplls));
12883         saved_state->crc_enabled = crtc_state->crc_enabled;
12884         if (IS_G4X(dev_priv) ||
12885             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
12886                 saved_state->wm = crtc_state->wm;
12887         /*
12888          * Save the slave bitmask which gets filled for master crtc state during
12889          * slave atomic check call.
12890          */
12891         if (is_trans_port_sync_master(crtc_state))
12892                 saved_state->sync_mode_slaves_mask =
12893                         crtc_state->sync_mode_slaves_mask;
12894
12895         memcpy(crtc_state, saved_state, sizeof(*crtc_state));
12896         kfree(saved_state);
12897
12898         intel_crtc_copy_uapi_to_hw_state(crtc_state);
12899
12900         return 0;
12901 }
12902
12903 static int
12904 intel_modeset_pipe_config(struct intel_crtc_state *pipe_config)
12905 {
12906         struct drm_crtc *crtc = pipe_config->uapi.crtc;
12907         struct drm_atomic_state *state = pipe_config->uapi.state;
12908         struct intel_encoder *encoder;
12909         struct drm_connector *connector;
12910         struct drm_connector_state *connector_state;
12911         int base_bpp, ret;
12912         int i;
12913         bool retry = true;
12914
12915         pipe_config->cpu_transcoder =
12916                 (enum transcoder) to_intel_crtc(crtc)->pipe;
12917
12918         /*
12919          * Sanitize sync polarity flags based on requested ones. If neither
12920          * positive or negative polarity is requested, treat this as meaning
12921          * negative polarity.
12922          */
12923         if (!(pipe_config->hw.adjusted_mode.flags &
12924               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
12925                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
12926
12927         if (!(pipe_config->hw.adjusted_mode.flags &
12928               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
12929                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
12930
12931         ret = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
12932                                         pipe_config);
12933         if (ret)
12934                 return ret;
12935
12936         base_bpp = pipe_config->pipe_bpp;
12937
12938         /*
12939          * Determine the real pipe dimensions. Note that stereo modes can
12940          * increase the actual pipe size due to the frame doubling and
12941          * insertion of additional space for blanks between the frame. This
12942          * is stored in the crtc timings. We use the requested mode to do this
12943          * computation to clearly distinguish it from the adjusted mode, which
12944          * can be changed by the connectors in the below retry loop.
12945          */
12946         drm_mode_get_hv_timing(&pipe_config->hw.mode,
12947                                &pipe_config->pipe_src_w,
12948                                &pipe_config->pipe_src_h);
12949
12950         for_each_new_connector_in_state(state, connector, connector_state, i) {
12951                 if (connector_state->crtc != crtc)
12952                         continue;
12953
12954                 encoder = to_intel_encoder(connector_state->best_encoder);
12955
12956                 if (!check_single_encoder_cloning(state, to_intel_crtc(crtc), encoder)) {
12957                         DRM_DEBUG_KMS("rejecting invalid cloning configuration\n");
12958                         return -EINVAL;
12959                 }
12960
12961                 /*
12962                  * Determine output_types before calling the .compute_config()
12963                  * hooks so that the hooks can use this information safely.
12964                  */
12965                 if (encoder->compute_output_type)
12966                         pipe_config->output_types |=
12967                                 BIT(encoder->compute_output_type(encoder, pipe_config,
12968                                                                  connector_state));
12969                 else
12970                         pipe_config->output_types |= BIT(encoder->type);
12971         }
12972
12973 encoder_retry:
12974         /* Ensure the port clock defaults are reset when retrying. */
12975         pipe_config->port_clock = 0;
12976         pipe_config->pixel_multiplier = 1;
12977
12978         /* Fill in default crtc timings, allow encoders to overwrite them. */
12979         drm_mode_set_crtcinfo(&pipe_config->hw.adjusted_mode,
12980                               CRTC_STEREO_DOUBLE);
12981
12982         /* Set the crtc_state defaults for trans_port_sync */
12983         pipe_config->master_transcoder = INVALID_TRANSCODER;
12984         ret = icl_add_sync_mode_crtcs(pipe_config);
12985         if (ret) {
12986                 DRM_DEBUG_KMS("Cannot assign Sync Mode CRTCs: %d\n",
12987                               ret);
12988                 return ret;
12989         }
12990
12991         /* Pass our mode to the connectors and the CRTC to give them a chance to
12992          * adjust it according to limitations or connector properties, and also
12993          * a chance to reject the mode entirely.
12994          */
12995         for_each_new_connector_in_state(state, connector, connector_state, i) {
12996                 if (connector_state->crtc != crtc)
12997                         continue;
12998
12999                 encoder = to_intel_encoder(connector_state->best_encoder);
13000                 ret = encoder->compute_config(encoder, pipe_config,
13001                                               connector_state);
13002                 if (ret < 0) {
13003                         if (ret != -EDEADLK)
13004                                 DRM_DEBUG_KMS("Encoder config failure: %d\n",
13005                                               ret);
13006                         return ret;
13007                 }
13008         }
13009
13010         /* Set default port clock if not overwritten by the encoder. Needs to be
13011          * done afterwards in case the encoder adjusts the mode. */
13012         if (!pipe_config->port_clock)
13013                 pipe_config->port_clock = pipe_config->hw.adjusted_mode.crtc_clock
13014                         * pipe_config->pixel_multiplier;
13015
13016         ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
13017         if (ret == -EDEADLK)
13018                 return ret;
13019         if (ret < 0) {
13020                 DRM_DEBUG_KMS("CRTC fixup failed\n");
13021                 return ret;
13022         }
13023
13024         if (ret == RETRY) {
13025                 if (WARN(!retry, "loop in pipe configuration computation\n"))
13026                         return -EINVAL;
13027
13028                 DRM_DEBUG_KMS("CRTC bw constrained, retrying\n");
13029                 retry = false;
13030                 goto encoder_retry;
13031         }
13032
13033         /* Dithering seems to not pass-through bits correctly when it should, so
13034          * only enable it on 6bpc panels and when its not a compliance
13035          * test requesting 6bpc video pattern.
13036          */
13037         pipe_config->dither = (pipe_config->pipe_bpp == 6*3) &&
13038                 !pipe_config->dither_force_disable;
13039         DRM_DEBUG_KMS("hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
13040                       base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
13041
13042         /*
13043          * Make drm_calc_timestamping_constants in
13044          * drm_atomic_helper_update_legacy_modeset_state() happy
13045          */
13046         pipe_config->uapi.adjusted_mode = pipe_config->hw.adjusted_mode;
13047
13048         return 0;
13049 }
13050
13051 bool intel_fuzzy_clock_check(int clock1, int clock2)
13052 {
13053         int diff;
13054
13055         if (clock1 == clock2)
13056                 return true;
13057
13058         if (!clock1 || !clock2)
13059                 return false;
13060
13061         diff = abs(clock1 - clock2);
13062
13063         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
13064                 return true;
13065
13066         return false;
13067 }
13068
13069 static bool
13070 intel_compare_m_n(unsigned int m, unsigned int n,
13071                   unsigned int m2, unsigned int n2,
13072                   bool exact)
13073 {
13074         if (m == m2 && n == n2)
13075                 return true;
13076
13077         if (exact || !m || !n || !m2 || !n2)
13078                 return false;
13079
13080         BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
13081
13082         if (n > n2) {
13083                 while (n > n2) {
13084                         m2 <<= 1;
13085                         n2 <<= 1;
13086                 }
13087         } else if (n < n2) {
13088                 while (n < n2) {
13089                         m <<= 1;
13090                         n <<= 1;
13091                 }
13092         }
13093
13094         if (n != n2)
13095                 return false;
13096
13097         return intel_fuzzy_clock_check(m, m2);
13098 }
13099
13100 static bool
13101 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
13102                        const struct intel_link_m_n *m2_n2,
13103                        bool exact)
13104 {
13105         return m_n->tu == m2_n2->tu &&
13106                 intel_compare_m_n(m_n->gmch_m, m_n->gmch_n,
13107                                   m2_n2->gmch_m, m2_n2->gmch_n, exact) &&
13108                 intel_compare_m_n(m_n->link_m, m_n->link_n,
13109                                   m2_n2->link_m, m2_n2->link_n, exact);
13110 }
13111
13112 static bool
13113 intel_compare_infoframe(const union hdmi_infoframe *a,
13114                         const union hdmi_infoframe *b)
13115 {
13116         return memcmp(a, b, sizeof(*a)) == 0;
13117 }
13118
13119 static void
13120 pipe_config_infoframe_mismatch(struct drm_i915_private *dev_priv,
13121                                bool fastset, const char *name,
13122                                const union hdmi_infoframe *a,
13123                                const union hdmi_infoframe *b)
13124 {
13125         if (fastset) {
13126                 if ((drm_debug & DRM_UT_KMS) == 0)
13127                         return;
13128
13129                 DRM_DEBUG_KMS("fastset mismatch in %s infoframe\n", name);
13130                 DRM_DEBUG_KMS("expected:\n");
13131                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, a);
13132                 DRM_DEBUG_KMS("found:\n");
13133                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, b);
13134         } else {
13135                 DRM_ERROR("mismatch in %s infoframe\n", name);
13136                 DRM_ERROR("expected:\n");
13137                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, a);
13138                 DRM_ERROR("found:\n");
13139                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, b);
13140         }
13141 }
13142
13143 static void __printf(4, 5)
13144 pipe_config_mismatch(bool fastset, const struct intel_crtc *crtc,
13145                      const char *name, const char *format, ...)
13146 {
13147         struct va_format vaf;
13148         va_list args;
13149
13150         va_start(args, format);
13151         vaf.fmt = format;
13152         vaf.va = &args;
13153
13154         if (fastset)
13155                 DRM_DEBUG_KMS("[CRTC:%d:%s] fastset mismatch in %s %pV\n",
13156                               crtc->base.base.id, crtc->base.name, name, &vaf);
13157         else
13158                 DRM_ERROR("[CRTC:%d:%s] mismatch in %s %pV\n",
13159                           crtc->base.base.id, crtc->base.name, name, &vaf);
13160
13161         va_end(args);
13162 }
13163
13164 static bool fastboot_enabled(struct drm_i915_private *dev_priv)
13165 {
13166         if (i915_modparams.fastboot != -1)
13167                 return i915_modparams.fastboot;
13168
13169         /* Enable fastboot by default on Skylake and newer */
13170         if (INTEL_GEN(dev_priv) >= 9)
13171                 return true;
13172
13173         /* Enable fastboot by default on VLV and CHV */
13174         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
13175                 return true;
13176
13177         /* Disabled by default on all others */
13178         return false;
13179 }
13180
13181 static bool
13182 intel_pipe_config_compare(const struct intel_crtc_state *current_config,
13183                           const struct intel_crtc_state *pipe_config,
13184                           bool fastset)
13185 {
13186         struct drm_i915_private *dev_priv = to_i915(current_config->uapi.crtc->dev);
13187         struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
13188         bool ret = true;
13189         u32 bp_gamma = 0;
13190         bool fixup_inherited = fastset &&
13191                 (current_config->hw.mode.private_flags & I915_MODE_FLAG_INHERITED) &&
13192                 !(pipe_config->hw.mode.private_flags & I915_MODE_FLAG_INHERITED);
13193
13194         if (fixup_inherited && !fastboot_enabled(dev_priv)) {
13195                 DRM_DEBUG_KMS("initial modeset and fastboot not set\n");
13196                 ret = false;
13197         }
13198
13199 #define PIPE_CONF_CHECK_X(name) do { \
13200         if (current_config->name != pipe_config->name) { \
13201                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13202                                      "(expected 0x%08x, found 0x%08x)", \
13203                                      current_config->name, \
13204                                      pipe_config->name); \
13205                 ret = false; \
13206         } \
13207 } while (0)
13208
13209 #define PIPE_CONF_CHECK_I(name) do { \
13210         if (current_config->name != pipe_config->name) { \
13211                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13212                                      "(expected %i, found %i)", \
13213                                      current_config->name, \
13214                                      pipe_config->name); \
13215                 ret = false; \
13216         } \
13217 } while (0)
13218
13219 #define PIPE_CONF_CHECK_BOOL(name) do { \
13220         if (current_config->name != pipe_config->name) { \
13221                 pipe_config_mismatch(fastset, crtc,  __stringify(name), \
13222                                      "(expected %s, found %s)", \
13223                                      yesno(current_config->name), \
13224                                      yesno(pipe_config->name)); \
13225                 ret = false; \
13226         } \
13227 } while (0)
13228
13229 /*
13230  * Checks state where we only read out the enabling, but not the entire
13231  * state itself (like full infoframes or ELD for audio). These states
13232  * require a full modeset on bootup to fix up.
13233  */
13234 #define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) do { \
13235         if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \
13236                 PIPE_CONF_CHECK_BOOL(name); \
13237         } else { \
13238                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13239                                      "unable to verify whether state matches exactly, forcing modeset (expected %s, found %s)", \
13240                                      yesno(current_config->name), \
13241                                      yesno(pipe_config->name)); \
13242                 ret = false; \
13243         } \
13244 } while (0)
13245
13246 #define PIPE_CONF_CHECK_P(name) do { \
13247         if (current_config->name != pipe_config->name) { \
13248                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13249                                      "(expected %p, found %p)", \
13250                                      current_config->name, \
13251                                      pipe_config->name); \
13252                 ret = false; \
13253         } \
13254 } while (0)
13255
13256 #define PIPE_CONF_CHECK_M_N(name) do { \
13257         if (!intel_compare_link_m_n(&current_config->name, \
13258                                     &pipe_config->name,\
13259                                     !fastset)) { \
13260                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13261                                      "(expected tu %i gmch %i/%i link %i/%i, " \
13262                                      "found tu %i, gmch %i/%i link %i/%i)", \
13263                                      current_config->name.tu, \
13264                                      current_config->name.gmch_m, \
13265                                      current_config->name.gmch_n, \
13266                                      current_config->name.link_m, \
13267                                      current_config->name.link_n, \
13268                                      pipe_config->name.tu, \
13269                                      pipe_config->name.gmch_m, \
13270                                      pipe_config->name.gmch_n, \
13271                                      pipe_config->name.link_m, \
13272                                      pipe_config->name.link_n); \
13273                 ret = false; \
13274         } \
13275 } while (0)
13276
13277 /* This is required for BDW+ where there is only one set of registers for
13278  * switching between high and low RR.
13279  * This macro can be used whenever a comparison has to be made between one
13280  * hw state and multiple sw state variables.
13281  */
13282 #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) do { \
13283         if (!intel_compare_link_m_n(&current_config->name, \
13284                                     &pipe_config->name, !fastset) && \
13285             !intel_compare_link_m_n(&current_config->alt_name, \
13286                                     &pipe_config->name, !fastset)) { \
13287                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13288                                      "(expected tu %i gmch %i/%i link %i/%i, " \
13289                                      "or tu %i gmch %i/%i link %i/%i, " \
13290                                      "found tu %i, gmch %i/%i link %i/%i)", \
13291                                      current_config->name.tu, \
13292                                      current_config->name.gmch_m, \
13293                                      current_config->name.gmch_n, \
13294                                      current_config->name.link_m, \
13295                                      current_config->name.link_n, \
13296                                      current_config->alt_name.tu, \
13297                                      current_config->alt_name.gmch_m, \
13298                                      current_config->alt_name.gmch_n, \
13299                                      current_config->alt_name.link_m, \
13300                                      current_config->alt_name.link_n, \
13301                                      pipe_config->name.tu, \
13302                                      pipe_config->name.gmch_m, \
13303                                      pipe_config->name.gmch_n, \
13304                                      pipe_config->name.link_m, \
13305                                      pipe_config->name.link_n); \
13306                 ret = false; \
13307         } \
13308 } while (0)
13309
13310 #define PIPE_CONF_CHECK_FLAGS(name, mask) do { \
13311         if ((current_config->name ^ pipe_config->name) & (mask)) { \
13312                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13313                                      "(%x) (expected %i, found %i)", \
13314                                      (mask), \
13315                                      current_config->name & (mask), \
13316                                      pipe_config->name & (mask)); \
13317                 ret = false; \
13318         } \
13319 } while (0)
13320
13321 #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) do { \
13322         if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
13323                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13324                                      "(expected %i, found %i)", \
13325                                      current_config->name, \
13326                                      pipe_config->name); \
13327                 ret = false; \
13328         } \
13329 } while (0)
13330
13331 #define PIPE_CONF_CHECK_INFOFRAME(name) do { \
13332         if (!intel_compare_infoframe(&current_config->infoframes.name, \
13333                                      &pipe_config->infoframes.name)) { \
13334                 pipe_config_infoframe_mismatch(dev_priv, fastset, __stringify(name), \
13335                                                &current_config->infoframes.name, \
13336                                                &pipe_config->infoframes.name); \
13337                 ret = false; \
13338         } \
13339 } while (0)
13340
13341 #define PIPE_CONF_CHECK_COLOR_LUT(name1, name2, bit_precision) do { \
13342         if (current_config->name1 != pipe_config->name1) { \
13343                 pipe_config_mismatch(fastset, crtc, __stringify(name1), \
13344                                 "(expected %i, found %i, won't compare lut values)", \
13345                                 current_config->name1, \
13346                                 pipe_config->name1); \
13347                 ret = false;\
13348         } else { \
13349                 if (!intel_color_lut_equal(current_config->name2, \
13350                                         pipe_config->name2, pipe_config->name1, \
13351                                         bit_precision)) { \
13352                         pipe_config_mismatch(fastset, crtc, __stringify(name2), \
13353                                         "hw_state doesn't match sw_state"); \
13354                         ret = false; \
13355                 } \
13356         } \
13357 } while (0)
13358
13359 #define PIPE_CONF_QUIRK(quirk) \
13360         ((current_config->quirks | pipe_config->quirks) & (quirk))
13361
13362         PIPE_CONF_CHECK_I(cpu_transcoder);
13363
13364         PIPE_CONF_CHECK_BOOL(has_pch_encoder);
13365         PIPE_CONF_CHECK_I(fdi_lanes);
13366         PIPE_CONF_CHECK_M_N(fdi_m_n);
13367
13368         PIPE_CONF_CHECK_I(lane_count);
13369         PIPE_CONF_CHECK_X(lane_lat_optim_mask);
13370
13371         if (INTEL_GEN(dev_priv) < 8) {
13372                 PIPE_CONF_CHECK_M_N(dp_m_n);
13373
13374                 if (current_config->has_drrs)
13375                         PIPE_CONF_CHECK_M_N(dp_m2_n2);
13376         } else
13377                 PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
13378
13379         PIPE_CONF_CHECK_X(output_types);
13380
13381         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hdisplay);
13382         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_htotal);
13383         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hblank_start);
13384         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hblank_end);
13385         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hsync_start);
13386         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hsync_end);
13387
13388         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vdisplay);
13389         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vtotal);
13390         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vblank_start);
13391         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vblank_end);
13392         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vsync_start);
13393         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vsync_end);
13394
13395         PIPE_CONF_CHECK_I(pixel_multiplier);
13396         PIPE_CONF_CHECK_I(output_format);
13397         PIPE_CONF_CHECK_I(dc3co_exitline);
13398         PIPE_CONF_CHECK_BOOL(has_hdmi_sink);
13399         if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
13400             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
13401                 PIPE_CONF_CHECK_BOOL(limited_color_range);
13402
13403         PIPE_CONF_CHECK_BOOL(hdmi_scrambling);
13404         PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio);
13405         PIPE_CONF_CHECK_BOOL(has_infoframe);
13406         PIPE_CONF_CHECK_BOOL(fec_enable);
13407
13408         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio);
13409
13410         PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13411                               DRM_MODE_FLAG_INTERLACE);
13412
13413         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
13414                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13415                                       DRM_MODE_FLAG_PHSYNC);
13416                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13417                                       DRM_MODE_FLAG_NHSYNC);
13418                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13419                                       DRM_MODE_FLAG_PVSYNC);
13420                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13421                                       DRM_MODE_FLAG_NVSYNC);
13422         }
13423
13424         PIPE_CONF_CHECK_X(gmch_pfit.control);
13425         /* pfit ratios are autocomputed by the hw on gen4+ */
13426         if (INTEL_GEN(dev_priv) < 4)
13427                 PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios);
13428         PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
13429
13430         /*
13431          * Changing the EDP transcoder input mux
13432          * (A_ONOFF vs. A_ON) requires a full modeset.
13433          */
13434         PIPE_CONF_CHECK_BOOL(pch_pfit.force_thru);
13435
13436         if (!fastset) {
13437                 PIPE_CONF_CHECK_I(pipe_src_w);
13438                 PIPE_CONF_CHECK_I(pipe_src_h);
13439
13440                 PIPE_CONF_CHECK_BOOL(pch_pfit.enabled);
13441                 if (current_config->pch_pfit.enabled) {
13442                         PIPE_CONF_CHECK_X(pch_pfit.pos);
13443                         PIPE_CONF_CHECK_X(pch_pfit.size);
13444                 }
13445
13446                 PIPE_CONF_CHECK_I(scaler_state.scaler_id);
13447                 PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate);
13448
13449                 PIPE_CONF_CHECK_X(gamma_mode);
13450                 if (IS_CHERRYVIEW(dev_priv))
13451                         PIPE_CONF_CHECK_X(cgm_mode);
13452                 else
13453                         PIPE_CONF_CHECK_X(csc_mode);
13454                 PIPE_CONF_CHECK_BOOL(gamma_enable);
13455                 PIPE_CONF_CHECK_BOOL(csc_enable);
13456
13457                 bp_gamma = intel_color_get_gamma_bit_precision(pipe_config);
13458                 if (bp_gamma)
13459                         PIPE_CONF_CHECK_COLOR_LUT(gamma_mode, hw.gamma_lut, bp_gamma);
13460
13461         }
13462
13463         PIPE_CONF_CHECK_BOOL(double_wide);
13464
13465         PIPE_CONF_CHECK_P(shared_dpll);
13466         PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
13467         PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
13468         PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
13469         PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
13470         PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
13471         PIPE_CONF_CHECK_X(dpll_hw_state.spll);
13472         PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
13473         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
13474         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
13475         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0);
13476         PIPE_CONF_CHECK_X(dpll_hw_state.ebb0);
13477         PIPE_CONF_CHECK_X(dpll_hw_state.ebb4);
13478         PIPE_CONF_CHECK_X(dpll_hw_state.pll0);
13479         PIPE_CONF_CHECK_X(dpll_hw_state.pll1);
13480         PIPE_CONF_CHECK_X(dpll_hw_state.pll2);
13481         PIPE_CONF_CHECK_X(dpll_hw_state.pll3);
13482         PIPE_CONF_CHECK_X(dpll_hw_state.pll6);
13483         PIPE_CONF_CHECK_X(dpll_hw_state.pll8);
13484         PIPE_CONF_CHECK_X(dpll_hw_state.pll9);
13485         PIPE_CONF_CHECK_X(dpll_hw_state.pll10);
13486         PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12);
13487         PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl);
13488         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1);
13489         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl);
13490         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0);
13491         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1);
13492         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf);
13493         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock);
13494         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc);
13495         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias);
13496         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias);
13497
13498         PIPE_CONF_CHECK_X(dsi_pll.ctrl);
13499         PIPE_CONF_CHECK_X(dsi_pll.div);
13500
13501         if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5)
13502                 PIPE_CONF_CHECK_I(pipe_bpp);
13503
13504         PIPE_CONF_CHECK_CLOCK_FUZZY(hw.adjusted_mode.crtc_clock);
13505         PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
13506
13507         PIPE_CONF_CHECK_I(min_voltage_level);
13508
13509         PIPE_CONF_CHECK_X(infoframes.enable);
13510         PIPE_CONF_CHECK_X(infoframes.gcp);
13511         PIPE_CONF_CHECK_INFOFRAME(avi);
13512         PIPE_CONF_CHECK_INFOFRAME(spd);
13513         PIPE_CONF_CHECK_INFOFRAME(hdmi);
13514         PIPE_CONF_CHECK_INFOFRAME(drm);
13515
13516         PIPE_CONF_CHECK_I(sync_mode_slaves_mask);
13517         PIPE_CONF_CHECK_I(master_transcoder);
13518
13519         PIPE_CONF_CHECK_I(dsc.compression_enable);
13520         PIPE_CONF_CHECK_I(dsc.dsc_split);
13521         PIPE_CONF_CHECK_I(dsc.compressed_bpp);
13522
13523         PIPE_CONF_CHECK_I(mst_master_transcoder);
13524
13525 #undef PIPE_CONF_CHECK_X
13526 #undef PIPE_CONF_CHECK_I
13527 #undef PIPE_CONF_CHECK_BOOL
13528 #undef PIPE_CONF_CHECK_BOOL_INCOMPLETE
13529 #undef PIPE_CONF_CHECK_P
13530 #undef PIPE_CONF_CHECK_FLAGS
13531 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
13532 #undef PIPE_CONF_CHECK_COLOR_LUT
13533 #undef PIPE_CONF_QUIRK
13534
13535         return ret;
13536 }
13537
13538 static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv,
13539                                            const struct intel_crtc_state *pipe_config)
13540 {
13541         if (pipe_config->has_pch_encoder) {
13542                 int fdi_dotclock = intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
13543                                                             &pipe_config->fdi_m_n);
13544                 int dotclock = pipe_config->hw.adjusted_mode.crtc_clock;
13545
13546                 /*
13547                  * FDI already provided one idea for the dotclock.
13548                  * Yell if the encoder disagrees.
13549                  */
13550                 WARN(!intel_fuzzy_clock_check(fdi_dotclock, dotclock),
13551                      "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
13552                      fdi_dotclock, dotclock);
13553         }
13554 }
13555
13556 static void verify_wm_state(struct intel_crtc *crtc,
13557                             struct intel_crtc_state *new_crtc_state)
13558 {
13559         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13560         struct skl_hw_state {
13561                 struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
13562                 struct skl_ddb_entry ddb_uv[I915_MAX_PLANES];
13563                 struct skl_ddb_allocation ddb;
13564                 struct skl_pipe_wm wm;
13565         } *hw;
13566         struct skl_ddb_allocation *sw_ddb;
13567         struct skl_pipe_wm *sw_wm;
13568         struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
13569         const enum pipe pipe = crtc->pipe;
13570         int plane, level, max_level = ilk_wm_max_level(dev_priv);
13571
13572         if (INTEL_GEN(dev_priv) < 9 || !new_crtc_state->hw.active)
13573                 return;
13574
13575         hw = kzalloc(sizeof(*hw), GFP_KERNEL);
13576         if (!hw)
13577                 return;
13578
13579         skl_pipe_wm_get_hw_state(crtc, &hw->wm);
13580         sw_wm = &new_crtc_state->wm.skl.optimal;
13581
13582         skl_pipe_ddb_get_hw_state(crtc, hw->ddb_y, hw->ddb_uv);
13583
13584         skl_ddb_get_hw_state(dev_priv, &hw->ddb);
13585         sw_ddb = &dev_priv->wm.skl_hw.ddb;
13586
13587         if (INTEL_GEN(dev_priv) >= 11 &&
13588             hw->ddb.enabled_slices != sw_ddb->enabled_slices)
13589                 DRM_ERROR("mismatch in DBUF Slices (expected %u, got %u)\n",
13590                           sw_ddb->enabled_slices,
13591                           hw->ddb.enabled_slices);
13592
13593         /* planes */
13594         for_each_universal_plane(dev_priv, pipe, plane) {
13595                 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
13596
13597                 hw_plane_wm = &hw->wm.planes[plane];
13598                 sw_plane_wm = &sw_wm->planes[plane];
13599
13600                 /* Watermarks */
13601                 for (level = 0; level <= max_level; level++) {
13602                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
13603                                                 &sw_plane_wm->wm[level]))
13604                                 continue;
13605
13606                         DRM_ERROR("mismatch in WM pipe %c plane %d level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
13607                                   pipe_name(pipe), plane + 1, level,
13608                                   sw_plane_wm->wm[level].plane_en,
13609                                   sw_plane_wm->wm[level].plane_res_b,
13610                                   sw_plane_wm->wm[level].plane_res_l,
13611                                   hw_plane_wm->wm[level].plane_en,
13612                                   hw_plane_wm->wm[level].plane_res_b,
13613                                   hw_plane_wm->wm[level].plane_res_l);
13614                 }
13615
13616                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
13617                                          &sw_plane_wm->trans_wm)) {
13618                         DRM_ERROR("mismatch in trans WM pipe %c plane %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
13619                                   pipe_name(pipe), plane + 1,
13620                                   sw_plane_wm->trans_wm.plane_en,
13621                                   sw_plane_wm->trans_wm.plane_res_b,
13622                                   sw_plane_wm->trans_wm.plane_res_l,
13623                                   hw_plane_wm->trans_wm.plane_en,
13624                                   hw_plane_wm->trans_wm.plane_res_b,
13625                                   hw_plane_wm->trans_wm.plane_res_l);
13626                 }
13627
13628                 /* DDB */
13629                 hw_ddb_entry = &hw->ddb_y[plane];
13630                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[plane];
13631
13632                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
13633                         DRM_ERROR("mismatch in DDB state pipe %c plane %d (expected (%u,%u), found (%u,%u))\n",
13634                                   pipe_name(pipe), plane + 1,
13635                                   sw_ddb_entry->start, sw_ddb_entry->end,
13636                                   hw_ddb_entry->start, hw_ddb_entry->end);
13637                 }
13638         }
13639
13640         /*
13641          * cursor
13642          * If the cursor plane isn't active, we may not have updated it's ddb
13643          * allocation. In that case since the ddb allocation will be updated
13644          * once the plane becomes visible, we can skip this check
13645          */
13646         if (1) {
13647                 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
13648
13649                 hw_plane_wm = &hw->wm.planes[PLANE_CURSOR];
13650                 sw_plane_wm = &sw_wm->planes[PLANE_CURSOR];
13651
13652                 /* Watermarks */
13653                 for (level = 0; level <= max_level; level++) {
13654                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
13655                                                 &sw_plane_wm->wm[level]))
13656                                 continue;
13657
13658                         DRM_ERROR("mismatch in WM pipe %c cursor level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
13659                                   pipe_name(pipe), level,
13660                                   sw_plane_wm->wm[level].plane_en,
13661                                   sw_plane_wm->wm[level].plane_res_b,
13662                                   sw_plane_wm->wm[level].plane_res_l,
13663                                   hw_plane_wm->wm[level].plane_en,
13664                                   hw_plane_wm->wm[level].plane_res_b,
13665                                   hw_plane_wm->wm[level].plane_res_l);
13666                 }
13667
13668                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
13669                                          &sw_plane_wm->trans_wm)) {
13670                         DRM_ERROR("mismatch in trans WM pipe %c cursor (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
13671                                   pipe_name(pipe),
13672                                   sw_plane_wm->trans_wm.plane_en,
13673                                   sw_plane_wm->trans_wm.plane_res_b,
13674                                   sw_plane_wm->trans_wm.plane_res_l,
13675                                   hw_plane_wm->trans_wm.plane_en,
13676                                   hw_plane_wm->trans_wm.plane_res_b,
13677                                   hw_plane_wm->trans_wm.plane_res_l);
13678                 }
13679
13680                 /* DDB */
13681                 hw_ddb_entry = &hw->ddb_y[PLANE_CURSOR];
13682                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR];
13683
13684                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
13685                         DRM_ERROR("mismatch in DDB state pipe %c cursor (expected (%u,%u), found (%u,%u))\n",
13686                                   pipe_name(pipe),
13687                                   sw_ddb_entry->start, sw_ddb_entry->end,
13688                                   hw_ddb_entry->start, hw_ddb_entry->end);
13689                 }
13690         }
13691
13692         kfree(hw);
13693 }
13694
13695 static void
13696 verify_connector_state(struct intel_atomic_state *state,
13697                        struct intel_crtc *crtc)
13698 {
13699         struct drm_connector *connector;
13700         struct drm_connector_state *new_conn_state;
13701         int i;
13702
13703         for_each_new_connector_in_state(&state->base, connector, new_conn_state, i) {
13704                 struct drm_encoder *encoder = connector->encoder;
13705                 struct intel_crtc_state *crtc_state = NULL;
13706
13707                 if (new_conn_state->crtc != &crtc->base)
13708                         continue;
13709
13710                 if (crtc)
13711                         crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
13712
13713                 intel_connector_verify_state(crtc_state, new_conn_state);
13714
13715                 I915_STATE_WARN(new_conn_state->best_encoder != encoder,
13716                      "connector's atomic encoder doesn't match legacy encoder\n");
13717         }
13718 }
13719
13720 static void
13721 verify_encoder_state(struct drm_i915_private *dev_priv, struct intel_atomic_state *state)
13722 {
13723         struct intel_encoder *encoder;
13724         struct drm_connector *connector;
13725         struct drm_connector_state *old_conn_state, *new_conn_state;
13726         int i;
13727
13728         for_each_intel_encoder(&dev_priv->drm, encoder) {
13729                 bool enabled = false, found = false;
13730                 enum pipe pipe;
13731
13732                 DRM_DEBUG_KMS("[ENCODER:%d:%s]\n",
13733                               encoder->base.base.id,
13734                               encoder->base.name);
13735
13736                 for_each_oldnew_connector_in_state(&state->base, connector, old_conn_state,
13737                                                    new_conn_state, i) {
13738                         if (old_conn_state->best_encoder == &encoder->base)
13739                                 found = true;
13740
13741                         if (new_conn_state->best_encoder != &encoder->base)
13742                                 continue;
13743                         found = enabled = true;
13744
13745                         I915_STATE_WARN(new_conn_state->crtc !=
13746                                         encoder->base.crtc,
13747                              "connector's crtc doesn't match encoder crtc\n");
13748                 }
13749
13750                 if (!found)
13751                         continue;
13752
13753                 I915_STATE_WARN(!!encoder->base.crtc != enabled,
13754                      "encoder's enabled state mismatch "
13755                      "(expected %i, found %i)\n",
13756                      !!encoder->base.crtc, enabled);
13757
13758                 if (!encoder->base.crtc) {
13759                         bool active;
13760
13761                         active = encoder->get_hw_state(encoder, &pipe);
13762                         I915_STATE_WARN(active,
13763                              "encoder detached but still enabled on pipe %c.\n",
13764                              pipe_name(pipe));
13765                 }
13766         }
13767 }
13768
13769 static void
13770 verify_crtc_state(struct intel_crtc *crtc,
13771                   struct intel_crtc_state *old_crtc_state,
13772                   struct intel_crtc_state *new_crtc_state)
13773 {
13774         struct drm_device *dev = crtc->base.dev;
13775         struct drm_i915_private *dev_priv = to_i915(dev);
13776         struct intel_encoder *encoder;
13777         struct intel_crtc_state *pipe_config = old_crtc_state;
13778         struct drm_atomic_state *state = old_crtc_state->uapi.state;
13779         bool active;
13780
13781         __drm_atomic_helper_crtc_destroy_state(&old_crtc_state->uapi);
13782         intel_crtc_free_hw_state(old_crtc_state);
13783         intel_crtc_state_reset(old_crtc_state, crtc);
13784         old_crtc_state->uapi.state = state;
13785
13786         DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.base.id, crtc->base.name);
13787
13788         active = dev_priv->display.get_pipe_config(crtc, pipe_config);
13789
13790         /* we keep both pipes enabled on 830 */
13791         if (IS_I830(dev_priv))
13792                 active = new_crtc_state->hw.active;
13793
13794         I915_STATE_WARN(new_crtc_state->hw.active != active,
13795                         "crtc active state doesn't match with hw state "
13796                         "(expected %i, found %i)\n",
13797                         new_crtc_state->hw.active, active);
13798
13799         I915_STATE_WARN(crtc->active != new_crtc_state->hw.active,
13800                         "transitional active state does not match atomic hw state "
13801                         "(expected %i, found %i)\n",
13802                         new_crtc_state->hw.active, crtc->active);
13803
13804         for_each_encoder_on_crtc(dev, &crtc->base, encoder) {
13805                 enum pipe pipe;
13806
13807                 active = encoder->get_hw_state(encoder, &pipe);
13808                 I915_STATE_WARN(active != new_crtc_state->hw.active,
13809                                 "[ENCODER:%i] active %i with crtc active %i\n",
13810                                 encoder->base.base.id, active,
13811                                 new_crtc_state->hw.active);
13812
13813                 I915_STATE_WARN(active && crtc->pipe != pipe,
13814                                 "Encoder connected to wrong pipe %c\n",
13815                                 pipe_name(pipe));
13816
13817                 if (active)
13818                         encoder->get_config(encoder, pipe_config);
13819         }
13820
13821         intel_crtc_compute_pixel_rate(pipe_config);
13822
13823         if (!new_crtc_state->hw.active)
13824                 return;
13825
13826         intel_pipe_config_sanity_check(dev_priv, pipe_config);
13827
13828         if (!intel_pipe_config_compare(new_crtc_state,
13829                                        pipe_config, false)) {
13830                 I915_STATE_WARN(1, "pipe state doesn't match!\n");
13831                 intel_dump_pipe_config(pipe_config, NULL, "[hw state]");
13832                 intel_dump_pipe_config(new_crtc_state, NULL, "[sw state]");
13833         }
13834 }
13835
13836 static void
13837 intel_verify_planes(struct intel_atomic_state *state)
13838 {
13839         struct intel_plane *plane;
13840         const struct intel_plane_state *plane_state;
13841         int i;
13842
13843         for_each_new_intel_plane_in_state(state, plane,
13844                                           plane_state, i)
13845                 assert_plane(plane, plane_state->planar_slave ||
13846                              plane_state->uapi.visible);
13847 }
13848
13849 static void
13850 verify_single_dpll_state(struct drm_i915_private *dev_priv,
13851                          struct intel_shared_dpll *pll,
13852                          struct intel_crtc *crtc,
13853                          struct intel_crtc_state *new_crtc_state)
13854 {
13855         struct intel_dpll_hw_state dpll_hw_state;
13856         unsigned int crtc_mask;
13857         bool active;
13858
13859         memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
13860
13861         DRM_DEBUG_KMS("%s\n", pll->info->name);
13862
13863         active = pll->info->funcs->get_hw_state(dev_priv, pll, &dpll_hw_state);
13864
13865         if (!(pll->info->flags & INTEL_DPLL_ALWAYS_ON)) {
13866                 I915_STATE_WARN(!pll->on && pll->active_mask,
13867                      "pll in active use but not on in sw tracking\n");
13868                 I915_STATE_WARN(pll->on && !pll->active_mask,
13869                      "pll is on but not used by any active crtc\n");
13870                 I915_STATE_WARN(pll->on != active,
13871                      "pll on state mismatch (expected %i, found %i)\n",
13872                      pll->on, active);
13873         }
13874
13875         if (!crtc) {
13876                 I915_STATE_WARN(pll->active_mask & ~pll->state.crtc_mask,
13877                                 "more active pll users than references: %x vs %x\n",
13878                                 pll->active_mask, pll->state.crtc_mask);
13879
13880                 return;
13881         }
13882
13883         crtc_mask = drm_crtc_mask(&crtc->base);
13884
13885         if (new_crtc_state->hw.active)
13886                 I915_STATE_WARN(!(pll->active_mask & crtc_mask),
13887                                 "pll active mismatch (expected pipe %c in active mask 0x%02x)\n",
13888                                 pipe_name(drm_crtc_index(&crtc->base)), pll->active_mask);
13889         else
13890                 I915_STATE_WARN(pll->active_mask & crtc_mask,
13891                                 "pll active mismatch (didn't expect pipe %c in active mask 0x%02x)\n",
13892                                 pipe_name(drm_crtc_index(&crtc->base)), pll->active_mask);
13893
13894         I915_STATE_WARN(!(pll->state.crtc_mask & crtc_mask),
13895                         "pll enabled crtcs mismatch (expected 0x%x in 0x%02x)\n",
13896                         crtc_mask, pll->state.crtc_mask);
13897
13898         I915_STATE_WARN(pll->on && memcmp(&pll->state.hw_state,
13899                                           &dpll_hw_state,
13900                                           sizeof(dpll_hw_state)),
13901                         "pll hw state mismatch\n");
13902 }
13903
13904 static void
13905 verify_shared_dpll_state(struct intel_crtc *crtc,
13906                          struct intel_crtc_state *old_crtc_state,
13907                          struct intel_crtc_state *new_crtc_state)
13908 {
13909         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13910
13911         if (new_crtc_state->shared_dpll)
13912                 verify_single_dpll_state(dev_priv, new_crtc_state->shared_dpll, crtc, new_crtc_state);
13913
13914         if (old_crtc_state->shared_dpll &&
13915             old_crtc_state->shared_dpll != new_crtc_state->shared_dpll) {
13916                 unsigned int crtc_mask = drm_crtc_mask(&crtc->base);
13917                 struct intel_shared_dpll *pll = old_crtc_state->shared_dpll;
13918
13919                 I915_STATE_WARN(pll->active_mask & crtc_mask,
13920                                 "pll active mismatch (didn't expect pipe %c in active mask)\n",
13921                                 pipe_name(drm_crtc_index(&crtc->base)));
13922                 I915_STATE_WARN(pll->state.crtc_mask & crtc_mask,
13923                                 "pll enabled crtcs mismatch (found %x in enabled mask)\n",
13924                                 pipe_name(drm_crtc_index(&crtc->base)));
13925         }
13926 }
13927
13928 static void
13929 intel_modeset_verify_crtc(struct intel_crtc *crtc,
13930                           struct intel_atomic_state *state,
13931                           struct intel_crtc_state *old_crtc_state,
13932                           struct intel_crtc_state *new_crtc_state)
13933 {
13934         if (!needs_modeset(new_crtc_state) && !new_crtc_state->update_pipe)
13935                 return;
13936
13937         verify_wm_state(crtc, new_crtc_state);
13938         verify_connector_state(state, crtc);
13939         verify_crtc_state(crtc, old_crtc_state, new_crtc_state);
13940         verify_shared_dpll_state(crtc, old_crtc_state, new_crtc_state);
13941 }
13942
13943 static void
13944 verify_disabled_dpll_state(struct drm_i915_private *dev_priv)
13945 {
13946         int i;
13947
13948         for (i = 0; i < dev_priv->num_shared_dpll; i++)
13949                 verify_single_dpll_state(dev_priv, &dev_priv->shared_dplls[i], NULL, NULL);
13950 }
13951
13952 static void
13953 intel_modeset_verify_disabled(struct drm_i915_private *dev_priv,
13954                               struct intel_atomic_state *state)
13955 {
13956         verify_encoder_state(dev_priv, state);
13957         verify_connector_state(state, NULL);
13958         verify_disabled_dpll_state(dev_priv);
13959 }
13960
13961 static void
13962 intel_crtc_update_active_timings(const struct intel_crtc_state *crtc_state)
13963 {
13964         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
13965         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13966         const struct drm_display_mode *adjusted_mode =
13967                 &crtc_state->hw.adjusted_mode;
13968
13969         drm_calc_timestamping_constants(&crtc->base, adjusted_mode);
13970
13971         /*
13972          * The scanline counter increments at the leading edge of hsync.
13973          *
13974          * On most platforms it starts counting from vtotal-1 on the
13975          * first active line. That means the scanline counter value is
13976          * always one less than what we would expect. Ie. just after
13977          * start of vblank, which also occurs at start of hsync (on the
13978          * last active line), the scanline counter will read vblank_start-1.
13979          *
13980          * On gen2 the scanline counter starts counting from 1 instead
13981          * of vtotal-1, so we have to subtract one (or rather add vtotal-1
13982          * to keep the value positive), instead of adding one.
13983          *
13984          * On HSW+ the behaviour of the scanline counter depends on the output
13985          * type. For DP ports it behaves like most other platforms, but on HDMI
13986          * there's an extra 1 line difference. So we need to add two instead of
13987          * one to the value.
13988          *
13989          * On VLV/CHV DSI the scanline counter would appear to increment
13990          * approx. 1/3 of a scanline before start of vblank. Unfortunately
13991          * that means we can't tell whether we're in vblank or not while
13992          * we're on that particular line. We must still set scanline_offset
13993          * to 1 so that the vblank timestamps come out correct when we query
13994          * the scanline counter from within the vblank interrupt handler.
13995          * However if queried just before the start of vblank we'll get an
13996          * answer that's slightly in the future.
13997          */
13998         if (IS_GEN(dev_priv, 2)) {
13999                 int vtotal;
14000
14001                 vtotal = adjusted_mode->crtc_vtotal;
14002                 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
14003                         vtotal /= 2;
14004
14005                 crtc->scanline_offset = vtotal - 1;
14006         } else if (HAS_DDI(dev_priv) &&
14007                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
14008                 crtc->scanline_offset = 2;
14009         } else {
14010                 crtc->scanline_offset = 1;
14011         }
14012 }
14013
14014 static void intel_modeset_clear_plls(struct intel_atomic_state *state)
14015 {
14016         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14017         struct intel_crtc_state *new_crtc_state;
14018         struct intel_crtc *crtc;
14019         int i;
14020
14021         if (!dev_priv->display.crtc_compute_clock)
14022                 return;
14023
14024         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
14025                 if (!needs_modeset(new_crtc_state))
14026                         continue;
14027
14028                 intel_release_shared_dplls(state, crtc);
14029         }
14030 }
14031
14032 /*
14033  * This implements the workaround described in the "notes" section of the mode
14034  * set sequence documentation. When going from no pipes or single pipe to
14035  * multiple pipes, and planes are enabled after the pipe, we need to wait at
14036  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
14037  */
14038 static int hsw_mode_set_planes_workaround(struct intel_atomic_state *state)
14039 {
14040         struct intel_crtc_state *crtc_state;
14041         struct intel_crtc *crtc;
14042         struct intel_crtc_state *first_crtc_state = NULL;
14043         struct intel_crtc_state *other_crtc_state = NULL;
14044         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
14045         int i;
14046
14047         /* look at all crtc's that are going to be enabled in during modeset */
14048         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
14049                 if (!crtc_state->hw.active ||
14050                     !needs_modeset(crtc_state))
14051                         continue;
14052
14053                 if (first_crtc_state) {
14054                         other_crtc_state = crtc_state;
14055                         break;
14056                 } else {
14057                         first_crtc_state = crtc_state;
14058                         first_pipe = crtc->pipe;
14059                 }
14060         }
14061
14062         /* No workaround needed? */
14063         if (!first_crtc_state)
14064                 return 0;
14065
14066         /* w/a possibly needed, check how many crtc's are already enabled. */
14067         for_each_intel_crtc(state->base.dev, crtc) {
14068                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
14069                 if (IS_ERR(crtc_state))
14070                         return PTR_ERR(crtc_state);
14071
14072                 crtc_state->hsw_workaround_pipe = INVALID_PIPE;
14073
14074                 if (!crtc_state->hw.active ||
14075                     needs_modeset(crtc_state))
14076                         continue;
14077
14078                 /* 2 or more enabled crtcs means no need for w/a */
14079                 if (enabled_pipe != INVALID_PIPE)
14080                         return 0;
14081
14082                 enabled_pipe = crtc->pipe;
14083         }
14084
14085         if (enabled_pipe != INVALID_PIPE)
14086                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
14087         else if (other_crtc_state)
14088                 other_crtc_state->hsw_workaround_pipe = first_pipe;
14089
14090         return 0;
14091 }
14092
14093 static int intel_modeset_checks(struct intel_atomic_state *state)
14094 {
14095         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14096         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14097         struct intel_crtc *crtc;
14098         int ret, i;
14099
14100         /* keep the current setting */
14101         if (!state->cdclk.force_min_cdclk_changed)
14102                 state->cdclk.force_min_cdclk = dev_priv->cdclk.force_min_cdclk;
14103
14104         state->modeset = true;
14105         state->active_pipes = dev_priv->active_pipes;
14106         state->cdclk.logical = dev_priv->cdclk.logical;
14107         state->cdclk.actual = dev_priv->cdclk.actual;
14108
14109         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14110                                             new_crtc_state, i) {
14111                 if (new_crtc_state->hw.active)
14112                         state->active_pipes |= BIT(crtc->pipe);
14113                 else
14114                         state->active_pipes &= ~BIT(crtc->pipe);
14115
14116                 if (old_crtc_state->hw.active != new_crtc_state->hw.active)
14117                         state->active_pipe_changes |= BIT(crtc->pipe);
14118         }
14119
14120         if (state->active_pipe_changes) {
14121                 ret = intel_atomic_lock_global_state(state);
14122                 if (ret)
14123                         return ret;
14124         }
14125
14126         ret = intel_modeset_calc_cdclk(state);
14127         if (ret)
14128                 return ret;
14129
14130         intel_modeset_clear_plls(state);
14131
14132         if (IS_HASWELL(dev_priv))
14133                 return hsw_mode_set_planes_workaround(state);
14134
14135         return 0;
14136 }
14137
14138 /*
14139  * Handle calculation of various watermark data at the end of the atomic check
14140  * phase.  The code here should be run after the per-crtc and per-plane 'check'
14141  * handlers to ensure that all derived state has been updated.
14142  */
14143 static int calc_watermark_data(struct intel_atomic_state *state)
14144 {
14145         struct drm_device *dev = state->base.dev;
14146         struct drm_i915_private *dev_priv = to_i915(dev);
14147
14148         /* Is there platform-specific watermark information to calculate? */
14149         if (dev_priv->display.compute_global_watermarks)
14150                 return dev_priv->display.compute_global_watermarks(state);
14151
14152         return 0;
14153 }
14154
14155 static void intel_crtc_check_fastset(const struct intel_crtc_state *old_crtc_state,
14156                                      struct intel_crtc_state *new_crtc_state)
14157 {
14158         if (!intel_pipe_config_compare(old_crtc_state, new_crtc_state, true))
14159                 return;
14160
14161         new_crtc_state->uapi.mode_changed = false;
14162         new_crtc_state->update_pipe = true;
14163 }
14164
14165 static void intel_crtc_copy_fastset(const struct intel_crtc_state *old_crtc_state,
14166                                     struct intel_crtc_state *new_crtc_state)
14167 {
14168         /*
14169          * If we're not doing the full modeset we want to
14170          * keep the current M/N values as they may be
14171          * sufficiently different to the computed values
14172          * to cause problems.
14173          *
14174          * FIXME: should really copy more fuzzy state here
14175          */
14176         new_crtc_state->fdi_m_n = old_crtc_state->fdi_m_n;
14177         new_crtc_state->dp_m_n = old_crtc_state->dp_m_n;
14178         new_crtc_state->dp_m2_n2 = old_crtc_state->dp_m2_n2;
14179         new_crtc_state->has_drrs = old_crtc_state->has_drrs;
14180 }
14181
14182 static int intel_crtc_add_planes_to_state(struct intel_atomic_state *state,
14183                                           struct intel_crtc *crtc,
14184                                           u8 plane_ids_mask)
14185 {
14186         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14187         struct intel_plane *plane;
14188
14189         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
14190                 struct intel_plane_state *plane_state;
14191
14192                 if ((plane_ids_mask & BIT(plane->id)) == 0)
14193                         continue;
14194
14195                 plane_state = intel_atomic_get_plane_state(state, plane);
14196                 if (IS_ERR(plane_state))
14197                         return PTR_ERR(plane_state);
14198         }
14199
14200         return 0;
14201 }
14202
14203 static bool active_planes_affects_min_cdclk(struct drm_i915_private *dev_priv)
14204 {
14205         /* See {hsw,vlv,ivb}_plane_ratio() */
14206         return IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv) ||
14207                 IS_CHERRYVIEW(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
14208                 IS_IVYBRIDGE(dev_priv);
14209 }
14210
14211 static int intel_atomic_check_planes(struct intel_atomic_state *state,
14212                                      bool *need_modeset)
14213 {
14214         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14215         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14216         struct intel_plane_state *plane_state;
14217         struct intel_plane *plane;
14218         struct intel_crtc *crtc;
14219         int i, ret;
14220
14221         ret = icl_add_linked_planes(state);
14222         if (ret)
14223                 return ret;
14224
14225         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
14226                 ret = intel_plane_atomic_check(state, plane);
14227                 if (ret) {
14228                         DRM_DEBUG_ATOMIC("[PLANE:%d:%s] atomic driver check failed\n",
14229                                          plane->base.base.id, plane->base.name);
14230                         return ret;
14231                 }
14232         }
14233
14234         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14235                                             new_crtc_state, i) {
14236                 u8 old_active_planes, new_active_planes;
14237
14238                 ret = icl_check_nv12_planes(new_crtc_state);
14239                 if (ret)
14240                         return ret;
14241
14242                 /*
14243                  * On some platforms the number of active planes affects
14244                  * the planes' minimum cdclk calculation. Add such planes
14245                  * to the state before we compute the minimum cdclk.
14246                  */
14247                 if (!active_planes_affects_min_cdclk(dev_priv))
14248                         continue;
14249
14250                 old_active_planes = old_crtc_state->active_planes & ~BIT(PLANE_CURSOR);
14251                 new_active_planes = new_crtc_state->active_planes & ~BIT(PLANE_CURSOR);
14252
14253                 if (hweight8(old_active_planes) == hweight8(new_active_planes))
14254                         continue;
14255
14256                 ret = intel_crtc_add_planes_to_state(state, crtc, new_active_planes);
14257                 if (ret)
14258                         return ret;
14259         }
14260
14261         /*
14262          * active_planes bitmask has been updated, and potentially
14263          * affected planes are part of the state. We can now
14264          * compute the minimum cdclk for each plane.
14265          */
14266         for_each_new_intel_plane_in_state(state, plane, plane_state, i)
14267                 *need_modeset |= intel_plane_calc_min_cdclk(state, plane);
14268
14269         return 0;
14270 }
14271
14272 static int intel_atomic_check_crtcs(struct intel_atomic_state *state)
14273 {
14274         struct intel_crtc_state *crtc_state;
14275         struct intel_crtc *crtc;
14276         int i;
14277
14278         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
14279                 int ret = intel_crtc_atomic_check(state, crtc);
14280                 if (ret) {
14281                         DRM_DEBUG_ATOMIC("[CRTC:%d:%s] atomic driver check failed\n",
14282                                          crtc->base.base.id, crtc->base.name);
14283                         return ret;
14284                 }
14285         }
14286
14287         return 0;
14288 }
14289
14290 static bool intel_cpu_transcoder_needs_modeset(struct intel_atomic_state *state,
14291                                                enum transcoder transcoder)
14292 {
14293         struct intel_crtc_state *new_crtc_state;
14294         struct intel_crtc *crtc;
14295         int i;
14296
14297         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
14298                 if (new_crtc_state->cpu_transcoder == transcoder)
14299                         return needs_modeset(new_crtc_state);
14300
14301         return false;
14302 }
14303
14304 /**
14305  * intel_atomic_check - validate state object
14306  * @dev: drm device
14307  * @_state: state to validate
14308  */
14309 static int intel_atomic_check(struct drm_device *dev,
14310                               struct drm_atomic_state *_state)
14311 {
14312         struct drm_i915_private *dev_priv = to_i915(dev);
14313         struct intel_atomic_state *state = to_intel_atomic_state(_state);
14314         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14315         struct intel_crtc *crtc;
14316         int ret, i;
14317         bool any_ms = false;
14318
14319         /* Catch I915_MODE_FLAG_INHERITED */
14320         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14321                                             new_crtc_state, i) {
14322                 if (new_crtc_state->hw.mode.private_flags !=
14323                     old_crtc_state->hw.mode.private_flags)
14324                         new_crtc_state->uapi.mode_changed = true;
14325         }
14326
14327         ret = drm_atomic_helper_check_modeset(dev, &state->base);
14328         if (ret)
14329                 goto fail;
14330
14331         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14332                                             new_crtc_state, i) {
14333                 if (!needs_modeset(new_crtc_state)) {
14334                         /* Light copy */
14335                         intel_crtc_copy_uapi_to_hw_state_nomodeset(new_crtc_state);
14336
14337                         continue;
14338                 }
14339
14340                 if (!new_crtc_state->uapi.enable) {
14341                         intel_crtc_copy_uapi_to_hw_state(new_crtc_state);
14342                         continue;
14343                 }
14344
14345                 ret = intel_crtc_prepare_cleared_state(new_crtc_state);
14346                 if (ret)
14347                         goto fail;
14348
14349                 ret = intel_modeset_pipe_config(new_crtc_state);
14350                 if (ret)
14351                         goto fail;
14352
14353                 intel_crtc_check_fastset(old_crtc_state, new_crtc_state);
14354         }
14355
14356         /**
14357          * Check if fastset is allowed by external dependencies like other
14358          * pipes and transcoders.
14359          *
14360          * Right now it only forces a fullmodeset when the MST master
14361          * transcoder did not changed but the pipe of the master transcoder
14362          * needs a fullmodeset so all slaves also needs to do a fullmodeset.
14363          */
14364         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
14365                 enum transcoder master = new_crtc_state->mst_master_transcoder;
14366
14367                 if (!new_crtc_state->hw.enable ||
14368                     needs_modeset(new_crtc_state) ||
14369                     !intel_dp_mst_is_slave_trans(new_crtc_state))
14370                         continue;
14371
14372                 if (intel_cpu_transcoder_needs_modeset(state, master)) {
14373                         new_crtc_state->uapi.mode_changed = true;
14374                         new_crtc_state->update_pipe = false;
14375                 }
14376         }
14377
14378         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14379                                             new_crtc_state, i) {
14380                 if (needs_modeset(new_crtc_state)) {
14381                         any_ms = true;
14382                         continue;
14383                 }
14384
14385                 if (!new_crtc_state->update_pipe)
14386                         continue;
14387
14388                 intel_crtc_copy_fastset(old_crtc_state, new_crtc_state);
14389         }
14390
14391         if (any_ms && !check_digital_port_conflicts(state)) {
14392                 DRM_DEBUG_KMS("rejecting conflicting digital port configuration\n");
14393                 ret = EINVAL;
14394                 goto fail;
14395         }
14396
14397         ret = drm_dp_mst_atomic_check(&state->base);
14398         if (ret)
14399                 goto fail;
14400
14401         any_ms |= state->cdclk.force_min_cdclk_changed;
14402
14403         ret = intel_atomic_check_planes(state, &any_ms);
14404         if (ret)
14405                 goto fail;
14406
14407         if (any_ms) {
14408                 ret = intel_modeset_checks(state);
14409                 if (ret)
14410                         goto fail;
14411         } else {
14412                 state->cdclk.logical = dev_priv->cdclk.logical;
14413         }
14414
14415         ret = intel_atomic_check_crtcs(state);
14416         if (ret)
14417                 goto fail;
14418
14419         intel_fbc_choose_crtc(dev_priv, state);
14420         ret = calc_watermark_data(state);
14421         if (ret)
14422                 goto fail;
14423
14424         ret = intel_bw_atomic_check(state);
14425         if (ret)
14426                 goto fail;
14427
14428         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14429                                             new_crtc_state, i) {
14430                 if (!needs_modeset(new_crtc_state) &&
14431                     !new_crtc_state->update_pipe)
14432                         continue;
14433
14434                 intel_dump_pipe_config(new_crtc_state, state,
14435                                        needs_modeset(new_crtc_state) ?
14436                                        "[modeset]" : "[fastset]");
14437         }
14438
14439         return 0;
14440
14441  fail:
14442         if (ret == -EDEADLK)
14443                 return ret;
14444
14445         /*
14446          * FIXME would probably be nice to know which crtc specifically
14447          * caused the failure, in cases where we can pinpoint it.
14448          */
14449         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14450                                             new_crtc_state, i)
14451                 intel_dump_pipe_config(new_crtc_state, state, "[failed]");
14452
14453         return ret;
14454 }
14455
14456 static int intel_atomic_prepare_commit(struct intel_atomic_state *state)
14457 {
14458         return drm_atomic_helper_prepare_planes(state->base.dev,
14459                                                 &state->base);
14460 }
14461
14462 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
14463 {
14464         struct drm_device *dev = crtc->base.dev;
14465         struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)];
14466
14467         if (!vblank->max_vblank_count)
14468                 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
14469
14470         return crtc->base.funcs->get_vblank_counter(&crtc->base);
14471 }
14472
14473 void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
14474                                   struct intel_crtc_state *crtc_state)
14475 {
14476         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14477
14478         if (!IS_GEN(dev_priv, 2) || crtc_state->active_planes)
14479                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
14480
14481         if (crtc_state->has_pch_encoder) {
14482                 enum pipe pch_transcoder =
14483                         intel_crtc_pch_transcoder(crtc);
14484
14485                 intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
14486         }
14487 }
14488
14489 static void intel_pipe_fastset(const struct intel_crtc_state *old_crtc_state,
14490                                const struct intel_crtc_state *new_crtc_state)
14491 {
14492         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
14493         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14494
14495         /*
14496          * Update pipe size and adjust fitter if needed: the reason for this is
14497          * that in compute_mode_changes we check the native mode (not the pfit
14498          * mode) to see if we can flip rather than do a full mode set. In the
14499          * fastboot case, we'll flip, but if we don't update the pipesrc and
14500          * pfit state, we'll end up with a big fb scanned out into the wrong
14501          * sized surface.
14502          */
14503         intel_set_pipe_src_size(new_crtc_state);
14504
14505         /* on skylake this is done by detaching scalers */
14506         if (INTEL_GEN(dev_priv) >= 9) {
14507                 skl_detach_scalers(new_crtc_state);
14508
14509                 if (new_crtc_state->pch_pfit.enabled)
14510                         skl_pfit_enable(new_crtc_state);
14511         } else if (HAS_PCH_SPLIT(dev_priv)) {
14512                 if (new_crtc_state->pch_pfit.enabled)
14513                         ilk_pfit_enable(new_crtc_state);
14514                 else if (old_crtc_state->pch_pfit.enabled)
14515                         ilk_pfit_disable(old_crtc_state);
14516         }
14517
14518         if (INTEL_GEN(dev_priv) >= 11)
14519                 icl_set_pipe_chicken(crtc);
14520 }
14521
14522 static void commit_pipe_config(struct intel_atomic_state *state,
14523                                struct intel_crtc_state *old_crtc_state,
14524                                struct intel_crtc_state *new_crtc_state)
14525 {
14526         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
14527         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14528         bool modeset = needs_modeset(new_crtc_state);
14529
14530         /*
14531          * During modesets pipe configuration was programmed as the
14532          * CRTC was enabled.
14533          */
14534         if (!modeset) {
14535                 if (new_crtc_state->uapi.color_mgmt_changed ||
14536                     new_crtc_state->update_pipe)
14537                         intel_color_commit(new_crtc_state);
14538
14539                 if (INTEL_GEN(dev_priv) >= 9)
14540                         skl_detach_scalers(new_crtc_state);
14541
14542                 if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
14543                         bdw_set_pipemisc(new_crtc_state);
14544
14545                 if (new_crtc_state->update_pipe)
14546                         intel_pipe_fastset(old_crtc_state, new_crtc_state);
14547         }
14548
14549         if (dev_priv->display.atomic_update_watermarks)
14550                 dev_priv->display.atomic_update_watermarks(state, crtc);
14551 }
14552
14553 static void intel_update_crtc(struct intel_crtc *crtc,
14554                               struct intel_atomic_state *state,
14555                               struct intel_crtc_state *old_crtc_state,
14556                               struct intel_crtc_state *new_crtc_state)
14557 {
14558         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14559         bool modeset = needs_modeset(new_crtc_state);
14560         struct intel_plane_state *new_plane_state =
14561                 intel_atomic_get_new_plane_state(state,
14562                                                  to_intel_plane(crtc->base.primary));
14563
14564         if (modeset) {
14565                 intel_crtc_update_active_timings(new_crtc_state);
14566
14567                 dev_priv->display.crtc_enable(state, crtc);
14568
14569                 /* vblanks work again, re-enable pipe CRC. */
14570                 intel_crtc_enable_pipe_crc(crtc);
14571         } else {
14572                 if (new_crtc_state->preload_luts &&
14573                     (new_crtc_state->uapi.color_mgmt_changed ||
14574                      new_crtc_state->update_pipe))
14575                         intel_color_load_luts(new_crtc_state);
14576
14577                 intel_pre_plane_update(state, crtc);
14578
14579                 if (new_crtc_state->update_pipe)
14580                         intel_encoders_update_pipe(state, crtc);
14581         }
14582
14583         if (new_crtc_state->update_pipe && !new_crtc_state->enable_fbc)
14584                 intel_fbc_disable(crtc);
14585         else if (new_plane_state)
14586                 intel_fbc_enable(crtc, new_crtc_state, new_plane_state);
14587
14588         /* Perform vblank evasion around commit operation */
14589         intel_pipe_update_start(new_crtc_state);
14590
14591         commit_pipe_config(state, old_crtc_state, new_crtc_state);
14592
14593         if (INTEL_GEN(dev_priv) >= 9)
14594                 skl_update_planes_on_crtc(state, crtc);
14595         else
14596                 i9xx_update_planes_on_crtc(state, crtc);
14597
14598         intel_pipe_update_end(new_crtc_state);
14599
14600         /*
14601          * We usually enable FIFO underrun interrupts as part of the
14602          * CRTC enable sequence during modesets.  But when we inherit a
14603          * valid pipe configuration from the BIOS we need to take care
14604          * of enabling them on the CRTC's first fastset.
14605          */
14606         if (new_crtc_state->update_pipe && !modeset &&
14607             old_crtc_state->hw.mode.private_flags & I915_MODE_FLAG_INHERITED)
14608                 intel_crtc_arm_fifo_underrun(crtc, new_crtc_state);
14609 }
14610
14611 static struct intel_crtc *intel_get_slave_crtc(const struct intel_crtc_state *new_crtc_state)
14612 {
14613         struct drm_i915_private *dev_priv = to_i915(new_crtc_state->uapi.crtc->dev);
14614         enum transcoder slave_transcoder;
14615
14616         WARN_ON(!is_power_of_2(new_crtc_state->sync_mode_slaves_mask));
14617
14618         slave_transcoder = ffs(new_crtc_state->sync_mode_slaves_mask) - 1;
14619         return intel_get_crtc_for_pipe(dev_priv,
14620                                        (enum pipe)slave_transcoder);
14621 }
14622
14623 static void intel_old_crtc_state_disables(struct intel_atomic_state *state,
14624                                           struct intel_crtc_state *old_crtc_state,
14625                                           struct intel_crtc_state *new_crtc_state,
14626                                           struct intel_crtc *crtc)
14627 {
14628         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14629
14630         intel_crtc_disable_planes(state, crtc);
14631
14632         /*
14633          * We need to disable pipe CRC before disabling the pipe,
14634          * or we race against vblank off.
14635          */
14636         intel_crtc_disable_pipe_crc(crtc);
14637
14638         dev_priv->display.crtc_disable(state, crtc);
14639         crtc->active = false;
14640         intel_fbc_disable(crtc);
14641         intel_disable_shared_dpll(old_crtc_state);
14642
14643         /* FIXME unify this for all platforms */
14644         if (!new_crtc_state->hw.active &&
14645             !HAS_GMCH(dev_priv) &&
14646             dev_priv->display.initial_watermarks)
14647                 dev_priv->display.initial_watermarks(state, crtc);
14648 }
14649
14650 static void intel_commit_modeset_disables(struct intel_atomic_state *state)
14651 {
14652         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
14653         struct intel_crtc *crtc;
14654         u32 handled = 0;
14655         int i;
14656
14657         /* Only disable port sync and MST slaves */
14658         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14659                                             new_crtc_state, i) {
14660                 if (!needs_modeset(new_crtc_state))
14661                         continue;
14662
14663                 if (!old_crtc_state->hw.active)
14664                         continue;
14665
14666                 /* In case of Transcoder port Sync master slave CRTCs can be
14667                  * assigned in any order and we need to make sure that
14668                  * slave CRTCs are disabled first and then master CRTC since
14669                  * Slave vblanks are masked till Master Vblanks.
14670                  */
14671                 if (!is_trans_port_sync_slave(old_crtc_state) &&
14672                     !intel_dp_mst_is_slave_trans(old_crtc_state))
14673                         continue;
14674
14675                 intel_pre_plane_update(state, crtc);
14676                 intel_old_crtc_state_disables(state, old_crtc_state,
14677                                               new_crtc_state, crtc);
14678                 handled |= BIT(crtc->pipe);
14679         }
14680
14681         /* Disable everything else left on */
14682         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14683                                             new_crtc_state, i) {
14684                 if (!needs_modeset(new_crtc_state) ||
14685                     (handled & BIT(crtc->pipe)))
14686                         continue;
14687
14688                 intel_pre_plane_update(state, crtc);
14689                 if (old_crtc_state->hw.active)
14690                         intel_old_crtc_state_disables(state, old_crtc_state,
14691                                                       new_crtc_state, crtc);
14692         }
14693 }
14694
14695 static void intel_commit_modeset_enables(struct intel_atomic_state *state)
14696 {
14697         struct intel_crtc *crtc;
14698         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14699         int i;
14700
14701         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
14702                 if (!new_crtc_state->hw.active)
14703                         continue;
14704
14705                 intel_update_crtc(crtc, state, old_crtc_state,
14706                                   new_crtc_state);
14707         }
14708 }
14709
14710 static void intel_crtc_enable_trans_port_sync(struct intel_crtc *crtc,
14711                                               struct intel_atomic_state *state,
14712                                               struct intel_crtc_state *new_crtc_state)
14713 {
14714         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14715
14716         intel_crtc_update_active_timings(new_crtc_state);
14717         dev_priv->display.crtc_enable(state, crtc);
14718         intel_crtc_enable_pipe_crc(crtc);
14719 }
14720
14721 static void intel_set_dp_tp_ctl_normal(struct intel_crtc *crtc,
14722                                        struct intel_atomic_state *state)
14723 {
14724         struct drm_connector *uninitialized_var(conn);
14725         struct drm_connector_state *conn_state;
14726         struct intel_dp *intel_dp;
14727         int i;
14728
14729         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
14730                 if (conn_state->crtc == &crtc->base)
14731                         break;
14732         }
14733         intel_dp = enc_to_intel_dp(&intel_attached_encoder(conn)->base);
14734         intel_dp_stop_link_train(intel_dp);
14735 }
14736
14737 /*
14738  * TODO: This is only called from port sync and it is identical to what will be
14739  * executed again in intel_update_crtc() over port sync pipes
14740  */
14741 static void intel_post_crtc_enable_updates(struct intel_crtc *crtc,
14742                                            struct intel_atomic_state *state)
14743 {
14744         struct intel_crtc_state *new_crtc_state =
14745                 intel_atomic_get_new_crtc_state(state, crtc);
14746         struct intel_crtc_state *old_crtc_state =
14747                 intel_atomic_get_old_crtc_state(state, crtc);
14748         struct intel_plane_state *new_plane_state =
14749                 intel_atomic_get_new_plane_state(state,
14750                                                  to_intel_plane(crtc->base.primary));
14751         bool modeset = needs_modeset(new_crtc_state);
14752
14753         if (new_crtc_state->update_pipe && !new_crtc_state->enable_fbc)
14754                 intel_fbc_disable(crtc);
14755         else if (new_plane_state)
14756                 intel_fbc_enable(crtc, new_crtc_state, new_plane_state);
14757
14758         /* Perform vblank evasion around commit operation */
14759         intel_pipe_update_start(new_crtc_state);
14760         commit_pipe_config(state, old_crtc_state, new_crtc_state);
14761         skl_update_planes_on_crtc(state, crtc);
14762         intel_pipe_update_end(new_crtc_state);
14763
14764         /*
14765          * We usually enable FIFO underrun interrupts as part of the
14766          * CRTC enable sequence during modesets.  But when we inherit a
14767          * valid pipe configuration from the BIOS we need to take care
14768          * of enabling them on the CRTC's first fastset.
14769          */
14770         if (new_crtc_state->update_pipe && !modeset &&
14771             old_crtc_state->hw.mode.private_flags & I915_MODE_FLAG_INHERITED)
14772                 intel_crtc_arm_fifo_underrun(crtc, new_crtc_state);
14773 }
14774
14775 static void intel_update_trans_port_sync_crtcs(struct intel_crtc *crtc,
14776                                                struct intel_atomic_state *state,
14777                                                struct intel_crtc_state *old_crtc_state,
14778                                                struct intel_crtc_state *new_crtc_state)
14779 {
14780         struct intel_crtc *slave_crtc = intel_get_slave_crtc(new_crtc_state);
14781         struct intel_crtc_state *new_slave_crtc_state =
14782                 intel_atomic_get_new_crtc_state(state, slave_crtc);
14783         struct intel_crtc_state *old_slave_crtc_state =
14784                 intel_atomic_get_old_crtc_state(state, slave_crtc);
14785
14786         WARN_ON(!slave_crtc || !new_slave_crtc_state ||
14787                 !old_slave_crtc_state);
14788
14789         DRM_DEBUG_KMS("Updating Transcoder Port Sync Master CRTC = %d %s and Slave CRTC %d %s\n",
14790                       crtc->base.base.id, crtc->base.name, slave_crtc->base.base.id,
14791                       slave_crtc->base.name);
14792
14793         /* Enable seq for slave with with DP_TP_CTL left Idle until the
14794          * master is ready
14795          */
14796         intel_crtc_enable_trans_port_sync(slave_crtc,
14797                                           state,
14798                                           new_slave_crtc_state);
14799
14800         /* Enable seq for master with with DP_TP_CTL left Idle */
14801         intel_crtc_enable_trans_port_sync(crtc,
14802                                           state,
14803                                           new_crtc_state);
14804
14805         /* Set Slave's DP_TP_CTL to Normal */
14806         intel_set_dp_tp_ctl_normal(slave_crtc,
14807                                    state);
14808
14809         /* Set Master's DP_TP_CTL To Normal */
14810         usleep_range(200, 400);
14811         intel_set_dp_tp_ctl_normal(crtc,
14812                                    state);
14813
14814         /* Now do the post crtc enable for all master and slaves */
14815         intel_post_crtc_enable_updates(slave_crtc,
14816                                        state);
14817         intel_post_crtc_enable_updates(crtc,
14818                                        state);
14819 }
14820
14821 static void skl_commit_modeset_enables(struct intel_atomic_state *state)
14822 {
14823         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14824         struct intel_crtc *crtc;
14825         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14826         u8 hw_enabled_slices = dev_priv->wm.skl_hw.ddb.enabled_slices;
14827         u8 required_slices = state->wm_results.ddb.enabled_slices;
14828         struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
14829         const u8 num_pipes = INTEL_NUM_PIPES(dev_priv);
14830         u8 update_pipes = 0, modeset_pipes = 0;
14831         int i;
14832
14833         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
14834                 if (!new_crtc_state->hw.active)
14835                         continue;
14836
14837                 /* ignore allocations for crtc's that have been turned off. */
14838                 if (!needs_modeset(new_crtc_state)) {
14839                         entries[i] = old_crtc_state->wm.skl.ddb;
14840                         update_pipes |= BIT(crtc->pipe);
14841                 } else {
14842                         modeset_pipes |= BIT(crtc->pipe);
14843                 }
14844         }
14845
14846         /* If 2nd DBuf slice required, enable it here */
14847         if (INTEL_GEN(dev_priv) >= 11 && required_slices > hw_enabled_slices)
14848                 icl_dbuf_slices_update(dev_priv, required_slices);
14849
14850         /*
14851          * Whenever the number of active pipes changes, we need to make sure we
14852          * update the pipes in the right order so that their ddb allocations
14853          * never overlap with each other between CRTC updates. Otherwise we'll
14854          * cause pipe underruns and other bad stuff.
14855          *
14856          * So first lets enable all pipes that do not need a fullmodeset as
14857          * those don't have any external dependency.
14858          */
14859         while (update_pipes) {
14860                 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14861                                                     new_crtc_state, i) {
14862                         enum pipe pipe = crtc->pipe;
14863
14864                         if ((update_pipes & BIT(pipe)) == 0)
14865                                 continue;
14866
14867                         if (skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
14868                                                         entries, num_pipes, i))
14869                                 continue;
14870
14871                         entries[i] = new_crtc_state->wm.skl.ddb;
14872                         update_pipes &= ~BIT(pipe);
14873
14874                         intel_update_crtc(crtc, state, old_crtc_state,
14875                                           new_crtc_state);
14876
14877                         /*
14878                          * If this is an already active pipe, it's DDB changed,
14879                          * and this isn't the last pipe that needs updating
14880                          * then we need to wait for a vblank to pass for the
14881                          * new ddb allocation to take effect.
14882                          */
14883                         if (!skl_ddb_entry_equal(&new_crtc_state->wm.skl.ddb,
14884                                                  &old_crtc_state->wm.skl.ddb) &&
14885                             (update_pipes | modeset_pipes))
14886                                 intel_wait_for_vblank(dev_priv, pipe);
14887                 }
14888         }
14889
14890         /*
14891          * Enable all pipes that needs a modeset and do not depends on other
14892          * pipes
14893          */
14894         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14895                                             new_crtc_state, i) {
14896                 enum pipe pipe = crtc->pipe;
14897
14898                 if ((modeset_pipes & BIT(pipe)) == 0)
14899                         continue;
14900
14901                 if (intel_dp_mst_is_slave_trans(new_crtc_state) ||
14902                     is_trans_port_sync_slave(new_crtc_state))
14903                         continue;
14904
14905                 WARN_ON(skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
14906                                                     entries, num_pipes, i));
14907
14908                 entries[i] = new_crtc_state->wm.skl.ddb;
14909                 modeset_pipes &= ~BIT(pipe);
14910
14911                 if (is_trans_port_sync_mode(new_crtc_state)) {
14912                         struct intel_crtc *slave_crtc;
14913
14914                         intel_update_trans_port_sync_crtcs(crtc, state,
14915                                                            old_crtc_state,
14916                                                            new_crtc_state);
14917
14918                         slave_crtc = intel_get_slave_crtc(new_crtc_state);
14919                         /* TODO: update entries[] of slave */
14920                         modeset_pipes &= ~BIT(slave_crtc->pipe);
14921
14922                 } else {
14923                         intel_update_crtc(crtc, state, old_crtc_state,
14924                                           new_crtc_state);
14925                 }
14926         }
14927
14928         /*
14929          * Finally enable all pipes that needs a modeset and depends on
14930          * other pipes, right now it is only MST slaves as both port sync slave
14931          * and master are enabled together
14932          */
14933         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14934                                             new_crtc_state, i) {
14935                 enum pipe pipe = crtc->pipe;
14936
14937                 if ((modeset_pipes & BIT(pipe)) == 0)
14938                         continue;
14939
14940                 WARN_ON(skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
14941                                                     entries, num_pipes, i));
14942
14943                 entries[i] = new_crtc_state->wm.skl.ddb;
14944                 modeset_pipes &= ~BIT(pipe);
14945
14946                 intel_update_crtc(crtc, state, old_crtc_state, new_crtc_state);
14947         }
14948
14949         WARN_ON(modeset_pipes);
14950
14951         /* If 2nd DBuf slice is no more required disable it */
14952         if (INTEL_GEN(dev_priv) >= 11 && required_slices < hw_enabled_slices)
14953                 icl_dbuf_slices_update(dev_priv, required_slices);
14954 }
14955
14956 static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv)
14957 {
14958         struct intel_atomic_state *state, *next;
14959         struct llist_node *freed;
14960
14961         freed = llist_del_all(&dev_priv->atomic_helper.free_list);
14962         llist_for_each_entry_safe(state, next, freed, freed)
14963                 drm_atomic_state_put(&state->base);
14964 }
14965
14966 static void intel_atomic_helper_free_state_worker(struct work_struct *work)
14967 {
14968         struct drm_i915_private *dev_priv =
14969                 container_of(work, typeof(*dev_priv), atomic_helper.free_work);
14970
14971         intel_atomic_helper_free_state(dev_priv);
14972 }
14973
14974 static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state)
14975 {
14976         struct wait_queue_entry wait_fence, wait_reset;
14977         struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev);
14978
14979         init_wait_entry(&wait_fence, 0);
14980         init_wait_entry(&wait_reset, 0);
14981         for (;;) {
14982                 prepare_to_wait(&intel_state->commit_ready.wait,
14983                                 &wait_fence, TASK_UNINTERRUPTIBLE);
14984                 prepare_to_wait(bit_waitqueue(&dev_priv->gt.reset.flags,
14985                                               I915_RESET_MODESET),
14986                                 &wait_reset, TASK_UNINTERRUPTIBLE);
14987
14988
14989                 if (i915_sw_fence_done(&intel_state->commit_ready) ||
14990                     test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags))
14991                         break;
14992
14993                 schedule();
14994         }
14995         finish_wait(&intel_state->commit_ready.wait, &wait_fence);
14996         finish_wait(bit_waitqueue(&dev_priv->gt.reset.flags,
14997                                   I915_RESET_MODESET),
14998                     &wait_reset);
14999 }
15000
15001 static void intel_atomic_cleanup_work(struct work_struct *work)
15002 {
15003         struct drm_atomic_state *state =
15004                 container_of(work, struct drm_atomic_state, commit_work);
15005         struct drm_i915_private *i915 = to_i915(state->dev);
15006
15007         drm_atomic_helper_cleanup_planes(&i915->drm, state);
15008         drm_atomic_helper_commit_cleanup_done(state);
15009         drm_atomic_state_put(state);
15010
15011         intel_atomic_helper_free_state(i915);
15012 }
15013
15014 static void intel_atomic_commit_tail(struct intel_atomic_state *state)
15015 {
15016         struct drm_device *dev = state->base.dev;
15017         struct drm_i915_private *dev_priv = to_i915(dev);
15018         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
15019         struct intel_crtc *crtc;
15020         u64 put_domains[I915_MAX_PIPES] = {};
15021         intel_wakeref_t wakeref = 0;
15022         int i;
15023
15024         intel_atomic_commit_fence_wait(state);
15025
15026         drm_atomic_helper_wait_for_dependencies(&state->base);
15027
15028         if (state->modeset)
15029                 wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
15030
15031         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
15032                                             new_crtc_state, i) {
15033                 if (needs_modeset(new_crtc_state) ||
15034                     new_crtc_state->update_pipe) {
15035
15036                         put_domains[crtc->pipe] =
15037                                 modeset_get_crtc_power_domains(new_crtc_state);
15038                 }
15039         }
15040
15041         intel_commit_modeset_disables(state);
15042
15043         /* FIXME: Eventually get rid of our crtc->config pointer */
15044         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
15045                 crtc->config = new_crtc_state;
15046
15047         if (state->modeset) {
15048                 drm_atomic_helper_update_legacy_modeset_state(dev, &state->base);
15049
15050                 intel_set_cdclk_pre_plane_update(dev_priv,
15051                                                  &state->cdclk.actual,
15052                                                  &dev_priv->cdclk.actual,
15053                                                  state->cdclk.pipe);
15054
15055                 /*
15056                  * SKL workaround: bspec recommends we disable the SAGV when we
15057                  * have more then one pipe enabled
15058                  */
15059                 if (!intel_can_enable_sagv(state))
15060                         intel_disable_sagv(dev_priv);
15061
15062                 intel_modeset_verify_disabled(dev_priv, state);
15063         }
15064
15065         /* Complete the events for pipes that have now been disabled */
15066         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
15067                 bool modeset = needs_modeset(new_crtc_state);
15068
15069                 /* Complete events for now disable pipes here. */
15070                 if (modeset && !new_crtc_state->hw.active && new_crtc_state->uapi.event) {
15071                         spin_lock_irq(&dev->event_lock);
15072                         drm_crtc_send_vblank_event(&crtc->base,
15073                                                    new_crtc_state->uapi.event);
15074                         spin_unlock_irq(&dev->event_lock);
15075
15076                         new_crtc_state->uapi.event = NULL;
15077                 }
15078         }
15079
15080         if (state->modeset)
15081                 intel_encoders_update_prepare(state);
15082
15083         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
15084         dev_priv->display.commit_modeset_enables(state);
15085
15086         if (state->modeset) {
15087                 intel_encoders_update_complete(state);
15088
15089                 intel_set_cdclk_post_plane_update(dev_priv,
15090                                                   &state->cdclk.actual,
15091                                                   &dev_priv->cdclk.actual,
15092                                                   state->cdclk.pipe);
15093         }
15094
15095         /* FIXME: We should call drm_atomic_helper_commit_hw_done() here
15096          * already, but still need the state for the delayed optimization. To
15097          * fix this:
15098          * - wrap the optimization/post_plane_update stuff into a per-crtc work.
15099          * - schedule that vblank worker _before_ calling hw_done
15100          * - at the start of commit_tail, cancel it _synchrously
15101          * - switch over to the vblank wait helper in the core after that since
15102          *   we don't need out special handling any more.
15103          */
15104         drm_atomic_helper_wait_for_flip_done(dev, &state->base);
15105
15106         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
15107                 if (new_crtc_state->hw.active &&
15108                     !needs_modeset(new_crtc_state) &&
15109                     !new_crtc_state->preload_luts &&
15110                     (new_crtc_state->uapi.color_mgmt_changed ||
15111                      new_crtc_state->update_pipe))
15112                         intel_color_load_luts(new_crtc_state);
15113         }
15114
15115         /*
15116          * Now that the vblank has passed, we can go ahead and program the
15117          * optimal watermarks on platforms that need two-step watermark
15118          * programming.
15119          *
15120          * TODO: Move this (and other cleanup) to an async worker eventually.
15121          */
15122         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
15123                                             new_crtc_state, i) {
15124                 /*
15125                  * Gen2 reports pipe underruns whenever all planes are disabled.
15126                  * So re-enable underrun reporting after some planes get enabled.
15127                  *
15128                  * We do this before .optimize_watermarks() so that we have a
15129                  * chance of catching underruns with the intermediate watermarks
15130                  * vs. the new plane configuration.
15131                  */
15132                 if (IS_GEN(dev_priv, 2) && planes_enabling(old_crtc_state, new_crtc_state))
15133                         intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
15134
15135                 if (dev_priv->display.optimize_watermarks)
15136                         dev_priv->display.optimize_watermarks(state, crtc);
15137         }
15138
15139         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
15140                 intel_post_plane_update(state, crtc);
15141
15142                 if (put_domains[i])
15143                         modeset_put_power_domains(dev_priv, put_domains[i]);
15144
15145                 intel_modeset_verify_crtc(crtc, state, old_crtc_state, new_crtc_state);
15146         }
15147
15148         /* Underruns don't always raise interrupts, so check manually */
15149         intel_check_cpu_fifo_underruns(dev_priv);
15150         intel_check_pch_fifo_underruns(dev_priv);
15151
15152         if (state->modeset)
15153                 intel_verify_planes(state);
15154
15155         if (state->modeset && intel_can_enable_sagv(state))
15156                 intel_enable_sagv(dev_priv);
15157
15158         drm_atomic_helper_commit_hw_done(&state->base);
15159
15160         if (state->modeset) {
15161                 /* As one of the primary mmio accessors, KMS has a high
15162                  * likelihood of triggering bugs in unclaimed access. After we
15163                  * finish modesetting, see if an error has been flagged, and if
15164                  * so enable debugging for the next modeset - and hope we catch
15165                  * the culprit.
15166                  */
15167                 intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore);
15168                 intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET, wakeref);
15169         }
15170         intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
15171
15172         /*
15173          * Defer the cleanup of the old state to a separate worker to not
15174          * impede the current task (userspace for blocking modesets) that
15175          * are executed inline. For out-of-line asynchronous modesets/flips,
15176          * deferring to a new worker seems overkill, but we would place a
15177          * schedule point (cond_resched()) here anyway to keep latencies
15178          * down.
15179          */
15180         INIT_WORK(&state->base.commit_work, intel_atomic_cleanup_work);
15181         queue_work(system_highpri_wq, &state->base.commit_work);
15182 }
15183
15184 static void intel_atomic_commit_work(struct work_struct *work)
15185 {
15186         struct intel_atomic_state *state =
15187                 container_of(work, struct intel_atomic_state, base.commit_work);
15188
15189         intel_atomic_commit_tail(state);
15190 }
15191
15192 static int __i915_sw_fence_call
15193 intel_atomic_commit_ready(struct i915_sw_fence *fence,
15194                           enum i915_sw_fence_notify notify)
15195 {
15196         struct intel_atomic_state *state =
15197                 container_of(fence, struct intel_atomic_state, commit_ready);
15198
15199         switch (notify) {
15200         case FENCE_COMPLETE:
15201                 /* we do blocking waits in the worker, nothing to do here */
15202                 break;
15203         case FENCE_FREE:
15204                 {
15205                         struct intel_atomic_helper *helper =
15206                                 &to_i915(state->base.dev)->atomic_helper;
15207
15208                         if (llist_add(&state->freed, &helper->free_list))
15209                                 schedule_work(&helper->free_work);
15210                         break;
15211                 }
15212         }
15213
15214         return NOTIFY_DONE;
15215 }
15216
15217 static void intel_atomic_track_fbs(struct intel_atomic_state *state)
15218 {
15219         struct intel_plane_state *old_plane_state, *new_plane_state;
15220         struct intel_plane *plane;
15221         int i;
15222
15223         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
15224                                              new_plane_state, i)
15225                 intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->hw.fb),
15226                                         to_intel_frontbuffer(new_plane_state->hw.fb),
15227                                         plane->frontbuffer_bit);
15228 }
15229
15230 static void assert_global_state_locked(struct drm_i915_private *dev_priv)
15231 {
15232         struct intel_crtc *crtc;
15233
15234         for_each_intel_crtc(&dev_priv->drm, crtc)
15235                 drm_modeset_lock_assert_held(&crtc->base.mutex);
15236 }
15237
15238 static int intel_atomic_commit(struct drm_device *dev,
15239                                struct drm_atomic_state *_state,
15240                                bool nonblock)
15241 {
15242         struct intel_atomic_state *state = to_intel_atomic_state(_state);
15243         struct drm_i915_private *dev_priv = to_i915(dev);
15244         int ret = 0;
15245
15246         state->wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
15247
15248         drm_atomic_state_get(&state->base);
15249         i915_sw_fence_init(&state->commit_ready,
15250                            intel_atomic_commit_ready);
15251
15252         /*
15253          * The intel_legacy_cursor_update() fast path takes care
15254          * of avoiding the vblank waits for simple cursor
15255          * movement and flips. For cursor on/off and size changes,
15256          * we want to perform the vblank waits so that watermark
15257          * updates happen during the correct frames. Gen9+ have
15258          * double buffered watermarks and so shouldn't need this.
15259          *
15260          * Unset state->legacy_cursor_update before the call to
15261          * drm_atomic_helper_setup_commit() because otherwise
15262          * drm_atomic_helper_wait_for_flip_done() is a noop and
15263          * we get FIFO underruns because we didn't wait
15264          * for vblank.
15265          *
15266          * FIXME doing watermarks and fb cleanup from a vblank worker
15267          * (assuming we had any) would solve these problems.
15268          */
15269         if (INTEL_GEN(dev_priv) < 9 && state->base.legacy_cursor_update) {
15270                 struct intel_crtc_state *new_crtc_state;
15271                 struct intel_crtc *crtc;
15272                 int i;
15273
15274                 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
15275                         if (new_crtc_state->wm.need_postvbl_update ||
15276                             new_crtc_state->update_wm_post)
15277                                 state->base.legacy_cursor_update = false;
15278         }
15279
15280         ret = intel_atomic_prepare_commit(state);
15281         if (ret) {
15282                 DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret);
15283                 i915_sw_fence_commit(&state->commit_ready);
15284                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
15285                 return ret;
15286         }
15287
15288         ret = drm_atomic_helper_setup_commit(&state->base, nonblock);
15289         if (!ret)
15290                 ret = drm_atomic_helper_swap_state(&state->base, true);
15291
15292         if (ret) {
15293                 i915_sw_fence_commit(&state->commit_ready);
15294
15295                 drm_atomic_helper_cleanup_planes(dev, &state->base);
15296                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
15297                 return ret;
15298         }
15299         dev_priv->wm.distrust_bios_wm = false;
15300         intel_shared_dpll_swap_state(state);
15301         intel_atomic_track_fbs(state);
15302
15303         if (state->global_state_changed) {
15304                 assert_global_state_locked(dev_priv);
15305
15306                 memcpy(dev_priv->min_cdclk, state->min_cdclk,
15307                        sizeof(state->min_cdclk));
15308                 memcpy(dev_priv->min_voltage_level, state->min_voltage_level,
15309                        sizeof(state->min_voltage_level));
15310                 dev_priv->active_pipes = state->active_pipes;
15311                 dev_priv->cdclk.force_min_cdclk = state->cdclk.force_min_cdclk;
15312
15313                 intel_cdclk_swap_state(state);
15314         }
15315
15316         drm_atomic_state_get(&state->base);
15317         INIT_WORK(&state->base.commit_work, intel_atomic_commit_work);
15318
15319         i915_sw_fence_commit(&state->commit_ready);
15320         if (nonblock && state->modeset) {
15321                 queue_work(dev_priv->modeset_wq, &state->base.commit_work);
15322         } else if (nonblock) {
15323                 queue_work(dev_priv->flip_wq, &state->base.commit_work);
15324         } else {
15325                 if (state->modeset)
15326                         flush_workqueue(dev_priv->modeset_wq);
15327                 intel_atomic_commit_tail(state);
15328         }
15329
15330         return 0;
15331 }
15332
15333 struct wait_rps_boost {
15334         struct wait_queue_entry wait;
15335
15336         struct drm_crtc *crtc;
15337         struct i915_request *request;
15338 };
15339
15340 static int do_rps_boost(struct wait_queue_entry *_wait,
15341                         unsigned mode, int sync, void *key)
15342 {
15343         struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait);
15344         struct i915_request *rq = wait->request;
15345
15346         /*
15347          * If we missed the vblank, but the request is already running it
15348          * is reasonable to assume that it will complete before the next
15349          * vblank without our intervention, so leave RPS alone.
15350          */
15351         if (!i915_request_started(rq))
15352                 intel_rps_boost(rq);
15353         i915_request_put(rq);
15354
15355         drm_crtc_vblank_put(wait->crtc);
15356
15357         list_del(&wait->wait.entry);
15358         kfree(wait);
15359         return 1;
15360 }
15361
15362 static void add_rps_boost_after_vblank(struct drm_crtc *crtc,
15363                                        struct dma_fence *fence)
15364 {
15365         struct wait_rps_boost *wait;
15366
15367         if (!dma_fence_is_i915(fence))
15368                 return;
15369
15370         if (INTEL_GEN(to_i915(crtc->dev)) < 6)
15371                 return;
15372
15373         if (drm_crtc_vblank_get(crtc))
15374                 return;
15375
15376         wait = kmalloc(sizeof(*wait), GFP_KERNEL);
15377         if (!wait) {
15378                 drm_crtc_vblank_put(crtc);
15379                 return;
15380         }
15381
15382         wait->request = to_request(dma_fence_get(fence));
15383         wait->crtc = crtc;
15384
15385         wait->wait.func = do_rps_boost;
15386         wait->wait.flags = 0;
15387
15388         add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait);
15389 }
15390
15391 static int intel_plane_pin_fb(struct intel_plane_state *plane_state)
15392 {
15393         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
15394         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
15395         struct drm_framebuffer *fb = plane_state->hw.fb;
15396         struct i915_vma *vma;
15397
15398         if (plane->id == PLANE_CURSOR &&
15399             INTEL_INFO(dev_priv)->display.cursor_needs_physical) {
15400                 struct drm_i915_gem_object *obj = intel_fb_obj(fb);
15401                 const int align = intel_cursor_alignment(dev_priv);
15402                 int err;
15403
15404                 err = i915_gem_object_attach_phys(obj, align);
15405                 if (err)
15406                         return err;
15407         }
15408
15409         vma = intel_pin_and_fence_fb_obj(fb,
15410                                          &plane_state->view,
15411                                          intel_plane_uses_fence(plane_state),
15412                                          &plane_state->flags);
15413         if (IS_ERR(vma))
15414                 return PTR_ERR(vma);
15415
15416         plane_state->vma = vma;
15417
15418         return 0;
15419 }
15420
15421 static void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
15422 {
15423         struct i915_vma *vma;
15424
15425         vma = fetch_and_zero(&old_plane_state->vma);
15426         if (vma)
15427                 intel_unpin_fb_vma(vma, old_plane_state->flags);
15428 }
15429
15430 static void fb_obj_bump_render_priority(struct drm_i915_gem_object *obj)
15431 {
15432         struct i915_sched_attr attr = {
15433                 .priority = I915_USER_PRIORITY(I915_PRIORITY_DISPLAY),
15434         };
15435
15436         i915_gem_object_wait_priority(obj, 0, &attr);
15437 }
15438
15439 /**
15440  * intel_prepare_plane_fb - Prepare fb for usage on plane
15441  * @plane: drm plane to prepare for
15442  * @_new_plane_state: the plane state being prepared
15443  *
15444  * Prepares a framebuffer for usage on a display plane.  Generally this
15445  * involves pinning the underlying object and updating the frontbuffer tracking
15446  * bits.  Some older platforms need special physical address handling for
15447  * cursor planes.
15448  *
15449  * Returns 0 on success, negative error code on failure.
15450  */
15451 int
15452 intel_prepare_plane_fb(struct drm_plane *plane,
15453                        struct drm_plane_state *_new_plane_state)
15454 {
15455         struct intel_plane_state *new_plane_state =
15456                 to_intel_plane_state(_new_plane_state);
15457         struct intel_atomic_state *intel_state =
15458                 to_intel_atomic_state(new_plane_state->uapi.state);
15459         struct drm_i915_private *dev_priv = to_i915(plane->dev);
15460         struct drm_framebuffer *fb = new_plane_state->hw.fb;
15461         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
15462         struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb);
15463         int ret;
15464
15465         if (old_obj) {
15466                 struct intel_crtc_state *crtc_state =
15467                         intel_atomic_get_new_crtc_state(intel_state,
15468                                                         to_intel_crtc(plane->state->crtc));
15469
15470                 /* Big Hammer, we also need to ensure that any pending
15471                  * MI_WAIT_FOR_EVENT inside a user batch buffer on the
15472                  * current scanout is retired before unpinning the old
15473                  * framebuffer. Note that we rely on userspace rendering
15474                  * into the buffer attached to the pipe they are waiting
15475                  * on. If not, userspace generates a GPU hang with IPEHR
15476                  * point to the MI_WAIT_FOR_EVENT.
15477                  *
15478                  * This should only fail upon a hung GPU, in which case we
15479                  * can safely continue.
15480                  */
15481                 if (needs_modeset(crtc_state)) {
15482                         ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
15483                                                               old_obj->base.resv, NULL,
15484                                                               false, 0,
15485                                                               GFP_KERNEL);
15486                         if (ret < 0)
15487                                 return ret;
15488                 }
15489         }
15490
15491         if (new_plane_state->uapi.fence) { /* explicit fencing */
15492                 ret = i915_sw_fence_await_dma_fence(&intel_state->commit_ready,
15493                                                     new_plane_state->uapi.fence,
15494                                                     I915_FENCE_TIMEOUT,
15495                                                     GFP_KERNEL);
15496                 if (ret < 0)
15497                         return ret;
15498         }
15499
15500         if (!obj)
15501                 return 0;
15502
15503         ret = i915_gem_object_pin_pages(obj);
15504         if (ret)
15505                 return ret;
15506
15507         ret = intel_plane_pin_fb(new_plane_state);
15508
15509         i915_gem_object_unpin_pages(obj);
15510         if (ret)
15511                 return ret;
15512
15513         fb_obj_bump_render_priority(obj);
15514         i915_gem_object_flush_frontbuffer(obj, ORIGIN_DIRTYFB);
15515
15516         if (!new_plane_state->uapi.fence) { /* implicit fencing */
15517                 struct dma_fence *fence;
15518
15519                 ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
15520                                                       obj->base.resv, NULL,
15521                                                       false, I915_FENCE_TIMEOUT,
15522                                                       GFP_KERNEL);
15523                 if (ret < 0)
15524                         return ret;
15525
15526                 fence = dma_resv_get_excl_rcu(obj->base.resv);
15527                 if (fence) {
15528                         add_rps_boost_after_vblank(new_plane_state->hw.crtc,
15529                                                    fence);
15530                         dma_fence_put(fence);
15531                 }
15532         } else {
15533                 add_rps_boost_after_vblank(new_plane_state->hw.crtc,
15534                                            new_plane_state->uapi.fence);
15535         }
15536
15537         /*
15538          * We declare pageflips to be interactive and so merit a small bias
15539          * towards upclocking to deliver the frame on time. By only changing
15540          * the RPS thresholds to sample more regularly and aim for higher
15541          * clocks we can hopefully deliver low power workloads (like kodi)
15542          * that are not quite steady state without resorting to forcing
15543          * maximum clocks following a vblank miss (see do_rps_boost()).
15544          */
15545         if (!intel_state->rps_interactive) {
15546                 intel_rps_mark_interactive(&dev_priv->gt.rps, true);
15547                 intel_state->rps_interactive = true;
15548         }
15549
15550         return 0;
15551 }
15552
15553 /**
15554  * intel_cleanup_plane_fb - Cleans up an fb after plane use
15555  * @plane: drm plane to clean up for
15556  * @_old_plane_state: the state from the previous modeset
15557  *
15558  * Cleans up a framebuffer that has just been removed from a plane.
15559  */
15560 void
15561 intel_cleanup_plane_fb(struct drm_plane *plane,
15562                        struct drm_plane_state *_old_plane_state)
15563 {
15564         struct intel_plane_state *old_plane_state =
15565                 to_intel_plane_state(_old_plane_state);
15566         struct intel_atomic_state *intel_state =
15567                 to_intel_atomic_state(old_plane_state->uapi.state);
15568         struct drm_i915_private *dev_priv = to_i915(plane->dev);
15569
15570         if (intel_state->rps_interactive) {
15571                 intel_rps_mark_interactive(&dev_priv->gt.rps, false);
15572                 intel_state->rps_interactive = false;
15573         }
15574
15575         /* Should only be called after a successful intel_prepare_plane_fb()! */
15576         intel_plane_unpin_fb(old_plane_state);
15577 }
15578
15579 /**
15580  * intel_plane_destroy - destroy a plane
15581  * @plane: plane to destroy
15582  *
15583  * Common destruction function for all types of planes (primary, cursor,
15584  * sprite).
15585  */
15586 void intel_plane_destroy(struct drm_plane *plane)
15587 {
15588         drm_plane_cleanup(plane);
15589         kfree(to_intel_plane(plane));
15590 }
15591
15592 static bool i8xx_plane_format_mod_supported(struct drm_plane *_plane,
15593                                             u32 format, u64 modifier)
15594 {
15595         switch (modifier) {
15596         case DRM_FORMAT_MOD_LINEAR:
15597         case I915_FORMAT_MOD_X_TILED:
15598                 break;
15599         default:
15600                 return false;
15601         }
15602
15603         switch (format) {
15604         case DRM_FORMAT_C8:
15605         case DRM_FORMAT_RGB565:
15606         case DRM_FORMAT_XRGB1555:
15607         case DRM_FORMAT_XRGB8888:
15608                 return modifier == DRM_FORMAT_MOD_LINEAR ||
15609                         modifier == I915_FORMAT_MOD_X_TILED;
15610         default:
15611                 return false;
15612         }
15613 }
15614
15615 static bool i965_plane_format_mod_supported(struct drm_plane *_plane,
15616                                             u32 format, u64 modifier)
15617 {
15618         switch (modifier) {
15619         case DRM_FORMAT_MOD_LINEAR:
15620         case I915_FORMAT_MOD_X_TILED:
15621                 break;
15622         default:
15623                 return false;
15624         }
15625
15626         switch (format) {
15627         case DRM_FORMAT_C8:
15628         case DRM_FORMAT_RGB565:
15629         case DRM_FORMAT_XRGB8888:
15630         case DRM_FORMAT_XBGR8888:
15631         case DRM_FORMAT_ARGB8888:
15632         case DRM_FORMAT_ABGR8888:
15633         case DRM_FORMAT_XRGB2101010:
15634         case DRM_FORMAT_XBGR2101010:
15635         case DRM_FORMAT_ARGB2101010:
15636         case DRM_FORMAT_ABGR2101010:
15637         case DRM_FORMAT_XBGR16161616F:
15638                 return modifier == DRM_FORMAT_MOD_LINEAR ||
15639                         modifier == I915_FORMAT_MOD_X_TILED;
15640         default:
15641                 return false;
15642         }
15643 }
15644
15645 static bool intel_cursor_format_mod_supported(struct drm_plane *_plane,
15646                                               u32 format, u64 modifier)
15647 {
15648         return modifier == DRM_FORMAT_MOD_LINEAR &&
15649                 format == DRM_FORMAT_ARGB8888;
15650 }
15651
15652 static const struct drm_plane_funcs i965_plane_funcs = {
15653         .update_plane = drm_atomic_helper_update_plane,
15654         .disable_plane = drm_atomic_helper_disable_plane,
15655         .destroy = intel_plane_destroy,
15656         .atomic_duplicate_state = intel_plane_duplicate_state,
15657         .atomic_destroy_state = intel_plane_destroy_state,
15658         .format_mod_supported = i965_plane_format_mod_supported,
15659 };
15660
15661 static const struct drm_plane_funcs i8xx_plane_funcs = {
15662         .update_plane = drm_atomic_helper_update_plane,
15663         .disable_plane = drm_atomic_helper_disable_plane,
15664         .destroy = intel_plane_destroy,
15665         .atomic_duplicate_state = intel_plane_duplicate_state,
15666         .atomic_destroy_state = intel_plane_destroy_state,
15667         .format_mod_supported = i8xx_plane_format_mod_supported,
15668 };
15669
15670 static int
15671 intel_legacy_cursor_update(struct drm_plane *_plane,
15672                            struct drm_crtc *_crtc,
15673                            struct drm_framebuffer *fb,
15674                            int crtc_x, int crtc_y,
15675                            unsigned int crtc_w, unsigned int crtc_h,
15676                            u32 src_x, u32 src_y,
15677                            u32 src_w, u32 src_h,
15678                            struct drm_modeset_acquire_ctx *ctx)
15679 {
15680         struct intel_plane *plane = to_intel_plane(_plane);
15681         struct intel_crtc *crtc = to_intel_crtc(_crtc);
15682         struct intel_plane_state *old_plane_state =
15683                 to_intel_plane_state(plane->base.state);
15684         struct intel_plane_state *new_plane_state;
15685         struct intel_crtc_state *crtc_state =
15686                 to_intel_crtc_state(crtc->base.state);
15687         struct intel_crtc_state *new_crtc_state;
15688         int ret;
15689
15690         /*
15691          * When crtc is inactive or there is a modeset pending,
15692          * wait for it to complete in the slowpath
15693          */
15694         if (!crtc_state->hw.active || needs_modeset(crtc_state) ||
15695             crtc_state->update_pipe)
15696                 goto slow;
15697
15698         /*
15699          * Don't do an async update if there is an outstanding commit modifying
15700          * the plane.  This prevents our async update's changes from getting
15701          * overridden by a previous synchronous update's state.
15702          */
15703         if (old_plane_state->uapi.commit &&
15704             !try_wait_for_completion(&old_plane_state->uapi.commit->hw_done))
15705                 goto slow;
15706
15707         /*
15708          * If any parameters change that may affect watermarks,
15709          * take the slowpath. Only changing fb or position should be
15710          * in the fastpath.
15711          */
15712         if (old_plane_state->uapi.crtc != &crtc->base ||
15713             old_plane_state->uapi.src_w != src_w ||
15714             old_plane_state->uapi.src_h != src_h ||
15715             old_plane_state->uapi.crtc_w != crtc_w ||
15716             old_plane_state->uapi.crtc_h != crtc_h ||
15717             !old_plane_state->uapi.fb != !fb)
15718                 goto slow;
15719
15720         new_plane_state = to_intel_plane_state(intel_plane_duplicate_state(&plane->base));
15721         if (!new_plane_state)
15722                 return -ENOMEM;
15723
15724         new_crtc_state = to_intel_crtc_state(intel_crtc_duplicate_state(&crtc->base));
15725         if (!new_crtc_state) {
15726                 ret = -ENOMEM;
15727                 goto out_free;
15728         }
15729
15730         drm_atomic_set_fb_for_plane(&new_plane_state->uapi, fb);
15731
15732         new_plane_state->uapi.src_x = src_x;
15733         new_plane_state->uapi.src_y = src_y;
15734         new_plane_state->uapi.src_w = src_w;
15735         new_plane_state->uapi.src_h = src_h;
15736         new_plane_state->uapi.crtc_x = crtc_x;
15737         new_plane_state->uapi.crtc_y = crtc_y;
15738         new_plane_state->uapi.crtc_w = crtc_w;
15739         new_plane_state->uapi.crtc_h = crtc_h;
15740
15741         ret = intel_plane_atomic_check_with_state(crtc_state, new_crtc_state,
15742                                                   old_plane_state, new_plane_state);
15743         if (ret)
15744                 goto out_free;
15745
15746         ret = intel_plane_pin_fb(new_plane_state);
15747         if (ret)
15748                 goto out_free;
15749
15750         intel_frontbuffer_flush(to_intel_frontbuffer(new_plane_state->hw.fb),
15751                                 ORIGIN_FLIP);
15752         intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->hw.fb),
15753                                 to_intel_frontbuffer(new_plane_state->hw.fb),
15754                                 plane->frontbuffer_bit);
15755
15756         /* Swap plane state */
15757         plane->base.state = &new_plane_state->uapi;
15758
15759         /*
15760          * We cannot swap crtc_state as it may be in use by an atomic commit or
15761          * page flip that's running simultaneously. If we swap crtc_state and
15762          * destroy the old state, we will cause a use-after-free there.
15763          *
15764          * Only update active_planes, which is needed for our internal
15765          * bookkeeping. Either value will do the right thing when updating
15766          * planes atomically. If the cursor was part of the atomic update then
15767          * we would have taken the slowpath.
15768          */
15769         crtc_state->active_planes = new_crtc_state->active_planes;
15770
15771         if (new_plane_state->uapi.visible)
15772                 intel_update_plane(plane, crtc_state, new_plane_state);
15773         else
15774                 intel_disable_plane(plane, crtc_state);
15775
15776         intel_plane_unpin_fb(old_plane_state);
15777
15778 out_free:
15779         if (new_crtc_state)
15780                 intel_crtc_destroy_state(&crtc->base, &new_crtc_state->uapi);
15781         if (ret)
15782                 intel_plane_destroy_state(&plane->base, &new_plane_state->uapi);
15783         else
15784                 intel_plane_destroy_state(&plane->base, &old_plane_state->uapi);
15785         return ret;
15786
15787 slow:
15788         return drm_atomic_helper_update_plane(&plane->base, &crtc->base, fb,
15789                                               crtc_x, crtc_y, crtc_w, crtc_h,
15790                                               src_x, src_y, src_w, src_h, ctx);
15791 }
15792
15793 static const struct drm_plane_funcs intel_cursor_plane_funcs = {
15794         .update_plane = intel_legacy_cursor_update,
15795         .disable_plane = drm_atomic_helper_disable_plane,
15796         .destroy = intel_plane_destroy,
15797         .atomic_duplicate_state = intel_plane_duplicate_state,
15798         .atomic_destroy_state = intel_plane_destroy_state,
15799         .format_mod_supported = intel_cursor_format_mod_supported,
15800 };
15801
15802 static bool i9xx_plane_has_fbc(struct drm_i915_private *dev_priv,
15803                                enum i9xx_plane_id i9xx_plane)
15804 {
15805         if (!HAS_FBC(dev_priv))
15806                 return false;
15807
15808         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
15809                 return i9xx_plane == PLANE_A; /* tied to pipe A */
15810         else if (IS_IVYBRIDGE(dev_priv))
15811                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B ||
15812                         i9xx_plane == PLANE_C;
15813         else if (INTEL_GEN(dev_priv) >= 4)
15814                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B;
15815         else
15816                 return i9xx_plane == PLANE_A;
15817 }
15818
15819 static struct intel_plane *
15820 intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
15821 {
15822         struct intel_plane *plane;
15823         const struct drm_plane_funcs *plane_funcs;
15824         unsigned int supported_rotations;
15825         unsigned int possible_crtcs;
15826         const u32 *formats;
15827         int num_formats;
15828         int ret, zpos;
15829
15830         if (INTEL_GEN(dev_priv) >= 9)
15831                 return skl_universal_plane_create(dev_priv, pipe,
15832                                                   PLANE_PRIMARY);
15833
15834         plane = intel_plane_alloc();
15835         if (IS_ERR(plane))
15836                 return plane;
15837
15838         plane->pipe = pipe;
15839         /*
15840          * On gen2/3 only plane A can do FBC, but the panel fitter and LVDS
15841          * port is hooked to pipe B. Hence we want plane A feeding pipe B.
15842          */
15843         if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4)
15844                 plane->i9xx_plane = (enum i9xx_plane_id) !pipe;
15845         else
15846                 plane->i9xx_plane = (enum i9xx_plane_id) pipe;
15847         plane->id = PLANE_PRIMARY;
15848         plane->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, plane->id);
15849
15850         plane->has_fbc = i9xx_plane_has_fbc(dev_priv, plane->i9xx_plane);
15851         if (plane->has_fbc) {
15852                 struct intel_fbc *fbc = &dev_priv->fbc;
15853
15854                 fbc->possible_framebuffer_bits |= plane->frontbuffer_bit;
15855         }
15856
15857         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
15858                 formats = vlv_primary_formats;
15859                 num_formats = ARRAY_SIZE(vlv_primary_formats);
15860         } else if (INTEL_GEN(dev_priv) >= 4) {
15861                 /*
15862                  * WaFP16GammaEnabling:ivb
15863                  * "Workaround : When using the 64-bit format, the plane
15864                  *  output on each color channel has one quarter amplitude.
15865                  *  It can be brought up to full amplitude by using pipe
15866                  *  gamma correction or pipe color space conversion to
15867                  *  multiply the plane output by four."
15868                  *
15869                  * There is no dedicated plane gamma for the primary plane,
15870                  * and using the pipe gamma/csc could conflict with other
15871                  * planes, so we choose not to expose fp16 on IVB primary
15872                  * planes. HSW primary planes no longer have this problem.
15873                  */
15874                 if (IS_IVYBRIDGE(dev_priv)) {
15875                         formats = ivb_primary_formats;
15876                         num_formats = ARRAY_SIZE(ivb_primary_formats);
15877                 } else {
15878                         formats = i965_primary_formats;
15879                         num_formats = ARRAY_SIZE(i965_primary_formats);
15880                 }
15881         } else {
15882                 formats = i8xx_primary_formats;
15883                 num_formats = ARRAY_SIZE(i8xx_primary_formats);
15884         }
15885
15886         if (INTEL_GEN(dev_priv) >= 4)
15887                 plane_funcs = &i965_plane_funcs;
15888         else
15889                 plane_funcs = &i8xx_plane_funcs;
15890
15891         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
15892                 plane->min_cdclk = vlv_plane_min_cdclk;
15893         else if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
15894                 plane->min_cdclk = hsw_plane_min_cdclk;
15895         else if (IS_IVYBRIDGE(dev_priv))
15896                 plane->min_cdclk = ivb_plane_min_cdclk;
15897         else
15898                 plane->min_cdclk = i9xx_plane_min_cdclk;
15899
15900         plane->max_stride = i9xx_plane_max_stride;
15901         plane->update_plane = i9xx_update_plane;
15902         plane->disable_plane = i9xx_disable_plane;
15903         plane->get_hw_state = i9xx_plane_get_hw_state;
15904         plane->check_plane = i9xx_plane_check;
15905
15906         possible_crtcs = BIT(pipe);
15907
15908         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
15909                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
15910                                                possible_crtcs, plane_funcs,
15911                                                formats, num_formats,
15912                                                i9xx_format_modifiers,
15913                                                DRM_PLANE_TYPE_PRIMARY,
15914                                                "primary %c", pipe_name(pipe));
15915         else
15916                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
15917                                                possible_crtcs, plane_funcs,
15918                                                formats, num_formats,
15919                                                i9xx_format_modifiers,
15920                                                DRM_PLANE_TYPE_PRIMARY,
15921                                                "plane %c",
15922                                                plane_name(plane->i9xx_plane));
15923         if (ret)
15924                 goto fail;
15925
15926         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
15927                 supported_rotations =
15928                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180 |
15929                         DRM_MODE_REFLECT_X;
15930         } else if (INTEL_GEN(dev_priv) >= 4) {
15931                 supported_rotations =
15932                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180;
15933         } else {
15934                 supported_rotations = DRM_MODE_ROTATE_0;
15935         }
15936
15937         if (INTEL_GEN(dev_priv) >= 4)
15938                 drm_plane_create_rotation_property(&plane->base,
15939                                                    DRM_MODE_ROTATE_0,
15940                                                    supported_rotations);
15941
15942         zpos = 0;
15943         drm_plane_create_zpos_immutable_property(&plane->base, zpos);
15944
15945         drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs);
15946
15947         return plane;
15948
15949 fail:
15950         intel_plane_free(plane);
15951
15952         return ERR_PTR(ret);
15953 }
15954
15955 static struct intel_plane *
15956 intel_cursor_plane_create(struct drm_i915_private *dev_priv,
15957                           enum pipe pipe)
15958 {
15959         unsigned int possible_crtcs;
15960         struct intel_plane *cursor;
15961         int ret, zpos;
15962
15963         cursor = intel_plane_alloc();
15964         if (IS_ERR(cursor))
15965                 return cursor;
15966
15967         cursor->pipe = pipe;
15968         cursor->i9xx_plane = (enum i9xx_plane_id) pipe;
15969         cursor->id = PLANE_CURSOR;
15970         cursor->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, cursor->id);
15971
15972         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
15973                 cursor->max_stride = i845_cursor_max_stride;
15974                 cursor->update_plane = i845_update_cursor;
15975                 cursor->disable_plane = i845_disable_cursor;
15976                 cursor->get_hw_state = i845_cursor_get_hw_state;
15977                 cursor->check_plane = i845_check_cursor;
15978         } else {
15979                 cursor->max_stride = i9xx_cursor_max_stride;
15980                 cursor->update_plane = i9xx_update_cursor;
15981                 cursor->disable_plane = i9xx_disable_cursor;
15982                 cursor->get_hw_state = i9xx_cursor_get_hw_state;
15983                 cursor->check_plane = i9xx_check_cursor;
15984         }
15985
15986         cursor->cursor.base = ~0;
15987         cursor->cursor.cntl = ~0;
15988
15989         if (IS_I845G(dev_priv) || IS_I865G(dev_priv) || HAS_CUR_FBC(dev_priv))
15990                 cursor->cursor.size = ~0;
15991
15992         possible_crtcs = BIT(pipe);
15993
15994         ret = drm_universal_plane_init(&dev_priv->drm, &cursor->base,
15995                                        possible_crtcs, &intel_cursor_plane_funcs,
15996                                        intel_cursor_formats,
15997                                        ARRAY_SIZE(intel_cursor_formats),
15998                                        cursor_format_modifiers,
15999                                        DRM_PLANE_TYPE_CURSOR,
16000                                        "cursor %c", pipe_name(pipe));
16001         if (ret)
16002                 goto fail;
16003
16004         if (INTEL_GEN(dev_priv) >= 4)
16005                 drm_plane_create_rotation_property(&cursor->base,
16006                                                    DRM_MODE_ROTATE_0,
16007                                                    DRM_MODE_ROTATE_0 |
16008                                                    DRM_MODE_ROTATE_180);
16009
16010         zpos = RUNTIME_INFO(dev_priv)->num_sprites[pipe] + 1;
16011         drm_plane_create_zpos_immutable_property(&cursor->base, zpos);
16012
16013         drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs);
16014
16015         return cursor;
16016
16017 fail:
16018         intel_plane_free(cursor);
16019
16020         return ERR_PTR(ret);
16021 }
16022
16023 #define INTEL_CRTC_FUNCS \
16024         .gamma_set = drm_atomic_helper_legacy_gamma_set, \
16025         .set_config = drm_atomic_helper_set_config, \
16026         .destroy = intel_crtc_destroy, \
16027         .page_flip = drm_atomic_helper_page_flip, \
16028         .atomic_duplicate_state = intel_crtc_duplicate_state, \
16029         .atomic_destroy_state = intel_crtc_destroy_state, \
16030         .set_crc_source = intel_crtc_set_crc_source, \
16031         .verify_crc_source = intel_crtc_verify_crc_source, \
16032         .get_crc_sources = intel_crtc_get_crc_sources
16033
16034 static const struct drm_crtc_funcs bdw_crtc_funcs = {
16035         INTEL_CRTC_FUNCS,
16036
16037         .get_vblank_counter = g4x_get_vblank_counter,
16038         .enable_vblank = bdw_enable_vblank,
16039         .disable_vblank = bdw_disable_vblank,
16040 };
16041
16042 static const struct drm_crtc_funcs ilk_crtc_funcs = {
16043         INTEL_CRTC_FUNCS,
16044
16045         .get_vblank_counter = g4x_get_vblank_counter,
16046         .enable_vblank = ilk_enable_vblank,
16047         .disable_vblank = ilk_disable_vblank,
16048 };
16049
16050 static const struct drm_crtc_funcs g4x_crtc_funcs = {
16051         INTEL_CRTC_FUNCS,
16052
16053         .get_vblank_counter = g4x_get_vblank_counter,
16054         .enable_vblank = i965_enable_vblank,
16055         .disable_vblank = i965_disable_vblank,
16056 };
16057
16058 static const struct drm_crtc_funcs i965_crtc_funcs = {
16059         INTEL_CRTC_FUNCS,
16060
16061         .get_vblank_counter = i915_get_vblank_counter,
16062         .enable_vblank = i965_enable_vblank,
16063         .disable_vblank = i965_disable_vblank,
16064 };
16065
16066 static const struct drm_crtc_funcs i915gm_crtc_funcs = {
16067         INTEL_CRTC_FUNCS,
16068
16069         .get_vblank_counter = i915_get_vblank_counter,
16070         .enable_vblank = i915gm_enable_vblank,
16071         .disable_vblank = i915gm_disable_vblank,
16072 };
16073
16074 static const struct drm_crtc_funcs i915_crtc_funcs = {
16075         INTEL_CRTC_FUNCS,
16076
16077         .get_vblank_counter = i915_get_vblank_counter,
16078         .enable_vblank = i8xx_enable_vblank,
16079         .disable_vblank = i8xx_disable_vblank,
16080 };
16081
16082 static const struct drm_crtc_funcs i8xx_crtc_funcs = {
16083         INTEL_CRTC_FUNCS,
16084
16085         /* no hw vblank counter */
16086         .enable_vblank = i8xx_enable_vblank,
16087         .disable_vblank = i8xx_disable_vblank,
16088 };
16089
16090 static struct intel_crtc *intel_crtc_alloc(void)
16091 {
16092         struct intel_crtc_state *crtc_state;
16093         struct intel_crtc *crtc;
16094
16095         crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
16096         if (!crtc)
16097                 return ERR_PTR(-ENOMEM);
16098
16099         crtc_state = intel_crtc_state_alloc(crtc);
16100         if (!crtc_state) {
16101                 kfree(crtc);
16102                 return ERR_PTR(-ENOMEM);
16103         }
16104
16105         crtc->base.state = &crtc_state->uapi;
16106         crtc->config = crtc_state;
16107
16108         return crtc;
16109 }
16110
16111 static void intel_crtc_free(struct intel_crtc *crtc)
16112 {
16113         intel_crtc_destroy_state(&crtc->base, crtc->base.state);
16114         kfree(crtc);
16115 }
16116
16117 static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
16118 {
16119         struct intel_plane *primary, *cursor;
16120         const struct drm_crtc_funcs *funcs;
16121         struct intel_crtc *crtc;
16122         int sprite, ret;
16123
16124         crtc = intel_crtc_alloc();
16125         if (IS_ERR(crtc))
16126                 return PTR_ERR(crtc);
16127
16128         crtc->pipe = pipe;
16129         crtc->num_scalers = RUNTIME_INFO(dev_priv)->num_scalers[pipe];
16130
16131         primary = intel_primary_plane_create(dev_priv, pipe);
16132         if (IS_ERR(primary)) {
16133                 ret = PTR_ERR(primary);
16134                 goto fail;
16135         }
16136         crtc->plane_ids_mask |= BIT(primary->id);
16137
16138         for_each_sprite(dev_priv, pipe, sprite) {
16139                 struct intel_plane *plane;
16140
16141                 plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
16142                 if (IS_ERR(plane)) {
16143                         ret = PTR_ERR(plane);
16144                         goto fail;
16145                 }
16146                 crtc->plane_ids_mask |= BIT(plane->id);
16147         }
16148
16149         cursor = intel_cursor_plane_create(dev_priv, pipe);
16150         if (IS_ERR(cursor)) {
16151                 ret = PTR_ERR(cursor);
16152                 goto fail;
16153         }
16154         crtc->plane_ids_mask |= BIT(cursor->id);
16155
16156         if (HAS_GMCH(dev_priv)) {
16157                 if (IS_CHERRYVIEW(dev_priv) ||
16158                     IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
16159                         funcs = &g4x_crtc_funcs;
16160                 else if (IS_GEN(dev_priv, 4))
16161                         funcs = &i965_crtc_funcs;
16162                 else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv))
16163                         funcs = &i915gm_crtc_funcs;
16164                 else if (IS_GEN(dev_priv, 3))
16165                         funcs = &i915_crtc_funcs;
16166                 else
16167                         funcs = &i8xx_crtc_funcs;
16168         } else {
16169                 if (INTEL_GEN(dev_priv) >= 8)
16170                         funcs = &bdw_crtc_funcs;
16171                 else
16172                         funcs = &ilk_crtc_funcs;
16173         }
16174
16175         ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base,
16176                                         &primary->base, &cursor->base,
16177                                         funcs, "pipe %c", pipe_name(pipe));
16178         if (ret)
16179                 goto fail;
16180
16181         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->pipe_to_crtc_mapping) ||
16182                dev_priv->pipe_to_crtc_mapping[pipe] != NULL);
16183         dev_priv->pipe_to_crtc_mapping[pipe] = crtc;
16184
16185         if (INTEL_GEN(dev_priv) < 9) {
16186                 enum i9xx_plane_id i9xx_plane = primary->i9xx_plane;
16187
16188                 BUG_ON(i9xx_plane >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
16189                        dev_priv->plane_to_crtc_mapping[i9xx_plane] != NULL);
16190                 dev_priv->plane_to_crtc_mapping[i9xx_plane] = crtc;
16191         }
16192
16193         intel_color_init(crtc);
16194
16195         WARN_ON(drm_crtc_index(&crtc->base) != crtc->pipe);
16196
16197         return 0;
16198
16199 fail:
16200         intel_crtc_free(crtc);
16201
16202         return ret;
16203 }
16204
16205 int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
16206                                       struct drm_file *file)
16207 {
16208         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
16209         struct drm_crtc *drmmode_crtc;
16210         struct intel_crtc *crtc;
16211
16212         drmmode_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
16213         if (!drmmode_crtc)
16214                 return -ENOENT;
16215
16216         crtc = to_intel_crtc(drmmode_crtc);
16217         pipe_from_crtc_id->pipe = crtc->pipe;
16218
16219         return 0;
16220 }
16221
16222 static u32 intel_encoder_possible_clones(struct intel_encoder *encoder)
16223 {
16224         struct drm_device *dev = encoder->base.dev;
16225         struct intel_encoder *source_encoder;
16226         u32 possible_clones = 0;
16227
16228         for_each_intel_encoder(dev, source_encoder) {
16229                 if (encoders_cloneable(encoder, source_encoder))
16230                         possible_clones |= drm_encoder_mask(&source_encoder->base);
16231         }
16232
16233         return possible_clones;
16234 }
16235
16236 static u32 intel_encoder_possible_crtcs(struct intel_encoder *encoder)
16237 {
16238         struct drm_device *dev = encoder->base.dev;
16239         struct intel_crtc *crtc;
16240         u32 possible_crtcs = 0;
16241
16242         for_each_intel_crtc(dev, crtc) {
16243                 if (encoder->pipe_mask & BIT(crtc->pipe))
16244                         possible_crtcs |= drm_crtc_mask(&crtc->base);
16245         }
16246
16247         return possible_crtcs;
16248 }
16249
16250 static bool ilk_has_edp_a(struct drm_i915_private *dev_priv)
16251 {
16252         if (!IS_MOBILE(dev_priv))
16253                 return false;
16254
16255         if ((I915_READ(DP_A) & DP_DETECTED) == 0)
16256                 return false;
16257
16258         if (IS_GEN(dev_priv, 5) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
16259                 return false;
16260
16261         return true;
16262 }
16263
16264 static bool intel_ddi_crt_present(struct drm_i915_private *dev_priv)
16265 {
16266         if (INTEL_GEN(dev_priv) >= 9)
16267                 return false;
16268
16269         if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
16270                 return false;
16271
16272         if (HAS_PCH_LPT_H(dev_priv) &&
16273             I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
16274                 return false;
16275
16276         /* DDI E can't be used if DDI A requires 4 lanes */
16277         if (I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
16278                 return false;
16279
16280         if (!dev_priv->vbt.int_crt_support)
16281                 return false;
16282
16283         return true;
16284 }
16285
16286 void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv)
16287 {
16288         int pps_num;
16289         int pps_idx;
16290
16291         if (HAS_DDI(dev_priv))
16292                 return;
16293         /*
16294          * This w/a is needed at least on CPT/PPT, but to be sure apply it
16295          * everywhere where registers can be write protected.
16296          */
16297         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
16298                 pps_num = 2;
16299         else
16300                 pps_num = 1;
16301
16302         for (pps_idx = 0; pps_idx < pps_num; pps_idx++) {
16303                 u32 val = I915_READ(PP_CONTROL(pps_idx));
16304
16305                 val = (val & ~PANEL_UNLOCK_MASK) | PANEL_UNLOCK_REGS;
16306                 I915_WRITE(PP_CONTROL(pps_idx), val);
16307         }
16308 }
16309
16310 static void intel_pps_init(struct drm_i915_private *dev_priv)
16311 {
16312         if (HAS_PCH_SPLIT(dev_priv) || IS_GEN9_LP(dev_priv))
16313                 dev_priv->pps_mmio_base = PCH_PPS_BASE;
16314         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
16315                 dev_priv->pps_mmio_base = VLV_PPS_BASE;
16316         else
16317                 dev_priv->pps_mmio_base = PPS_BASE;
16318
16319         intel_pps_unlock_regs_wa(dev_priv);
16320 }
16321
16322 static void intel_setup_outputs(struct drm_i915_private *dev_priv)
16323 {
16324         struct intel_encoder *encoder;
16325         bool dpd_is_edp = false;
16326
16327         intel_pps_init(dev_priv);
16328
16329         if (!HAS_DISPLAY(dev_priv) || !INTEL_DISPLAY_ENABLED(dev_priv))
16330                 return;
16331
16332         if (INTEL_GEN(dev_priv) >= 12) {
16333                 intel_ddi_init(dev_priv, PORT_A);
16334                 intel_ddi_init(dev_priv, PORT_B);
16335                 intel_ddi_init(dev_priv, PORT_D);
16336                 intel_ddi_init(dev_priv, PORT_E);
16337                 intel_ddi_init(dev_priv, PORT_F);
16338                 intel_ddi_init(dev_priv, PORT_G);
16339                 intel_ddi_init(dev_priv, PORT_H);
16340                 intel_ddi_init(dev_priv, PORT_I);
16341                 icl_dsi_init(dev_priv);
16342         } else if (IS_ELKHARTLAKE(dev_priv)) {
16343                 intel_ddi_init(dev_priv, PORT_A);
16344                 intel_ddi_init(dev_priv, PORT_B);
16345                 intel_ddi_init(dev_priv, PORT_C);
16346                 intel_ddi_init(dev_priv, PORT_D);
16347                 icl_dsi_init(dev_priv);
16348         } else if (IS_GEN(dev_priv, 11)) {
16349                 intel_ddi_init(dev_priv, PORT_A);
16350                 intel_ddi_init(dev_priv, PORT_B);
16351                 intel_ddi_init(dev_priv, PORT_C);
16352                 intel_ddi_init(dev_priv, PORT_D);
16353                 intel_ddi_init(dev_priv, PORT_E);
16354                 /*
16355                  * On some ICL SKUs port F is not present. No strap bits for
16356                  * this, so rely on VBT.
16357                  * Work around broken VBTs on SKUs known to have no port F.
16358                  */
16359                 if (IS_ICL_WITH_PORT_F(dev_priv) &&
16360                     intel_bios_is_port_present(dev_priv, PORT_F))
16361                         intel_ddi_init(dev_priv, PORT_F);
16362
16363                 icl_dsi_init(dev_priv);
16364         } else if (IS_GEN9_LP(dev_priv)) {
16365                 /*
16366                  * FIXME: Broxton doesn't support port detection via the
16367                  * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to
16368                  * detect the ports.
16369                  */
16370                 intel_ddi_init(dev_priv, PORT_A);
16371                 intel_ddi_init(dev_priv, PORT_B);
16372                 intel_ddi_init(dev_priv, PORT_C);
16373
16374                 vlv_dsi_init(dev_priv);
16375         } else if (HAS_DDI(dev_priv)) {
16376                 int found;
16377
16378                 if (intel_ddi_crt_present(dev_priv))
16379                         intel_crt_init(dev_priv);
16380
16381                 /*
16382                  * Haswell uses DDI functions to detect digital outputs.
16383                  * On SKL pre-D0 the strap isn't connected, so we assume
16384                  * it's there.
16385                  */
16386                 found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
16387                 /* WaIgnoreDDIAStrap: skl */
16388                 if (found || IS_GEN9_BC(dev_priv))
16389                         intel_ddi_init(dev_priv, PORT_A);
16390
16391                 /* DDI B, C, D, and F detection is indicated by the SFUSE_STRAP
16392                  * register */
16393                 found = I915_READ(SFUSE_STRAP);
16394
16395                 if (found & SFUSE_STRAP_DDIB_DETECTED)
16396                         intel_ddi_init(dev_priv, PORT_B);
16397                 if (found & SFUSE_STRAP_DDIC_DETECTED)
16398                         intel_ddi_init(dev_priv, PORT_C);
16399                 if (found & SFUSE_STRAP_DDID_DETECTED)
16400                         intel_ddi_init(dev_priv, PORT_D);
16401                 if (found & SFUSE_STRAP_DDIF_DETECTED)
16402                         intel_ddi_init(dev_priv, PORT_F);
16403                 /*
16404                  * On SKL we don't have a way to detect DDI-E so we rely on VBT.
16405                  */
16406                 if (IS_GEN9_BC(dev_priv) &&
16407                     intel_bios_is_port_present(dev_priv, PORT_E))
16408                         intel_ddi_init(dev_priv, PORT_E);
16409
16410         } else if (HAS_PCH_SPLIT(dev_priv)) {
16411                 int found;
16412
16413                 /*
16414                  * intel_edp_init_connector() depends on this completing first,
16415                  * to prevent the registration of both eDP and LVDS and the
16416                  * incorrect sharing of the PPS.
16417                  */
16418                 intel_lvds_init(dev_priv);
16419                 intel_crt_init(dev_priv);
16420
16421                 dpd_is_edp = intel_dp_is_port_edp(dev_priv, PORT_D);
16422
16423                 if (ilk_has_edp_a(dev_priv))
16424                         intel_dp_init(dev_priv, DP_A, PORT_A);
16425
16426                 if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) {
16427                         /* PCH SDVOB multiplex with HDMIB */
16428                         found = intel_sdvo_init(dev_priv, PCH_SDVOB, PORT_B);
16429                         if (!found)
16430                                 intel_hdmi_init(dev_priv, PCH_HDMIB, PORT_B);
16431                         if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
16432                                 intel_dp_init(dev_priv, PCH_DP_B, PORT_B);
16433                 }
16434
16435                 if (I915_READ(PCH_HDMIC) & SDVO_DETECTED)
16436                         intel_hdmi_init(dev_priv, PCH_HDMIC, PORT_C);
16437
16438                 if (!dpd_is_edp && I915_READ(PCH_HDMID) & SDVO_DETECTED)
16439                         intel_hdmi_init(dev_priv, PCH_HDMID, PORT_D);
16440
16441                 if (I915_READ(PCH_DP_C) & DP_DETECTED)
16442                         intel_dp_init(dev_priv, PCH_DP_C, PORT_C);
16443
16444                 if (I915_READ(PCH_DP_D) & DP_DETECTED)
16445                         intel_dp_init(dev_priv, PCH_DP_D, PORT_D);
16446         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
16447                 bool has_edp, has_port;
16448
16449                 if (IS_VALLEYVIEW(dev_priv) && dev_priv->vbt.int_crt_support)
16450                         intel_crt_init(dev_priv);
16451
16452                 /*
16453                  * The DP_DETECTED bit is the latched state of the DDC
16454                  * SDA pin at boot. However since eDP doesn't require DDC
16455                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
16456                  * eDP ports may have been muxed to an alternate function.
16457                  * Thus we can't rely on the DP_DETECTED bit alone to detect
16458                  * eDP ports. Consult the VBT as well as DP_DETECTED to
16459                  * detect eDP ports.
16460                  *
16461                  * Sadly the straps seem to be missing sometimes even for HDMI
16462                  * ports (eg. on Voyo V3 - CHT x7-Z8700), so check both strap
16463                  * and VBT for the presence of the port. Additionally we can't
16464                  * trust the port type the VBT declares as we've seen at least
16465                  * HDMI ports that the VBT claim are DP or eDP.
16466                  */
16467                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_B);
16468                 has_port = intel_bios_is_port_present(dev_priv, PORT_B);
16469                 if (I915_READ(VLV_DP_B) & DP_DETECTED || has_port)
16470                         has_edp &= intel_dp_init(dev_priv, VLV_DP_B, PORT_B);
16471                 if ((I915_READ(VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp)
16472                         intel_hdmi_init(dev_priv, VLV_HDMIB, PORT_B);
16473
16474                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_C);
16475                 has_port = intel_bios_is_port_present(dev_priv, PORT_C);
16476                 if (I915_READ(VLV_DP_C) & DP_DETECTED || has_port)
16477                         has_edp &= intel_dp_init(dev_priv, VLV_DP_C, PORT_C);
16478                 if ((I915_READ(VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp)
16479                         intel_hdmi_init(dev_priv, VLV_HDMIC, PORT_C);
16480
16481                 if (IS_CHERRYVIEW(dev_priv)) {
16482                         /*
16483                          * eDP not supported on port D,
16484                          * so no need to worry about it
16485                          */
16486                         has_port = intel_bios_is_port_present(dev_priv, PORT_D);
16487                         if (I915_READ(CHV_DP_D) & DP_DETECTED || has_port)
16488                                 intel_dp_init(dev_priv, CHV_DP_D, PORT_D);
16489                         if (I915_READ(CHV_HDMID) & SDVO_DETECTED || has_port)
16490                                 intel_hdmi_init(dev_priv, CHV_HDMID, PORT_D);
16491                 }
16492
16493                 vlv_dsi_init(dev_priv);
16494         } else if (IS_PINEVIEW(dev_priv)) {
16495                 intel_lvds_init(dev_priv);
16496                 intel_crt_init(dev_priv);
16497         } else if (IS_GEN_RANGE(dev_priv, 3, 4)) {
16498                 bool found = false;
16499
16500                 if (IS_MOBILE(dev_priv))
16501                         intel_lvds_init(dev_priv);
16502
16503                 intel_crt_init(dev_priv);
16504
16505                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
16506                         DRM_DEBUG_KMS("probing SDVOB\n");
16507                         found = intel_sdvo_init(dev_priv, GEN3_SDVOB, PORT_B);
16508                         if (!found && IS_G4X(dev_priv)) {
16509                                 DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
16510                                 intel_hdmi_init(dev_priv, GEN4_HDMIB, PORT_B);
16511                         }
16512
16513                         if (!found && IS_G4X(dev_priv))
16514                                 intel_dp_init(dev_priv, DP_B, PORT_B);
16515                 }
16516
16517                 /* Before G4X SDVOC doesn't have its own detect register */
16518
16519                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
16520                         DRM_DEBUG_KMS("probing SDVOC\n");
16521                         found = intel_sdvo_init(dev_priv, GEN3_SDVOC, PORT_C);
16522                 }
16523
16524                 if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) {
16525
16526                         if (IS_G4X(dev_priv)) {
16527                                 DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
16528                                 intel_hdmi_init(dev_priv, GEN4_HDMIC, PORT_C);
16529                         }
16530                         if (IS_G4X(dev_priv))
16531                                 intel_dp_init(dev_priv, DP_C, PORT_C);
16532                 }
16533
16534                 if (IS_G4X(dev_priv) && (I915_READ(DP_D) & DP_DETECTED))
16535                         intel_dp_init(dev_priv, DP_D, PORT_D);
16536
16537                 if (SUPPORTS_TV(dev_priv))
16538                         intel_tv_init(dev_priv);
16539         } else if (IS_GEN(dev_priv, 2)) {
16540                 if (IS_I85X(dev_priv))
16541                         intel_lvds_init(dev_priv);
16542
16543                 intel_crt_init(dev_priv);
16544                 intel_dvo_init(dev_priv);
16545         }
16546
16547         intel_psr_init(dev_priv);
16548
16549         for_each_intel_encoder(&dev_priv->drm, encoder) {
16550                 encoder->base.possible_crtcs =
16551                         intel_encoder_possible_crtcs(encoder);
16552                 encoder->base.possible_clones =
16553                         intel_encoder_possible_clones(encoder);
16554         }
16555
16556         intel_init_pch_refclk(dev_priv);
16557
16558         drm_helper_move_panel_connectors_to_head(&dev_priv->drm);
16559 }
16560
16561 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
16562 {
16563         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
16564
16565         drm_framebuffer_cleanup(fb);
16566         intel_frontbuffer_put(intel_fb->frontbuffer);
16567
16568         kfree(intel_fb);
16569 }
16570
16571 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
16572                                                 struct drm_file *file,
16573                                                 unsigned int *handle)
16574 {
16575         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
16576
16577         if (obj->userptr.mm) {
16578                 DRM_DEBUG("attempting to use a userptr for a framebuffer, denied\n");
16579                 return -EINVAL;
16580         }
16581
16582         return drm_gem_handle_create(file, &obj->base, handle);
16583 }
16584
16585 static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
16586                                         struct drm_file *file,
16587                                         unsigned flags, unsigned color,
16588                                         struct drm_clip_rect *clips,
16589                                         unsigned num_clips)
16590 {
16591         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
16592
16593         i915_gem_object_flush_if_display(obj);
16594         intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB);
16595
16596         return 0;
16597 }
16598
16599 static const struct drm_framebuffer_funcs intel_fb_funcs = {
16600         .destroy = intel_user_framebuffer_destroy,
16601         .create_handle = intel_user_framebuffer_create_handle,
16602         .dirty = intel_user_framebuffer_dirty,
16603 };
16604
16605 static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
16606                                   struct drm_i915_gem_object *obj,
16607                                   struct drm_mode_fb_cmd2 *mode_cmd)
16608 {
16609         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
16610         struct drm_framebuffer *fb = &intel_fb->base;
16611         u32 max_stride;
16612         unsigned int tiling, stride;
16613         int ret = -EINVAL;
16614         int i;
16615
16616         intel_fb->frontbuffer = intel_frontbuffer_get(obj);
16617         if (!intel_fb->frontbuffer)
16618                 return -ENOMEM;
16619
16620         i915_gem_object_lock(obj);
16621         tiling = i915_gem_object_get_tiling(obj);
16622         stride = i915_gem_object_get_stride(obj);
16623         i915_gem_object_unlock(obj);
16624
16625         if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
16626                 /*
16627                  * If there's a fence, enforce that
16628                  * the fb modifier and tiling mode match.
16629                  */
16630                 if (tiling != I915_TILING_NONE &&
16631                     tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
16632                         DRM_DEBUG_KMS("tiling_mode doesn't match fb modifier\n");
16633                         goto err;
16634                 }
16635         } else {
16636                 if (tiling == I915_TILING_X) {
16637                         mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
16638                 } else if (tiling == I915_TILING_Y) {
16639                         DRM_DEBUG_KMS("No Y tiling for legacy addfb\n");
16640                         goto err;
16641                 }
16642         }
16643
16644         if (!drm_any_plane_has_format(&dev_priv->drm,
16645                                       mode_cmd->pixel_format,
16646                                       mode_cmd->modifier[0])) {
16647                 struct drm_format_name_buf format_name;
16648
16649                 DRM_DEBUG_KMS("unsupported pixel format %s / modifier 0x%llx\n",
16650                               drm_get_format_name(mode_cmd->pixel_format,
16651                                                   &format_name),
16652                               mode_cmd->modifier[0]);
16653                 goto err;
16654         }
16655
16656         /*
16657          * gen2/3 display engine uses the fence if present,
16658          * so the tiling mode must match the fb modifier exactly.
16659          */
16660         if (INTEL_GEN(dev_priv) < 4 &&
16661             tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
16662                 DRM_DEBUG_KMS("tiling_mode must match fb modifier exactly on gen2/3\n");
16663                 goto err;
16664         }
16665
16666         max_stride = intel_fb_max_stride(dev_priv, mode_cmd->pixel_format,
16667                                          mode_cmd->modifier[0]);
16668         if (mode_cmd->pitches[0] > max_stride) {
16669                 DRM_DEBUG_KMS("%s pitch (%u) must be at most %d\n",
16670                               mode_cmd->modifier[0] != DRM_FORMAT_MOD_LINEAR ?
16671                               "tiled" : "linear",
16672                               mode_cmd->pitches[0], max_stride);
16673                 goto err;
16674         }
16675
16676         /*
16677          * If there's a fence, enforce that
16678          * the fb pitch and fence stride match.
16679          */
16680         if (tiling != I915_TILING_NONE && mode_cmd->pitches[0] != stride) {
16681                 DRM_DEBUG_KMS("pitch (%d) must match tiling stride (%d)\n",
16682                               mode_cmd->pitches[0], stride);
16683                 goto err;
16684         }
16685
16686         /* FIXME need to adjust LINOFF/TILEOFF accordingly. */
16687         if (mode_cmd->offsets[0] != 0)
16688                 goto err;
16689
16690         drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd);
16691
16692         for (i = 0; i < fb->format->num_planes; i++) {
16693                 u32 stride_alignment;
16694
16695                 if (mode_cmd->handles[i] != mode_cmd->handles[0]) {
16696                         DRM_DEBUG_KMS("bad plane %d handle\n", i);
16697                         goto err;
16698                 }
16699
16700                 stride_alignment = intel_fb_stride_alignment(fb, i);
16701                 if (fb->pitches[i] & (stride_alignment - 1)) {
16702                         DRM_DEBUG_KMS("plane %d pitch (%d) must be at least %u byte aligned\n",
16703                                       i, fb->pitches[i], stride_alignment);
16704                         goto err;
16705                 }
16706
16707                 if (is_gen12_ccs_plane(fb, i)) {
16708                         int ccs_aux_stride = gen12_ccs_aux_stride(fb, i);
16709
16710                         if (fb->pitches[i] != ccs_aux_stride) {
16711                                 DRM_DEBUG_KMS("ccs aux plane %d pitch (%d) must be %d\n",
16712                                               i,
16713                                               fb->pitches[i], ccs_aux_stride);
16714                                 goto err;
16715                         }
16716                 }
16717
16718                 fb->obj[i] = &obj->base;
16719         }
16720
16721         ret = intel_fill_fb_info(dev_priv, fb);
16722         if (ret)
16723                 goto err;
16724
16725         ret = drm_framebuffer_init(&dev_priv->drm, fb, &intel_fb_funcs);
16726         if (ret) {
16727                 DRM_ERROR("framebuffer init failed %d\n", ret);
16728                 goto err;
16729         }
16730
16731         return 0;
16732
16733 err:
16734         intel_frontbuffer_put(intel_fb->frontbuffer);
16735         return ret;
16736 }
16737
16738 static struct drm_framebuffer *
16739 intel_user_framebuffer_create(struct drm_device *dev,
16740                               struct drm_file *filp,
16741                               const struct drm_mode_fb_cmd2 *user_mode_cmd)
16742 {
16743         struct drm_framebuffer *fb;
16744         struct drm_i915_gem_object *obj;
16745         struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
16746
16747         obj = i915_gem_object_lookup(filp, mode_cmd.handles[0]);
16748         if (!obj)
16749                 return ERR_PTR(-ENOENT);
16750
16751         fb = intel_framebuffer_create(obj, &mode_cmd);
16752         i915_gem_object_put(obj);
16753
16754         return fb;
16755 }
16756
16757 static void intel_atomic_state_free(struct drm_atomic_state *state)
16758 {
16759         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
16760
16761         drm_atomic_state_default_release(state);
16762
16763         i915_sw_fence_fini(&intel_state->commit_ready);
16764
16765         kfree(state);
16766 }
16767
16768 static enum drm_mode_status
16769 intel_mode_valid(struct drm_device *dev,
16770                  const struct drm_display_mode *mode)
16771 {
16772         struct drm_i915_private *dev_priv = to_i915(dev);
16773         int hdisplay_max, htotal_max;
16774         int vdisplay_max, vtotal_max;
16775
16776         /*
16777          * Can't reject DBLSCAN here because Xorg ddxen can add piles
16778          * of DBLSCAN modes to the output's mode list when they detect
16779          * the scaling mode property on the connector. And they don't
16780          * ask the kernel to validate those modes in any way until
16781          * modeset time at which point the client gets a protocol error.
16782          * So in order to not upset those clients we silently ignore the
16783          * DBLSCAN flag on such connectors. For other connectors we will
16784          * reject modes with the DBLSCAN flag in encoder->compute_config().
16785          * And we always reject DBLSCAN modes in connector->mode_valid()
16786          * as we never want such modes on the connector's mode list.
16787          */
16788
16789         if (mode->vscan > 1)
16790                 return MODE_NO_VSCAN;
16791
16792         if (mode->flags & DRM_MODE_FLAG_HSKEW)
16793                 return MODE_H_ILLEGAL;
16794
16795         if (mode->flags & (DRM_MODE_FLAG_CSYNC |
16796                            DRM_MODE_FLAG_NCSYNC |
16797                            DRM_MODE_FLAG_PCSYNC))
16798                 return MODE_HSYNC;
16799
16800         if (mode->flags & (DRM_MODE_FLAG_BCAST |
16801                            DRM_MODE_FLAG_PIXMUX |
16802                            DRM_MODE_FLAG_CLKDIV2))
16803                 return MODE_BAD;
16804
16805         /* Transcoder timing limits */
16806         if (INTEL_GEN(dev_priv) >= 11) {
16807                 hdisplay_max = 16384;
16808                 vdisplay_max = 8192;
16809                 htotal_max = 16384;
16810                 vtotal_max = 8192;
16811         } else if (INTEL_GEN(dev_priv) >= 9 ||
16812                    IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
16813                 hdisplay_max = 8192; /* FDI max 4096 handled elsewhere */
16814                 vdisplay_max = 4096;
16815                 htotal_max = 8192;
16816                 vtotal_max = 8192;
16817         } else if (INTEL_GEN(dev_priv) >= 3) {
16818                 hdisplay_max = 4096;
16819                 vdisplay_max = 4096;
16820                 htotal_max = 8192;
16821                 vtotal_max = 8192;
16822         } else {
16823                 hdisplay_max = 2048;
16824                 vdisplay_max = 2048;
16825                 htotal_max = 4096;
16826                 vtotal_max = 4096;
16827         }
16828
16829         if (mode->hdisplay > hdisplay_max ||
16830             mode->hsync_start > htotal_max ||
16831             mode->hsync_end > htotal_max ||
16832             mode->htotal > htotal_max)
16833                 return MODE_H_ILLEGAL;
16834
16835         if (mode->vdisplay > vdisplay_max ||
16836             mode->vsync_start > vtotal_max ||
16837             mode->vsync_end > vtotal_max ||
16838             mode->vtotal > vtotal_max)
16839                 return MODE_V_ILLEGAL;
16840
16841         if (INTEL_GEN(dev_priv) >= 5) {
16842                 if (mode->hdisplay < 64 ||
16843                     mode->htotal - mode->hdisplay < 32)
16844                         return MODE_H_ILLEGAL;
16845
16846                 if (mode->vtotal - mode->vdisplay < 5)
16847                         return MODE_V_ILLEGAL;
16848         } else {
16849                 if (mode->htotal - mode->hdisplay < 32)
16850                         return MODE_H_ILLEGAL;
16851
16852                 if (mode->vtotal - mode->vdisplay < 3)
16853                         return MODE_V_ILLEGAL;
16854         }
16855
16856         return MODE_OK;
16857 }
16858
16859 enum drm_mode_status
16860 intel_mode_valid_max_plane_size(struct drm_i915_private *dev_priv,
16861                                 const struct drm_display_mode *mode)
16862 {
16863         int plane_width_max, plane_height_max;
16864
16865         /*
16866          * intel_mode_valid() should be
16867          * sufficient on older platforms.
16868          */
16869         if (INTEL_GEN(dev_priv) < 9)
16870                 return MODE_OK;
16871
16872         /*
16873          * Most people will probably want a fullscreen
16874          * plane so let's not advertize modes that are
16875          * too big for that.
16876          */
16877         if (INTEL_GEN(dev_priv) >= 11) {
16878                 plane_width_max = 5120;
16879                 plane_height_max = 4320;
16880         } else {
16881                 plane_width_max = 5120;
16882                 plane_height_max = 4096;
16883         }
16884
16885         if (mode->hdisplay > plane_width_max)
16886                 return MODE_H_ILLEGAL;
16887
16888         if (mode->vdisplay > plane_height_max)
16889                 return MODE_V_ILLEGAL;
16890
16891         return MODE_OK;
16892 }
16893
16894 static const struct drm_mode_config_funcs intel_mode_funcs = {
16895         .fb_create = intel_user_framebuffer_create,
16896         .get_format_info = intel_get_format_info,
16897         .output_poll_changed = intel_fbdev_output_poll_changed,
16898         .mode_valid = intel_mode_valid,
16899         .atomic_check = intel_atomic_check,
16900         .atomic_commit = intel_atomic_commit,
16901         .atomic_state_alloc = intel_atomic_state_alloc,
16902         .atomic_state_clear = intel_atomic_state_clear,
16903         .atomic_state_free = intel_atomic_state_free,
16904 };
16905
16906 /**
16907  * intel_init_display_hooks - initialize the display modesetting hooks
16908  * @dev_priv: device private
16909  */
16910 void intel_init_display_hooks(struct drm_i915_private *dev_priv)
16911 {
16912         intel_init_cdclk_hooks(dev_priv);
16913
16914         if (INTEL_GEN(dev_priv) >= 9) {
16915                 dev_priv->display.get_pipe_config = hsw_get_pipe_config;
16916                 dev_priv->display.get_initial_plane_config =
16917                         skl_get_initial_plane_config;
16918                 dev_priv->display.crtc_compute_clock = hsw_crtc_compute_clock;
16919                 dev_priv->display.crtc_enable = hsw_crtc_enable;
16920                 dev_priv->display.crtc_disable = hsw_crtc_disable;
16921         } else if (HAS_DDI(dev_priv)) {
16922                 dev_priv->display.get_pipe_config = hsw_get_pipe_config;
16923                 dev_priv->display.get_initial_plane_config =
16924                         i9xx_get_initial_plane_config;
16925                 dev_priv->display.crtc_compute_clock =
16926                         hsw_crtc_compute_clock;
16927                 dev_priv->display.crtc_enable = hsw_crtc_enable;
16928                 dev_priv->display.crtc_disable = hsw_crtc_disable;
16929         } else if (HAS_PCH_SPLIT(dev_priv)) {
16930                 dev_priv->display.get_pipe_config = ilk_get_pipe_config;
16931                 dev_priv->display.get_initial_plane_config =
16932                         i9xx_get_initial_plane_config;
16933                 dev_priv->display.crtc_compute_clock =
16934                         ilk_crtc_compute_clock;
16935                 dev_priv->display.crtc_enable = ilk_crtc_enable;
16936                 dev_priv->display.crtc_disable = ilk_crtc_disable;
16937         } else if (IS_CHERRYVIEW(dev_priv)) {
16938                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
16939                 dev_priv->display.get_initial_plane_config =
16940                         i9xx_get_initial_plane_config;
16941                 dev_priv->display.crtc_compute_clock = chv_crtc_compute_clock;
16942                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
16943                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
16944         } else if (IS_VALLEYVIEW(dev_priv)) {
16945                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
16946                 dev_priv->display.get_initial_plane_config =
16947                         i9xx_get_initial_plane_config;
16948                 dev_priv->display.crtc_compute_clock = vlv_crtc_compute_clock;
16949                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
16950                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
16951         } else if (IS_G4X(dev_priv)) {
16952                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
16953                 dev_priv->display.get_initial_plane_config =
16954                         i9xx_get_initial_plane_config;
16955                 dev_priv->display.crtc_compute_clock = g4x_crtc_compute_clock;
16956                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
16957                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
16958         } else if (IS_PINEVIEW(dev_priv)) {
16959                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
16960                 dev_priv->display.get_initial_plane_config =
16961                         i9xx_get_initial_plane_config;
16962                 dev_priv->display.crtc_compute_clock = pnv_crtc_compute_clock;
16963                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
16964                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
16965         } else if (!IS_GEN(dev_priv, 2)) {
16966                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
16967                 dev_priv->display.get_initial_plane_config =
16968                         i9xx_get_initial_plane_config;
16969                 dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
16970                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
16971                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
16972         } else {
16973                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
16974                 dev_priv->display.get_initial_plane_config =
16975                         i9xx_get_initial_plane_config;
16976                 dev_priv->display.crtc_compute_clock = i8xx_crtc_compute_clock;
16977                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
16978                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
16979         }
16980
16981         if (IS_GEN(dev_priv, 5)) {
16982                 dev_priv->display.fdi_link_train = ilk_fdi_link_train;
16983         } else if (IS_GEN(dev_priv, 6)) {
16984                 dev_priv->display.fdi_link_train = gen6_fdi_link_train;
16985         } else if (IS_IVYBRIDGE(dev_priv)) {
16986                 /* FIXME: detect B0+ stepping and use auto training */
16987                 dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
16988         }
16989
16990         if (INTEL_GEN(dev_priv) >= 9)
16991                 dev_priv->display.commit_modeset_enables = skl_commit_modeset_enables;
16992         else
16993                 dev_priv->display.commit_modeset_enables = intel_commit_modeset_enables;
16994
16995 }
16996
16997 void intel_modeset_init_hw(struct drm_i915_private *i915)
16998 {
16999         intel_update_cdclk(i915);
17000         intel_dump_cdclk_state(&i915->cdclk.hw, "Current CDCLK");
17001         i915->cdclk.logical = i915->cdclk.actual = i915->cdclk.hw;
17002 }
17003
17004 /*
17005  * Calculate what we think the watermarks should be for the state we've read
17006  * out of the hardware and then immediately program those watermarks so that
17007  * we ensure the hardware settings match our internal state.
17008  *
17009  * We can calculate what we think WM's should be by creating a duplicate of the
17010  * current state (which was constructed during hardware readout) and running it
17011  * through the atomic check code to calculate new watermark values in the
17012  * state object.
17013  */
17014 static void sanitize_watermarks(struct drm_device *dev)
17015 {
17016         struct drm_i915_private *dev_priv = to_i915(dev);
17017         struct drm_atomic_state *state;
17018         struct intel_atomic_state *intel_state;
17019         struct intel_crtc *crtc;
17020         struct intel_crtc_state *crtc_state;
17021         struct drm_modeset_acquire_ctx ctx;
17022         int ret;
17023         int i;
17024
17025         /* Only supported on platforms that use atomic watermark design */
17026         if (!dev_priv->display.optimize_watermarks)
17027                 return;
17028
17029         /*
17030          * We need to hold connection_mutex before calling duplicate_state so
17031          * that the connector loop is protected.
17032          */
17033         drm_modeset_acquire_init(&ctx, 0);
17034 retry:
17035         ret = drm_modeset_lock_all_ctx(dev, &ctx);
17036         if (ret == -EDEADLK) {
17037                 drm_modeset_backoff(&ctx);
17038                 goto retry;
17039         } else if (WARN_ON(ret)) {
17040                 goto fail;
17041         }
17042
17043         state = drm_atomic_helper_duplicate_state(dev, &ctx);
17044         if (WARN_ON(IS_ERR(state)))
17045                 goto fail;
17046
17047         intel_state = to_intel_atomic_state(state);
17048
17049         /*
17050          * Hardware readout is the only time we don't want to calculate
17051          * intermediate watermarks (since we don't trust the current
17052          * watermarks).
17053          */
17054         if (!HAS_GMCH(dev_priv))
17055                 intel_state->skip_intermediate_wm = true;
17056
17057         ret = intel_atomic_check(dev, state);
17058         if (ret) {
17059                 /*
17060                  * If we fail here, it means that the hardware appears to be
17061                  * programmed in a way that shouldn't be possible, given our
17062                  * understanding of watermark requirements.  This might mean a
17063                  * mistake in the hardware readout code or a mistake in the
17064                  * watermark calculations for a given platform.  Raise a WARN
17065                  * so that this is noticeable.
17066                  *
17067                  * If this actually happens, we'll have to just leave the
17068                  * BIOS-programmed watermarks untouched and hope for the best.
17069                  */
17070                 WARN(true, "Could not determine valid watermarks for inherited state\n");
17071                 goto put_state;
17072         }
17073
17074         /* Write calculated watermark values back */
17075         for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
17076                 crtc_state->wm.need_postvbl_update = true;
17077                 dev_priv->display.optimize_watermarks(intel_state, crtc);
17078
17079                 to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
17080         }
17081
17082 put_state:
17083         drm_atomic_state_put(state);
17084 fail:
17085         drm_modeset_drop_locks(&ctx);
17086         drm_modeset_acquire_fini(&ctx);
17087 }
17088
17089 static void intel_update_fdi_pll_freq(struct drm_i915_private *dev_priv)
17090 {
17091         if (IS_GEN(dev_priv, 5)) {
17092                 u32 fdi_pll_clk =
17093                         I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK;
17094
17095                 dev_priv->fdi_pll_freq = (fdi_pll_clk + 2) * 10000;
17096         } else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) {
17097                 dev_priv->fdi_pll_freq = 270000;
17098         } else {
17099                 return;
17100         }
17101
17102         DRM_DEBUG_DRIVER("FDI PLL freq=%d\n", dev_priv->fdi_pll_freq);
17103 }
17104
17105 static int intel_initial_commit(struct drm_device *dev)
17106 {
17107         struct drm_atomic_state *state = NULL;
17108         struct drm_modeset_acquire_ctx ctx;
17109         struct intel_crtc *crtc;
17110         int ret = 0;
17111
17112         state = drm_atomic_state_alloc(dev);
17113         if (!state)
17114                 return -ENOMEM;
17115
17116         drm_modeset_acquire_init(&ctx, 0);
17117
17118 retry:
17119         state->acquire_ctx = &ctx;
17120
17121         for_each_intel_crtc(dev, crtc) {
17122                 struct intel_crtc_state *crtc_state =
17123                         intel_atomic_get_crtc_state(state, crtc);
17124
17125                 if (IS_ERR(crtc_state)) {
17126                         ret = PTR_ERR(crtc_state);
17127                         goto out;
17128                 }
17129
17130                 if (crtc_state->hw.active) {
17131                         ret = drm_atomic_add_affected_planes(state, &crtc->base);
17132                         if (ret)
17133                                 goto out;
17134
17135                         /*
17136                          * FIXME hack to force a LUT update to avoid the
17137                          * plane update forcing the pipe gamma on without
17138                          * having a proper LUT loaded. Remove once we
17139                          * have readout for pipe gamma enable.
17140                          */
17141                         crtc_state->uapi.color_mgmt_changed = true;
17142                 }
17143         }
17144
17145         ret = drm_atomic_commit(state);
17146
17147 out:
17148         if (ret == -EDEADLK) {
17149                 drm_atomic_state_clear(state);
17150                 drm_modeset_backoff(&ctx);
17151                 goto retry;
17152         }
17153
17154         drm_atomic_state_put(state);
17155
17156         drm_modeset_drop_locks(&ctx);
17157         drm_modeset_acquire_fini(&ctx);
17158
17159         return ret;
17160 }
17161
17162 static void intel_mode_config_init(struct drm_i915_private *i915)
17163 {
17164         struct drm_mode_config *mode_config = &i915->drm.mode_config;
17165
17166         drm_mode_config_init(&i915->drm);
17167
17168         mode_config->min_width = 0;
17169         mode_config->min_height = 0;
17170
17171         mode_config->preferred_depth = 24;
17172         mode_config->prefer_shadow = 1;
17173
17174         mode_config->allow_fb_modifiers = true;
17175
17176         mode_config->funcs = &intel_mode_funcs;
17177
17178         /*
17179          * Maximum framebuffer dimensions, chosen to match
17180          * the maximum render engine surface size on gen4+.
17181          */
17182         if (INTEL_GEN(i915) >= 7) {
17183                 mode_config->max_width = 16384;
17184                 mode_config->max_height = 16384;
17185         } else if (INTEL_GEN(i915) >= 4) {
17186                 mode_config->max_width = 8192;
17187                 mode_config->max_height = 8192;
17188         } else if (IS_GEN(i915, 3)) {
17189                 mode_config->max_width = 4096;
17190                 mode_config->max_height = 4096;
17191         } else {
17192                 mode_config->max_width = 2048;
17193                 mode_config->max_height = 2048;
17194         }
17195
17196         if (IS_I845G(i915) || IS_I865G(i915)) {
17197                 mode_config->cursor_width = IS_I845G(i915) ? 64 : 512;
17198                 mode_config->cursor_height = 1023;
17199         } else if (IS_GEN(i915, 2)) {
17200                 mode_config->cursor_width = 64;
17201                 mode_config->cursor_height = 64;
17202         } else {
17203                 mode_config->cursor_width = 256;
17204                 mode_config->cursor_height = 256;
17205         }
17206 }
17207
17208 int intel_modeset_init(struct drm_i915_private *i915)
17209 {
17210         struct drm_device *dev = &i915->drm;
17211         enum pipe pipe;
17212         struct intel_crtc *crtc;
17213         int ret;
17214
17215         i915->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0);
17216         i915->flip_wq = alloc_workqueue("i915_flip", WQ_HIGHPRI |
17217                                         WQ_UNBOUND, WQ_UNBOUND_MAX_ACTIVE);
17218
17219         intel_mode_config_init(i915);
17220
17221         ret = intel_bw_init(i915);
17222         if (ret)
17223                 return ret;
17224
17225         init_llist_head(&i915->atomic_helper.free_list);
17226         INIT_WORK(&i915->atomic_helper.free_work,
17227                   intel_atomic_helper_free_state_worker);
17228
17229         intel_init_quirks(i915);
17230
17231         intel_fbc_init(i915);
17232
17233         intel_init_pm(i915);
17234
17235         intel_panel_sanitize_ssc(i915);
17236
17237         intel_gmbus_setup(i915);
17238
17239         DRM_DEBUG_KMS("%d display pipe%s available.\n",
17240                       INTEL_NUM_PIPES(i915),
17241                       INTEL_NUM_PIPES(i915) > 1 ? "s" : "");
17242
17243         if (HAS_DISPLAY(i915) && INTEL_DISPLAY_ENABLED(i915)) {
17244                 for_each_pipe(i915, pipe) {
17245                         ret = intel_crtc_init(i915, pipe);
17246                         if (ret) {
17247                                 drm_mode_config_cleanup(dev);
17248                                 return ret;
17249                         }
17250                 }
17251         }
17252
17253         intel_shared_dpll_init(dev);
17254         intel_update_fdi_pll_freq(i915);
17255
17256         intel_update_czclk(i915);
17257         intel_modeset_init_hw(i915);
17258
17259         intel_hdcp_component_init(i915);
17260
17261         if (i915->max_cdclk_freq == 0)
17262                 intel_update_max_cdclk(i915);
17263
17264         /* Just disable it once at startup */
17265         intel_vga_disable(i915);
17266         intel_setup_outputs(i915);
17267
17268         drm_modeset_lock_all(dev);
17269         intel_modeset_setup_hw_state(dev, dev->mode_config.acquire_ctx);
17270         drm_modeset_unlock_all(dev);
17271
17272         for_each_intel_crtc(dev, crtc) {
17273                 struct intel_initial_plane_config plane_config = {};
17274
17275                 if (!crtc->active)
17276                         continue;
17277
17278                 /*
17279                  * Note that reserving the BIOS fb up front prevents us
17280                  * from stuffing other stolen allocations like the ring
17281                  * on top.  This prevents some ugliness at boot time, and
17282                  * can even allow for smooth boot transitions if the BIOS
17283                  * fb is large enough for the active pipe configuration.
17284                  */
17285                 i915->display.get_initial_plane_config(crtc, &plane_config);
17286
17287                 /*
17288                  * If the fb is shared between multiple heads, we'll
17289                  * just get the first one.
17290                  */
17291                 intel_find_initial_plane_obj(crtc, &plane_config);
17292         }
17293
17294         /*
17295          * Make sure hardware watermarks really match the state we read out.
17296          * Note that we need to do this after reconstructing the BIOS fb's
17297          * since the watermark calculation done here will use pstate->fb.
17298          */
17299         if (!HAS_GMCH(i915))
17300                 sanitize_watermarks(dev);
17301
17302         /*
17303          * Force all active planes to recompute their states. So that on
17304          * mode_setcrtc after probe, all the intel_plane_state variables
17305          * are already calculated and there is no assert_plane warnings
17306          * during bootup.
17307          */
17308         ret = intel_initial_commit(dev);
17309         if (ret)
17310                 DRM_DEBUG_KMS("Initial commit in probe failed.\n");
17311
17312         return 0;
17313 }
17314
17315 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
17316 {
17317         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
17318         /* 640x480@60Hz, ~25175 kHz */
17319         struct dpll clock = {
17320                 .m1 = 18,
17321                 .m2 = 7,
17322                 .p1 = 13,
17323                 .p2 = 4,
17324                 .n = 2,
17325         };
17326         u32 dpll, fp;
17327         int i;
17328
17329         WARN_ON(i9xx_calc_dpll_params(48000, &clock) != 25154);
17330
17331         DRM_DEBUG_KMS("enabling pipe %c due to force quirk (vco=%d dot=%d)\n",
17332                       pipe_name(pipe), clock.vco, clock.dot);
17333
17334         fp = i9xx_dpll_compute_fp(&clock);
17335         dpll = DPLL_DVO_2X_MODE |
17336                 DPLL_VGA_MODE_DIS |
17337                 ((clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT) |
17338                 PLL_P2_DIVIDE_BY_4 |
17339                 PLL_REF_INPUT_DREFCLK |
17340                 DPLL_VCO_ENABLE;
17341
17342         I915_WRITE(FP0(pipe), fp);
17343         I915_WRITE(FP1(pipe), fp);
17344
17345         I915_WRITE(HTOTAL(pipe), (640 - 1) | ((800 - 1) << 16));
17346         I915_WRITE(HBLANK(pipe), (640 - 1) | ((800 - 1) << 16));
17347         I915_WRITE(HSYNC(pipe), (656 - 1) | ((752 - 1) << 16));
17348         I915_WRITE(VTOTAL(pipe), (480 - 1) | ((525 - 1) << 16));
17349         I915_WRITE(VBLANK(pipe), (480 - 1) | ((525 - 1) << 16));
17350         I915_WRITE(VSYNC(pipe), (490 - 1) | ((492 - 1) << 16));
17351         I915_WRITE(PIPESRC(pipe), ((640 - 1) << 16) | (480 - 1));
17352
17353         /*
17354          * Apparently we need to have VGA mode enabled prior to changing
17355          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
17356          * dividers, even though the register value does change.
17357          */
17358         I915_WRITE(DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS);
17359         I915_WRITE(DPLL(pipe), dpll);
17360
17361         /* Wait for the clocks to stabilize. */
17362         POSTING_READ(DPLL(pipe));
17363         udelay(150);
17364
17365         /* The pixel multiplier can only be updated once the
17366          * DPLL is enabled and the clocks are stable.
17367          *
17368          * So write it again.
17369          */
17370         I915_WRITE(DPLL(pipe), dpll);
17371
17372         /* We do this three times for luck */
17373         for (i = 0; i < 3 ; i++) {
17374                 I915_WRITE(DPLL(pipe), dpll);
17375                 POSTING_READ(DPLL(pipe));
17376                 udelay(150); /* wait for warmup */
17377         }
17378
17379         I915_WRITE(PIPECONF(pipe), PIPECONF_ENABLE | PIPECONF_PROGRESSIVE);
17380         POSTING_READ(PIPECONF(pipe));
17381
17382         intel_wait_for_pipe_scanline_moving(crtc);
17383 }
17384
17385 void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
17386 {
17387         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
17388
17389         DRM_DEBUG_KMS("disabling pipe %c due to force quirk\n",
17390                       pipe_name(pipe));
17391
17392         WARN_ON(I915_READ(DSPCNTR(PLANE_A)) & DISPLAY_PLANE_ENABLE);
17393         WARN_ON(I915_READ(DSPCNTR(PLANE_B)) & DISPLAY_PLANE_ENABLE);
17394         WARN_ON(I915_READ(DSPCNTR(PLANE_C)) & DISPLAY_PLANE_ENABLE);
17395         WARN_ON(I915_READ(CURCNTR(PIPE_A)) & MCURSOR_MODE);
17396         WARN_ON(I915_READ(CURCNTR(PIPE_B)) & MCURSOR_MODE);
17397
17398         I915_WRITE(PIPECONF(pipe), 0);
17399         POSTING_READ(PIPECONF(pipe));
17400
17401         intel_wait_for_pipe_scanline_stopped(crtc);
17402
17403         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
17404         POSTING_READ(DPLL(pipe));
17405 }
17406
17407 static void
17408 intel_sanitize_plane_mapping(struct drm_i915_private *dev_priv)
17409 {
17410         struct intel_crtc *crtc;
17411
17412         if (INTEL_GEN(dev_priv) >= 4)
17413                 return;
17414
17415         for_each_intel_crtc(&dev_priv->drm, crtc) {
17416                 struct intel_plane *plane =
17417                         to_intel_plane(crtc->base.primary);
17418                 struct intel_crtc *plane_crtc;
17419                 enum pipe pipe;
17420
17421                 if (!plane->get_hw_state(plane, &pipe))
17422                         continue;
17423
17424                 if (pipe == crtc->pipe)
17425                         continue;
17426
17427                 DRM_DEBUG_KMS("[PLANE:%d:%s] attached to the wrong pipe, disabling plane\n",
17428                               plane->base.base.id, plane->base.name);
17429
17430                 plane_crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
17431                 intel_plane_disable_noatomic(plane_crtc, plane);
17432         }
17433 }
17434
17435 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
17436 {
17437         struct drm_device *dev = crtc->base.dev;
17438         struct intel_encoder *encoder;
17439
17440         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
17441                 return true;
17442
17443         return false;
17444 }
17445
17446 static struct intel_connector *intel_encoder_find_connector(struct intel_encoder *encoder)
17447 {
17448         struct drm_device *dev = encoder->base.dev;
17449         struct intel_connector *connector;
17450
17451         for_each_connector_on_encoder(dev, &encoder->base, connector)
17452                 return connector;
17453
17454         return NULL;
17455 }
17456
17457 static bool has_pch_trancoder(struct drm_i915_private *dev_priv,
17458                               enum pipe pch_transcoder)
17459 {
17460         return HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
17461                 (HAS_PCH_LPT_H(dev_priv) && pch_transcoder == PIPE_A);
17462 }
17463
17464 static void intel_sanitize_frame_start_delay(const struct intel_crtc_state *crtc_state)
17465 {
17466         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
17467         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
17468         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
17469
17470         if (INTEL_GEN(dev_priv) >= 9 ||
17471             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
17472                 i915_reg_t reg = CHICKEN_TRANS(cpu_transcoder);
17473                 u32 val;
17474
17475                 if (transcoder_is_dsi(cpu_transcoder))
17476                         return;
17477
17478                 val = I915_READ(reg);
17479                 val &= ~HSW_FRAME_START_DELAY_MASK;
17480                 val |= HSW_FRAME_START_DELAY(0);
17481                 I915_WRITE(reg, val);
17482         } else {
17483                 i915_reg_t reg = PIPECONF(cpu_transcoder);
17484                 u32 val;
17485
17486                 val = I915_READ(reg);
17487                 val &= ~PIPECONF_FRAME_START_DELAY_MASK;
17488                 val |= PIPECONF_FRAME_START_DELAY(0);
17489                 I915_WRITE(reg, val);
17490         }
17491
17492         if (!crtc_state->has_pch_encoder)
17493                 return;
17494
17495         if (HAS_PCH_IBX(dev_priv)) {
17496                 i915_reg_t reg = PCH_TRANSCONF(crtc->pipe);
17497                 u32 val;
17498
17499                 val = I915_READ(reg);
17500                 val &= ~TRANS_FRAME_START_DELAY_MASK;
17501                 val |= TRANS_FRAME_START_DELAY(0);
17502                 I915_WRITE(reg, val);
17503         } else {
17504                 enum pipe pch_transcoder = intel_crtc_pch_transcoder(crtc);
17505                 i915_reg_t reg = TRANS_CHICKEN2(pch_transcoder);
17506                 u32 val;
17507
17508                 val = I915_READ(reg);
17509                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
17510                 val |= TRANS_CHICKEN2_FRAME_START_DELAY(0);
17511                 I915_WRITE(reg, val);
17512         }
17513 }
17514
17515 static void intel_sanitize_crtc(struct intel_crtc *crtc,
17516                                 struct drm_modeset_acquire_ctx *ctx)
17517 {
17518         struct drm_device *dev = crtc->base.dev;
17519         struct drm_i915_private *dev_priv = to_i915(dev);
17520         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
17521
17522         if (crtc_state->hw.active) {
17523                 struct intel_plane *plane;
17524
17525                 /* Clear any frame start delays used for debugging left by the BIOS */
17526                 intel_sanitize_frame_start_delay(crtc_state);
17527
17528                 /* Disable everything but the primary plane */
17529                 for_each_intel_plane_on_crtc(dev, crtc, plane) {
17530                         const struct intel_plane_state *plane_state =
17531                                 to_intel_plane_state(plane->base.state);
17532
17533                         if (plane_state->uapi.visible &&
17534                             plane->base.type != DRM_PLANE_TYPE_PRIMARY)
17535                                 intel_plane_disable_noatomic(crtc, plane);
17536                 }
17537
17538                 /*
17539                  * Disable any background color set by the BIOS, but enable the
17540                  * gamma and CSC to match how we program our planes.
17541                  */
17542                 if (INTEL_GEN(dev_priv) >= 9)
17543                         I915_WRITE(SKL_BOTTOM_COLOR(crtc->pipe),
17544                                    SKL_BOTTOM_COLOR_GAMMA_ENABLE |
17545                                    SKL_BOTTOM_COLOR_CSC_ENABLE);
17546         }
17547
17548         /* Adjust the state of the output pipe according to whether we
17549          * have active connectors/encoders. */
17550         if (crtc_state->hw.active && !intel_crtc_has_encoders(crtc))
17551                 intel_crtc_disable_noatomic(crtc, ctx);
17552
17553         if (crtc_state->hw.active || HAS_GMCH(dev_priv)) {
17554                 /*
17555                  * We start out with underrun reporting disabled to avoid races.
17556                  * For correct bookkeeping mark this on active crtcs.
17557                  *
17558                  * Also on gmch platforms we dont have any hardware bits to
17559                  * disable the underrun reporting. Which means we need to start
17560                  * out with underrun reporting disabled also on inactive pipes,
17561                  * since otherwise we'll complain about the garbage we read when
17562                  * e.g. coming up after runtime pm.
17563                  *
17564                  * No protection against concurrent access is required - at
17565                  * worst a fifo underrun happens which also sets this to false.
17566                  */
17567                 crtc->cpu_fifo_underrun_disabled = true;
17568                 /*
17569                  * We track the PCH trancoder underrun reporting state
17570                  * within the crtc. With crtc for pipe A housing the underrun
17571                  * reporting state for PCH transcoder A, crtc for pipe B housing
17572                  * it for PCH transcoder B, etc. LPT-H has only PCH transcoder A,
17573                  * and marking underrun reporting as disabled for the non-existing
17574                  * PCH transcoders B and C would prevent enabling the south
17575                  * error interrupt (see cpt_can_enable_serr_int()).
17576                  */
17577                 if (has_pch_trancoder(dev_priv, crtc->pipe))
17578                         crtc->pch_fifo_underrun_disabled = true;
17579         }
17580 }
17581
17582 static bool has_bogus_dpll_config(const struct intel_crtc_state *crtc_state)
17583 {
17584         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
17585
17586         /*
17587          * Some SNB BIOSen (eg. ASUS K53SV) are known to misprogram
17588          * the hardware when a high res displays plugged in. DPLL P
17589          * divider is zero, and the pipe timings are bonkers. We'll
17590          * try to disable everything in that case.
17591          *
17592          * FIXME would be nice to be able to sanitize this state
17593          * without several WARNs, but for now let's take the easy
17594          * road.
17595          */
17596         return IS_GEN(dev_priv, 6) &&
17597                 crtc_state->hw.active &&
17598                 crtc_state->shared_dpll &&
17599                 crtc_state->port_clock == 0;
17600 }
17601
17602 static void intel_sanitize_encoder(struct intel_encoder *encoder)
17603 {
17604         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
17605         struct intel_connector *connector;
17606         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
17607         struct intel_crtc_state *crtc_state = crtc ?
17608                 to_intel_crtc_state(crtc->base.state) : NULL;
17609
17610         /* We need to check both for a crtc link (meaning that the
17611          * encoder is active and trying to read from a pipe) and the
17612          * pipe itself being active. */
17613         bool has_active_crtc = crtc_state &&
17614                 crtc_state->hw.active;
17615
17616         if (crtc_state && has_bogus_dpll_config(crtc_state)) {
17617                 DRM_DEBUG_KMS("BIOS has misprogrammed the hardware. Disabling pipe %c\n",
17618                               pipe_name(crtc->pipe));
17619                 has_active_crtc = false;
17620         }
17621
17622         connector = intel_encoder_find_connector(encoder);
17623         if (connector && !has_active_crtc) {
17624                 DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n",
17625                               encoder->base.base.id,
17626                               encoder->base.name);
17627
17628                 /* Connector is active, but has no active pipe. This is
17629                  * fallout from our resume register restoring. Disable
17630                  * the encoder manually again. */
17631                 if (crtc_state) {
17632                         struct drm_encoder *best_encoder;
17633
17634                         DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n",
17635                                       encoder->base.base.id,
17636                                       encoder->base.name);
17637
17638                         /* avoid oopsing in case the hooks consult best_encoder */
17639                         best_encoder = connector->base.state->best_encoder;
17640                         connector->base.state->best_encoder = &encoder->base;
17641
17642                         if (encoder->disable)
17643                                 encoder->disable(encoder, crtc_state,
17644                                                  connector->base.state);
17645                         if (encoder->post_disable)
17646                                 encoder->post_disable(encoder, crtc_state,
17647                                                       connector->base.state);
17648
17649                         connector->base.state->best_encoder = best_encoder;
17650                 }
17651                 encoder->base.crtc = NULL;
17652
17653                 /* Inconsistent output/port/pipe state happens presumably due to
17654                  * a bug in one of the get_hw_state functions. Or someplace else
17655                  * in our code, like the register restore mess on resume. Clamp
17656                  * things to off as a safer default. */
17657
17658                 connector->base.dpms = DRM_MODE_DPMS_OFF;
17659                 connector->base.encoder = NULL;
17660         }
17661
17662         /* notify opregion of the sanitized encoder state */
17663         intel_opregion_notify_encoder(encoder, connector && has_active_crtc);
17664
17665         if (INTEL_GEN(dev_priv) >= 11)
17666                 icl_sanitize_encoder_pll_mapping(encoder);
17667 }
17668
17669 /* FIXME read out full plane state for all planes */
17670 static void readout_plane_state(struct drm_i915_private *dev_priv)
17671 {
17672         struct intel_plane *plane;
17673         struct intel_crtc *crtc;
17674
17675         for_each_intel_plane(&dev_priv->drm, plane) {
17676                 struct intel_plane_state *plane_state =
17677                         to_intel_plane_state(plane->base.state);
17678                 struct intel_crtc_state *crtc_state;
17679                 enum pipe pipe = PIPE_A;
17680                 bool visible;
17681
17682                 visible = plane->get_hw_state(plane, &pipe);
17683
17684                 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
17685                 crtc_state = to_intel_crtc_state(crtc->base.state);
17686
17687                 intel_set_plane_visible(crtc_state, plane_state, visible);
17688
17689                 DRM_DEBUG_KMS("[PLANE:%d:%s] hw state readout: %s, pipe %c\n",
17690                               plane->base.base.id, plane->base.name,
17691                               enableddisabled(visible), pipe_name(pipe));
17692         }
17693
17694         for_each_intel_crtc(&dev_priv->drm, crtc) {
17695                 struct intel_crtc_state *crtc_state =
17696                         to_intel_crtc_state(crtc->base.state);
17697
17698                 fixup_active_planes(crtc_state);
17699         }
17700 }
17701
17702 static void intel_modeset_readout_hw_state(struct drm_device *dev)
17703 {
17704         struct drm_i915_private *dev_priv = to_i915(dev);
17705         enum pipe pipe;
17706         struct intel_crtc *crtc;
17707         struct intel_encoder *encoder;
17708         struct intel_connector *connector;
17709         struct drm_connector_list_iter conn_iter;
17710         int i;
17711
17712         dev_priv->active_pipes = 0;
17713
17714         for_each_intel_crtc(dev, crtc) {
17715                 struct intel_crtc_state *crtc_state =
17716                         to_intel_crtc_state(crtc->base.state);
17717
17718                 __drm_atomic_helper_crtc_destroy_state(&crtc_state->uapi);
17719                 intel_crtc_free_hw_state(crtc_state);
17720                 intel_crtc_state_reset(crtc_state, crtc);
17721
17722                 crtc_state->hw.active = crtc_state->hw.enable =
17723                         dev_priv->display.get_pipe_config(crtc, crtc_state);
17724
17725                 crtc->base.enabled = crtc_state->hw.enable;
17726                 crtc->active = crtc_state->hw.active;
17727
17728                 if (crtc_state->hw.active)
17729                         dev_priv->active_pipes |= BIT(crtc->pipe);
17730
17731                 DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n",
17732                               crtc->base.base.id, crtc->base.name,
17733                               enableddisabled(crtc_state->hw.active));
17734         }
17735
17736         readout_plane_state(dev_priv);
17737
17738         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
17739                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
17740
17741                 pll->on = pll->info->funcs->get_hw_state(dev_priv, pll,
17742                                                         &pll->state.hw_state);
17743
17744                 if (IS_ELKHARTLAKE(dev_priv) && pll->on &&
17745                     pll->info->id == DPLL_ID_EHL_DPLL4) {
17746                         pll->wakeref = intel_display_power_get(dev_priv,
17747                                                                POWER_DOMAIN_DPLL_DC_OFF);
17748                 }
17749
17750                 pll->state.crtc_mask = 0;
17751                 for_each_intel_crtc(dev, crtc) {
17752                         struct intel_crtc_state *crtc_state =
17753                                 to_intel_crtc_state(crtc->base.state);
17754
17755                         if (crtc_state->hw.active &&
17756                             crtc_state->shared_dpll == pll)
17757                                 pll->state.crtc_mask |= 1 << crtc->pipe;
17758                 }
17759                 pll->active_mask = pll->state.crtc_mask;
17760
17761                 DRM_DEBUG_KMS("%s hw state readout: crtc_mask 0x%08x, on %i\n",
17762                               pll->info->name, pll->state.crtc_mask, pll->on);
17763         }
17764
17765         for_each_intel_encoder(dev, encoder) {
17766                 pipe = 0;
17767
17768                 if (encoder->get_hw_state(encoder, &pipe)) {
17769                         struct intel_crtc_state *crtc_state;
17770
17771                         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
17772                         crtc_state = to_intel_crtc_state(crtc->base.state);
17773
17774                         encoder->base.crtc = &crtc->base;
17775                         encoder->get_config(encoder, crtc_state);
17776                 } else {
17777                         encoder->base.crtc = NULL;
17778                 }
17779
17780                 DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
17781                               encoder->base.base.id, encoder->base.name,
17782                               enableddisabled(encoder->base.crtc),
17783                               pipe_name(pipe));
17784         }
17785
17786         drm_connector_list_iter_begin(dev, &conn_iter);
17787         for_each_intel_connector_iter(connector, &conn_iter) {
17788                 if (connector->get_hw_state(connector)) {
17789                         struct intel_crtc_state *crtc_state;
17790                         struct intel_crtc *crtc;
17791
17792                         connector->base.dpms = DRM_MODE_DPMS_ON;
17793
17794                         encoder = connector->encoder;
17795                         connector->base.encoder = &encoder->base;
17796
17797                         crtc = to_intel_crtc(encoder->base.crtc);
17798                         crtc_state = crtc ? to_intel_crtc_state(crtc->base.state) : NULL;
17799
17800                         if (crtc_state && crtc_state->hw.active) {
17801                                 /*
17802                                  * This has to be done during hardware readout
17803                                  * because anything calling .crtc_disable may
17804                                  * rely on the connector_mask being accurate.
17805                                  */
17806                                 crtc_state->uapi.connector_mask |=
17807                                         drm_connector_mask(&connector->base);
17808                                 crtc_state->uapi.encoder_mask |=
17809                                         drm_encoder_mask(&encoder->base);
17810                         }
17811                 } else {
17812                         connector->base.dpms = DRM_MODE_DPMS_OFF;
17813                         connector->base.encoder = NULL;
17814                 }
17815                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n",
17816                               connector->base.base.id, connector->base.name,
17817                               enableddisabled(connector->base.encoder));
17818         }
17819         drm_connector_list_iter_end(&conn_iter);
17820
17821         for_each_intel_crtc(dev, crtc) {
17822                 struct intel_bw_state *bw_state =
17823                         to_intel_bw_state(dev_priv->bw_obj.state);
17824                 struct intel_crtc_state *crtc_state =
17825                         to_intel_crtc_state(crtc->base.state);
17826                 struct intel_plane *plane;
17827                 int min_cdclk = 0;
17828
17829                 if (crtc_state->hw.active) {
17830                         struct drm_display_mode *mode = &crtc_state->hw.mode;
17831
17832                         intel_mode_from_pipe_config(&crtc_state->hw.adjusted_mode,
17833                                                     crtc_state);
17834
17835                         *mode = crtc_state->hw.adjusted_mode;
17836                         mode->hdisplay = crtc_state->pipe_src_w;
17837                         mode->vdisplay = crtc_state->pipe_src_h;
17838
17839                         /*
17840                          * The initial mode needs to be set in order to keep
17841                          * the atomic core happy. It wants a valid mode if the
17842                          * crtc's enabled, so we do the above call.
17843                          *
17844                          * But we don't set all the derived state fully, hence
17845                          * set a flag to indicate that a full recalculation is
17846                          * needed on the next commit.
17847                          */
17848                         mode->private_flags = I915_MODE_FLAG_INHERITED;
17849
17850                         intel_crtc_compute_pixel_rate(crtc_state);
17851
17852                         intel_crtc_update_active_timings(crtc_state);
17853
17854                         intel_crtc_copy_hw_to_uapi_state(crtc_state);
17855                 }
17856
17857                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
17858                         const struct intel_plane_state *plane_state =
17859                                 to_intel_plane_state(plane->base.state);
17860
17861                         /*
17862                          * FIXME don't have the fb yet, so can't
17863                          * use intel_plane_data_rate() :(
17864                          */
17865                         if (plane_state->uapi.visible)
17866                                 crtc_state->data_rate[plane->id] =
17867                                         4 * crtc_state->pixel_rate;
17868                         /*
17869                          * FIXME don't have the fb yet, so can't
17870                          * use plane->min_cdclk() :(
17871                          */
17872                         if (plane_state->uapi.visible && plane->min_cdclk) {
17873                                 if (crtc_state->double_wide ||
17874                                     INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
17875                                         crtc_state->min_cdclk[plane->id] =
17876                                                 DIV_ROUND_UP(crtc_state->pixel_rate, 2);
17877                                 else
17878                                         crtc_state->min_cdclk[plane->id] =
17879                                                 crtc_state->pixel_rate;
17880                         }
17881                         DRM_DEBUG_KMS("[PLANE:%d:%s] min_cdclk %d kHz\n",
17882                                       plane->base.base.id, plane->base.name,
17883                                       crtc_state->min_cdclk[plane->id]);
17884                 }
17885
17886                 if (crtc_state->hw.active) {
17887                         min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
17888                         if (WARN_ON(min_cdclk < 0))
17889                                 min_cdclk = 0;
17890                 }
17891
17892                 dev_priv->min_cdclk[crtc->pipe] = min_cdclk;
17893                 dev_priv->min_voltage_level[crtc->pipe] =
17894                         crtc_state->min_voltage_level;
17895
17896                 intel_bw_crtc_update(bw_state, crtc_state);
17897
17898                 intel_pipe_config_sanity_check(dev_priv, crtc_state);
17899         }
17900 }
17901
17902 static void
17903 get_encoder_power_domains(struct drm_i915_private *dev_priv)
17904 {
17905         struct intel_encoder *encoder;
17906
17907         for_each_intel_encoder(&dev_priv->drm, encoder) {
17908                 struct intel_crtc_state *crtc_state;
17909
17910                 if (!encoder->get_power_domains)
17911                         continue;
17912
17913                 /*
17914                  * MST-primary and inactive encoders don't have a crtc state
17915                  * and neither of these require any power domain references.
17916                  */
17917                 if (!encoder->base.crtc)
17918                         continue;
17919
17920                 crtc_state = to_intel_crtc_state(encoder->base.crtc->state);
17921                 encoder->get_power_domains(encoder, crtc_state);
17922         }
17923 }
17924
17925 static void intel_early_display_was(struct drm_i915_private *dev_priv)
17926 {
17927         /*
17928          * Display WA #1185 WaDisableDARBFClkGating:cnl,glk,icl,ehl,tgl
17929          * Also known as Wa_14010480278.
17930          */
17931         if (IS_GEN_RANGE(dev_priv, 10, 12) || IS_GEMINILAKE(dev_priv))
17932                 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
17933                            DARBF_GATING_DIS);
17934
17935         if (IS_HASWELL(dev_priv)) {
17936                 /*
17937                  * WaRsPkgCStateDisplayPMReq:hsw
17938                  * System hang if this isn't done before disabling all planes!
17939                  */
17940                 I915_WRITE(CHICKEN_PAR1_1,
17941                            I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
17942         }
17943 }
17944
17945 static void ibx_sanitize_pch_hdmi_port(struct drm_i915_private *dev_priv,
17946                                        enum port port, i915_reg_t hdmi_reg)
17947 {
17948         u32 val = I915_READ(hdmi_reg);
17949
17950         if (val & SDVO_ENABLE ||
17951             (val & SDVO_PIPE_SEL_MASK) == SDVO_PIPE_SEL(PIPE_A))
17952                 return;
17953
17954         DRM_DEBUG_KMS("Sanitizing transcoder select for HDMI %c\n",
17955                       port_name(port));
17956
17957         val &= ~SDVO_PIPE_SEL_MASK;
17958         val |= SDVO_PIPE_SEL(PIPE_A);
17959
17960         I915_WRITE(hdmi_reg, val);
17961 }
17962
17963 static void ibx_sanitize_pch_dp_port(struct drm_i915_private *dev_priv,
17964                                      enum port port, i915_reg_t dp_reg)
17965 {
17966         u32 val = I915_READ(dp_reg);
17967
17968         if (val & DP_PORT_EN ||
17969             (val & DP_PIPE_SEL_MASK) == DP_PIPE_SEL(PIPE_A))
17970                 return;
17971
17972         DRM_DEBUG_KMS("Sanitizing transcoder select for DP %c\n",
17973                       port_name(port));
17974
17975         val &= ~DP_PIPE_SEL_MASK;
17976         val |= DP_PIPE_SEL(PIPE_A);
17977
17978         I915_WRITE(dp_reg, val);
17979 }
17980
17981 static void ibx_sanitize_pch_ports(struct drm_i915_private *dev_priv)
17982 {
17983         /*
17984          * The BIOS may select transcoder B on some of the PCH
17985          * ports even it doesn't enable the port. This would trip
17986          * assert_pch_dp_disabled() and assert_pch_hdmi_disabled().
17987          * Sanitize the transcoder select bits to prevent that. We
17988          * assume that the BIOS never actually enabled the port,
17989          * because if it did we'd actually have to toggle the port
17990          * on and back off to make the transcoder A select stick
17991          * (see. intel_dp_link_down(), intel_disable_hdmi(),
17992          * intel_disable_sdvo()).
17993          */
17994         ibx_sanitize_pch_dp_port(dev_priv, PORT_B, PCH_DP_B);
17995         ibx_sanitize_pch_dp_port(dev_priv, PORT_C, PCH_DP_C);
17996         ibx_sanitize_pch_dp_port(dev_priv, PORT_D, PCH_DP_D);
17997
17998         /* PCH SDVOB multiplex with HDMIB */
17999         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_B, PCH_HDMIB);
18000         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_C, PCH_HDMIC);
18001         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_D, PCH_HDMID);
18002 }
18003
18004 /* Scan out the current hw modeset state,
18005  * and sanitizes it to the current state
18006  */
18007 static void
18008 intel_modeset_setup_hw_state(struct drm_device *dev,
18009                              struct drm_modeset_acquire_ctx *ctx)
18010 {
18011         struct drm_i915_private *dev_priv = to_i915(dev);
18012         struct intel_encoder *encoder;
18013         struct intel_crtc *crtc;
18014         intel_wakeref_t wakeref;
18015         int i;
18016
18017         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
18018
18019         intel_early_display_was(dev_priv);
18020         intel_modeset_readout_hw_state(dev);
18021
18022         /* HW state is read out, now we need to sanitize this mess. */
18023
18024         /* Sanitize the TypeC port mode upfront, encoders depend on this */
18025         for_each_intel_encoder(dev, encoder) {
18026                 enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
18027
18028                 /* We need to sanitize only the MST primary port. */
18029                 if (encoder->type != INTEL_OUTPUT_DP_MST &&
18030                     intel_phy_is_tc(dev_priv, phy))
18031                         intel_tc_port_sanitize(enc_to_dig_port(&encoder->base));
18032         }
18033
18034         get_encoder_power_domains(dev_priv);
18035
18036         if (HAS_PCH_IBX(dev_priv))
18037                 ibx_sanitize_pch_ports(dev_priv);
18038
18039         /*
18040          * intel_sanitize_plane_mapping() may need to do vblank
18041          * waits, so we need vblank interrupts restored beforehand.
18042          */
18043         for_each_intel_crtc(&dev_priv->drm, crtc) {
18044                 struct intel_crtc_state *crtc_state =
18045                         to_intel_crtc_state(crtc->base.state);
18046
18047                 drm_crtc_vblank_reset(&crtc->base);
18048
18049                 if (crtc_state->hw.active)
18050                         intel_crtc_vblank_on(crtc_state);
18051         }
18052
18053         intel_sanitize_plane_mapping(dev_priv);
18054
18055         for_each_intel_encoder(dev, encoder)
18056                 intel_sanitize_encoder(encoder);
18057
18058         for_each_intel_crtc(&dev_priv->drm, crtc) {
18059                 struct intel_crtc_state *crtc_state =
18060                         to_intel_crtc_state(crtc->base.state);
18061
18062                 intel_sanitize_crtc(crtc, ctx);
18063                 intel_dump_pipe_config(crtc_state, NULL, "[setup_hw_state]");
18064         }
18065
18066         intel_modeset_update_connector_atomic_state(dev);
18067
18068         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
18069                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
18070
18071                 if (!pll->on || pll->active_mask)
18072                         continue;
18073
18074                 DRM_DEBUG_KMS("%s enabled but not in use, disabling\n",
18075                               pll->info->name);
18076
18077                 pll->info->funcs->disable(dev_priv, pll);
18078                 pll->on = false;
18079         }
18080
18081         if (IS_G4X(dev_priv)) {
18082                 g4x_wm_get_hw_state(dev_priv);
18083                 g4x_wm_sanitize(dev_priv);
18084         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
18085                 vlv_wm_get_hw_state(dev_priv);
18086                 vlv_wm_sanitize(dev_priv);
18087         } else if (INTEL_GEN(dev_priv) >= 9) {
18088                 skl_wm_get_hw_state(dev_priv);
18089         } else if (HAS_PCH_SPLIT(dev_priv)) {
18090                 ilk_wm_get_hw_state(dev_priv);
18091         }
18092
18093         for_each_intel_crtc(dev, crtc) {
18094                 struct intel_crtc_state *crtc_state =
18095                         to_intel_crtc_state(crtc->base.state);
18096                 u64 put_domains;
18097
18098                 put_domains = modeset_get_crtc_power_domains(crtc_state);
18099                 if (WARN_ON(put_domains))
18100                         modeset_put_power_domains(dev_priv, put_domains);
18101         }
18102
18103         intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref);
18104 }
18105
18106 void intel_display_resume(struct drm_device *dev)
18107 {
18108         struct drm_i915_private *dev_priv = to_i915(dev);
18109         struct drm_atomic_state *state = dev_priv->modeset_restore_state;
18110         struct drm_modeset_acquire_ctx ctx;
18111         int ret;
18112
18113         dev_priv->modeset_restore_state = NULL;
18114         if (state)
18115                 state->acquire_ctx = &ctx;
18116
18117         drm_modeset_acquire_init(&ctx, 0);
18118
18119         while (1) {
18120                 ret = drm_modeset_lock_all_ctx(dev, &ctx);
18121                 if (ret != -EDEADLK)
18122                         break;
18123
18124                 drm_modeset_backoff(&ctx);
18125         }
18126
18127         if (!ret)
18128                 ret = __intel_display_resume(dev, state, &ctx);
18129
18130         intel_enable_ipc(dev_priv);
18131         drm_modeset_drop_locks(&ctx);
18132         drm_modeset_acquire_fini(&ctx);
18133
18134         if (ret)
18135                 DRM_ERROR("Restoring old state failed with %i\n", ret);
18136         if (state)
18137                 drm_atomic_state_put(state);
18138 }
18139
18140 static void intel_hpd_poll_fini(struct drm_i915_private *i915)
18141 {
18142         struct intel_connector *connector;
18143         struct drm_connector_list_iter conn_iter;
18144
18145         /* Kill all the work that may have been queued by hpd. */
18146         drm_connector_list_iter_begin(&i915->drm, &conn_iter);
18147         for_each_intel_connector_iter(connector, &conn_iter) {
18148                 if (connector->modeset_retry_work.func)
18149                         cancel_work_sync(&connector->modeset_retry_work);
18150                 if (connector->hdcp.shim) {
18151                         cancel_delayed_work_sync(&connector->hdcp.check_work);
18152                         cancel_work_sync(&connector->hdcp.prop_work);
18153                 }
18154         }
18155         drm_connector_list_iter_end(&conn_iter);
18156 }
18157
18158 void intel_modeset_driver_remove(struct drm_i915_private *i915)
18159 {
18160         flush_workqueue(i915->flip_wq);
18161         flush_workqueue(i915->modeset_wq);
18162
18163         flush_work(&i915->atomic_helper.free_work);
18164         WARN_ON(!llist_empty(&i915->atomic_helper.free_list));
18165
18166         /*
18167          * Interrupts and polling as the first thing to avoid creating havoc.
18168          * Too much stuff here (turning of connectors, ...) would
18169          * experience fancy races otherwise.
18170          */
18171         intel_irq_uninstall(i915);
18172
18173         /*
18174          * Due to the hpd irq storm handling the hotplug work can re-arm the
18175          * poll handlers. Hence disable polling after hpd handling is shut down.
18176          */
18177         intel_hpd_poll_fini(i915);
18178
18179         /*
18180          * MST topology needs to be suspended so we don't have any calls to
18181          * fbdev after it's finalized. MST will be destroyed later as part of
18182          * drm_mode_config_cleanup()
18183          */
18184         intel_dp_mst_suspend(i915);
18185
18186         /* poll work can call into fbdev, hence clean that up afterwards */
18187         intel_fbdev_fini(i915);
18188
18189         intel_unregister_dsm_handler();
18190
18191         intel_fbc_global_disable(i915);
18192
18193         /* flush any delayed tasks or pending work */
18194         flush_scheduled_work();
18195
18196         intel_hdcp_component_fini(i915);
18197
18198         drm_mode_config_cleanup(&i915->drm);
18199
18200         intel_overlay_cleanup(i915);
18201
18202         intel_gmbus_teardown(i915);
18203
18204         intel_bw_cleanup(i915);
18205
18206         destroy_workqueue(i915->flip_wq);
18207         destroy_workqueue(i915->modeset_wq);
18208
18209         intel_fbc_cleanup_cfb(i915);
18210 }
18211
18212 #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
18213
18214 struct intel_display_error_state {
18215
18216         u32 power_well_driver;
18217
18218         struct intel_cursor_error_state {
18219                 u32 control;
18220                 u32 position;
18221                 u32 base;
18222                 u32 size;
18223         } cursor[I915_MAX_PIPES];
18224
18225         struct intel_pipe_error_state {
18226                 bool power_domain_on;
18227                 u32 source;
18228                 u32 stat;
18229         } pipe[I915_MAX_PIPES];
18230
18231         struct intel_plane_error_state {
18232                 u32 control;
18233                 u32 stride;
18234                 u32 size;
18235                 u32 pos;
18236                 u32 addr;
18237                 u32 surface;
18238                 u32 tile_offset;
18239         } plane[I915_MAX_PIPES];
18240
18241         struct intel_transcoder_error_state {
18242                 bool available;
18243                 bool power_domain_on;
18244                 enum transcoder cpu_transcoder;
18245
18246                 u32 conf;
18247
18248                 u32 htotal;
18249                 u32 hblank;
18250                 u32 hsync;
18251                 u32 vtotal;
18252                 u32 vblank;
18253                 u32 vsync;
18254         } transcoder[5];
18255 };
18256
18257 struct intel_display_error_state *
18258 intel_display_capture_error_state(struct drm_i915_private *dev_priv)
18259 {
18260         struct intel_display_error_state *error;
18261         int transcoders[] = {
18262                 TRANSCODER_A,
18263                 TRANSCODER_B,
18264                 TRANSCODER_C,
18265                 TRANSCODER_D,
18266                 TRANSCODER_EDP,
18267         };
18268         int i;
18269
18270         BUILD_BUG_ON(ARRAY_SIZE(transcoders) != ARRAY_SIZE(error->transcoder));
18271
18272         if (!HAS_DISPLAY(dev_priv) || !INTEL_DISPLAY_ENABLED(dev_priv))
18273                 return NULL;
18274
18275         error = kzalloc(sizeof(*error), GFP_ATOMIC);
18276         if (error == NULL)
18277                 return NULL;
18278
18279         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
18280                 error->power_well_driver = I915_READ(HSW_PWR_WELL_CTL2);
18281
18282         for_each_pipe(dev_priv, i) {
18283                 error->pipe[i].power_domain_on =
18284                         __intel_display_power_is_enabled(dev_priv,
18285                                                          POWER_DOMAIN_PIPE(i));
18286                 if (!error->pipe[i].power_domain_on)
18287                         continue;
18288
18289                 error->cursor[i].control = I915_READ(CURCNTR(i));
18290                 error->cursor[i].position = I915_READ(CURPOS(i));
18291                 error->cursor[i].base = I915_READ(CURBASE(i));
18292
18293                 error->plane[i].control = I915_READ(DSPCNTR(i));
18294                 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
18295                 if (INTEL_GEN(dev_priv) <= 3) {
18296                         error->plane[i].size = I915_READ(DSPSIZE(i));
18297                         error->plane[i].pos = I915_READ(DSPPOS(i));
18298                 }
18299                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
18300                         error->plane[i].addr = I915_READ(DSPADDR(i));
18301                 if (INTEL_GEN(dev_priv) >= 4) {
18302                         error->plane[i].surface = I915_READ(DSPSURF(i));
18303                         error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
18304                 }
18305
18306                 error->pipe[i].source = I915_READ(PIPESRC(i));
18307
18308                 if (HAS_GMCH(dev_priv))
18309                         error->pipe[i].stat = I915_READ(PIPESTAT(i));
18310         }
18311
18312         for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
18313                 enum transcoder cpu_transcoder = transcoders[i];
18314
18315                 if (!INTEL_INFO(dev_priv)->trans_offsets[cpu_transcoder])
18316                         continue;
18317
18318                 error->transcoder[i].available = true;
18319                 error->transcoder[i].power_domain_on =
18320                         __intel_display_power_is_enabled(dev_priv,
18321                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder));
18322                 if (!error->transcoder[i].power_domain_on)
18323                         continue;
18324
18325                 error->transcoder[i].cpu_transcoder = cpu_transcoder;
18326
18327                 error->transcoder[i].conf = I915_READ(PIPECONF(cpu_transcoder));
18328                 error->transcoder[i].htotal = I915_READ(HTOTAL(cpu_transcoder));
18329                 error->transcoder[i].hblank = I915_READ(HBLANK(cpu_transcoder));
18330                 error->transcoder[i].hsync = I915_READ(HSYNC(cpu_transcoder));
18331                 error->transcoder[i].vtotal = I915_READ(VTOTAL(cpu_transcoder));
18332                 error->transcoder[i].vblank = I915_READ(VBLANK(cpu_transcoder));
18333                 error->transcoder[i].vsync = I915_READ(VSYNC(cpu_transcoder));
18334         }
18335
18336         return error;
18337 }
18338
18339 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
18340
18341 void
18342 intel_display_print_error_state(struct drm_i915_error_state_buf *m,
18343                                 struct intel_display_error_state *error)
18344 {
18345         struct drm_i915_private *dev_priv = m->i915;
18346         int i;
18347
18348         if (!error)
18349                 return;
18350
18351         err_printf(m, "Num Pipes: %d\n", INTEL_NUM_PIPES(dev_priv));
18352         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
18353                 err_printf(m, "PWR_WELL_CTL2: %08x\n",
18354                            error->power_well_driver);
18355         for_each_pipe(dev_priv, i) {
18356                 err_printf(m, "Pipe [%d]:\n", i);
18357                 err_printf(m, "  Power: %s\n",
18358                            onoff(error->pipe[i].power_domain_on));
18359                 err_printf(m, "  SRC: %08x\n", error->pipe[i].source);
18360                 err_printf(m, "  STAT: %08x\n", error->pipe[i].stat);
18361
18362                 err_printf(m, "Plane [%d]:\n", i);
18363                 err_printf(m, "  CNTR: %08x\n", error->plane[i].control);
18364                 err_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
18365                 if (INTEL_GEN(dev_priv) <= 3) {
18366                         err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
18367                         err_printf(m, "  POS: %08x\n", error->plane[i].pos);
18368                 }
18369                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
18370                         err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
18371                 if (INTEL_GEN(dev_priv) >= 4) {
18372                         err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
18373                         err_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
18374                 }
18375
18376                 err_printf(m, "Cursor [%d]:\n", i);
18377                 err_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
18378                 err_printf(m, "  POS: %08x\n", error->cursor[i].position);
18379                 err_printf(m, "  BASE: %08x\n", error->cursor[i].base);
18380         }
18381
18382         for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
18383                 if (!error->transcoder[i].available)
18384                         continue;
18385
18386                 err_printf(m, "CPU transcoder: %s\n",
18387                            transcoder_name(error->transcoder[i].cpu_transcoder));
18388                 err_printf(m, "  Power: %s\n",
18389                            onoff(error->transcoder[i].power_domain_on));
18390                 err_printf(m, "  CONF: %08x\n", error->transcoder[i].conf);
18391                 err_printf(m, "  HTOTAL: %08x\n", error->transcoder[i].htotal);
18392                 err_printf(m, "  HBLANK: %08x\n", error->transcoder[i].hblank);
18393                 err_printf(m, "  HSYNC: %08x\n", error->transcoder[i].hsync);
18394                 err_printf(m, "  VTOTAL: %08x\n", error->transcoder[i].vtotal);
18395                 err_printf(m, "  VBLANK: %08x\n", error->transcoder[i].vblank);
18396                 err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
18397         }
18398 }
18399
18400 #endif