drivers/gpu/drm/i915/display/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/i2c.h>
  28 #include <linux/input.h>
  29 #include <linux/intel-iommu.h>
  30 #include <linux/kernel.h>
  31 #include <linux/module.h>
  32 #include <linux/dma-resv.h>
  33 #include <linux/slab.h>
  34
  35 #include <drm/drm_atomic.h>
  36 #include <drm/drm_atomic_helper.h>
  37 #include <drm/drm_atomic_uapi.h>
  38 #include <drm/drm_dp_helper.h>
  39 #include <drm/drm_edid.h>
  40 #include <drm/drm_fourcc.h>
  41 #include <drm/drm_plane_helper.h>
  42 #include <drm/drm_probe_helper.h>
  43 #include <drm/drm_rect.h>
  44 #include <drm/i915_drm.h>
  45
  46 #include "display/intel_crt.h"
  47 #include "display/intel_ddi.h"
  48 #include "display/intel_dp.h"
  49 #include "display/intel_dp_mst.h"
  50 #include "display/intel_dsi.h"
  51 #include "display/intel_dvo.h"
  52 #include "display/intel_gmbus.h"
  53 #include "display/intel_hdmi.h"
  54 #include "display/intel_lvds.h"
  55 #include "display/intel_sdvo.h"
  56 #include "display/intel_tv.h"
  57 #include "display/intel_vdsc.h"
  58
  59 #include "gt/intel_rps.h"
  60
  61 #include "i915_drv.h"
  62 #include "i915_trace.h"
  63 #include "intel_acpi.h"
  64 #include "intel_atomic.h"
  65 #include "intel_atomic_plane.h"
  66 #include "intel_bw.h"
  67 #include "intel_cdclk.h"
  68 #include "intel_color.h"
  69 #include "intel_display_types.h"
  70 #include "intel_dp_link_training.h"
  71 #include "intel_fbc.h"
  72 #include "intel_fbdev.h"
  73 #include "intel_fifo_underrun.h"
  74 #include "intel_frontbuffer.h"
  75 #include "intel_hdcp.h"
  76 #include "intel_hotplug.h"
  77 #include "intel_overlay.h"
  78 #include "intel_pipe_crc.h"
  79 #include "intel_pm.h"
  80 #include "intel_psr.h"
  81 #include "intel_quirks.h"
  82 #include "intel_sideband.h"
  83 #include "intel_sprite.h"
  84 #include "intel_tc.h"
  85 #include "intel_vga.h"
  86
  87 /* Primary plane formats for gen <= 3 */
  88 static const u32 i8xx_primary_formats[] = {
  89         DRM_FORMAT_C8,
  90         DRM_FORMAT_XRGB1555,
  91         DRM_FORMAT_RGB565,
  92         DRM_FORMAT_XRGB8888,
  93 };
  94
  95 /* Primary plane formats for ivb (no fp16 due to hw issue) */
  96 static const u32 ivb_primary_formats[] = {
  97         DRM_FORMAT_C8,
  98         DRM_FORMAT_RGB565,
  99         DRM_FORMAT_XRGB8888,
 100         DRM_FORMAT_XBGR8888,
 101         DRM_FORMAT_XRGB2101010,
 102         DRM_FORMAT_XBGR2101010,
 103 };
 104
 105 /* Primary plane formats for gen >= 4, except ivb */
 106 static const u32 i965_primary_formats[] = {
 107         DRM_FORMAT_C8,
 108         DRM_FORMAT_RGB565,
 109         DRM_FORMAT_XRGB8888,
 110         DRM_FORMAT_XBGR8888,
 111         DRM_FORMAT_XRGB2101010,
 112         DRM_FORMAT_XBGR2101010,
 113         DRM_FORMAT_XBGR16161616F,
 114 };
 115
 116 /* Primary plane formats for vlv/chv */
 117 static const u32 vlv_primary_formats[] = {
 118         DRM_FORMAT_C8,
 119         DRM_FORMAT_RGB565,
 120         DRM_FORMAT_XRGB8888,
 121         DRM_FORMAT_XBGR8888,
 122         DRM_FORMAT_ARGB8888,
 123         DRM_FORMAT_ABGR8888,
 124         DRM_FORMAT_XRGB2101010,
 125         DRM_FORMAT_XBGR2101010,
 126         DRM_FORMAT_ARGB2101010,
 127         DRM_FORMAT_ABGR2101010,
 128         DRM_FORMAT_XBGR16161616F,
 129 };
 130
 131 static const u64 i9xx_format_modifiers[] = {
 132         I915_FORMAT_MOD_X_TILED,
 133         DRM_FORMAT_MOD_LINEAR,
 134         DRM_FORMAT_MOD_INVALID
 135 };
 136
 137 /* Cursor formats */
 138 static const u32 intel_cursor_formats[] = {
 139         DRM_FORMAT_ARGB8888,
 140 };
 141
 142 static const u64 cursor_format_modifiers[] = {
 143         DRM_FORMAT_MOD_LINEAR,
 144         DRM_FORMAT_MOD_INVALID
 145 };
 146
 147 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
 148                                 struct intel_crtc_state *pipe_config);
 149 static void ilk_pch_clock_get(struct intel_crtc *crtc,
 150                               struct intel_crtc_state *pipe_config);
 151
 152 static int intel_framebuffer_init(struct intel_framebuffer *ifb,
 153                                   struct drm_i915_gem_object *obj,
 154                                   struct drm_mode_fb_cmd2 *mode_cmd);
 155 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state);
 156 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state);
 157 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
 158                                          const struct intel_link_m_n *m_n,
 159                                          const struct intel_link_m_n *m2_n2);
 160 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state);
 161 static void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state);
 162 static void hsw_set_pipeconf(const struct intel_crtc_state *crtc_state);
 163 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state);
 164 static void vlv_prepare_pll(struct intel_crtc *crtc,
 165                             const struct intel_crtc_state *pipe_config);
 166 static void chv_prepare_pll(struct intel_crtc *crtc,
 167                             const struct intel_crtc_state *pipe_config);
 168 static void skl_pfit_enable(const struct intel_crtc_state *crtc_state);
 169 static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state);
 170 static void intel_modeset_setup_hw_state(struct drm_device *dev,
 171                                          struct drm_modeset_acquire_ctx *ctx);
 172 static struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc);
 173
 174 struct intel_limit {
 175         struct {
 176                 int min, max;
 177         } dot, vco, n, m, m1, m2, p, p1;
 178
 179         struct {
 180                 int dot_limit;
 181                 int p2_slow, p2_fast;
 182         } p2;
 183 };
 184
 185 /* returns HPLL frequency in kHz */
 186 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv)
 187 {
 188         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 189
 190         /* Obtain SKU information */
 191         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
 192                 CCK_FUSE_HPLL_FREQ_MASK;
 193
 194         return vco_freq[hpll_freq] * 1000;
 195 }
 196
 197 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
 198                       const char *name, u32 reg, int ref_freq)
 199 {
 200         u32 val;
 201         int divider;
 202
 203         val = vlv_cck_read(dev_priv, reg);
 204         divider = val & CCK_FREQUENCY_VALUES;
 205
 206         WARN((val & CCK_FREQUENCY_STATUS) !=
 207              (divider << CCK_FREQUENCY_STATUS_SHIFT),
 208              "%s change in progress\n", name);
 209
 210         return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1);
 211 }
 212
 213 int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 214                            const char *name, u32 reg)
 215 {
 216         int hpll;
 217
 218         vlv_cck_get(dev_priv);
 219
 220         if (dev_priv->hpll_freq == 0)
 221                 dev_priv->hpll_freq = vlv_get_hpll_vco(dev_priv);
 222
 223         hpll = vlv_get_cck_clock(dev_priv, name, reg, dev_priv->hpll_freq);
 224
 225         vlv_cck_put(dev_priv);
 226
 227         return hpll;
 228 }
 229
 230 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 231 {
 232         if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
 233                 return;
 234
 235         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
 236                                                       CCK_CZ_CLOCK_CONTROL);
 237
 238         DRM_DEBUG_DRIVER("CZ clock rate: %d kHz\n", dev_priv->czclk_freq);
 239 }
 240
 241 static inline u32 /* units of 100MHz */
 242 intel_fdi_link_freq(struct drm_i915_private *dev_priv,
 243                     const struct intel_crtc_state *pipe_config)
 244 {
 245         if (HAS_DDI(dev_priv))
 246                 return pipe_config->port_clock; /* SPLL */
 247         else
 248                 return dev_priv->fdi_pll_freq;
 249 }
 250
 251 static const struct intel_limit intel_limits_i8xx_dac = {
 252         .dot = { .min = 25000, .max = 350000 },
 253         .vco = { .min = 908000, .max = 1512000 },
 254         .n = { .min = 2, .max = 16 },
 255         .m = { .min = 96, .max = 140 },
 256         .m1 = { .min = 18, .max = 26 },
 257         .m2 = { .min = 6, .max = 16 },
 258         .p = { .min = 4, .max = 128 },
 259         .p1 = { .min = 2, .max = 33 },
 260         .p2 = { .dot_limit = 165000,
 261                 .p2_slow = 4, .p2_fast = 2 },
 262 };
 263
 264 static const struct intel_limit intel_limits_i8xx_dvo = {
 265         .dot = { .min = 25000, .max = 350000 },
 266         .vco = { .min = 908000, .max = 1512000 },
 267         .n = { .min = 2, .max = 16 },
 268         .m = { .min = 96, .max = 140 },
 269         .m1 = { .min = 18, .max = 26 },
 270         .m2 = { .min = 6, .max = 16 },
 271         .p = { .min = 4, .max = 128 },
 272         .p1 = { .min = 2, .max = 33 },
 273         .p2 = { .dot_limit = 165000,
 274                 .p2_slow = 4, .p2_fast = 4 },
 275 };
 276
 277 static const struct intel_limit intel_limits_i8xx_lvds = {
 278         .dot = { .min = 25000, .max = 350000 },
 279         .vco = { .min = 908000, .max = 1512000 },
 280         .n = { .min = 2, .max = 16 },
 281         .m = { .min = 96, .max = 140 },
 282         .m1 = { .min = 18, .max = 26 },
 283         .m2 = { .min = 6, .max = 16 },
 284         .p = { .min = 4, .max = 128 },
 285         .p1 = { .min = 1, .max = 6 },
 286         .p2 = { .dot_limit = 165000,
 287                 .p2_slow = 14, .p2_fast = 7 },
 288 };
 289
 290 static const struct intel_limit intel_limits_i9xx_sdvo = {
 291         .dot = { .min = 20000, .max = 400000 },
 292         .vco = { .min = 1400000, .max = 2800000 },
 293         .n = { .min = 1, .max = 6 },
 294         .m = { .min = 70, .max = 120 },
 295         .m1 = { .min = 8, .max = 18 },
 296         .m2 = { .min = 3, .max = 7 },
 297         .p = { .min = 5, .max = 80 },
 298         .p1 = { .min = 1, .max = 8 },
 299         .p2 = { .dot_limit = 200000,
 300                 .p2_slow = 10, .p2_fast = 5 },
 301 };
 302
 303 static const struct intel_limit intel_limits_i9xx_lvds = {
 304         .dot = { .min = 20000, .max = 400000 },
 305         .vco = { .min = 1400000, .max = 2800000 },
 306         .n = { .min = 1, .max = 6 },
 307         .m = { .min = 70, .max = 120 },
 308         .m1 = { .min = 8, .max = 18 },
 309         .m2 = { .min = 3, .max = 7 },
 310         .p = { .min = 7, .max = 98 },
 311         .p1 = { .min = 1, .max = 8 },
 312         .p2 = { .dot_limit = 112000,
 313                 .p2_slow = 14, .p2_fast = 7 },
 314 };
 315
 316
 317 static const struct intel_limit intel_limits_g4x_sdvo = {
 318         .dot = { .min = 25000, .max = 270000 },
 319         .vco = { .min = 1750000, .max = 3500000},
 320         .n = { .min = 1, .max = 4 },
 321         .m = { .min = 104, .max = 138 },
 322         .m1 = { .min = 17, .max = 23 },
 323         .m2 = { .min = 5, .max = 11 },
 324         .p = { .min = 10, .max = 30 },
 325         .p1 = { .min = 1, .max = 3},
 326         .p2 = { .dot_limit = 270000,
 327                 .p2_slow = 10,
 328                 .p2_fast = 10
 329         },
 330 };
 331
 332 static const struct intel_limit intel_limits_g4x_hdmi = {
 333         .dot = { .min = 22000, .max = 400000 },
 334         .vco = { .min = 1750000, .max = 3500000},
 335         .n = { .min = 1, .max = 4 },
 336         .m = { .min = 104, .max = 138 },
 337         .m1 = { .min = 16, .max = 23 },
 338         .m2 = { .min = 5, .max = 11 },
 339         .p = { .min = 5, .max = 80 },
 340         .p1 = { .min = 1, .max = 8},
 341         .p2 = { .dot_limit = 165000,
 342                 .p2_slow = 10, .p2_fast = 5 },
 343 };
 344
 345 static const struct intel_limit intel_limits_g4x_single_channel_lvds = {
 346         .dot = { .min = 20000, .max = 115000 },
 347         .vco = { .min = 1750000, .max = 3500000 },
 348         .n = { .min = 1, .max = 3 },
 349         .m = { .min = 104, .max = 138 },
 350         .m1 = { .min = 17, .max = 23 },
 351         .m2 = { .min = 5, .max = 11 },
 352         .p = { .min = 28, .max = 112 },
 353         .p1 = { .min = 2, .max = 8 },
 354         .p2 = { .dot_limit = 0,
 355                 .p2_slow = 14, .p2_fast = 14
 356         },
 357 };
 358
 359 static const struct intel_limit intel_limits_g4x_dual_channel_lvds = {
 360         .dot = { .min = 80000, .max = 224000 },
 361         .vco = { .min = 1750000, .max = 3500000 },
 362         .n = { .min = 1, .max = 3 },
 363         .m = { .min = 104, .max = 138 },
 364         .m1 = { .min = 17, .max = 23 },
 365         .m2 = { .min = 5, .max = 11 },
 366         .p = { .min = 14, .max = 42 },
 367         .p1 = { .min = 2, .max = 6 },
 368         .p2 = { .dot_limit = 0,
 369                 .p2_slow = 7, .p2_fast = 7
 370         },
 371 };
 372
 373 static const struct intel_limit pnv_limits_sdvo = {
 374         .dot = { .min = 20000, .max = 400000},
 375         .vco = { .min = 1700000, .max = 3500000 },
 376         /* Pineview's Ncounter is a ring counter */
 377         .n = { .min = 3, .max = 6 },
 378         .m = { .min = 2, .max = 256 },
 379         /* Pineview only has one combined m divider, which we treat as m2. */
 380         .m1 = { .min = 0, .max = 0 },
 381         .m2 = { .min = 0, .max = 254 },
 382         .p = { .min = 5, .max = 80 },
 383         .p1 = { .min = 1, .max = 8 },
 384         .p2 = { .dot_limit = 200000,
 385                 .p2_slow = 10, .p2_fast = 5 },
 386 };
 387
 388 static const struct intel_limit pnv_limits_lvds = {
 389         .dot = { .min = 20000, .max = 400000 },
 390         .vco = { .min = 1700000, .max = 3500000 },
 391         .n = { .min = 3, .max = 6 },
 392         .m = { .min = 2, .max = 256 },
 393         .m1 = { .min = 0, .max = 0 },
 394         .m2 = { .min = 0, .max = 254 },
 395         .p = { .min = 7, .max = 112 },
 396         .p1 = { .min = 1, .max = 8 },
 397         .p2 = { .dot_limit = 112000,
 398                 .p2_slow = 14, .p2_fast = 14 },
 399 };
 400
 401 /* Ironlake / Sandybridge
 402  *
 403  * We calculate clock using (register_value + 2) for N/M1/M2, so here
 404  * the range value for them is (actual_value - 2).
 405  */
 406 static const struct intel_limit ilk_limits_dac = {
 407         .dot = { .min = 25000, .max = 350000 },
 408         .vco = { .min = 1760000, .max = 3510000 },
 409         .n = { .min = 1, .max = 5 },
 410         .m = { .min = 79, .max = 127 },
 411         .m1 = { .min = 12, .max = 22 },
 412         .m2 = { .min = 5, .max = 9 },
 413         .p = { .min = 5, .max = 80 },
 414         .p1 = { .min = 1, .max = 8 },
 415         .p2 = { .dot_limit = 225000,
 416                 .p2_slow = 10, .p2_fast = 5 },
 417 };
 418
 419 static const struct intel_limit ilk_limits_single_lvds = {
 420         .dot = { .min = 25000, .max = 350000 },
 421         .vco = { .min = 1760000, .max = 3510000 },
 422         .n = { .min = 1, .max = 3 },
 423         .m = { .min = 79, .max = 118 },
 424         .m1 = { .min = 12, .max = 22 },
 425         .m2 = { .min = 5, .max = 9 },
 426         .p = { .min = 28, .max = 112 },
 427         .p1 = { .min = 2, .max = 8 },
 428         .p2 = { .dot_limit = 225000,
 429                 .p2_slow = 14, .p2_fast = 14 },
 430 };
 431
 432 static const struct intel_limit ilk_limits_dual_lvds = {
 433         .dot = { .min = 25000, .max = 350000 },
 434         .vco = { .min = 1760000, .max = 3510000 },
 435         .n = { .min = 1, .max = 3 },
 436         .m = { .min = 79, .max = 127 },
 437         .m1 = { .min = 12, .max = 22 },
 438         .m2 = { .min = 5, .max = 9 },
 439         .p = { .min = 14, .max = 56 },
 440         .p1 = { .min = 2, .max = 8 },
 441         .p2 = { .dot_limit = 225000,
 442                 .p2_slow = 7, .p2_fast = 7 },
 443 };
 444
 445 /* LVDS 100mhz refclk limits. */
 446 static const struct intel_limit ilk_limits_single_lvds_100m = {
 447         .dot = { .min = 25000, .max = 350000 },
 448         .vco = { .min = 1760000, .max = 3510000 },
 449         .n = { .min = 1, .max = 2 },
 450         .m = { .min = 79, .max = 126 },
 451         .m1 = { .min = 12, .max = 22 },
 452         .m2 = { .min = 5, .max = 9 },
 453         .p = { .min = 28, .max = 112 },
 454         .p1 = { .min = 2, .max = 8 },
 455         .p2 = { .dot_limit = 225000,
 456                 .p2_slow = 14, .p2_fast = 14 },
 457 };
 458
 459 static const struct intel_limit ilk_limits_dual_lvds_100m = {
 460         .dot = { .min = 25000, .max = 350000 },
 461         .vco = { .min = 1760000, .max = 3510000 },
 462         .n = { .min = 1, .max = 3 },
 463         .m = { .min = 79, .max = 126 },
 464         .m1 = { .min = 12, .max = 22 },
 465         .m2 = { .min = 5, .max = 9 },
 466         .p = { .min = 14, .max = 42 },
 467         .p1 = { .min = 2, .max = 6 },
 468         .p2 = { .dot_limit = 225000,
 469                 .p2_slow = 7, .p2_fast = 7 },
 470 };
 471
 472 static const struct intel_limit intel_limits_vlv = {
 473          /*
 474           * These are the data rate limits (measured in fast clocks)
 475           * since those are the strictest limits we have. The fast
 476           * clock and actual rate limits are more relaxed, so checking
 477           * them would make no difference.
 478           */
 479         .dot = { .min = 25000 * 5, .max = 270000 * 5 },
 480         .vco = { .min = 4000000, .max = 6000000 },
 481         .n = { .min = 1, .max = 7 },
 482         .m1 = { .min = 2, .max = 3 },
 483         .m2 = { .min = 11, .max = 156 },
 484         .p1 = { .min = 2, .max = 3 },
 485         .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */
 486 };
 487
 488 static const struct intel_limit intel_limits_chv = {
 489         /*
 490          * These are the data rate limits (measured in fast clocks)
 491          * since those are the strictest limits we have.  The fast
 492          * clock and actual rate limits are more relaxed, so checking
 493          * them would make no difference.
 494          */
 495         .dot = { .min = 25000 * 5, .max = 540000 * 5},
 496         .vco = { .min = 4800000, .max = 6480000 },
 497         .n = { .min = 1, .max = 1 },
 498         .m1 = { .min = 2, .max = 2 },
 499         .m2 = { .min = 24 << 22, .max = 175 << 22 },
 500         .p1 = { .min = 2, .max = 4 },
 501         .p2 = { .p2_slow = 1, .p2_fast = 14 },
 502 };
 503
 504 static const struct intel_limit intel_limits_bxt = {
 505         /* FIXME: find real dot limits */
 506         .dot = { .min = 0, .max = INT_MAX },
 507         .vco = { .min = 4800000, .max = 6700000 },
 508         .n = { .min = 1, .max = 1 },
 509         .m1 = { .min = 2, .max = 2 },
 510         /* FIXME: find real m2 limits */
 511         .m2 = { .min = 2 << 22, .max = 255 << 22 },
 512         .p1 = { .min = 2, .max = 4 },
 513         .p2 = { .p2_slow = 1, .p2_fast = 20 },
 514 };
 515
 516 /* WA Display #0827: Gen9:all */
 517 static void
 518 skl_wa_827(struct drm_i915_private *dev_priv, enum pipe pipe, bool enable)
 519 {
 520         if (enable)
 521                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 522                            I915_READ(CLKGATE_DIS_PSL(pipe)) |
 523                            DUPS1_GATING_DIS | DUPS2_GATING_DIS);
 524         else
 525                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 526                            I915_READ(CLKGATE_DIS_PSL(pipe)) &
 527                            ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS));
 528 }
 529
 530 /* Wa_2006604312:icl */
 531 static void
 532 icl_wa_scalerclkgating(struct drm_i915_private *dev_priv, enum pipe pipe,
 533                        bool enable)
 534 {
 535         if (enable)
 536                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 537                            I915_READ(CLKGATE_DIS_PSL(pipe)) | DPFR_GATING_DIS);
 538         else
 539                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 540                            I915_READ(CLKGATE_DIS_PSL(pipe)) & ~DPFR_GATING_DIS);
 541 }
 542
 543 static bool
 544 needs_modeset(const struct intel_crtc_state *state)
 545 {
 546         return drm_atomic_crtc_needs_modeset(&state->uapi);
 547 }
 548
 549 bool
 550 is_trans_port_sync_mode(const struct intel_crtc_state *crtc_state)
 551 {
 552         return (crtc_state->master_transcoder != INVALID_TRANSCODER ||
 553                 crtc_state->sync_mode_slaves_mask);
 554 }
 555
 556 static bool
 557 is_trans_port_sync_master(const struct intel_crtc_state *crtc_state)
 558 {
 559         return (crtc_state->master_transcoder == INVALID_TRANSCODER &&
 560                 crtc_state->sync_mode_slaves_mask);
 561 }
 562
 563 static bool
 564 is_trans_port_sync_slave(const struct intel_crtc_state *crtc_state)
 565 {
 566         return crtc_state->master_transcoder != INVALID_TRANSCODER;
 567 }
 568
 569 /*
 570  * Platform specific helpers to calculate the port PLL loopback- (clock.m),
 571  * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast
 572  * (clock.dot) clock rates. This fast dot clock is fed to the port's IO logic.
 573  * The helpers' return value is the rate of the clock that is fed to the
 574  * display engine's pipe which can be the above fast dot clock rate or a
 575  * divided-down version of it.
 576  */
 577 /* m1 is reserved as 0 in Pineview, n is a ring counter */
 578 static int pnv_calc_dpll_params(int refclk, struct dpll *clock)
 579 {
 580         clock->m = clock->m2 + 2;
 581         clock->p = clock->p1 * clock->p2;
 582         if (WARN_ON(clock->n == 0 || clock->p == 0))
 583                 return 0;
 584         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 585         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 586
 587         return clock->dot;
 588 }
 589
 590 static u32 i9xx_dpll_compute_m(struct dpll *dpll)
 591 {
 592         return 5 * (dpll->m1 + 2) + (dpll->m2 + 2);
 593 }
 594
 595 static int i9xx_calc_dpll_params(int refclk, struct dpll *clock)
 596 {
 597         clock->m = i9xx_dpll_compute_m(clock);
 598         clock->p = clock->p1 * clock->p2;
 599         if (WARN_ON(clock->n + 2 == 0 || clock->p == 0))
 600                 return 0;
 601         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n + 2);
 602         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 603
 604         return clock->dot;
 605 }
 606
 607 static int vlv_calc_dpll_params(int refclk, struct dpll *clock)
 608 {
 609         clock->m = clock->m1 * clock->m2;
 610         clock->p = clock->p1 * clock->p2;
 611         if (WARN_ON(clock->n == 0 || clock->p == 0))
 612                 return 0;
 613         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 614         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 615
 616         return clock->dot / 5;
 617 }
 618
 619 int chv_calc_dpll_params(int refclk, struct dpll *clock)
 620 {
 621         clock->m = clock->m1 * clock->m2;
 622         clock->p = clock->p1 * clock->p2;
 623         if (WARN_ON(clock->n == 0 || clock->p == 0))
 624                 return 0;
 625         clock->vco = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(refclk, clock->m),
 626                                            clock->n << 22);
 627         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 628
 629         return clock->dot / 5;
 630 }
 631
 632 #define INTELPllInvalid(s)   do { /* DRM_DEBUG(s); */ return false; } while (0)
 633
 634 /*
 635  * Returns whether the given set of divisors are valid for a given refclk with
 636  * the given connectors.
 637  */
 638 static bool intel_PLL_is_valid(struct drm_i915_private *dev_priv,
 639                                const struct intel_limit *limit,
 640                                const struct dpll *clock)
 641 {
 642         if (clock->n   < limit->n.min   || limit->n.max   < clock->n)
 643                 INTELPllInvalid("n out of range\n");
 644         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
 645                 INTELPllInvalid("p1 out of range\n");
 646         if (clock->m2  < limit->m2.min  || limit->m2.max  < clock->m2)
 647                 INTELPllInvalid("m2 out of range\n");
 648         if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
 649                 INTELPllInvalid("m1 out of range\n");
 650
 651         if (!IS_PINEVIEW(dev_priv) && !IS_VALLEYVIEW(dev_priv) &&
 652             !IS_CHERRYVIEW(dev_priv) && !IS_GEN9_LP(dev_priv))
 653                 if (clock->m1 <= clock->m2)
 654                         INTELPllInvalid("m1 <= m2\n");
 655
 656         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
 657             !IS_GEN9_LP(dev_priv)) {
 658                 if (clock->p < limit->p.min || limit->p.max < clock->p)
 659                         INTELPllInvalid("p out of range\n");
 660                 if (clock->m < limit->m.min || limit->m.max < clock->m)
 661                         INTELPllInvalid("m out of range\n");
 662         }
 663
 664         if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
 665                 INTELPllInvalid("vco out of range\n");
 666         /* XXX: We may need to be checking "Dot clock" depending on the multiplier,
 667          * connector, etc., rather than just a single range.
 668          */
 669         if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
 670                 INTELPllInvalid("dot out of range\n");
 671
 672         return true;
 673 }
 674
 675 static int
 676 i9xx_select_p2_div(const struct intel_limit *limit,
 677                    const struct intel_crtc_state *crtc_state,
 678                    int target)
 679 {
 680         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 681
 682         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 683                 /*
 684                  * For LVDS just rely on its current settings for dual-channel.
 685                  * We haven't figured out how to reliably set up different
 686                  * single/dual channel state, if we even can.
 687                  */
 688                 if (intel_is_dual_link_lvds(dev_priv))
 689                         return limit->p2.p2_fast;
 690                 else
 691                         return limit->p2.p2_slow;
 692         } else {
 693                 if (target < limit->p2.dot_limit)
 694                         return limit->p2.p2_slow;
 695                 else
 696                         return limit->p2.p2_fast;
 697         }
 698 }
 699
 700 /*
 701  * Returns a set of divisors for the desired target clock with the given
 702  * refclk, or FALSE.  The returned values represent the clock equation:
 703  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 704  *
 705  * Target and reference clocks are specified in kHz.
 706  *
 707  * If match_clock is provided, then best_clock P divider must match the P
 708  * divider from @match_clock used for LVDS downclocking.
 709  */
 710 static bool
 711 i9xx_find_best_dpll(const struct intel_limit *limit,
 712                     struct intel_crtc_state *crtc_state,
 713                     int target, int refclk, struct dpll *match_clock,
 714                     struct dpll *best_clock)
 715 {
 716         struct drm_device *dev = crtc_state->uapi.crtc->dev;
 717         struct dpll clock;
 718         int err = target;
 719
 720         memset(best_clock, 0, sizeof(*best_clock));
 721
 722         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 723
 724         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 725              clock.m1++) {
 726                 for (clock.m2 = limit->m2.min;
 727                      clock.m2 <= limit->m2.max; clock.m2++) {
 728                         if (clock.m2 >= clock.m1)
 729                                 break;
 730                         for (clock.n = limit->n.min;
 731                              clock.n <= limit->n.max; clock.n++) {
 732                                 for (clock.p1 = limit->p1.min;
 733                                         clock.p1 <= limit->p1.max; clock.p1++) {
 734                                         int this_err;
 735
 736                                         i9xx_calc_dpll_params(refclk, &clock);
 737                                         if (!intel_PLL_is_valid(to_i915(dev),
 738                                                                 limit,
 739                                                                 &clock))
 740                                                 continue;
 741                                         if (match_clock &&
 742                                             clock.p != match_clock->p)
 743                                                 continue;
 744
 745                                         this_err = abs(clock.dot - target);
 746                                         if (this_err < err) {
 747                                                 *best_clock = clock;
 748                                                 err = this_err;
 749                                         }
 750                                 }
 751                         }
 752                 }
 753         }
 754
 755         return (err != target);
 756 }
 757
 758 /*
 759  * Returns a set of divisors for the desired target clock with the given
 760  * refclk, or FALSE.  The returned values represent the clock equation:
 761  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 762  *
 763  * Target and reference clocks are specified in kHz.
 764  *
 765  * If match_clock is provided, then best_clock P divider must match the P
 766  * divider from @match_clock used for LVDS downclocking.
 767  */
 768 static bool
 769 pnv_find_best_dpll(const struct intel_limit *limit,
 770                    struct intel_crtc_state *crtc_state,
 771                    int target, int refclk, struct dpll *match_clock,
 772                    struct dpll *best_clock)
 773 {
 774         struct drm_device *dev = crtc_state->uapi.crtc->dev;
 775         struct dpll clock;
 776         int err = target;
 777
 778         memset(best_clock, 0, sizeof(*best_clock));
 779
 780         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 781
 782         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 783              clock.m1++) {
 784                 for (clock.m2 = limit->m2.min;
 785                      clock.m2 <= limit->m2.max; clock.m2++) {
 786                         for (clock.n = limit->n.min;
 787                              clock.n <= limit->n.max; clock.n++) {
 788                                 for (clock.p1 = limit->p1.min;
 789                                         clock.p1 <= limit->p1.max; clock.p1++) {
 790                                         int this_err;
 791
 792                                         pnv_calc_dpll_params(refclk, &clock);
 793                                         if (!intel_PLL_is_valid(to_i915(dev),
 794                                                                 limit,
 795                                                                 &clock))
 796                                                 continue;
 797                                         if (match_clock &&
 798                                             clock.p != match_clock->p)
 799                                                 continue;
 800
 801                                         this_err = abs(clock.dot - target);
 802                                         if (this_err < err) {
 803                                                 *best_clock = clock;
 804                                                 err = this_err;
 805                                         }
 806                                 }
 807                         }
 808                 }
 809         }
 810
 811         return (err != target);
 812 }
 813
 814 /*
 815  * Returns a set of divisors for the desired target clock with the given
 816  * refclk, or FALSE.  The returned values represent the clock equation:
 817  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 818  *
 819  * Target and reference clocks are specified in kHz.
 820  *
 821  * If match_clock is provided, then best_clock P divider must match the P
 822  * divider from @match_clock used for LVDS downclocking.
 823  */
 824 static bool
 825 g4x_find_best_dpll(const struct intel_limit *limit,
 826                    struct intel_crtc_state *crtc_state,
 827                    int target, int refclk, struct dpll *match_clock,
 828                    struct dpll *best_clock)
 829 {
 830         struct drm_device *dev = crtc_state->uapi.crtc->dev;
 831         struct dpll clock;
 832         int max_n;
 833         bool found = false;
 834         /* approximately equals target * 0.00585 */
 835         int err_most = (target >> 8) + (target >> 9);
 836
 837         memset(best_clock, 0, sizeof(*best_clock));
 838
 839         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 840
 841         max_n = limit->n.max;
 842         /* based on hardware requirement, prefer smaller n to precision */
 843         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 844                 /* based on hardware requirement, prefere larger m1,m2 */
 845                 for (clock.m1 = limit->m1.max;
 846                      clock.m1 >= limit->m1.min; clock.m1--) {
 847                         for (clock.m2 = limit->m2.max;
 848                              clock.m2 >= limit->m2.min; clock.m2--) {
 849                                 for (clock.p1 = limit->p1.max;
 850                                      clock.p1 >= limit->p1.min; clock.p1--) {
 851                                         int this_err;
 852
 853                                         i9xx_calc_dpll_params(refclk, &clock);
 854                                         if (!intel_PLL_is_valid(to_i915(dev),
 855                                                                 limit,
 856                                                                 &clock))
 857                                                 continue;
 858
 859                                         this_err = abs(clock.dot - target);
 860                                         if (this_err < err_most) {
 861                                                 *best_clock = clock;
 862                                                 err_most = this_err;
 863                                                 max_n = clock.n;
 864                                                 found = true;
 865                                         }
 866                                 }
 867                         }
 868                 }
 869         }
 870         return found;
 871 }
 872
 873 /*
 874  * Check if the calculated PLL configuration is more optimal compared to the
 875  * best configuration and error found so far. Return the calculated error.
 876  */
 877 static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
 878                                const struct dpll *calculated_clock,
 879                                const struct dpll *best_clock,
 880                                unsigned int best_error_ppm,
 881                                unsigned int *error_ppm)
 882 {
 883         /*
 884          * For CHV ignore the error and consider only the P value.
 885          * Prefer a bigger P value based on HW requirements.
 886          */
 887         if (IS_CHERRYVIEW(to_i915(dev))) {
 888                 *error_ppm = 0;
 889
 890                 return calculated_clock->p > best_clock->p;
 891         }
 892
 893         if (WARN_ON_ONCE(!target_freq))
 894                 return false;
 895
 896         *error_ppm = div_u64(1000000ULL *
 897                                 abs(target_freq - calculated_clock->dot),
 898                              target_freq);
 899         /*
 900          * Prefer a better P value over a better (smaller) error if the error
 901          * is small. Ensure this preference for future configurations too by
 902          * setting the error to 0.
 903          */
 904         if (*error_ppm < 100 && calculated_clock->p > best_clock->p) {
 905                 *error_ppm = 0;
 906
 907                 return true;
 908         }
 909
 910         return *error_ppm + 10 < best_error_ppm;
 911 }
 912
 913 /*
 914  * Returns a set of divisors for the desired target clock with the given
 915  * refclk, or FALSE.  The returned values represent the clock equation:
 916  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 917  */
 918 static bool
 919 vlv_find_best_dpll(const struct intel_limit *limit,
 920                    struct intel_crtc_state *crtc_state,
 921                    int target, int refclk, struct dpll *match_clock,
 922                    struct dpll *best_clock)
 923 {
 924         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 925         struct drm_device *dev = crtc->base.dev;
 926         struct dpll clock;
 927         unsigned int bestppm = 1000000;
 928         /* min update 19.2 MHz */
 929         int max_n = min(limit->n.max, refclk / 19200);
 930         bool found = false;
 931
 932         target *= 5; /* fast clock */
 933
 934         memset(best_clock, 0, sizeof(*best_clock));
 935
 936         /* based on hardware requirement, prefer smaller n to precision */
 937         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 938                 for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 939                         for (clock.p2 = limit->p2.p2_fast; clock.p2 >= limit->p2.p2_slow;
 940                              clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 941                                 clock.p = clock.p1 * clock.p2;
 942                                 /* based on hardware requirement, prefer bigger m1,m2 values */
 943                                 for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) {
 944                                         unsigned int ppm;
 945
 946                                         clock.m2 = DIV_ROUND_CLOSEST(target * clock.p * clock.n,
 947                                                                      refclk * clock.m1);
 948
 949                                         vlv_calc_dpll_params(refclk, &clock);
 950
 951                                         if (!intel_PLL_is_valid(to_i915(dev),
 952                                                                 limit,
 953                                                                 &clock))
 954                                                 continue;
 955
 956                                         if (!vlv_PLL_is_optimal(dev, target,
 957                                                                 &clock,
 958                                                                 best_clock,
 959                                                                 bestppm, &ppm))
 960                                                 continue;
 961
 962                                         *best_clock = clock;
 963                                         bestppm = ppm;
 964                                         found = true;
 965                                 }
 966                         }
 967                 }
 968         }
 969
 970         return found;
 971 }
 972
 973 /*
 974  * Returns a set of divisors for the desired target clock with the given
 975  * refclk, or FALSE.  The returned values represent the clock equation:
 976  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 977  */
 978 static bool
 979 chv_find_best_dpll(const struct intel_limit *limit,
 980                    struct intel_crtc_state *crtc_state,
 981                    int target, int refclk, struct dpll *match_clock,
 982                    struct dpll *best_clock)
 983 {
 984         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 985         struct drm_device *dev = crtc->base.dev;
 986         unsigned int best_error_ppm;
 987         struct dpll clock;
 988         u64 m2;
 989         int found = false;
 990
 991         memset(best_clock, 0, sizeof(*best_clock));
 992         best_error_ppm = 1000000;
 993
 994         /*
 995          * Based on hardware doc, the n always set to 1, and m1 always
 996          * set to 2.  If requires to support 200Mhz refclk, we need to
 997          * revisit this because n may not 1 anymore.
 998          */
 999         clock.n = 1, clock.m1 = 2;
1000         target *= 5;    /* fast clock */
1001
1002         for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
1003                 for (clock.p2 = limit->p2.p2_fast;
1004                                 clock.p2 >= limit->p2.p2_slow;
1005                                 clock.p2 -= clock.p2 > 10 ? 2 : 1) {
1006                         unsigned int error_ppm;
1007
1008                         clock.p = clock.p1 * clock.p2;
1009
1010                         m2 = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(target, clock.p * clock.n) << 22,
1011                                                    refclk * clock.m1);
1012
1013                         if (m2 > INT_MAX/clock.m1)
1014                                 continue;
1015
1016                         clock.m2 = m2;
1017
1018                         chv_calc_dpll_params(refclk, &clock);
1019
1020                         if (!intel_PLL_is_valid(to_i915(dev), limit, &clock))
1021                                 continue;
1022
1023                         if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock,
1024                                                 best_error_ppm, &error_ppm))
1025                                 continue;
1026
1027                         *best_clock = clock;
1028                         best_error_ppm = error_ppm;
1029                         found = true;
1030                 }
1031         }
1032
1033         return found;
1034 }
1035
1036 bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state,
1037                         struct dpll *best_clock)
1038 {
1039         int refclk = 100000;
1040         const struct intel_limit *limit = &intel_limits_bxt;
1041
1042         return chv_find_best_dpll(limit, crtc_state,
1043                                   crtc_state->port_clock, refclk,
1044                                   NULL, best_clock);
1045 }
1046
1047 static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv,
1048                                     enum pipe pipe)
1049 {
1050         i915_reg_t reg = PIPEDSL(pipe);
1051         u32 line1, line2;
1052         u32 line_mask;
1053
1054         if (IS_GEN(dev_priv, 2))
1055                 line_mask = DSL_LINEMASK_GEN2;
1056         else
1057                 line_mask = DSL_LINEMASK_GEN3;
1058
1059         line1 = I915_READ(reg) & line_mask;
1060         msleep(5);
1061         line2 = I915_READ(reg) & line_mask;
1062
1063         return line1 != line2;
1064 }
1065
1066 static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state)
1067 {
1068         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1069         enum pipe pipe = crtc->pipe;
1070
1071         /* Wait for the display line to settle/start moving */
1072         if (wait_for(pipe_scanline_is_moving(dev_priv, pipe) == state, 100))
1073                 DRM_ERROR("pipe %c scanline %s wait timed out\n",
1074                           pipe_name(pipe), onoff(state));
1075 }
1076
1077 static void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc)
1078 {
1079         wait_for_pipe_scanline_moving(crtc, false);
1080 }
1081
1082 static void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
1083 {
1084         wait_for_pipe_scanline_moving(crtc, true);
1085 }
1086
1087 static void
1088 intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
1089 {
1090         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
1091         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1092
1093         if (INTEL_GEN(dev_priv) >= 4) {
1094                 enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1095                 i915_reg_t reg = PIPECONF(cpu_transcoder);
1096
1097                 /* Wait for the Pipe State to go off */
1098                 if (intel_de_wait_for_clear(dev_priv, reg,
1099                                             I965_PIPECONF_ACTIVE, 100))
1100                         WARN(1, "pipe_off wait timed out\n");
1101         } else {
1102                 intel_wait_for_pipe_scanline_stopped(crtc);
1103         }
1104 }
1105
1106 /* Only for pre-ILK configs */
1107 void assert_pll(struct drm_i915_private *dev_priv,
1108                 enum pipe pipe, bool state)
1109 {
1110         u32 val;
1111         bool cur_state;
1112
1113         val = I915_READ(DPLL(pipe));
1114         cur_state = !!(val & DPLL_VCO_ENABLE);
1115         I915_STATE_WARN(cur_state != state,
1116              "PLL state assertion failure (expected %s, current %s)\n",
1117                         onoff(state), onoff(cur_state));
1118 }
1119
1120 /* XXX: the dsi pll is shared between MIPI DSI ports */
1121 void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state)
1122 {
1123         u32 val;
1124         bool cur_state;
1125
1126         vlv_cck_get(dev_priv);
1127         val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL);
1128         vlv_cck_put(dev_priv);
1129
1130         cur_state = val & DSI_PLL_VCO_EN;
1131         I915_STATE_WARN(cur_state != state,
1132              "DSI PLL state assertion failure (expected %s, current %s)\n",
1133                         onoff(state), onoff(cur_state));
1134 }
1135
1136 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
1137                           enum pipe pipe, bool state)
1138 {
1139         bool cur_state;
1140
1141         if (HAS_DDI(dev_priv)) {
1142                 /*
1143                  * DDI does not have a specific FDI_TX register.
1144                  *
1145                  * FDI is never fed from EDP transcoder
1146                  * so pipe->transcoder cast is fine here.
1147                  */
1148                 enum transcoder cpu_transcoder = (enum transcoder)pipe;
1149                 u32 val = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
1150                 cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
1151         } else {
1152                 u32 val = I915_READ(FDI_TX_CTL(pipe));
1153                 cur_state = !!(val & FDI_TX_ENABLE);
1154         }
1155         I915_STATE_WARN(cur_state != state,
1156              "FDI TX state assertion failure (expected %s, current %s)\n",
1157                         onoff(state), onoff(cur_state));
1158 }
1159 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
1160 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
1161
1162 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
1163                           enum pipe pipe, bool state)
1164 {
1165         u32 val;
1166         bool cur_state;
1167
1168         val = I915_READ(FDI_RX_CTL(pipe));
1169         cur_state = !!(val & FDI_RX_ENABLE);
1170         I915_STATE_WARN(cur_state != state,
1171              "FDI RX state assertion failure (expected %s, current %s)\n",
1172                         onoff(state), onoff(cur_state));
1173 }
1174 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
1175 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
1176
1177 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
1178                                       enum pipe pipe)
1179 {
1180         u32 val;
1181
1182         /* ILK FDI PLL is always enabled */
1183         if (IS_GEN(dev_priv, 5))
1184                 return;
1185
1186         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
1187         if (HAS_DDI(dev_priv))
1188                 return;
1189
1190         val = I915_READ(FDI_TX_CTL(pipe));
1191         I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
1192 }
1193
1194 void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
1195                        enum pipe pipe, bool state)
1196 {
1197         u32 val;
1198         bool cur_state;
1199
1200         val = I915_READ(FDI_RX_CTL(pipe));
1201         cur_state = !!(val & FDI_RX_PLL_ENABLE);
1202         I915_STATE_WARN(cur_state != state,
1203              "FDI RX PLL assertion failure (expected %s, current %s)\n",
1204                         onoff(state), onoff(cur_state));
1205 }
1206
1207 void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe)
1208 {
1209         i915_reg_t pp_reg;
1210         u32 val;
1211         enum pipe panel_pipe = INVALID_PIPE;
1212         bool locked = true;
1213
1214         if (WARN_ON(HAS_DDI(dev_priv)))
1215                 return;
1216
1217         if (HAS_PCH_SPLIT(dev_priv)) {
1218                 u32 port_sel;
1219
1220                 pp_reg = PP_CONTROL(0);
1221                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1222
1223                 switch (port_sel) {
1224                 case PANEL_PORT_SELECT_LVDS:
1225                         intel_lvds_port_enabled(dev_priv, PCH_LVDS, &panel_pipe);
1226                         break;
1227                 case PANEL_PORT_SELECT_DPA:
1228                         intel_dp_port_enabled(dev_priv, DP_A, PORT_A, &panel_pipe);
1229                         break;
1230                 case PANEL_PORT_SELECT_DPC:
1231                         intel_dp_port_enabled(dev_priv, PCH_DP_C, PORT_C, &panel_pipe);
1232                         break;
1233                 case PANEL_PORT_SELECT_DPD:
1234                         intel_dp_port_enabled(dev_priv, PCH_DP_D, PORT_D, &panel_pipe);
1235                         break;
1236                 default:
1237                         MISSING_CASE(port_sel);
1238                         break;
1239                 }
1240         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1241                 /* presumably write lock depends on pipe, not port select */
1242                 pp_reg = PP_CONTROL(pipe);
1243                 panel_pipe = pipe;
1244         } else {
1245                 u32 port_sel;
1246
1247                 pp_reg = PP_CONTROL(0);
1248                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1249
1250                 WARN_ON(port_sel != PANEL_PORT_SELECT_LVDS);
1251                 intel_lvds_port_enabled(dev_priv, LVDS, &panel_pipe);
1252         }
1253
1254         val = I915_READ(pp_reg);
1255         if (!(val & PANEL_POWER_ON) ||
1256             ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
1257                 locked = false;
1258
1259         I915_STATE_WARN(panel_pipe == pipe && locked,
1260              "panel assertion failure, pipe %c regs locked\n",
1261              pipe_name(pipe));
1262 }
1263
1264 void assert_pipe(struct drm_i915_private *dev_priv,
1265                  enum transcoder cpu_transcoder, bool state)
1266 {
1267         bool cur_state;
1268         enum intel_display_power_domain power_domain;
1269         intel_wakeref_t wakeref;
1270
1271         /* we keep both pipes enabled on 830 */
1272         if (IS_I830(dev_priv))
1273                 state = true;
1274
1275         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
1276         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
1277         if (wakeref) {
1278                 u32 val = I915_READ(PIPECONF(cpu_transcoder));
1279                 cur_state = !!(val & PIPECONF_ENABLE);
1280
1281                 intel_display_power_put(dev_priv, power_domain, wakeref);
1282         } else {
1283                 cur_state = false;
1284         }
1285
1286         I915_STATE_WARN(cur_state != state,
1287                         "transcoder %s assertion failure (expected %s, current %s)\n",
1288                         transcoder_name(cpu_transcoder),
1289                         onoff(state), onoff(cur_state));
1290 }
1291
1292 static void assert_plane(struct intel_plane *plane, bool state)
1293 {
1294         enum pipe pipe;
1295         bool cur_state;
1296
1297         cur_state = plane->get_hw_state(plane, &pipe);
1298
1299         I915_STATE_WARN(cur_state != state,
1300                         "%s assertion failure (expected %s, current %s)\n",
1301                         plane->base.name, onoff(state), onoff(cur_state));
1302 }
1303
1304 #define assert_plane_enabled(p) assert_plane(p, true)
1305 #define assert_plane_disabled(p) assert_plane(p, false)
1306
1307 static void assert_planes_disabled(struct intel_crtc *crtc)
1308 {
1309         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1310         struct intel_plane *plane;
1311
1312         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
1313                 assert_plane_disabled(plane);
1314 }
1315
1316 static void assert_vblank_disabled(struct drm_crtc *crtc)
1317 {
1318         if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
1319                 drm_crtc_vblank_put(crtc);
1320 }
1321
1322 void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
1323                                     enum pipe pipe)
1324 {
1325         u32 val;
1326         bool enabled;
1327
1328         val = I915_READ(PCH_TRANSCONF(pipe));
1329         enabled = !!(val & TRANS_ENABLE);
1330         I915_STATE_WARN(enabled,
1331              "transcoder assertion failed, should be off on pipe %c but is still active\n",
1332              pipe_name(pipe));
1333 }
1334
1335 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
1336                                    enum pipe pipe, enum port port,
1337                                    i915_reg_t dp_reg)
1338 {
1339         enum pipe port_pipe;
1340         bool state;
1341
1342         state = intel_dp_port_enabled(dev_priv, dp_reg, port, &port_pipe);
1343
1344         I915_STATE_WARN(state && port_pipe == pipe,
1345                         "PCH DP %c enabled on transcoder %c, should be disabled\n",
1346                         port_name(port), pipe_name(pipe));
1347
1348         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1349                         "IBX PCH DP %c still using transcoder B\n",
1350                         port_name(port));
1351 }
1352
1353 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
1354                                      enum pipe pipe, enum port port,
1355                                      i915_reg_t hdmi_reg)
1356 {
1357         enum pipe port_pipe;
1358         bool state;
1359
1360         state = intel_sdvo_port_enabled(dev_priv, hdmi_reg, &port_pipe);
1361
1362         I915_STATE_WARN(state && port_pipe == pipe,
1363                         "PCH HDMI %c enabled on transcoder %c, should be disabled\n",
1364                         port_name(port), pipe_name(pipe));
1365
1366         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1367                         "IBX PCH HDMI %c still using transcoder B\n",
1368                         port_name(port));
1369 }
1370
1371 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
1372                                       enum pipe pipe)
1373 {
1374         enum pipe port_pipe;
1375
1376         assert_pch_dp_disabled(dev_priv, pipe, PORT_B, PCH_DP_B);
1377         assert_pch_dp_disabled(dev_priv, pipe, PORT_C, PCH_DP_C);
1378         assert_pch_dp_disabled(dev_priv, pipe, PORT_D, PCH_DP_D);
1379
1380         I915_STATE_WARN(intel_crt_port_enabled(dev_priv, PCH_ADPA, &port_pipe) &&
1381                         port_pipe == pipe,
1382                         "PCH VGA enabled on transcoder %c, should be disabled\n",
1383                         pipe_name(pipe));
1384
1385         I915_STATE_WARN(intel_lvds_port_enabled(dev_priv, PCH_LVDS, &port_pipe) &&
1386                         port_pipe == pipe,
1387                         "PCH LVDS enabled on transcoder %c, should be disabled\n",
1388                         pipe_name(pipe));
1389
1390         /* PCH SDVOB multiplex with HDMIB */
1391         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_B, PCH_HDMIB);
1392         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_C, PCH_HDMIC);
1393         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_D, PCH_HDMID);
1394 }
1395
1396 static void _vlv_enable_pll(struct intel_crtc *crtc,
1397                             const struct intel_crtc_state *pipe_config)
1398 {
1399         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1400         enum pipe pipe = crtc->pipe;
1401
1402         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1403         POSTING_READ(DPLL(pipe));
1404         udelay(150);
1405
1406         if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1))
1407                 DRM_ERROR("DPLL %d failed to lock\n", pipe);
1408 }
1409
1410 static void vlv_enable_pll(struct intel_crtc *crtc,
1411                            const struct intel_crtc_state *pipe_config)
1412 {
1413         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1414         enum pipe pipe = crtc->pipe;
1415
1416         assert_pipe_disabled(dev_priv, pipe_config->cpu_transcoder);
1417
1418         /* PLL is protected by panel, make sure we can write it */
1419         assert_panel_unlocked(dev_priv, pipe);
1420
1421         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1422                 _vlv_enable_pll(crtc, pipe_config);
1423
1424         I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1425         POSTING_READ(DPLL_MD(pipe));
1426 }
1427
1428
1429 static void _chv_enable_pll(struct intel_crtc *crtc,
1430                             const struct intel_crtc_state *pipe_config)
1431 {
1432         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1433         enum pipe pipe = crtc->pipe;
1434         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1435         u32 tmp;
1436
1437         vlv_dpio_get(dev_priv);
1438
1439         /* Enable back the 10bit clock to display controller */
1440         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1441         tmp |= DPIO_DCLKP_EN;
1442         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp);
1443
1444         vlv_dpio_put(dev_priv);
1445
1446         /*
1447          * Need to wait > 100ns between dclkp clock enable bit and PLL enable.
1448          */
1449         udelay(1);
1450
1451         /* Enable PLL */
1452         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1453
1454         /* Check PLL is locked */
1455         if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1))
1456                 DRM_ERROR("PLL %d failed to lock\n", pipe);
1457 }
1458
1459 static void chv_enable_pll(struct intel_crtc *crtc,
1460                            const struct intel_crtc_state *pipe_config)
1461 {
1462         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1463         enum pipe pipe = crtc->pipe;
1464
1465         assert_pipe_disabled(dev_priv, pipe_config->cpu_transcoder);
1466
1467         /* PLL is protected by panel, make sure we can write it */
1468         assert_panel_unlocked(dev_priv, pipe);
1469
1470         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1471                 _chv_enable_pll(crtc, pipe_config);
1472
1473         if (pipe != PIPE_A) {
1474                 /*
1475                  * WaPixelRepeatModeFixForC0:chv
1476                  *
1477                  * DPLLCMD is AWOL. Use chicken bits to propagate
1478                  * the value from DPLLBMD to either pipe B or C.
1479                  */
1480                 I915_WRITE(CBR4_VLV, CBR_DPLLBMD_PIPE(pipe));
1481                 I915_WRITE(DPLL_MD(PIPE_B), pipe_config->dpll_hw_state.dpll_md);
1482                 I915_WRITE(CBR4_VLV, 0);
1483                 dev_priv->chv_dpll_md[pipe] = pipe_config->dpll_hw_state.dpll_md;
1484
1485                 /*
1486                  * DPLLB VGA mode also seems to cause problems.
1487                  * We should always have it disabled.
1488                  */
1489                 WARN_ON((I915_READ(DPLL(PIPE_B)) & DPLL_VGA_MODE_DIS) == 0);
1490         } else {
1491                 I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1492                 POSTING_READ(DPLL_MD(pipe));
1493         }
1494 }
1495
1496 static bool i9xx_has_pps(struct drm_i915_private *dev_priv)
1497 {
1498         if (IS_I830(dev_priv))
1499                 return false;
1500
1501         return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
1502 }
1503
1504 static void i9xx_enable_pll(struct intel_crtc *crtc,
1505                             const struct intel_crtc_state *crtc_state)
1506 {
1507         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1508         i915_reg_t reg = DPLL(crtc->pipe);
1509         u32 dpll = crtc_state->dpll_hw_state.dpll;
1510         int i;
1511
1512         assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder);
1513
1514         /* PLL is protected by panel, make sure we can write it */
1515         if (i9xx_has_pps(dev_priv))
1516                 assert_panel_unlocked(dev_priv, crtc->pipe);
1517
1518         /*
1519          * Apparently we need to have VGA mode enabled prior to changing
1520          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
1521          * dividers, even though the register value does change.
1522          */
1523         I915_WRITE(reg, dpll & ~DPLL_VGA_MODE_DIS);
1524         I915_WRITE(reg, dpll);
1525
1526         /* Wait for the clocks to stabilize. */
1527         POSTING_READ(reg);
1528         udelay(150);
1529
1530         if (INTEL_GEN(dev_priv) >= 4) {
1531                 I915_WRITE(DPLL_MD(crtc->pipe),
1532                            crtc_state->dpll_hw_state.dpll_md);
1533         } else {
1534                 /* The pixel multiplier can only be updated once the
1535                  * DPLL is enabled and the clocks are stable.
1536                  *
1537                  * So write it again.
1538                  */
1539                 I915_WRITE(reg, dpll);
1540         }
1541
1542         /* We do this three times for luck */
1543         for (i = 0; i < 3; i++) {
1544                 I915_WRITE(reg, dpll);
1545                 POSTING_READ(reg);
1546                 udelay(150); /* wait for warmup */
1547         }
1548 }
1549
1550 static void i9xx_disable_pll(const struct intel_crtc_state *crtc_state)
1551 {
1552         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1553         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1554         enum pipe pipe = crtc->pipe;
1555
1556         /* Don't disable pipe or pipe PLLs if needed */
1557         if (IS_I830(dev_priv))
1558                 return;
1559
1560         /* Make sure the pipe isn't still relying on us */
1561         assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder);
1562
1563         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
1564         POSTING_READ(DPLL(pipe));
1565 }
1566
1567 static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1568 {
1569         u32 val;
1570
1571         /* Make sure the pipe isn't still relying on us */
1572         assert_pipe_disabled(dev_priv, (enum transcoder)pipe);
1573
1574         val = DPLL_INTEGRATED_REF_CLK_VLV |
1575                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1576         if (pipe != PIPE_A)
1577                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1578
1579         I915_WRITE(DPLL(pipe), val);
1580         POSTING_READ(DPLL(pipe));
1581 }
1582
1583 static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1584 {
1585         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1586         u32 val;
1587
1588         /* Make sure the pipe isn't still relying on us */
1589         assert_pipe_disabled(dev_priv, (enum transcoder)pipe);
1590
1591         val = DPLL_SSC_REF_CLK_CHV |
1592                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1593         if (pipe != PIPE_A)
1594                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1595
1596         I915_WRITE(DPLL(pipe), val);
1597         POSTING_READ(DPLL(pipe));
1598
1599         vlv_dpio_get(dev_priv);
1600
1601         /* Disable 10bit clock to display controller */
1602         val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1603         val &= ~DPIO_DCLKP_EN;
1604         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
1605
1606         vlv_dpio_put(dev_priv);
1607 }
1608
1609 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
1610                          struct intel_digital_port *dport,
1611                          unsigned int expected_mask)
1612 {
1613         u32 port_mask;
1614         i915_reg_t dpll_reg;
1615
1616         switch (dport->base.port) {
1617         case PORT_B:
1618                 port_mask = DPLL_PORTB_READY_MASK;
1619                 dpll_reg = DPLL(0);
1620                 break;
1621         case PORT_C:
1622                 port_mask = DPLL_PORTC_READY_MASK;
1623                 dpll_reg = DPLL(0);
1624                 expected_mask <<= 4;
1625                 break;
1626         case PORT_D:
1627                 port_mask = DPLL_PORTD_READY_MASK;
1628                 dpll_reg = DPIO_PHY_STATUS;
1629                 break;
1630         default:
1631                 BUG();
1632         }
1633
1634         if (intel_de_wait_for_register(dev_priv, dpll_reg,
1635                                        port_mask, expected_mask, 1000))
1636                 WARN(1, "timed out waiting for [ENCODER:%d:%s] port ready: got 0x%x, expected 0x%x\n",
1637                      dport->base.base.base.id, dport->base.base.name,
1638                      I915_READ(dpll_reg) & port_mask, expected_mask);
1639 }
1640
1641 static void ilk_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
1642 {
1643         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1644         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1645         enum pipe pipe = crtc->pipe;
1646         i915_reg_t reg;
1647         u32 val, pipeconf_val;
1648
1649         /* Make sure PCH DPLL is enabled */
1650         assert_shared_dpll_enabled(dev_priv, crtc_state->shared_dpll);
1651
1652         /* FDI must be feeding us bits for PCH ports */
1653         assert_fdi_tx_enabled(dev_priv, pipe);
1654         assert_fdi_rx_enabled(dev_priv, pipe);
1655
1656         if (HAS_PCH_CPT(dev_priv)) {
1657                 reg = TRANS_CHICKEN2(pipe);
1658                 val = I915_READ(reg);
1659                 /*
1660                  * Workaround: Set the timing override bit
1661                  * before enabling the pch transcoder.
1662                  */
1663                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1664                 /* Configure frame start delay to match the CPU */
1665                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
1666                 val |= TRANS_CHICKEN2_FRAME_START_DELAY(0);
1667                 I915_WRITE(reg, val);
1668         }
1669
1670         reg = PCH_TRANSCONF(pipe);
1671         val = I915_READ(reg);
1672         pipeconf_val = I915_READ(PIPECONF(pipe));
1673
1674         if (HAS_PCH_IBX(dev_priv)) {
1675                 /* Configure frame start delay to match the CPU */
1676                 val &= ~TRANS_FRAME_START_DELAY_MASK;
1677                 val |= TRANS_FRAME_START_DELAY(0);
1678
1679                 /*
1680                  * Make the BPC in transcoder be consistent with
1681                  * that in pipeconf reg. For HDMI we must use 8bpc
1682                  * here for both 8bpc and 12bpc.
1683                  */
1684                 val &= ~PIPECONF_BPC_MASK;
1685                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
1686                         val |= PIPECONF_8BPC;
1687                 else
1688                         val |= pipeconf_val & PIPECONF_BPC_MASK;
1689         }
1690
1691         val &= ~TRANS_INTERLACE_MASK;
1692         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK) {
1693                 if (HAS_PCH_IBX(dev_priv) &&
1694                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
1695                         val |= TRANS_LEGACY_INTERLACED_ILK;
1696                 else
1697                         val |= TRANS_INTERLACED;
1698         } else {
1699                 val |= TRANS_PROGRESSIVE;
1700         }
1701
1702         I915_WRITE(reg, val | TRANS_ENABLE);
1703         if (intel_de_wait_for_set(dev_priv, reg, TRANS_STATE_ENABLE, 100))
1704                 DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe));
1705 }
1706
1707 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1708                                       enum transcoder cpu_transcoder)
1709 {
1710         u32 val, pipeconf_val;
1711
1712         /* FDI must be feeding us bits for PCH ports */
1713         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
1714         assert_fdi_rx_enabled(dev_priv, PIPE_A);
1715
1716         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1717         /* Workaround: set timing override bit. */
1718         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1719         /* Configure frame start delay to match the CPU */
1720         val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
1721         val |= TRANS_CHICKEN2_FRAME_START_DELAY(0);
1722         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1723
1724         val = TRANS_ENABLE;
1725         pipeconf_val = I915_READ(PIPECONF(cpu_transcoder));
1726
1727         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
1728             PIPECONF_INTERLACED_ILK)
1729                 val |= TRANS_INTERLACED;
1730         else
1731                 val |= TRANS_PROGRESSIVE;
1732
1733         I915_WRITE(LPT_TRANSCONF, val);
1734         if (intel_de_wait_for_set(dev_priv, LPT_TRANSCONF,
1735                                   TRANS_STATE_ENABLE, 100))
1736                 DRM_ERROR("Failed to enable PCH transcoder\n");
1737 }
1738
1739 static void ilk_disable_pch_transcoder(struct drm_i915_private *dev_priv,
1740                                        enum pipe pipe)
1741 {
1742         i915_reg_t reg;
1743         u32 val;
1744
1745         /* FDI relies on the transcoder */
1746         assert_fdi_tx_disabled(dev_priv, pipe);
1747         assert_fdi_rx_disabled(dev_priv, pipe);
1748
1749         /* Ports must be off as well */
1750         assert_pch_ports_disabled(dev_priv, pipe);
1751
1752         reg = PCH_TRANSCONF(pipe);
1753         val = I915_READ(reg);
1754         val &= ~TRANS_ENABLE;
1755         I915_WRITE(reg, val);
1756         /* wait for PCH transcoder off, transcoder state */
1757         if (intel_de_wait_for_clear(dev_priv, reg, TRANS_STATE_ENABLE, 50))
1758                 DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
1759
1760         if (HAS_PCH_CPT(dev_priv)) {
1761                 /* Workaround: Clear the timing override chicken bit again. */
1762                 reg = TRANS_CHICKEN2(pipe);
1763                 val = I915_READ(reg);
1764                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1765                 I915_WRITE(reg, val);
1766         }
1767 }
1768
1769 void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
1770 {
1771         u32 val;
1772
1773         val = I915_READ(LPT_TRANSCONF);
1774         val &= ~TRANS_ENABLE;
1775         I915_WRITE(LPT_TRANSCONF, val);
1776         /* wait for PCH transcoder off, transcoder state */
1777         if (intel_de_wait_for_clear(dev_priv, LPT_TRANSCONF,
1778                                     TRANS_STATE_ENABLE, 50))
1779                 DRM_ERROR("Failed to disable PCH transcoder\n");
1780
1781         /* Workaround: clear timing override bit. */
1782         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1783         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1784         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1785 }
1786
1787 enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
1788 {
1789         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1790
1791         if (HAS_PCH_LPT(dev_priv))
1792                 return PIPE_A;
1793         else
1794                 return crtc->pipe;
1795 }
1796
1797 static u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
1798 {
1799         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1800
1801         /*
1802          * On i965gm the hardware frame counter reads
1803          * zero when the TV encoder is enabled :(
1804          */
1805         if (IS_I965GM(dev_priv) &&
1806             (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
1807                 return 0;
1808
1809         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1810                 return 0xffffffff; /* full 32 bit counter */
1811         else if (INTEL_GEN(dev_priv) >= 3)
1812                 return 0xffffff; /* only 24 bits of frame count */
1813         else
1814                 return 0; /* Gen2 doesn't have a hardware frame counter */
1815 }
1816
1817 static void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
1818 {
1819         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1820
1821         assert_vblank_disabled(&crtc->base);
1822         drm_crtc_set_max_vblank_count(&crtc->base,
1823                                       intel_crtc_max_vblank_count(crtc_state));
1824         drm_crtc_vblank_on(&crtc->base);
1825 }
1826
1827 void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
1828 {
1829         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1830
1831         drm_crtc_vblank_off(&crtc->base);
1832         assert_vblank_disabled(&crtc->base);
1833 }
1834
1835 static void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state)
1836 {
1837         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
1838         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1839         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
1840         enum pipe pipe = crtc->pipe;
1841         i915_reg_t reg;
1842         u32 val;
1843
1844         DRM_DEBUG_KMS("enabling pipe %c\n", pipe_name(pipe));
1845
1846         assert_planes_disabled(crtc);
1847
1848         /*
1849          * A pipe without a PLL won't actually be able to drive bits from
1850          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
1851          * need the check.
1852          */
1853         if (HAS_GMCH(dev_priv)) {
1854                 if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
1855                         assert_dsi_pll_enabled(dev_priv);
1856                 else
1857                         assert_pll_enabled(dev_priv, pipe);
1858         } else {
1859                 if (new_crtc_state->has_pch_encoder) {
1860                         /* if driving the PCH, we need FDI enabled */
1861                         assert_fdi_rx_pll_enabled(dev_priv,
1862                                                   intel_crtc_pch_transcoder(crtc));
1863                         assert_fdi_tx_pll_enabled(dev_priv,
1864                                                   (enum pipe) cpu_transcoder);
1865                 }
1866                 /* FIXME: assert CPU port conditions for SNB+ */
1867         }
1868
1869         trace_intel_pipe_enable(crtc);
1870
1871         reg = PIPECONF(cpu_transcoder);
1872         val = I915_READ(reg);
1873         if (val & PIPECONF_ENABLE) {
1874                 /* we keep both pipes enabled on 830 */
1875                 WARN_ON(!IS_I830(dev_priv));
1876                 return;
1877         }
1878
1879         I915_WRITE(reg, val | PIPECONF_ENABLE);
1880         POSTING_READ(reg);
1881
1882         /*
1883          * Until the pipe starts PIPEDSL reads will return a stale value,
1884          * which causes an apparent vblank timestamp jump when PIPEDSL
1885          * resets to its proper value. That also messes up the frame count
1886          * when it's derived from the timestamps. So let's wait for the
1887          * pipe to start properly before we call drm_crtc_vblank_on()
1888          */
1889         if (intel_crtc_max_vblank_count(new_crtc_state) == 0)
1890                 intel_wait_for_pipe_scanline_moving(crtc);
1891 }
1892
1893 void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state)
1894 {
1895         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
1896         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1897         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1898         enum pipe pipe = crtc->pipe;
1899         i915_reg_t reg;
1900         u32 val;
1901
1902         DRM_DEBUG_KMS("disabling pipe %c\n", pipe_name(pipe));
1903
1904         /*
1905          * Make sure planes won't keep trying to pump pixels to us,
1906          * or we might hang the display.
1907          */
1908         assert_planes_disabled(crtc);
1909
1910         trace_intel_pipe_disable(crtc);
1911
1912         reg = PIPECONF(cpu_transcoder);
1913         val = I915_READ(reg);
1914         if ((val & PIPECONF_ENABLE) == 0)
1915                 return;
1916
1917         /*
1918          * Double wide has implications for planes
1919          * so best keep it disabled when not needed.
1920          */
1921         if (old_crtc_state->double_wide)
1922                 val &= ~PIPECONF_DOUBLE_WIDE;
1923
1924         /* Don't disable pipe or pipe PLLs if needed */
1925         if (!IS_I830(dev_priv))
1926                 val &= ~PIPECONF_ENABLE;
1927
1928         I915_WRITE(reg, val);
1929         if ((val & PIPECONF_ENABLE) == 0)
1930                 intel_wait_for_pipe_off(old_crtc_state);
1931 }
1932
1933 static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv)
1934 {
1935         return IS_GEN(dev_priv, 2) ? 2048 : 4096;
1936 }
1937
1938 static bool is_ccs_plane(const struct drm_framebuffer *fb, int plane)
1939 {
1940         if (!is_ccs_modifier(fb->modifier))
1941                 return false;
1942
1943         return plane >= fb->format->num_planes / 2;
1944 }
1945
1946 static bool is_gen12_ccs_modifier(u64 modifier)
1947 {
1948         return modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS;
1949 }
1950
1951 static bool is_gen12_ccs_plane(const struct drm_framebuffer *fb, int plane)
1952 {
1953         return is_gen12_ccs_modifier(fb->modifier) && is_ccs_plane(fb, plane);
1954 }
1955
1956 static bool is_aux_plane(const struct drm_framebuffer *fb, int plane)
1957 {
1958         if (is_ccs_modifier(fb->modifier))
1959                 return is_ccs_plane(fb, plane);
1960
1961         return plane == 1;
1962 }
1963
1964 static int main_to_ccs_plane(const struct drm_framebuffer *fb, int main_plane)
1965 {
1966         WARN_ON(!is_ccs_modifier(fb->modifier) ||
1967                 (main_plane && main_plane >= fb->format->num_planes / 2));
1968
1969         return fb->format->num_planes / 2 + main_plane;
1970 }
1971
1972 static int ccs_to_main_plane(const struct drm_framebuffer *fb, int ccs_plane)
1973 {
1974         WARN_ON(!is_ccs_modifier(fb->modifier) ||
1975                 ccs_plane < fb->format->num_planes / 2);
1976
1977         return ccs_plane - fb->format->num_planes / 2;
1978 }
1979
1980 /* Return either the main plane's CCS or - if not a CCS FB - UV plane */
1981 static int
1982 intel_main_to_aux_plane(const struct drm_framebuffer *fb, int main_plane)
1983 {
1984         if (is_ccs_modifier(fb->modifier))
1985                 return main_to_ccs_plane(fb, main_plane);
1986
1987         return 1;
1988 }
1989
1990 bool
1991 intel_format_info_is_yuv_semiplanar(const struct drm_format_info *info,
1992                                     uint64_t modifier)
1993 {
1994         return info->is_yuv &&
1995                info->num_planes == (is_ccs_modifier(modifier) ? 4 : 2);
1996 }
1997
1998 static unsigned int
1999 intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane)
2000 {
2001         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2002         unsigned int cpp = fb->format->cpp[color_plane];
2003
2004         switch (fb->modifier) {
2005         case DRM_FORMAT_MOD_LINEAR:
2006                 return intel_tile_size(dev_priv);
2007         case I915_FORMAT_MOD_X_TILED:
2008                 if (IS_GEN(dev_priv, 2))
2009                         return 128;
2010                 else
2011                         return 512;
2012         case I915_FORMAT_MOD_Y_TILED_CCS:
2013                 if (is_ccs_plane(fb, color_plane))
2014                         return 128;
2015                 /* fall through */
2016         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
2017                 if (is_ccs_plane(fb, color_plane))
2018                         return 64;
2019                 /* fall through */
2020         case I915_FORMAT_MOD_Y_TILED:
2021                 if (IS_GEN(dev_priv, 2) || HAS_128_BYTE_Y_TILING(dev_priv))
2022                         return 128;
2023                 else
2024                         return 512;
2025         case I915_FORMAT_MOD_Yf_TILED_CCS:
2026                 if (is_ccs_plane(fb, color_plane))
2027                         return 128;
2028                 /* fall through */
2029         case I915_FORMAT_MOD_Yf_TILED:
2030                 switch (cpp) {
2031                 case 1:
2032                         return 64;
2033                 case 2:
2034                 case 4:
2035                         return 128;
2036                 case 8:
2037                 case 16:
2038                         return 256;
2039                 default:
2040                         MISSING_CASE(cpp);
2041                         return cpp;
2042                 }
2043                 break;
2044         default:
2045                 MISSING_CASE(fb->modifier);
2046                 return cpp;
2047         }
2048 }
2049
2050 static unsigned int
2051 intel_tile_height(const struct drm_framebuffer *fb, int color_plane)
2052 {
2053         if (is_gen12_ccs_plane(fb, color_plane))
2054                 return 1;
2055
2056         return intel_tile_size(to_i915(fb->dev)) /
2057                 intel_tile_width_bytes(fb, color_plane);
2058 }
2059
2060 /* Return the tile dimensions in pixel units */
2061 static void intel_tile_dims(const struct drm_framebuffer *fb, int color_plane,
2062                             unsigned int *tile_width,
2063                             unsigned int *tile_height)
2064 {
2065         unsigned int tile_width_bytes = intel_tile_width_bytes(fb, color_plane);
2066         unsigned int cpp = fb->format->cpp[color_plane];
2067
2068         *tile_width = tile_width_bytes / cpp;
2069         *tile_height = intel_tile_height(fb, color_plane);
2070 }
2071
2072 unsigned int
2073 intel_fb_align_height(const struct drm_framebuffer *fb,
2074                       int color_plane, unsigned int height)
2075 {
2076         unsigned int tile_height = intel_tile_height(fb, color_plane);
2077
2078         return ALIGN(height, tile_height);
2079 }
2080
2081 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info)
2082 {
2083         unsigned int size = 0;
2084         int i;
2085
2086         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
2087                 size += rot_info->plane[i].width * rot_info->plane[i].height;
2088
2089         return size;
2090 }
2091
2092 unsigned int intel_remapped_info_size(const struct intel_remapped_info *rem_info)
2093 {
2094         unsigned int size = 0;
2095         int i;
2096
2097         for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++)
2098                 size += rem_info->plane[i].width * rem_info->plane[i].height;
2099
2100         return size;
2101 }
2102
2103 static void
2104 intel_fill_fb_ggtt_view(struct i915_ggtt_view *view,
2105                         const struct drm_framebuffer *fb,
2106                         unsigned int rotation)
2107 {
2108         view->type = I915_GGTT_VIEW_NORMAL;
2109         if (drm_rotation_90_or_270(rotation)) {
2110                 view->type = I915_GGTT_VIEW_ROTATED;
2111                 view->rotated = to_intel_framebuffer(fb)->rot_info;
2112         }
2113 }
2114
2115 static unsigned int intel_cursor_alignment(const struct drm_i915_private *dev_priv)
2116 {
2117         if (IS_I830(dev_priv))
2118                 return 16 * 1024;
2119         else if (IS_I85X(dev_priv))
2120                 return 256;
2121         else if (IS_I845G(dev_priv) || IS_I865G(dev_priv))
2122                 return 32;
2123         else
2124                 return 4 * 1024;
2125 }
2126
2127 static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv)
2128 {
2129         if (INTEL_GEN(dev_priv) >= 9)
2130                 return 256 * 1024;
2131         else if (IS_I965G(dev_priv) || IS_I965GM(dev_priv) ||
2132                  IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2133                 return 128 * 1024;
2134         else if (INTEL_GEN(dev_priv) >= 4)
2135                 return 4 * 1024;
2136         else
2137                 return 0;
2138 }
2139
2140 static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb,
2141                                          int color_plane)
2142 {
2143         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2144
2145         /* AUX_DIST needs only 4K alignment */
2146         if (is_aux_plane(fb, color_plane))
2147                 return 4096;
2148
2149         switch (fb->modifier) {
2150         case DRM_FORMAT_MOD_LINEAR:
2151                 return intel_linear_alignment(dev_priv);
2152         case I915_FORMAT_MOD_X_TILED:
2153                 if (INTEL_GEN(dev_priv) >= 9)
2154                         return 256 * 1024;
2155                 return 0;
2156         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
2157                 return 16 * 1024;
2158         case I915_FORMAT_MOD_Y_TILED_CCS:
2159         case I915_FORMAT_MOD_Yf_TILED_CCS:
2160         case I915_FORMAT_MOD_Y_TILED:
2161         case I915_FORMAT_MOD_Yf_TILED:
2162                 return 1 * 1024 * 1024;
2163         default:
2164                 MISSING_CASE(fb->modifier);
2165                 return 0;
2166         }
2167 }
2168
2169 static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
2170 {
2171         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2172         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2173
2174         return INTEL_GEN(dev_priv) < 4 ||
2175                 (plane->has_fbc &&
2176                  plane_state->view.type == I915_GGTT_VIEW_NORMAL);
2177 }
2178
2179 struct i915_vma *
2180 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
2181                            const struct i915_ggtt_view *view,
2182                            bool uses_fence,
2183                            unsigned long *out_flags)
2184 {
2185         struct drm_device *dev = fb->dev;
2186         struct drm_i915_private *dev_priv = to_i915(dev);
2187         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2188         intel_wakeref_t wakeref;
2189         struct i915_vma *vma;
2190         unsigned int pinctl;
2191         u32 alignment;
2192
2193         if (WARN_ON(!i915_gem_object_is_framebuffer(obj)))
2194                 return ERR_PTR(-EINVAL);
2195
2196         alignment = intel_surf_alignment(fb, 0);
2197         if (WARN_ON(alignment && !is_power_of_2(alignment)))
2198                 return ERR_PTR(-EINVAL);
2199
2200         /* Note that the w/a also requires 64 PTE of padding following the
2201          * bo. We currently fill all unused PTE with the shadow page and so
2202          * we should always have valid PTE following the scanout preventing
2203          * the VT-d warning.
2204          */
2205         if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024)
2206                 alignment = 256 * 1024;
2207
2208         /*
2209          * Global gtt pte registers are special registers which actually forward
2210          * writes to a chunk of system memory. Which means that there is no risk
2211          * that the register values disappear as soon as we call
2212          * intel_runtime_pm_put(), so it is correct to wrap only the
2213          * pin/unpin/fence and not more.
2214          */
2215         wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
2216
2217         atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
2218
2219         /*
2220          * Valleyview is definitely limited to scanning out the first
2221          * 512MiB. Lets presume this behaviour was inherited from the
2222          * g4x display engine and that all earlier gen are similarly
2223          * limited. Testing suggests that it is a little more
2224          * complicated than this. For example, Cherryview appears quite
2225          * happy to scanout from anywhere within its global aperture.
2226          */
2227         pinctl = 0;
2228         if (HAS_GMCH(dev_priv))
2229                 pinctl |= PIN_MAPPABLE;
2230
2231         vma = i915_gem_object_pin_to_display_plane(obj,
2232                                                    alignment, view, pinctl);
2233         if (IS_ERR(vma))
2234                 goto err;
2235
2236         if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
2237                 int ret;
2238
2239                 /*
2240                  * Install a fence for tiled scan-out. Pre-i965 always needs a
2241                  * fence, whereas 965+ only requires a fence if using
2242                  * framebuffer compression.  For simplicity, we always, when
2243                  * possible, install a fence as the cost is not that onerous.
2244                  *
2245                  * If we fail to fence the tiled scanout, then either the
2246                  * modeset will reject the change (which is highly unlikely as
2247                  * the affected systems, all but one, do not have unmappable
2248                  * space) or we will not be able to enable full powersaving
2249                  * techniques (also likely not to apply due to various limits
2250                  * FBC and the like impose on the size of the buffer, which
2251                  * presumably we violated anyway with this unmappable buffer).
2252                  * Anyway, it is presumably better to stumble onwards with
2253                  * something and try to run the system in a "less than optimal"
2254                  * mode that matches the user configuration.
2255                  */
2256                 ret = i915_vma_pin_fence(vma);
2257                 if (ret != 0 && INTEL_GEN(dev_priv) < 4) {
2258                         i915_gem_object_unpin_from_display_plane(vma);
2259                         vma = ERR_PTR(ret);
2260                         goto err;
2261                 }
2262
2263                 if (ret == 0 && vma->fence)
2264                         *out_flags |= PLANE_HAS_FENCE;
2265         }
2266
2267         i915_vma_get(vma);
2268 err:
2269         atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
2270         intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
2271         return vma;
2272 }
2273
2274 void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
2275 {
2276         i915_gem_object_lock(vma->obj);
2277         if (flags & PLANE_HAS_FENCE)
2278                 i915_vma_unpin_fence(vma);
2279         i915_gem_object_unpin_from_display_plane(vma);
2280         i915_gem_object_unlock(vma->obj);
2281
2282         i915_vma_put(vma);
2283 }
2284
2285 static int intel_fb_pitch(const struct drm_framebuffer *fb, int color_plane,
2286                           unsigned int rotation)
2287 {
2288         if (drm_rotation_90_or_270(rotation))
2289                 return to_intel_framebuffer(fb)->rotated[color_plane].pitch;
2290         else
2291                 return fb->pitches[color_plane];
2292 }
2293
2294 /*
2295  * Convert the x/y offsets into a linear offset.
2296  * Only valid with 0/180 degree rotation, which is fine since linear
2297  * offset is only used with linear buffers on pre-hsw and tiled buffers
2298  * with gen2/3, and 90/270 degree rotations isn't supported on any of them.
2299  */
2300 u32 intel_fb_xy_to_linear(int x, int y,
2301                           const struct intel_plane_state *state,
2302                           int color_plane)
2303 {
2304         const struct drm_framebuffer *fb = state->hw.fb;
2305         unsigned int cpp = fb->format->cpp[color_plane];
2306         unsigned int pitch = state->color_plane[color_plane].stride;
2307
2308         return y * pitch + x * cpp;
2309 }
2310
2311 /*
2312  * Add the x/y offsets derived from fb->offsets[] to the user
2313  * specified plane src x/y offsets. The resulting x/y offsets
2314  * specify the start of scanout from the beginning of the gtt mapping.
2315  */
2316 void intel_add_fb_offsets(int *x, int *y,
2317                           const struct intel_plane_state *state,
2318                           int color_plane)
2319
2320 {
2321         *x += state->color_plane[color_plane].x;
2322         *y += state->color_plane[color_plane].y;
2323 }
2324
2325 static u32 intel_adjust_tile_offset(int *x, int *y,
2326                                     unsigned int tile_width,
2327                                     unsigned int tile_height,
2328                                     unsigned int tile_size,
2329                                     unsigned int pitch_tiles,
2330                                     u32 old_offset,
2331                                     u32 new_offset)
2332 {
2333         unsigned int pitch_pixels = pitch_tiles * tile_width;
2334         unsigned int tiles;
2335
2336         WARN_ON(old_offset & (tile_size - 1));
2337         WARN_ON(new_offset & (tile_size - 1));
2338         WARN_ON(new_offset > old_offset);
2339
2340         tiles = (old_offset - new_offset) / tile_size;
2341
2342         *y += tiles / pitch_tiles * tile_height;
2343         *x += tiles % pitch_tiles * tile_width;
2344
2345         /* minimize x in case it got needlessly big */
2346         *y += *x / pitch_pixels * tile_height;
2347         *x %= pitch_pixels;
2348
2349         return new_offset;
2350 }
2351
2352 static bool is_surface_linear(const struct drm_framebuffer *fb, int color_plane)
2353 {
2354         return fb->modifier == DRM_FORMAT_MOD_LINEAR ||
2355                is_gen12_ccs_plane(fb, color_plane);
2356 }
2357
2358 static u32 intel_adjust_aligned_offset(int *x, int *y,
2359                                        const struct drm_framebuffer *fb,
2360                                        int color_plane,
2361                                        unsigned int rotation,
2362                                        unsigned int pitch,
2363                                        u32 old_offset, u32 new_offset)
2364 {
2365         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2366         unsigned int cpp = fb->format->cpp[color_plane];
2367
2368         WARN_ON(new_offset > old_offset);
2369
2370         if (!is_surface_linear(fb, color_plane)) {
2371                 unsigned int tile_size, tile_width, tile_height;
2372                 unsigned int pitch_tiles;
2373
2374                 tile_size = intel_tile_size(dev_priv);
2375                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2376
2377                 if (drm_rotation_90_or_270(rotation)) {
2378                         pitch_tiles = pitch / tile_height;
2379                         swap(tile_width, tile_height);
2380                 } else {
2381                         pitch_tiles = pitch / (tile_width * cpp);
2382                 }
2383
2384                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2385                                          tile_size, pitch_tiles,
2386                                          old_offset, new_offset);
2387         } else {
2388                 old_offset += *y * pitch + *x * cpp;
2389
2390                 *y = (old_offset - new_offset) / pitch;
2391                 *x = ((old_offset - new_offset) - *y * pitch) / cpp;
2392         }
2393
2394         return new_offset;
2395 }
2396
2397 /*
2398  * Adjust the tile offset by moving the difference into
2399  * the x/y offsets.
2400  */
2401 static u32 intel_plane_adjust_aligned_offset(int *x, int *y,
2402                                              const struct intel_plane_state *state,
2403                                              int color_plane,
2404                                              u32 old_offset, u32 new_offset)
2405 {
2406         return intel_adjust_aligned_offset(x, y, state->hw.fb, color_plane,
2407                                            state->hw.rotation,
2408                                            state->color_plane[color_plane].stride,
2409                                            old_offset, new_offset);
2410 }
2411
2412 /*
2413  * Computes the aligned offset to the base tile and adjusts
2414  * x, y. bytes per pixel is assumed to be a power-of-two.
2415  *
2416  * In the 90/270 rotated case, x and y are assumed
2417  * to be already rotated to match the rotated GTT view, and
2418  * pitch is the tile_height aligned framebuffer height.
2419  *
2420  * This function is used when computing the derived information
2421  * under intel_framebuffer, so using any of that information
2422  * here is not allowed. Anything under drm_framebuffer can be
2423  * used. This is why the user has to pass in the pitch since it
2424  * is specified in the rotated orientation.
2425  */
2426 static u32 intel_compute_aligned_offset(struct drm_i915_private *dev_priv,
2427                                         int *x, int *y,
2428                                         const struct drm_framebuffer *fb,
2429                                         int color_plane,
2430                                         unsigned int pitch,
2431                                         unsigned int rotation,
2432                                         u32 alignment)
2433 {
2434         unsigned int cpp = fb->format->cpp[color_plane];
2435         u32 offset, offset_aligned;
2436
2437         if (!is_surface_linear(fb, color_plane)) {
2438                 unsigned int tile_size, tile_width, tile_height;
2439                 unsigned int tile_rows, tiles, pitch_tiles;
2440
2441                 tile_size = intel_tile_size(dev_priv);
2442                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2443
2444                 if (drm_rotation_90_or_270(rotation)) {
2445                         pitch_tiles = pitch / tile_height;
2446                         swap(tile_width, tile_height);
2447                 } else {
2448                         pitch_tiles = pitch / (tile_width * cpp);
2449                 }
2450
2451                 tile_rows = *y / tile_height;
2452                 *y %= tile_height;
2453
2454                 tiles = *x / tile_width;
2455                 *x %= tile_width;
2456
2457                 offset = (tile_rows * pitch_tiles + tiles) * tile_size;
2458
2459                 offset_aligned = offset;
2460                 if (alignment)
2461                         offset_aligned = rounddown(offset_aligned, alignment);
2462
2463                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2464                                          tile_size, pitch_tiles,
2465                                          offset, offset_aligned);
2466         } else {
2467                 offset = *y * pitch + *x * cpp;
2468                 offset_aligned = offset;
2469                 if (alignment) {
2470                         offset_aligned = rounddown(offset_aligned, alignment);
2471                         *y = (offset % alignment) / pitch;
2472                         *x = ((offset % alignment) - *y * pitch) / cpp;
2473                 } else {
2474                         *y = *x = 0;
2475                 }
2476         }
2477
2478         return offset_aligned;
2479 }
2480
2481 static u32 intel_plane_compute_aligned_offset(int *x, int *y,
2482                                               const struct intel_plane_state *state,
2483                                               int color_plane)
2484 {
2485         struct intel_plane *intel_plane = to_intel_plane(state->uapi.plane);
2486         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
2487         const struct drm_framebuffer *fb = state->hw.fb;
2488         unsigned int rotation = state->hw.rotation;
2489         int pitch = state->color_plane[color_plane].stride;
2490         u32 alignment;
2491
2492         if (intel_plane->id == PLANE_CURSOR)
2493                 alignment = intel_cursor_alignment(dev_priv);
2494         else
2495                 alignment = intel_surf_alignment(fb, color_plane);
2496
2497         return intel_compute_aligned_offset(dev_priv, x, y, fb, color_plane,
2498                                             pitch, rotation, alignment);
2499 }
2500
2501 /* Convert the fb->offset[] into x/y offsets */
2502 static int intel_fb_offset_to_xy(int *x, int *y,
2503                                  const struct drm_framebuffer *fb,
2504                                  int color_plane)
2505 {
2506         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2507         unsigned int height;
2508
2509         if (fb->modifier != DRM_FORMAT_MOD_LINEAR &&
2510             fb->offsets[color_plane] % intel_tile_size(dev_priv)) {
2511                 DRM_DEBUG_KMS("Misaligned offset 0x%08x for color plane %d\n",
2512                               fb->offsets[color_plane], color_plane);
2513                 return -EINVAL;
2514         }
2515
2516         height = drm_framebuffer_plane_height(fb->height, fb, color_plane);
2517         height = ALIGN(height, intel_tile_height(fb, color_plane));
2518
2519         /* Catch potential overflows early */
2520         if (add_overflows_t(u32, mul_u32_u32(height, fb->pitches[color_plane]),
2521                             fb->offsets[color_plane])) {
2522                 DRM_DEBUG_KMS("Bad offset 0x%08x or pitch %d for color plane %d\n",
2523                               fb->offsets[color_plane], fb->pitches[color_plane],
2524                               color_plane);
2525                 return -ERANGE;
2526         }
2527
2528         *x = 0;
2529         *y = 0;
2530
2531         intel_adjust_aligned_offset(x, y,
2532                                     fb, color_plane, DRM_MODE_ROTATE_0,
2533                                     fb->pitches[color_plane],
2534                                     fb->offsets[color_plane], 0);
2535
2536         return 0;
2537 }
2538
2539 static unsigned int intel_fb_modifier_to_tiling(u64 fb_modifier)
2540 {
2541         switch (fb_modifier) {
2542         case I915_FORMAT_MOD_X_TILED:
2543                 return I915_TILING_X;
2544         case I915_FORMAT_MOD_Y_TILED:
2545         case I915_FORMAT_MOD_Y_TILED_CCS:
2546         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
2547                 return I915_TILING_Y;
2548         default:
2549                 return I915_TILING_NONE;
2550         }
2551 }
2552
2553 /*
2554  * From the Sky Lake PRM:
2555  * "The Color Control Surface (CCS) contains the compression status of
2556  *  the cache-line pairs. The compression state of the cache-line pair
2557  *  is specified by 2 bits in the CCS. Each CCS cache-line represents
2558  *  an area on the main surface of 16 x16 sets of 128 byte Y-tiled
2559  *  cache-line-pairs. CCS is always Y tiled."
2560  *
2561  * Since cache line pairs refers to horizontally adjacent cache lines,
2562  * each cache line in the CCS corresponds to an area of 32x16 cache
2563  * lines on the main surface. Since each pixel is 4 bytes, this gives
2564  * us a ratio of one byte in the CCS for each 8x16 pixels in the
2565  * main surface.
2566  */
2567 static const struct drm_format_info skl_ccs_formats[] = {
2568         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
2569           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2570         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
2571           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2572         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
2573           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
2574         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
2575           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
2576 };
2577
2578 /*
2579  * Gen-12 compression uses 4 bits of CCS data for each cache line pair in the
2580  * main surface. And each 64B CCS cache line represents an area of 4x1 Y-tiles
2581  * in the main surface. With 4 byte pixels and each Y-tile having dimensions of
2582  * 32x32 pixels, the ratio turns out to 1B in the CCS for every 2x32 pixels in
2583  * the main surface.
2584  */
2585 static const struct drm_format_info gen12_ccs_formats[] = {
2586         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
2587           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2588           .hsub = 1, .vsub = 1, },
2589         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
2590           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2591           .hsub = 1, .vsub = 1, },
2592         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
2593           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2594           .hsub = 1, .vsub = 1, .has_alpha = true },
2595         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
2596           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2597           .hsub = 1, .vsub = 1, .has_alpha = true },
2598 };
2599
2600 static const struct drm_format_info *
2601 lookup_format_info(const struct drm_format_info formats[],
2602                    int num_formats, u32 format)
2603 {
2604         int i;
2605
2606         for (i = 0; i < num_formats; i++) {
2607                 if (formats[i].format == format)
2608                         return &formats[i];
2609         }
2610
2611         return NULL;
2612 }
2613
2614 static const struct drm_format_info *
2615 intel_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
2616 {
2617         switch (cmd->modifier[0]) {
2618         case I915_FORMAT_MOD_Y_TILED_CCS:
2619         case I915_FORMAT_MOD_Yf_TILED_CCS:
2620                 return lookup_format_info(skl_ccs_formats,
2621                                           ARRAY_SIZE(skl_ccs_formats),
2622                                           cmd->pixel_format);
2623         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
2624                 return lookup_format_info(gen12_ccs_formats,
2625                                           ARRAY_SIZE(gen12_ccs_formats),
2626                                           cmd->pixel_format);
2627         default:
2628                 return NULL;
2629         }
2630 }
2631
2632 bool is_ccs_modifier(u64 modifier)
2633 {
2634         return modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS ||
2635                modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
2636                modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
2637 }
2638
2639 static int gen12_ccs_aux_stride(struct drm_framebuffer *fb, int ccs_plane)
2640 {
2641         return DIV_ROUND_UP(fb->pitches[ccs_to_main_plane(fb, ccs_plane)],
2642                             512) * 64;
2643 }
2644
2645 u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv,
2646                               u32 pixel_format, u64 modifier)
2647 {
2648         struct intel_crtc *crtc;
2649         struct intel_plane *plane;
2650
2651         /*
2652          * We assume the primary plane for pipe A has
2653          * the highest stride limits of them all.
2654          */
2655         crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A);
2656         if (!crtc)
2657                 return 0;
2658
2659         plane = to_intel_plane(crtc->base.primary);
2660
2661         return plane->max_stride(plane, pixel_format, modifier,
2662                                  DRM_MODE_ROTATE_0);
2663 }
2664
2665 static
2666 u32 intel_fb_max_stride(struct drm_i915_private *dev_priv,
2667                         u32 pixel_format, u64 modifier)
2668 {
2669         /*
2670          * Arbitrary limit for gen4+ chosen to match the
2671          * render engine max stride.
2672          *
2673          * The new CCS hash mode makes remapping impossible
2674          */
2675         if (!is_ccs_modifier(modifier)) {
2676                 if (INTEL_GEN(dev_priv) >= 7)
2677                         return 256*1024;
2678                 else if (INTEL_GEN(dev_priv) >= 4)
2679                         return 128*1024;
2680         }
2681
2682         return intel_plane_fb_max_stride(dev_priv, pixel_format, modifier);
2683 }
2684
2685 static u32
2686 intel_fb_stride_alignment(const struct drm_framebuffer *fb, int color_plane)
2687 {
2688         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2689         u32 tile_width;
2690
2691         if (is_surface_linear(fb, color_plane)) {
2692                 u32 max_stride = intel_plane_fb_max_stride(dev_priv,
2693                                                            fb->format->format,
2694                                                            fb->modifier);
2695
2696                 /*
2697                  * To make remapping with linear generally feasible
2698                  * we need the stride to be page aligned.
2699                  */
2700                 if (fb->pitches[color_plane] > max_stride &&
2701                     !is_ccs_modifier(fb->modifier))
2702                         return intel_tile_size(dev_priv);
2703                 else
2704                         return 64;
2705         }
2706
2707         tile_width = intel_tile_width_bytes(fb, color_plane);
2708         if (is_ccs_modifier(fb->modifier) && color_plane == 0) {
2709                 /*
2710                  * Display WA #0531: skl,bxt,kbl,glk
2711                  *
2712                  * Render decompression and plane width > 3840
2713                  * combined with horizontal panning requires the
2714                  * plane stride to be a multiple of 4. We'll just
2715                  * require the entire fb to accommodate that to avoid
2716                  * potential runtime errors at plane configuration time.
2717                  */
2718                 if (IS_GEN(dev_priv, 9) && fb->width > 3840)
2719                         tile_width *= 4;
2720                 /*
2721                  * The main surface pitch must be padded to a multiple of four
2722                  * tile widths.
2723                  */
2724                 else if (INTEL_GEN(dev_priv) >= 12)
2725                         tile_width *= 4;
2726         }
2727         return tile_width;
2728 }
2729
2730 bool intel_plane_can_remap(const struct intel_plane_state *plane_state)
2731 {
2732         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2733         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2734         const struct drm_framebuffer *fb = plane_state->hw.fb;
2735         int i;
2736
2737         /* We don't want to deal with remapping with cursors */
2738         if (plane->id == PLANE_CURSOR)
2739                 return false;
2740
2741         /*
2742          * The display engine limits already match/exceed the
2743          * render engine limits, so not much point in remapping.
2744          * Would also need to deal with the fence POT alignment
2745          * and gen2 2KiB GTT tile size.
2746          */
2747         if (INTEL_GEN(dev_priv) < 4)
2748                 return false;
2749
2750         /*
2751          * The new CCS hash mode isn't compatible with remapping as
2752          * the virtual address of the pages affects the compressed data.
2753          */
2754         if (is_ccs_modifier(fb->modifier))
2755                 return false;
2756
2757         /* Linear needs a page aligned stride for remapping */
2758         if (fb->modifier == DRM_FORMAT_MOD_LINEAR) {
2759                 unsigned int alignment = intel_tile_size(dev_priv) - 1;
2760
2761                 for (i = 0; i < fb->format->num_planes; i++) {
2762                         if (fb->pitches[i] & alignment)
2763                                 return false;
2764                 }
2765         }
2766
2767         return true;
2768 }
2769
2770 static bool intel_plane_needs_remap(const struct intel_plane_state *plane_state)
2771 {
2772         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2773         const struct drm_framebuffer *fb = plane_state->hw.fb;
2774         unsigned int rotation = plane_state->hw.rotation;
2775         u32 stride, max_stride;
2776
2777         /*
2778          * No remapping for invisible planes since we don't have
2779          * an actual source viewport to remap.
2780          */
2781         if (!plane_state->uapi.visible)
2782                 return false;
2783
2784         if (!intel_plane_can_remap(plane_state))
2785                 return false;
2786
2787         /*
2788          * FIXME: aux plane limits on gen9+ are
2789          * unclear in Bspec, for now no checking.
2790          */
2791         stride = intel_fb_pitch(fb, 0, rotation);
2792         max_stride = plane->max_stride(plane, fb->format->format,
2793                                        fb->modifier, rotation);
2794
2795         return stride > max_stride;
2796 }
2797
2798 static void
2799 intel_fb_plane_get_subsampling(int *hsub, int *vsub,
2800                                const struct drm_framebuffer *fb,
2801                                int color_plane)
2802 {
2803         int main_plane;
2804
2805         if (color_plane == 0) {
2806                 *hsub = 1;
2807                 *vsub = 1;
2808
2809                 return;
2810         }
2811
2812         /*
2813          * TODO: Deduct the subsampling from the char block for all CCS
2814          * formats and planes.
2815          */
2816         if (!is_gen12_ccs_plane(fb, color_plane)) {
2817                 *hsub = fb->format->hsub;
2818                 *vsub = fb->format->vsub;
2819
2820                 return;
2821         }
2822
2823         main_plane = ccs_to_main_plane(fb, color_plane);
2824         *hsub = drm_format_info_block_width(fb->format, color_plane) /
2825                 drm_format_info_block_width(fb->format, main_plane);
2826
2827         /*
2828          * The min stride check in the core framebuffer_check() function
2829          * assumes that format->hsub applies to every plane except for the
2830          * first plane. That's incorrect for the CCS AUX plane of the first
2831          * plane, but for the above check to pass we must define the block
2832          * width with that subsampling applied to it. Adjust the width here
2833          * accordingly, so we can calculate the actual subsampling factor.
2834          */
2835         if (main_plane == 0)
2836                 *hsub *= fb->format->hsub;
2837
2838         *vsub = 32;
2839 }
2840 static int
2841 intel_fb_check_ccs_xy(struct drm_framebuffer *fb, int ccs_plane, int x, int y)
2842 {
2843         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2844         int main_plane;
2845         int hsub, vsub;
2846         int tile_width, tile_height;
2847         int ccs_x, ccs_y;
2848         int main_x, main_y;
2849
2850         if (!is_ccs_plane(fb, ccs_plane))
2851                 return 0;
2852
2853         intel_tile_dims(fb, ccs_plane, &tile_width, &tile_height);
2854         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, ccs_plane);
2855
2856         tile_width *= hsub;
2857         tile_height *= vsub;
2858
2859         ccs_x = (x * hsub) % tile_width;
2860         ccs_y = (y * vsub) % tile_height;
2861
2862         main_plane = ccs_to_main_plane(fb, ccs_plane);
2863         main_x = intel_fb->normal[main_plane].x % tile_width;
2864         main_y = intel_fb->normal[main_plane].y % tile_height;
2865
2866         /*
2867          * CCS doesn't have its own x/y offset register, so the intra CCS tile
2868          * x/y offsets must match between CCS and the main surface.
2869          */
2870         if (main_x != ccs_x || main_y != ccs_y) {
2871                 DRM_DEBUG_KMS("Bad CCS x/y (main %d,%d ccs %d,%d) full (main %d,%d ccs %d,%d)\n",
2872                               main_x, main_y,
2873                               ccs_x, ccs_y,
2874                               intel_fb->normal[main_plane].x,
2875                               intel_fb->normal[main_plane].y,
2876                               x, y);
2877                 return -EINVAL;
2878         }
2879
2880         return 0;
2881 }
2882
2883 static void
2884 intel_fb_plane_dims(int *w, int *h, struct drm_framebuffer *fb, int color_plane)
2885 {
2886         int hsub, vsub;
2887
2888         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, color_plane);
2889         *w = fb->width / hsub;
2890         *h = fb->height / vsub;
2891 }
2892
2893 /*
2894  * Setup the rotated view for an FB plane and return the size the GTT mapping
2895  * requires for this view.
2896  */
2897 static u32
2898 setup_fb_rotation(int plane, const struct intel_remapped_plane_info *plane_info,
2899                   u32 gtt_offset_rotated, int x, int y,
2900                   unsigned int width, unsigned int height,
2901                   unsigned int tile_size,
2902                   unsigned int tile_width, unsigned int tile_height,
2903                   struct drm_framebuffer *fb)
2904 {
2905         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2906         struct intel_rotation_info *rot_info = &intel_fb->rot_info;
2907         unsigned int pitch_tiles;
2908         struct drm_rect r;
2909
2910         /* Y or Yf modifiers required for 90/270 rotation */
2911         if (fb->modifier != I915_FORMAT_MOD_Y_TILED &&
2912             fb->modifier != I915_FORMAT_MOD_Yf_TILED)
2913                 return 0;
2914
2915         if (WARN_ON(plane >= ARRAY_SIZE(rot_info->plane)))
2916                 return 0;
2917
2918         rot_info->plane[plane] = *plane_info;
2919
2920         intel_fb->rotated[plane].pitch = plane_info->height * tile_height;
2921
2922         /* rotate the x/y offsets to match the GTT view */
2923         drm_rect_init(&r, x, y, width, height);
2924         drm_rect_rotate(&r,
2925                         plane_info->width * tile_width,
2926                         plane_info->height * tile_height,
2927                         DRM_MODE_ROTATE_270);
2928         x = r.x1;
2929         y = r.y1;
2930
2931         /* rotate the tile dimensions to match the GTT view */
2932         pitch_tiles = intel_fb->rotated[plane].pitch / tile_height;
2933         swap(tile_width, tile_height);
2934
2935         /*
2936          * We only keep the x/y offsets, so push all of the
2937          * gtt offset into the x/y offsets.
2938          */
2939         intel_adjust_tile_offset(&x, &y,
2940                                  tile_width, tile_height,
2941                                  tile_size, pitch_tiles,
2942                                  gtt_offset_rotated * tile_size, 0);
2943
2944         /*
2945          * First pixel of the framebuffer from
2946          * the start of the rotated gtt mapping.
2947          */
2948         intel_fb->rotated[plane].x = x;
2949         intel_fb->rotated[plane].y = y;
2950
2951         return plane_info->width * plane_info->height;
2952 }
2953
2954 static int
2955 intel_fill_fb_info(struct drm_i915_private *dev_priv,
2956                    struct drm_framebuffer *fb)
2957 {
2958         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2959         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2960         u32 gtt_offset_rotated = 0;
2961         unsigned int max_size = 0;
2962         int i, num_planes = fb->format->num_planes;
2963         unsigned int tile_size = intel_tile_size(dev_priv);
2964
2965         for (i = 0; i < num_planes; i++) {
2966                 unsigned int width, height;
2967                 unsigned int cpp, size;
2968                 u32 offset;
2969                 int x, y;
2970                 int ret;
2971
2972                 cpp = fb->format->cpp[i];
2973                 intel_fb_plane_dims(&width, &height, fb, i);
2974
2975                 ret = intel_fb_offset_to_xy(&x, &y, fb, i);
2976                 if (ret) {
2977                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2978                                       i, fb->offsets[i]);
2979                         return ret;
2980                 }
2981
2982                 ret = intel_fb_check_ccs_xy(fb, i, x, y);
2983                 if (ret)
2984                         return ret;
2985
2986                 /*
2987                  * The fence (if used) is aligned to the start of the object
2988                  * so having the framebuffer wrap around across the edge of the
2989                  * fenced region doesn't really work. We have no API to configure
2990                  * the fence start offset within the object (nor could we probably
2991                  * on gen2/3). So it's just easier if we just require that the
2992                  * fb layout agrees with the fence layout. We already check that the
2993                  * fb stride matches the fence stride elsewhere.
2994                  */
2995                 if (i == 0 && i915_gem_object_is_tiled(obj) &&
2996                     (x + width) * cpp > fb->pitches[i]) {
2997                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2998                                       i, fb->offsets[i]);
2999                         return -EINVAL;
3000                 }
3001
3002                 /*
3003                  * First pixel of the framebuffer from
3004                  * the start of the normal gtt mapping.
3005                  */
3006                 intel_fb->normal[i].x = x;
3007                 intel_fb->normal[i].y = y;
3008
3009                 offset = intel_compute_aligned_offset(dev_priv, &x, &y, fb, i,
3010                                                       fb->pitches[i],
3011                                                       DRM_MODE_ROTATE_0,
3012                                                       tile_size);
3013                 offset /= tile_size;
3014
3015                 if (!is_surface_linear(fb, i)) {
3016                         struct intel_remapped_plane_info plane_info;
3017                         unsigned int tile_width, tile_height;
3018
3019                         intel_tile_dims(fb, i, &tile_width, &tile_height);
3020
3021                         plane_info.offset = offset;
3022                         plane_info.stride = DIV_ROUND_UP(fb->pitches[i],
3023                                                          tile_width * cpp);
3024                         plane_info.width = DIV_ROUND_UP(x + width, tile_width);
3025                         plane_info.height = DIV_ROUND_UP(y + height,
3026                                                          tile_height);
3027
3028                         /* how many tiles does this plane need */
3029                         size = plane_info.stride * plane_info.height;
3030                         /*
3031                          * If the plane isn't horizontally tile aligned,
3032                          * we need one more tile.
3033                          */
3034                         if (x != 0)
3035                                 size++;
3036
3037                         gtt_offset_rotated +=
3038                                 setup_fb_rotation(i, &plane_info,
3039                                                   gtt_offset_rotated,
3040                                                   x, y, width, height,
3041                                                   tile_size,
3042                                                   tile_width, tile_height,
3043                                                   fb);
3044                 } else {
3045                         size = DIV_ROUND_UP((y + height) * fb->pitches[i] +
3046                                             x * cpp, tile_size);
3047                 }
3048
3049                 /* how many tiles in total needed in the bo */
3050                 max_size = max(max_size, offset + size);
3051         }
3052
3053         if (mul_u32_u32(max_size, tile_size) > obj->base.size) {
3054                 DRM_DEBUG_KMS("fb too big for bo (need %llu bytes, have %zu bytes)\n",
3055                               mul_u32_u32(max_size, tile_size), obj->base.size);
3056                 return -EINVAL;
3057         }
3058
3059         return 0;
3060 }
3061
3062 static void
3063 intel_plane_remap_gtt(struct intel_plane_state *plane_state)
3064 {
3065         struct drm_i915_private *dev_priv =
3066                 to_i915(plane_state->uapi.plane->dev);
3067         struct drm_framebuffer *fb = plane_state->hw.fb;
3068         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
3069         struct intel_rotation_info *info = &plane_state->view.rotated;
3070         unsigned int rotation = plane_state->hw.rotation;
3071         int i, num_planes = fb->format->num_planes;
3072         unsigned int tile_size = intel_tile_size(dev_priv);
3073         unsigned int src_x, src_y;
3074         unsigned int src_w, src_h;
3075         u32 gtt_offset = 0;
3076
3077         memset(&plane_state->view, 0, sizeof(plane_state->view));
3078         plane_state->view.type = drm_rotation_90_or_270(rotation) ?
3079                 I915_GGTT_VIEW_ROTATED : I915_GGTT_VIEW_REMAPPED;
3080
3081         src_x = plane_state->uapi.src.x1 >> 16;
3082         src_y = plane_state->uapi.src.y1 >> 16;
3083         src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
3084         src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
3085
3086         WARN_ON(is_ccs_modifier(fb->modifier));
3087
3088         /* Make src coordinates relative to the viewport */
3089         drm_rect_translate(&plane_state->uapi.src,
3090                            -(src_x << 16), -(src_y << 16));
3091
3092         /* Rotate src coordinates to match rotated GTT view */
3093         if (drm_rotation_90_or_270(rotation))
3094                 drm_rect_rotate(&plane_state->uapi.src,
3095                                 src_w << 16, src_h << 16,
3096                                 DRM_MODE_ROTATE_270);
3097
3098         for (i = 0; i < num_planes; i++) {
3099                 unsigned int hsub = i ? fb->format->hsub : 1;
3100                 unsigned int vsub = i ? fb->format->vsub : 1;
3101                 unsigned int cpp = fb->format->cpp[i];
3102                 unsigned int tile_width, tile_height;
3103                 unsigned int width, height;
3104                 unsigned int pitch_tiles;
3105                 unsigned int x, y;
3106                 u32 offset;
3107
3108                 intel_tile_dims(fb, i, &tile_width, &tile_height);
3109
3110                 x = src_x / hsub;
3111                 y = src_y / vsub;
3112                 width = src_w / hsub;
3113                 height = src_h / vsub;
3114
3115                 /*
3116                  * First pixel of the src viewport from the
3117                  * start of the normal gtt mapping.
3118                  */
3119                 x += intel_fb->normal[i].x;
3120                 y += intel_fb->normal[i].y;
3121
3122                 offset = intel_compute_aligned_offset(dev_priv, &x, &y,
3123                                                       fb, i, fb->pitches[i],
3124                                                       DRM_MODE_ROTATE_0, tile_size);
3125                 offset /= tile_size;
3126
3127                 WARN_ON(i >= ARRAY_SIZE(info->plane));
3128                 info->plane[i].offset = offset;
3129                 info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i],
3130                                                      tile_width * cpp);
3131                 info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
3132                 info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
3133
3134                 if (drm_rotation_90_or_270(rotation)) {
3135                         struct drm_rect r;
3136
3137                         /* rotate the x/y offsets to match the GTT view */
3138                         drm_rect_init(&r, x, y, width, height);
3139                         drm_rect_rotate(&r,
3140                                         info->plane[i].width * tile_width,
3141                                         info->plane[i].height * tile_height,
3142                                         DRM_MODE_ROTATE_270);
3143                         x = r.x1;
3144                         y = r.y1;
3145
3146                         pitch_tiles = info->plane[i].height;
3147                         plane_state->color_plane[i].stride = pitch_tiles * tile_height;
3148
3149                         /* rotate the tile dimensions to match the GTT view */
3150                         swap(tile_width, tile_height);
3151                 } else {
3152                         pitch_tiles = info->plane[i].width;
3153                         plane_state->color_plane[i].stride = pitch_tiles * tile_width * cpp;
3154                 }
3155
3156                 /*
3157                  * We only keep the x/y offsets, so push all of the
3158                  * gtt offset into the x/y offsets.
3159                  */
3160                 intel_adjust_tile_offset(&x, &y,
3161                                          tile_width, tile_height,
3162                                          tile_size, pitch_tiles,
3163                                          gtt_offset * tile_size, 0);
3164
3165                 gtt_offset += info->plane[i].width * info->plane[i].height;
3166
3167                 plane_state->color_plane[i].offset = 0;
3168                 plane_state->color_plane[i].x = x;
3169                 plane_state->color_plane[i].y = y;
3170         }
3171 }
3172
3173 static int
3174 intel_plane_compute_gtt(struct intel_plane_state *plane_state)
3175 {
3176         const struct intel_framebuffer *fb =
3177                 to_intel_framebuffer(plane_state->hw.fb);
3178         unsigned int rotation = plane_state->hw.rotation;
3179         int i, num_planes;
3180
3181         if (!fb)
3182                 return 0;
3183
3184         num_planes = fb->base.format->num_planes;
3185
3186         if (intel_plane_needs_remap(plane_state)) {
3187                 intel_plane_remap_gtt(plane_state);
3188
3189                 /*
3190                  * Sometimes even remapping can't overcome
3191                  * the stride limitations :( Can happen with
3192                  * big plane sizes and suitably misaligned
3193                  * offsets.
3194                  */
3195                 return intel_plane_check_stride(plane_state);
3196         }
3197
3198         intel_fill_fb_ggtt_view(&plane_state->view, &fb->base, rotation);
3199
3200         for (i = 0; i < num_planes; i++) {
3201                 plane_state->color_plane[i].stride = intel_fb_pitch(&fb->base, i, rotation);
3202                 plane_state->color_plane[i].offset = 0;
3203
3204                 if (drm_rotation_90_or_270(rotation)) {
3205                         plane_state->color_plane[i].x = fb->rotated[i].x;
3206                         plane_state->color_plane[i].y = fb->rotated[i].y;
3207                 } else {
3208                         plane_state->color_plane[i].x = fb->normal[i].x;
3209                         plane_state->color_plane[i].y = fb->normal[i].y;
3210                 }
3211         }
3212
3213         /* Rotate src coordinates to match rotated GTT view */
3214         if (drm_rotation_90_or_270(rotation))
3215                 drm_rect_rotate(&plane_state->uapi.src,
3216                                 fb->base.width << 16, fb->base.height << 16,
3217                                 DRM_MODE_ROTATE_270);
3218
3219         return intel_plane_check_stride(plane_state);
3220 }
3221
3222 static int i9xx_format_to_fourcc(int format)
3223 {
3224         switch (format) {
3225         case DISPPLANE_8BPP:
3226                 return DRM_FORMAT_C8;
3227         case DISPPLANE_BGRA555:
3228                 return DRM_FORMAT_ARGB1555;
3229         case DISPPLANE_BGRX555:
3230                 return DRM_FORMAT_XRGB1555;
3231         case DISPPLANE_BGRX565:
3232                 return DRM_FORMAT_RGB565;
3233         default:
3234         case DISPPLANE_BGRX888:
3235                 return DRM_FORMAT_XRGB8888;
3236         case DISPPLANE_RGBX888:
3237                 return DRM_FORMAT_XBGR8888;
3238         case DISPPLANE_BGRA888:
3239                 return DRM_FORMAT_ARGB8888;
3240         case DISPPLANE_RGBA888:
3241                 return DRM_FORMAT_ABGR8888;
3242         case DISPPLANE_BGRX101010:
3243                 return DRM_FORMAT_XRGB2101010;
3244         case DISPPLANE_RGBX101010:
3245                 return DRM_FORMAT_XBGR2101010;
3246         case DISPPLANE_BGRA101010:
3247                 return DRM_FORMAT_ARGB2101010;
3248         case DISPPLANE_RGBA101010:
3249                 return DRM_FORMAT_ABGR2101010;
3250         case DISPPLANE_RGBX161616:
3251                 return DRM_FORMAT_XBGR16161616F;
3252         }
3253 }
3254
3255 int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
3256 {
3257         switch (format) {
3258         case PLANE_CTL_FORMAT_RGB_565:
3259                 return DRM_FORMAT_RGB565;
3260         case PLANE_CTL_FORMAT_NV12:
3261                 return DRM_FORMAT_NV12;
3262         case PLANE_CTL_FORMAT_P010:
3263                 return DRM_FORMAT_P010;
3264         case PLANE_CTL_FORMAT_P012:
3265                 return DRM_FORMAT_P012;
3266         case PLANE_CTL_FORMAT_P016:
3267                 return DRM_FORMAT_P016;
3268         case PLANE_CTL_FORMAT_Y210:
3269                 return DRM_FORMAT_Y210;
3270         case PLANE_CTL_FORMAT_Y212:
3271                 return DRM_FORMAT_Y212;
3272         case PLANE_CTL_FORMAT_Y216:
3273                 return DRM_FORMAT_Y216;
3274         case PLANE_CTL_FORMAT_Y410:
3275                 return DRM_FORMAT_XVYU2101010;
3276         case PLANE_CTL_FORMAT_Y412:
3277                 return DRM_FORMAT_XVYU12_16161616;
3278         case PLANE_CTL_FORMAT_Y416:
3279                 return DRM_FORMAT_XVYU16161616;
3280         default:
3281         case PLANE_CTL_FORMAT_XRGB_8888:
3282                 if (rgb_order) {
3283                         if (alpha)
3284                                 return DRM_FORMAT_ABGR8888;
3285                         else
3286                                 return DRM_FORMAT_XBGR8888;
3287                 } else {
3288                         if (alpha)
3289                                 return DRM_FORMAT_ARGB8888;
3290                         else
3291                                 return DRM_FORMAT_XRGB8888;
3292                 }
3293         case PLANE_CTL_FORMAT_XRGB_2101010:
3294                 if (rgb_order) {
3295                         if (alpha)
3296                                 return DRM_FORMAT_ABGR2101010;
3297                         else
3298                                 return DRM_FORMAT_XBGR2101010;
3299                 } else {
3300                         if (alpha)
3301                                 return DRM_FORMAT_ARGB2101010;
3302                         else
3303                                 return DRM_FORMAT_XRGB2101010;
3304                 }
3305         case PLANE_CTL_FORMAT_XRGB_16161616F:
3306                 if (rgb_order) {
3307                         if (alpha)
3308                                 return DRM_FORMAT_ABGR16161616F;
3309                         else
3310                                 return DRM_FORMAT_XBGR16161616F;
3311                 } else {
3312                         if (alpha)
3313                                 return DRM_FORMAT_ARGB16161616F;
3314                         else
3315                                 return DRM_FORMAT_XRGB16161616F;
3316                 }
3317         }
3318 }
3319
3320 static bool
3321 intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
3322                               struct intel_initial_plane_config *plane_config)
3323 {
3324         struct drm_device *dev = crtc->base.dev;
3325         struct drm_i915_private *dev_priv = to_i915(dev);
3326         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
3327         struct drm_framebuffer *fb = &plane_config->fb->base;
3328         u32 base_aligned = round_down(plane_config->base, PAGE_SIZE);
3329         u32 size_aligned = round_up(plane_config->base + plane_config->size,
3330                                     PAGE_SIZE);
3331         struct drm_i915_gem_object *obj;
3332         bool ret = false;
3333
3334         size_aligned -= base_aligned;
3335
3336         if (plane_config->size == 0)
3337                 return false;
3338
3339         /* If the FB is too big, just don't use it since fbdev is not very
3340          * important and we should probably use that space with FBC or other
3341          * features. */
3342         if (size_aligned * 2 > dev_priv->stolen_usable_size)
3343                 return false;
3344
3345         switch (fb->modifier) {
3346         case DRM_FORMAT_MOD_LINEAR:
3347         case I915_FORMAT_MOD_X_TILED:
3348         case I915_FORMAT_MOD_Y_TILED:
3349                 break;
3350         default:
3351                 DRM_DEBUG_DRIVER("Unsupported modifier for initial FB: 0x%llx\n",
3352                                  fb->modifier);
3353                 return false;
3354         }
3355
3356         obj = i915_gem_object_create_stolen_for_preallocated(dev_priv,
3357                                                              base_aligned,
3358                                                              base_aligned,
3359                                                              size_aligned);
3360         if (IS_ERR(obj))
3361                 return false;
3362
3363         switch (plane_config->tiling) {
3364         case I915_TILING_NONE:
3365                 break;
3366         case I915_TILING_X:
3367         case I915_TILING_Y:
3368                 obj->tiling_and_stride = fb->pitches[0] | plane_config->tiling;
3369                 break;
3370         default:
3371                 MISSING_CASE(plane_config->tiling);
3372                 goto out;
3373         }
3374
3375         mode_cmd.pixel_format = fb->format->format;
3376         mode_cmd.width = fb->width;
3377         mode_cmd.height = fb->height;
3378         mode_cmd.pitches[0] = fb->pitches[0];
3379         mode_cmd.modifier[0] = fb->modifier;
3380         mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
3381
3382         if (intel_framebuffer_init(to_intel_framebuffer(fb), obj, &mode_cmd)) {
3383                 DRM_DEBUG_KMS("intel fb init failed\n");
3384                 goto out;
3385         }
3386
3387
3388         DRM_DEBUG_KMS("initial plane fb obj %p\n", obj);
3389         ret = true;
3390 out:
3391         i915_gem_object_put(obj);
3392         return ret;
3393 }
3394
3395 static void
3396 intel_set_plane_visible(struct intel_crtc_state *crtc_state,
3397                         struct intel_plane_state *plane_state,
3398                         bool visible)
3399 {
3400         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
3401
3402         plane_state->uapi.visible = visible;
3403
3404         if (visible)
3405                 crtc_state->uapi.plane_mask |= drm_plane_mask(&plane->base);
3406         else
3407                 crtc_state->uapi.plane_mask &= ~drm_plane_mask(&plane->base);
3408 }
3409
3410 static void fixup_active_planes(struct intel_crtc_state *crtc_state)
3411 {
3412         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3413         struct drm_plane *plane;
3414
3415         /*
3416          * Active_planes aliases if multiple "primary" or cursor planes
3417          * have been used on the same (or wrong) pipe. plane_mask uses
3418          * unique ids, hence we can use that to reconstruct active_planes.
3419          */
3420         crtc_state->active_planes = 0;
3421
3422         drm_for_each_plane_mask(plane, &dev_priv->drm,
3423                                 crtc_state->uapi.plane_mask)
3424                 crtc_state->active_planes |= BIT(to_intel_plane(plane)->id);
3425 }
3426
3427 static void intel_plane_disable_noatomic(struct intel_crtc *crtc,
3428                                          struct intel_plane *plane)
3429 {
3430         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3431         struct intel_crtc_state *crtc_state =
3432                 to_intel_crtc_state(crtc->base.state);
3433         struct intel_plane_state *plane_state =
3434                 to_intel_plane_state(plane->base.state);
3435
3436         DRM_DEBUG_KMS("Disabling [PLANE:%d:%s] on [CRTC:%d:%s]\n",
3437                       plane->base.base.id, plane->base.name,
3438                       crtc->base.base.id, crtc->base.name);
3439
3440         intel_set_plane_visible(crtc_state, plane_state, false);
3441         fixup_active_planes(crtc_state);
3442         crtc_state->data_rate[plane->id] = 0;
3443         crtc_state->min_cdclk[plane->id] = 0;
3444
3445         if (plane->id == PLANE_PRIMARY)
3446                 hsw_disable_ips(crtc_state);
3447
3448         /*
3449          * Vblank time updates from the shadow to live plane control register
3450          * are blocked if the memory self-refresh mode is active at that
3451          * moment. So to make sure the plane gets truly disabled, disable
3452          * first the self-refresh mode. The self-refresh enable bit in turn
3453          * will be checked/applied by the HW only at the next frame start
3454          * event which is after the vblank start event, so we need to have a
3455          * wait-for-vblank between disabling the plane and the pipe.
3456          */
3457         if (HAS_GMCH(dev_priv) &&
3458             intel_set_memory_cxsr(dev_priv, false))
3459                 intel_wait_for_vblank(dev_priv, crtc->pipe);
3460
3461         /*
3462          * Gen2 reports pipe underruns whenever all planes are disabled.
3463          * So disable underrun reporting before all the planes get disabled.
3464          */
3465         if (IS_GEN(dev_priv, 2) && !crtc_state->active_planes)
3466                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
3467
3468         intel_disable_plane(plane, crtc_state);
3469 }
3470
3471 static struct intel_frontbuffer *
3472 to_intel_frontbuffer(struct drm_framebuffer *fb)
3473 {
3474         return fb ? to_intel_framebuffer(fb)->frontbuffer : NULL;
3475 }
3476
3477 static void
3478 intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
3479                              struct intel_initial_plane_config *plane_config)
3480 {
3481         struct drm_device *dev = intel_crtc->base.dev;
3482         struct drm_i915_private *dev_priv = to_i915(dev);
3483         struct drm_crtc *c;
3484         struct drm_plane *primary = intel_crtc->base.primary;
3485         struct drm_plane_state *plane_state = primary->state;
3486         struct intel_plane *intel_plane = to_intel_plane(primary);
3487         struct intel_plane_state *intel_state =
3488                 to_intel_plane_state(plane_state);
3489         struct drm_framebuffer *fb;
3490
3491         if (!plane_config->fb)
3492                 return;
3493
3494         if (intel_alloc_initial_plane_obj(intel_crtc, plane_config)) {
3495                 fb = &plane_config->fb->base;
3496                 goto valid_fb;
3497         }
3498
3499         kfree(plane_config->fb);
3500
3501         /*
3502          * Failed to alloc the obj, check to see if we should share
3503          * an fb with another CRTC instead
3504          */
3505         for_each_crtc(dev, c) {
3506                 struct intel_plane_state *state;
3507
3508                 if (c == &intel_crtc->base)
3509                         continue;
3510
3511                 if (!to_intel_crtc(c)->active)
3512                         continue;
3513
3514                 state = to_intel_plane_state(c->primary->state);
3515                 if (!state->vma)
3516                         continue;
3517
3518                 if (intel_plane_ggtt_offset(state) == plane_config->base) {
3519                         fb = state->hw.fb;
3520                         drm_framebuffer_get(fb);
3521                         goto valid_fb;
3522                 }
3523         }
3524
3525         /*
3526          * We've failed to reconstruct the BIOS FB.  Current display state
3527          * indicates that the primary plane is visible, but has a NULL FB,
3528          * which will lead to problems later if we don't fix it up.  The
3529          * simplest solution is to just disable the primary plane now and
3530          * pretend the BIOS never had it enabled.
3531          */
3532         intel_plane_disable_noatomic(intel_crtc, intel_plane);
3533
3534         return;
3535
3536 valid_fb:
3537         intel_state->hw.rotation = plane_config->rotation;
3538         intel_fill_fb_ggtt_view(&intel_state->view, fb,
3539                                 intel_state->hw.rotation);
3540         intel_state->color_plane[0].stride =
3541                 intel_fb_pitch(fb, 0, intel_state->hw.rotation);
3542
3543         intel_state->vma =
3544                 intel_pin_and_fence_fb_obj(fb,
3545                                            &intel_state->view,
3546                                            intel_plane_uses_fence(intel_state),
3547                                            &intel_state->flags);
3548         if (IS_ERR(intel_state->vma)) {
3549                 DRM_ERROR("failed to pin boot fb on pipe %d: %li\n",
3550                           intel_crtc->pipe, PTR_ERR(intel_state->vma));
3551
3552                 intel_state->vma = NULL;
3553                 drm_framebuffer_put(fb);
3554                 return;
3555         }
3556
3557         intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB);
3558
3559         plane_state->src_x = 0;
3560         plane_state->src_y = 0;
3561         plane_state->src_w = fb->width << 16;
3562         plane_state->src_h = fb->height << 16;
3563
3564         plane_state->crtc_x = 0;
3565         plane_state->crtc_y = 0;
3566         plane_state->crtc_w = fb->width;
3567         plane_state->crtc_h = fb->height;
3568
3569         intel_state->uapi.src = drm_plane_state_src(plane_state);
3570         intel_state->uapi.dst = drm_plane_state_dest(plane_state);
3571
3572         if (plane_config->tiling)
3573                 dev_priv->preserve_bios_swizzle = true;
3574
3575         plane_state->fb = fb;
3576         plane_state->crtc = &intel_crtc->base;
3577         intel_plane_copy_uapi_to_hw_state(intel_state, intel_state);
3578
3579         atomic_or(to_intel_plane(primary)->frontbuffer_bit,
3580                   &to_intel_frontbuffer(fb)->bits);
3581 }
3582
3583 static int skl_max_plane_width(const struct drm_framebuffer *fb,
3584                                int color_plane,
3585                                unsigned int rotation)
3586 {
3587         int cpp = fb->format->cpp[color_plane];
3588
3589         switch (fb->modifier) {
3590         case DRM_FORMAT_MOD_LINEAR:
3591         case I915_FORMAT_MOD_X_TILED:
3592                 /*
3593                  * Validated limit is 4k, but has 5k should
3594                  * work apart from the following features:
3595                  * - Ytile (already limited to 4k)
3596                  * - FP16 (already limited to 4k)
3597                  * - render compression (already limited to 4k)
3598                  * - KVMR sprite and cursor (don't care)
3599                  * - horizontal panning (TODO verify this)
3600                  * - pipe and plane scaling (TODO verify this)
3601                  */
3602                 if (cpp == 8)
3603                         return 4096;
3604                 else
3605                         return 5120;
3606         case I915_FORMAT_MOD_Y_TILED_CCS:
3607         case I915_FORMAT_MOD_Yf_TILED_CCS:
3608                 /* FIXME AUX plane? */
3609         case I915_FORMAT_MOD_Y_TILED:
3610         case I915_FORMAT_MOD_Yf_TILED:
3611                 if (cpp == 8)
3612                         return 2048;
3613                 else
3614                         return 4096;
3615         default:
3616                 MISSING_CASE(fb->modifier);
3617                 return 2048;
3618         }
3619 }
3620
3621 static int glk_max_plane_width(const struct drm_framebuffer *fb,
3622                                int color_plane,
3623                                unsigned int rotation)
3624 {
3625         int cpp = fb->format->cpp[color_plane];
3626
3627         switch (fb->modifier) {
3628         case DRM_FORMAT_MOD_LINEAR:
3629         case I915_FORMAT_MOD_X_TILED:
3630                 if (cpp == 8)
3631                         return 4096;
3632                 else
3633                         return 5120;
3634         case I915_FORMAT_MOD_Y_TILED_CCS:
3635         case I915_FORMAT_MOD_Yf_TILED_CCS:
3636                 /* FIXME AUX plane? */
3637         case I915_FORMAT_MOD_Y_TILED:
3638         case I915_FORMAT_MOD_Yf_TILED:
3639                 if (cpp == 8)
3640                         return 2048;
3641                 else
3642                         return 5120;
3643         default:
3644                 MISSING_CASE(fb->modifier);
3645                 return 2048;
3646         }
3647 }
3648
3649 static int icl_max_plane_width(const struct drm_framebuffer *fb,
3650                                int color_plane,
3651                                unsigned int rotation)
3652 {
3653         return 5120;
3654 }
3655
3656 static int skl_max_plane_height(void)
3657 {
3658         return 4096;
3659 }
3660
3661 static int icl_max_plane_height(void)
3662 {
3663         return 4320;
3664 }
3665
3666 static bool skl_check_main_ccs_coordinates(struct intel_plane_state *plane_state,
3667                                            int main_x, int main_y, u32 main_offset)
3668 {
3669         const struct drm_framebuffer *fb = plane_state->hw.fb;
3670         int ccs_plane = main_to_ccs_plane(fb, 0);
3671         int aux_x = plane_state->color_plane[ccs_plane].x;
3672         int aux_y = plane_state->color_plane[ccs_plane].y;
3673         u32 aux_offset = plane_state->color_plane[ccs_plane].offset;
3674         u32 alignment = intel_surf_alignment(fb, ccs_plane);
3675         int hsub;
3676         int vsub;
3677
3678         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, ccs_plane);
3679         while (aux_offset >= main_offset && aux_y <= main_y) {
3680                 int x, y;
3681
3682                 if (aux_x == main_x && aux_y == main_y)
3683                         break;
3684
3685                 if (aux_offset == 0)
3686                         break;
3687
3688                 x = aux_x / hsub;
3689                 y = aux_y / vsub;
3690                 aux_offset = intel_plane_adjust_aligned_offset(&x, &y,
3691                                                                plane_state,
3692                                                                ccs_plane,
3693                                                                aux_offset,
3694                                                                aux_offset -
3695                                                                 alignment);
3696                 aux_x = x * hsub + aux_x % hsub;
3697                 aux_y = y * vsub + aux_y % vsub;
3698         }
3699
3700         if (aux_x != main_x || aux_y != main_y)
3701                 return false;
3702
3703         plane_state->color_plane[ccs_plane].offset = aux_offset;
3704         plane_state->color_plane[ccs_plane].x = aux_x;
3705         plane_state->color_plane[ccs_plane].y = aux_y;
3706
3707         return true;
3708 }
3709
3710 static int skl_check_main_surface(struct intel_plane_state *plane_state)
3711 {
3712         struct drm_i915_private *dev_priv = to_i915(plane_state->uapi.plane->dev);
3713         const struct drm_framebuffer *fb = plane_state->hw.fb;
3714         unsigned int rotation = plane_state->hw.rotation;
3715         int x = plane_state->uapi.src.x1 >> 16;
3716         int y = plane_state->uapi.src.y1 >> 16;
3717         int w = drm_rect_width(&plane_state->uapi.src) >> 16;
3718         int h = drm_rect_height(&plane_state->uapi.src) >> 16;
3719         int max_width;
3720         int max_height;
3721         u32 alignment;
3722         u32 offset;
3723         int aux_plane = intel_main_to_aux_plane(fb, 0);
3724         u32 aux_offset = plane_state->color_plane[aux_plane].offset;
3725
3726         if (INTEL_GEN(dev_priv) >= 11)
3727                 max_width = icl_max_plane_width(fb, 0, rotation);
3728         else if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
3729                 max_width = glk_max_plane_width(fb, 0, rotation);
3730         else
3731                 max_width = skl_max_plane_width(fb, 0, rotation);
3732
3733         if (INTEL_GEN(dev_priv) >= 11)
3734                 max_height = icl_max_plane_height();
3735         else
3736                 max_height = skl_max_plane_height();
3737
3738         if (w > max_width || h > max_height) {
3739                 DRM_DEBUG_KMS("requested Y/RGB source size %dx%d too big (limit %dx%d)\n",
3740                               w, h, max_width, max_height);
3741                 return -EINVAL;
3742         }
3743
3744         intel_add_fb_offsets(&x, &y, plane_state, 0);
3745         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 0);
3746         alignment = intel_surf_alignment(fb, 0);
3747         if (WARN_ON(alignment && !is_power_of_2(alignment)))
3748                 return -EINVAL;
3749
3750         /*
3751          * AUX surface offset is specified as the distance from the
3752          * main surface offset, and it must be non-negative. Make
3753          * sure that is what we will get.
3754          */
3755         if (offset > aux_offset)
3756                 offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3757                                                            offset, aux_offset & ~(alignment - 1));
3758
3759         /*
3760          * When using an X-tiled surface, the plane blows up
3761          * if the x offset + width exceed the stride.
3762          *
3763          * TODO: linear and Y-tiled seem fine, Yf untested,
3764          */
3765         if (fb->modifier == I915_FORMAT_MOD_X_TILED) {
3766                 int cpp = fb->format->cpp[0];
3767
3768                 while ((x + w) * cpp > plane_state->color_plane[0].stride) {
3769                         if (offset == 0) {
3770                                 DRM_DEBUG_KMS("Unable to find suitable display surface offset due to X-tiling\n");
3771                                 return -EINVAL;
3772                         }
3773
3774                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3775                                                                    offset, offset - alignment);
3776                 }
3777         }
3778
3779         /*
3780          * CCS AUX surface doesn't have its own x/y offsets, we must make sure
3781          * they match with the main surface x/y offsets.
3782          */
3783         if (is_ccs_modifier(fb->modifier)) {
3784                 while (!skl_check_main_ccs_coordinates(plane_state, x, y, offset)) {
3785                         if (offset == 0)
3786                                 break;
3787
3788                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3789                                                                    offset, offset - alignment);
3790                 }
3791
3792                 if (x != plane_state->color_plane[aux_plane].x ||
3793                     y != plane_state->color_plane[aux_plane].y) {
3794                         DRM_DEBUG_KMS("Unable to find suitable display surface offset due to CCS\n");
3795                         return -EINVAL;
3796                 }
3797         }
3798
3799         plane_state->color_plane[0].offset = offset;
3800         plane_state->color_plane[0].x = x;
3801         plane_state->color_plane[0].y = y;
3802
3803         /*
3804          * Put the final coordinates back so that the src
3805          * coordinate checks will see the right values.
3806          */
3807         drm_rect_translate_to(&plane_state->uapi.src,
3808                               x << 16, y << 16);
3809
3810         return 0;
3811 }
3812
3813 static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
3814 {
3815         const struct drm_framebuffer *fb = plane_state->hw.fb;
3816         unsigned int rotation = plane_state->hw.rotation;
3817         int max_width = skl_max_plane_width(fb, 1, rotation);
3818         int max_height = 4096;
3819         int x = plane_state->uapi.src.x1 >> 17;
3820         int y = plane_state->uapi.src.y1 >> 17;
3821         int w = drm_rect_width(&plane_state->uapi.src) >> 17;
3822         int h = drm_rect_height(&plane_state->uapi.src) >> 17;
3823         u32 offset;
3824
3825         intel_add_fb_offsets(&x, &y, plane_state, 1);
3826         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3827
3828         /* FIXME not quite sure how/if these apply to the chroma plane */
3829         if (w > max_width || h > max_height) {
3830                 DRM_DEBUG_KMS("CbCr source size %dx%d too big (limit %dx%d)\n",
3831                               w, h, max_width, max_height);
3832                 return -EINVAL;
3833         }
3834
3835         plane_state->color_plane[1].offset = offset;
3836         plane_state->color_plane[1].x = x;
3837         plane_state->color_plane[1].y = y;
3838
3839         return 0;
3840 }
3841
3842 static int skl_check_ccs_aux_surface(struct intel_plane_state *plane_state)
3843 {
3844         const struct drm_framebuffer *fb = plane_state->hw.fb;
3845         int src_x = plane_state->uapi.src.x1 >> 16;
3846         int src_y = plane_state->uapi.src.y1 >> 16;
3847         int hsub;
3848         int vsub;
3849         int x;
3850         int y;
3851         u32 offset;
3852
3853         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, 1);
3854         x = src_x / hsub;
3855         y = src_y / vsub;
3856         intel_add_fb_offsets(&x, &y, plane_state, 1);
3857         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3858
3859         plane_state->color_plane[1].offset = offset;
3860         plane_state->color_plane[1].x = x * hsub + src_x % hsub;
3861         plane_state->color_plane[1].y = y * vsub + src_y % vsub;
3862
3863         return 0;
3864 }
3865
3866 int skl_check_plane_surface(struct intel_plane_state *plane_state)
3867 {
3868         const struct drm_framebuffer *fb = plane_state->hw.fb;
3869         int ret;
3870
3871         ret = intel_plane_compute_gtt(plane_state);
3872         if (ret)
3873                 return ret;
3874
3875         if (!plane_state->uapi.visible)
3876                 return 0;
3877
3878         /*
3879          * Handle the AUX surface first since
3880          * the main surface setup depends on it.
3881          */
3882         if (intel_format_info_is_yuv_semiplanar(fb->format,
3883                                                 fb->modifier)) {
3884                 ret = skl_check_nv12_aux_surface(plane_state);
3885                 if (ret)
3886                         return ret;
3887         } else if (is_ccs_modifier(fb->modifier)) {
3888                 ret = skl_check_ccs_aux_surface(plane_state);
3889                 if (ret)
3890                         return ret;
3891         } else {
3892                 plane_state->color_plane[1].offset = ~0xfff;
3893                 plane_state->color_plane[1].x = 0;
3894                 plane_state->color_plane[1].y = 0;
3895         }
3896
3897         ret = skl_check_main_surface(plane_state);
3898         if (ret)
3899                 return ret;
3900
3901         return 0;
3902 }
3903
3904 static void i9xx_plane_ratio(const struct intel_crtc_state *crtc_state,
3905                              const struct intel_plane_state *plane_state,
3906                              unsigned int *num, unsigned int *den)
3907 {
3908         const struct drm_framebuffer *fb = plane_state->hw.fb;
3909         unsigned int cpp = fb->format->cpp[0];
3910
3911         /*
3912          * g4x bspec says 64bpp pixel rate can't exceed 80%
3913          * of cdclk when the sprite plane is enabled on the
3914          * same pipe. ilk/snb bspec says 64bpp pixel rate is
3915          * never allowed to exceed 80% of cdclk. Let's just go
3916          * with the ilk/snb limit always.
3917          */
3918         if (cpp == 8) {
3919                 *num = 10;
3920                 *den = 8;
3921         } else {
3922                 *num = 1;
3923                 *den = 1;
3924         }
3925 }
3926
3927 static int i9xx_plane_min_cdclk(const struct intel_crtc_state *crtc_state,
3928                                 const struct intel_plane_state *plane_state)
3929 {
3930         unsigned int pixel_rate;
3931         unsigned int num, den;
3932
3933         /*
3934          * Note that crtc_state->pixel_rate accounts for both
3935          * horizontal and vertical panel fitter downscaling factors.
3936          * Pre-HSW bspec tells us to only consider the horizontal
3937          * downscaling factor here. We ignore that and just consider
3938          * both for simplicity.
3939          */
3940         pixel_rate = crtc_state->pixel_rate;
3941
3942         i9xx_plane_ratio(crtc_state, plane_state, &num, &den);
3943
3944         /* two pixels per clock with double wide pipe */
3945         if (crtc_state->double_wide)
3946                 den *= 2;
3947
3948         return DIV_ROUND_UP(pixel_rate * num, den);
3949 }
3950
3951 unsigned int
3952 i9xx_plane_max_stride(struct intel_plane *plane,
3953                       u32 pixel_format, u64 modifier,
3954                       unsigned int rotation)
3955 {
3956         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3957
3958         if (!HAS_GMCH(dev_priv)) {
3959                 return 32*1024;
3960         } else if (INTEL_GEN(dev_priv) >= 4) {
3961                 if (modifier == I915_FORMAT_MOD_X_TILED)
3962                         return 16*1024;
3963                 else
3964                         return 32*1024;
3965         } else if (INTEL_GEN(dev_priv) >= 3) {
3966                 if (modifier == I915_FORMAT_MOD_X_TILED)
3967                         return 8*1024;
3968                 else
3969                         return 16*1024;
3970         } else {
3971                 if (plane->i9xx_plane == PLANE_C)
3972                         return 4*1024;
3973                 else
3974                         return 8*1024;
3975         }
3976 }
3977
3978 static u32 i9xx_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
3979 {
3980         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3981         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3982         u32 dspcntr = 0;
3983
3984         if (crtc_state->gamma_enable)
3985                 dspcntr |= DISPPLANE_GAMMA_ENABLE;
3986
3987         if (crtc_state->csc_enable)
3988                 dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
3989
3990         if (INTEL_GEN(dev_priv) < 5)
3991                 dspcntr |= DISPPLANE_SEL_PIPE(crtc->pipe);
3992
3993         return dspcntr;
3994 }
3995
3996 static u32 i9xx_plane_ctl(const struct intel_crtc_state *crtc_state,
3997                           const struct intel_plane_state *plane_state)
3998 {
3999         struct drm_i915_private *dev_priv =
4000                 to_i915(plane_state->uapi.plane->dev);
4001         const struct drm_framebuffer *fb = plane_state->hw.fb;
4002         unsigned int rotation = plane_state->hw.rotation;
4003         u32 dspcntr;
4004
4005         dspcntr = DISPLAY_PLANE_ENABLE;
4006
4007         if (IS_G4X(dev_priv) || IS_GEN(dev_priv, 5) ||
4008             IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
4009                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
4010
4011         switch (fb->format->format) {
4012         case DRM_FORMAT_C8:
4013                 dspcntr |= DISPPLANE_8BPP;
4014                 break;
4015         case DRM_FORMAT_XRGB1555:
4016                 dspcntr |= DISPPLANE_BGRX555;
4017                 break;
4018         case DRM_FORMAT_ARGB1555:
4019                 dspcntr |= DISPPLANE_BGRA555;
4020                 break;
4021         case DRM_FORMAT_RGB565:
4022                 dspcntr |= DISPPLANE_BGRX565;
4023                 break;
4024         case DRM_FORMAT_XRGB8888:
4025                 dspcntr |= DISPPLANE_BGRX888;
4026                 break;
4027         case DRM_FORMAT_XBGR8888:
4028                 dspcntr |= DISPPLANE_RGBX888;
4029                 break;
4030         case DRM_FORMAT_ARGB8888:
4031                 dspcntr |= DISPPLANE_BGRA888;
4032                 break;
4033         case DRM_FORMAT_ABGR8888:
4034                 dspcntr |= DISPPLANE_RGBA888;
4035                 break;
4036         case DRM_FORMAT_XRGB2101010:
4037                 dspcntr |= DISPPLANE_BGRX101010;
4038                 break;
4039         case DRM_FORMAT_XBGR2101010:
4040                 dspcntr |= DISPPLANE_RGBX101010;
4041                 break;
4042         case DRM_FORMAT_ARGB2101010:
4043                 dspcntr |= DISPPLANE_BGRA101010;
4044                 break;
4045         case DRM_FORMAT_ABGR2101010:
4046                 dspcntr |= DISPPLANE_RGBA101010;
4047                 break;
4048         case DRM_FORMAT_XBGR16161616F:
4049                 dspcntr |= DISPPLANE_RGBX161616;
4050                 break;
4051         default:
4052                 MISSING_CASE(fb->format->format);
4053                 return 0;
4054         }
4055
4056         if (INTEL_GEN(dev_priv) >= 4 &&
4057             fb->modifier == I915_FORMAT_MOD_X_TILED)
4058                 dspcntr |= DISPPLANE_TILED;
4059
4060         if (rotation & DRM_MODE_ROTATE_180)
4061                 dspcntr |= DISPPLANE_ROTATE_180;
4062
4063         if (rotation & DRM_MODE_REFLECT_X)
4064                 dspcntr |= DISPPLANE_MIRROR;
4065
4066         return dspcntr;
4067 }
4068
4069 int i9xx_check_plane_surface(struct intel_plane_state *plane_state)
4070 {
4071         struct drm_i915_private *dev_priv =
4072                 to_i915(plane_state->uapi.plane->dev);
4073         const struct drm_framebuffer *fb = plane_state->hw.fb;
4074         int src_x, src_y, src_w;
4075         u32 offset;
4076         int ret;
4077
4078         ret = intel_plane_compute_gtt(plane_state);
4079         if (ret)
4080                 return ret;
4081
4082         if (!plane_state->uapi.visible)
4083                 return 0;
4084
4085         src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4086         src_x = plane_state->uapi.src.x1 >> 16;
4087         src_y = plane_state->uapi.src.y1 >> 16;
4088
4089         /* Undocumented hardware limit on i965/g4x/vlv/chv */
4090         if (HAS_GMCH(dev_priv) && fb->format->cpp[0] == 8 && src_w > 2048)
4091                 return -EINVAL;
4092
4093         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
4094
4095         if (INTEL_GEN(dev_priv) >= 4)
4096                 offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
4097                                                             plane_state, 0);
4098         else
4099                 offset = 0;
4100
4101         /*
4102          * Put the final coordinates back so that the src
4103          * coordinate checks will see the right values.
4104          */
4105         drm_rect_translate_to(&plane_state->uapi.src,
4106                               src_x << 16, src_y << 16);
4107
4108         /* HSW/BDW do this automagically in hardware */
4109         if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) {
4110                 unsigned int rotation = plane_state->hw.rotation;
4111                 int src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4112                 int src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
4113
4114                 if (rotation & DRM_MODE_ROTATE_180) {
4115                         src_x += src_w - 1;
4116                         src_y += src_h - 1;
4117                 } else if (rotation & DRM_MODE_REFLECT_X) {
4118                         src_x += src_w - 1;
4119                 }
4120         }
4121
4122         plane_state->color_plane[0].offset = offset;
4123         plane_state->color_plane[0].x = src_x;
4124         plane_state->color_plane[0].y = src_y;
4125
4126         return 0;
4127 }
4128
4129 static bool i9xx_plane_has_windowing(struct intel_plane *plane)
4130 {
4131         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4132         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4133
4134         if (IS_CHERRYVIEW(dev_priv))
4135                 return i9xx_plane == PLANE_B;
4136         else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
4137                 return false;
4138         else if (IS_GEN(dev_priv, 4))
4139                 return i9xx_plane == PLANE_C;
4140         else
4141                 return i9xx_plane == PLANE_B ||
4142                         i9xx_plane == PLANE_C;
4143 }
4144
4145 static int
4146 i9xx_plane_check(struct intel_crtc_state *crtc_state,
4147                  struct intel_plane_state *plane_state)
4148 {
4149         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4150         int ret;
4151
4152         ret = chv_plane_check_rotation(plane_state);
4153         if (ret)
4154                 return ret;
4155
4156         ret = drm_atomic_helper_check_plane_state(&plane_state->uapi,
4157                                                   &crtc_state->uapi,
4158                                                   DRM_PLANE_HELPER_NO_SCALING,
4159                                                   DRM_PLANE_HELPER_NO_SCALING,
4160                                                   i9xx_plane_has_windowing(plane),
4161                                                   true);
4162         if (ret)
4163                 return ret;
4164
4165         ret = i9xx_check_plane_surface(plane_state);
4166         if (ret)
4167                 return ret;
4168
4169         if (!plane_state->uapi.visible)
4170                 return 0;
4171
4172         ret = intel_plane_check_src_coordinates(plane_state);
4173         if (ret)
4174                 return ret;
4175
4176         plane_state->ctl = i9xx_plane_ctl(crtc_state, plane_state);
4177
4178         return 0;
4179 }
4180
4181 static void i9xx_update_plane(struct intel_plane *plane,
4182                               const struct intel_crtc_state *crtc_state,
4183                               const struct intel_plane_state *plane_state)
4184 {
4185         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4186         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4187         u32 linear_offset;
4188         int x = plane_state->color_plane[0].x;
4189         int y = plane_state->color_plane[0].y;
4190         int crtc_x = plane_state->uapi.dst.x1;
4191         int crtc_y = plane_state->uapi.dst.y1;
4192         int crtc_w = drm_rect_width(&plane_state->uapi.dst);
4193         int crtc_h = drm_rect_height(&plane_state->uapi.dst);
4194         unsigned long irqflags;
4195         u32 dspaddr_offset;
4196         u32 dspcntr;
4197
4198         dspcntr = plane_state->ctl | i9xx_plane_ctl_crtc(crtc_state);
4199
4200         linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
4201
4202         if (INTEL_GEN(dev_priv) >= 4)
4203                 dspaddr_offset = plane_state->color_plane[0].offset;
4204         else
4205                 dspaddr_offset = linear_offset;
4206
4207         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
4208
4209         I915_WRITE_FW(DSPSTRIDE(i9xx_plane), plane_state->color_plane[0].stride);
4210
4211         if (INTEL_GEN(dev_priv) < 4) {
4212                 /*
4213                  * PLANE_A doesn't actually have a full window
4214                  * generator but let's assume we still need to
4215                  * program whatever is there.
4216                  */
4217                 I915_WRITE_FW(DSPPOS(i9xx_plane), (crtc_y << 16) | crtc_x);
4218                 I915_WRITE_FW(DSPSIZE(i9xx_plane),
4219                               ((crtc_h - 1) << 16) | (crtc_w - 1));
4220         } else if (IS_CHERRYVIEW(dev_priv) && i9xx_plane == PLANE_B) {
4221                 I915_WRITE_FW(PRIMPOS(i9xx_plane), (crtc_y << 16) | crtc_x);
4222                 I915_WRITE_FW(PRIMSIZE(i9xx_plane),
4223                               ((crtc_h - 1) << 16) | (crtc_w - 1));
4224                 I915_WRITE_FW(PRIMCNSTALPHA(i9xx_plane), 0);
4225         }
4226
4227         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
4228                 I915_WRITE_FW(DSPOFFSET(i9xx_plane), (y << 16) | x);
4229         } else if (INTEL_GEN(dev_priv) >= 4) {
4230                 I915_WRITE_FW(DSPLINOFF(i9xx_plane), linear_offset);
4231                 I915_WRITE_FW(DSPTILEOFF(i9xx_plane), (y << 16) | x);
4232         }
4233
4234         /*
4235          * The control register self-arms if the plane was previously
4236          * disabled. Try to make the plane enable atomic by writing
4237          * the control register just before the surface register.
4238          */
4239         I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr);
4240         if (INTEL_GEN(dev_priv) >= 4)
4241                 I915_WRITE_FW(DSPSURF(i9xx_plane),
4242                               intel_plane_ggtt_offset(plane_state) +
4243                               dspaddr_offset);
4244         else
4245                 I915_WRITE_FW(DSPADDR(i9xx_plane),
4246                               intel_plane_ggtt_offset(plane_state) +
4247                               dspaddr_offset);
4248
4249         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
4250 }
4251
4252 static void i9xx_disable_plane(struct intel_plane *plane,
4253                                const struct intel_crtc_state *crtc_state)
4254 {
4255         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4256         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4257         unsigned long irqflags;
4258         u32 dspcntr;
4259
4260         /*
4261          * DSPCNTR pipe gamma enable on g4x+ and pipe csc
4262          * enable on ilk+ affect the pipe bottom color as
4263          * well, so we must configure them even if the plane
4264          * is disabled.
4265          *
4266          * On pre-g4x there is no way to gamma correct the
4267          * pipe bottom color but we'll keep on doing this
4268          * anyway so that the crtc state readout works correctly.
4269          */
4270         dspcntr = i9xx_plane_ctl_crtc(crtc_state);
4271
4272         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
4273
4274         I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr);
4275         if (INTEL_GEN(dev_priv) >= 4)
4276                 I915_WRITE_FW(DSPSURF(i9xx_plane), 0);
4277         else
4278                 I915_WRITE_FW(DSPADDR(i9xx_plane), 0);
4279
4280         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
4281 }
4282
4283 static bool i9xx_plane_get_hw_state(struct intel_plane *plane,
4284                                     enum pipe *pipe)
4285 {
4286         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4287         enum intel_display_power_domain power_domain;
4288         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4289         intel_wakeref_t wakeref;
4290         bool ret;
4291         u32 val;
4292
4293         /*
4294          * Not 100% correct for planes that can move between pipes,
4295          * but that's only the case for gen2-4 which don't have any
4296          * display power wells.
4297          */
4298         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
4299         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4300         if (!wakeref)
4301                 return false;
4302
4303         val = I915_READ(DSPCNTR(i9xx_plane));
4304
4305         ret = val & DISPLAY_PLANE_ENABLE;
4306
4307         if (INTEL_GEN(dev_priv) >= 5)
4308                 *pipe = plane->pipe;
4309         else
4310                 *pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
4311                         DISPPLANE_SEL_PIPE_SHIFT;
4312
4313         intel_display_power_put(dev_priv, power_domain, wakeref);
4314
4315         return ret;
4316 }
4317
4318 static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
4319 {
4320         struct drm_device *dev = intel_crtc->base.dev;
4321         struct drm_i915_private *dev_priv = to_i915(dev);
4322
4323         I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, id), 0);
4324         I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, id), 0);
4325         I915_WRITE(SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0);
4326 }
4327
4328 /*
4329  * This function detaches (aka. unbinds) unused scalers in hardware
4330  */
4331 static void skl_detach_scalers(const struct intel_crtc_state *crtc_state)
4332 {
4333         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->uapi.crtc);
4334         const struct intel_crtc_scaler_state *scaler_state =
4335                 &crtc_state->scaler_state;
4336         int i;
4337
4338         /* loop through and disable scalers that aren't in use */
4339         for (i = 0; i < intel_crtc->num_scalers; i++) {
4340                 if (!scaler_state->scalers[i].in_use)
4341                         skl_detach_scaler(intel_crtc, i);
4342         }
4343 }
4344
4345 static unsigned int skl_plane_stride_mult(const struct drm_framebuffer *fb,
4346                                           int color_plane, unsigned int rotation)
4347 {
4348         /*
4349          * The stride is either expressed as a multiple of 64 bytes chunks for
4350          * linear buffers or in number of tiles for tiled buffers.
4351          */
4352         if (is_surface_linear(fb, color_plane))
4353                 return 64;
4354         else if (drm_rotation_90_or_270(rotation))
4355                 return intel_tile_height(fb, color_plane);
4356         else
4357                 return intel_tile_width_bytes(fb, color_plane);
4358 }
4359
4360 u32 skl_plane_stride(const struct intel_plane_state *plane_state,
4361                      int color_plane)
4362 {
4363         const struct drm_framebuffer *fb = plane_state->hw.fb;
4364         unsigned int rotation = plane_state->hw.rotation;
4365         u32 stride = plane_state->color_plane[color_plane].stride;
4366
4367         if (color_plane >= fb->format->num_planes)
4368                 return 0;
4369
4370         return stride / skl_plane_stride_mult(fb, color_plane, rotation);
4371 }
4372
4373 static u32 skl_plane_ctl_format(u32 pixel_format)
4374 {
4375         switch (pixel_format) {
4376         case DRM_FORMAT_C8:
4377                 return PLANE_CTL_FORMAT_INDEXED;
4378         case DRM_FORMAT_RGB565:
4379                 return PLANE_CTL_FORMAT_RGB_565;
4380         case DRM_FORMAT_XBGR8888:
4381         case DRM_FORMAT_ABGR8888:
4382                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX;
4383         case DRM_FORMAT_XRGB8888:
4384         case DRM_FORMAT_ARGB8888:
4385                 return PLANE_CTL_FORMAT_XRGB_8888;
4386         case DRM_FORMAT_XBGR2101010:
4387         case DRM_FORMAT_ABGR2101010:
4388                 return PLANE_CTL_FORMAT_XRGB_2101010 | PLANE_CTL_ORDER_RGBX;
4389         case DRM_FORMAT_XRGB2101010:
4390         case DRM_FORMAT_ARGB2101010:
4391                 return PLANE_CTL_FORMAT_XRGB_2101010;
4392         case DRM_FORMAT_XBGR16161616F:
4393         case DRM_FORMAT_ABGR16161616F:
4394                 return PLANE_CTL_FORMAT_XRGB_16161616F | PLANE_CTL_ORDER_RGBX;
4395         case DRM_FORMAT_XRGB16161616F:
4396         case DRM_FORMAT_ARGB16161616F:
4397                 return PLANE_CTL_FORMAT_XRGB_16161616F;
4398         case DRM_FORMAT_YUYV:
4399                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV;
4400         case DRM_FORMAT_YVYU:
4401                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YVYU;
4402         case DRM_FORMAT_UYVY:
4403                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_UYVY;
4404         case DRM_FORMAT_VYUY:
4405                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY;
4406         case DRM_FORMAT_NV12:
4407                 return PLANE_CTL_FORMAT_NV12;
4408         case DRM_FORMAT_P010:
4409                 return PLANE_CTL_FORMAT_P010;
4410         case DRM_FORMAT_P012:
4411                 return PLANE_CTL_FORMAT_P012;
4412         case DRM_FORMAT_P016:
4413                 return PLANE_CTL_FORMAT_P016;
4414         case DRM_FORMAT_Y210:
4415                 return PLANE_CTL_FORMAT_Y210;
4416         case DRM_FORMAT_Y212:
4417                 return PLANE_CTL_FORMAT_Y212;
4418         case DRM_FORMAT_Y216:
4419                 return PLANE_CTL_FORMAT_Y216;
4420         case DRM_FORMAT_XVYU2101010:
4421                 return PLANE_CTL_FORMAT_Y410;
4422         case DRM_FORMAT_XVYU12_16161616:
4423                 return PLANE_CTL_FORMAT_Y412;
4424         case DRM_FORMAT_XVYU16161616:
4425                 return PLANE_CTL_FORMAT_Y416;
4426         default:
4427                 MISSING_CASE(pixel_format);
4428         }
4429
4430         return 0;
4431 }
4432
4433 static u32 skl_plane_ctl_alpha(const struct intel_plane_state *plane_state)
4434 {
4435         if (!plane_state->hw.fb->format->has_alpha)
4436                 return PLANE_CTL_ALPHA_DISABLE;
4437
4438         switch (plane_state->hw.pixel_blend_mode) {
4439         case DRM_MODE_BLEND_PIXEL_NONE:
4440                 return PLANE_CTL_ALPHA_DISABLE;
4441         case DRM_MODE_BLEND_PREMULTI:
4442                 return PLANE_CTL_ALPHA_SW_PREMULTIPLY;
4443         case DRM_MODE_BLEND_COVERAGE:
4444                 return PLANE_CTL_ALPHA_HW_PREMULTIPLY;
4445         default:
4446                 MISSING_CASE(plane_state->hw.pixel_blend_mode);
4447                 return PLANE_CTL_ALPHA_DISABLE;
4448         }
4449 }
4450
4451 static u32 glk_plane_color_ctl_alpha(const struct intel_plane_state *plane_state)
4452 {
4453         if (!plane_state->hw.fb->format->has_alpha)
4454                 return PLANE_COLOR_ALPHA_DISABLE;
4455
4456         switch (plane_state->hw.pixel_blend_mode) {
4457         case DRM_MODE_BLEND_PIXEL_NONE:
4458                 return PLANE_COLOR_ALPHA_DISABLE;
4459         case DRM_MODE_BLEND_PREMULTI:
4460                 return PLANE_COLOR_ALPHA_SW_PREMULTIPLY;
4461         case DRM_MODE_BLEND_COVERAGE:
4462                 return PLANE_COLOR_ALPHA_HW_PREMULTIPLY;
4463         default:
4464                 MISSING_CASE(plane_state->hw.pixel_blend_mode);
4465                 return PLANE_COLOR_ALPHA_DISABLE;
4466         }
4467 }
4468
4469 static u32 skl_plane_ctl_tiling(u64 fb_modifier)
4470 {
4471         switch (fb_modifier) {
4472         case DRM_FORMAT_MOD_LINEAR:
4473                 break;
4474         case I915_FORMAT_MOD_X_TILED:
4475                 return PLANE_CTL_TILED_X;
4476         case I915_FORMAT_MOD_Y_TILED:
4477                 return PLANE_CTL_TILED_Y;
4478         case I915_FORMAT_MOD_Y_TILED_CCS:
4479                 return PLANE_CTL_TILED_Y | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
4480         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
4481                 return PLANE_CTL_TILED_Y |
4482                        PLANE_CTL_RENDER_DECOMPRESSION_ENABLE |
4483                        PLANE_CTL_CLEAR_COLOR_DISABLE;
4484         case I915_FORMAT_MOD_Yf_TILED:
4485                 return PLANE_CTL_TILED_YF;
4486         case I915_FORMAT_MOD_Yf_TILED_CCS:
4487                 return PLANE_CTL_TILED_YF | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
4488         default:
4489                 MISSING_CASE(fb_modifier);
4490         }
4491
4492         return 0;
4493 }
4494
4495 static u32 skl_plane_ctl_rotate(unsigned int rotate)
4496 {
4497         switch (rotate) {
4498         case DRM_MODE_ROTATE_0:
4499                 break;
4500         /*
4501          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
4502          * while i915 HW rotation is clockwise, thats why this swapping.
4503          */
4504         case DRM_MODE_ROTATE_90:
4505                 return PLANE_CTL_ROTATE_270;
4506         case DRM_MODE_ROTATE_180:
4507                 return PLANE_CTL_ROTATE_180;
4508         case DRM_MODE_ROTATE_270:
4509                 return PLANE_CTL_ROTATE_90;
4510         default:
4511                 MISSING_CASE(rotate);
4512         }
4513
4514         return 0;
4515 }
4516
4517 static u32 cnl_plane_ctl_flip(unsigned int reflect)
4518 {
4519         switch (reflect) {
4520         case 0:
4521                 break;
4522         case DRM_MODE_REFLECT_X:
4523                 return PLANE_CTL_FLIP_HORIZONTAL;
4524         case DRM_MODE_REFLECT_Y:
4525         default:
4526                 MISSING_CASE(reflect);
4527         }
4528
4529         return 0;
4530 }
4531
4532 u32 skl_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
4533 {
4534         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4535         u32 plane_ctl = 0;
4536
4537         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
4538                 return plane_ctl;
4539
4540         if (crtc_state->gamma_enable)
4541                 plane_ctl |= PLANE_CTL_PIPE_GAMMA_ENABLE;
4542
4543         if (crtc_state->csc_enable)
4544                 plane_ctl |= PLANE_CTL_PIPE_CSC_ENABLE;
4545
4546         return plane_ctl;
4547 }
4548
4549 u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
4550                   const struct intel_plane_state *plane_state)
4551 {
4552         struct drm_i915_private *dev_priv =
4553                 to_i915(plane_state->uapi.plane->dev);
4554         const struct drm_framebuffer *fb = plane_state->hw.fb;
4555         unsigned int rotation = plane_state->hw.rotation;
4556         const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
4557         u32 plane_ctl;
4558
4559         plane_ctl = PLANE_CTL_ENABLE;
4560
4561         if (INTEL_GEN(dev_priv) < 10 && !IS_GEMINILAKE(dev_priv)) {
4562                 plane_ctl |= skl_plane_ctl_alpha(plane_state);
4563                 plane_ctl |= PLANE_CTL_PLANE_GAMMA_DISABLE;
4564
4565                 if (plane_state->hw.color_encoding == DRM_COLOR_YCBCR_BT709)
4566                         plane_ctl |= PLANE_CTL_YUV_TO_RGB_CSC_FORMAT_BT709;
4567
4568                 if (plane_state->hw.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
4569                         plane_ctl |= PLANE_CTL_YUV_RANGE_CORRECTION_DISABLE;
4570         }
4571
4572         plane_ctl |= skl_plane_ctl_format(fb->format->format);
4573         plane_ctl |= skl_plane_ctl_tiling(fb->modifier);
4574         plane_ctl |= skl_plane_ctl_rotate(rotation & DRM_MODE_ROTATE_MASK);
4575
4576         if (INTEL_GEN(dev_priv) >= 10)
4577                 plane_ctl |= cnl_plane_ctl_flip(rotation &
4578                                                 DRM_MODE_REFLECT_MASK);
4579
4580         if (key->flags & I915_SET_COLORKEY_DESTINATION)
4581                 plane_ctl |= PLANE_CTL_KEY_ENABLE_DESTINATION;
4582         else if (key->flags & I915_SET_COLORKEY_SOURCE)
4583                 plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE;
4584
4585         return plane_ctl;
4586 }
4587
4588 u32 glk_plane_color_ctl_crtc(const struct intel_crtc_state *crtc_state)
4589 {
4590         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4591         u32 plane_color_ctl = 0;
4592
4593         if (INTEL_GEN(dev_priv) >= 11)
4594                 return plane_color_ctl;
4595
4596         if (crtc_state->gamma_enable)
4597                 plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE;
4598
4599         if (crtc_state->csc_enable)
4600                 plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE;
4601
4602         return plane_color_ctl;
4603 }
4604
4605 u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
4606                         const struct intel_plane_state *plane_state)
4607 {
4608         struct drm_i915_private *dev_priv =
4609                 to_i915(plane_state->uapi.plane->dev);
4610         const struct drm_framebuffer *fb = plane_state->hw.fb;
4611         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4612         u32 plane_color_ctl = 0;
4613
4614         plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE;
4615         plane_color_ctl |= glk_plane_color_ctl_alpha(plane_state);
4616
4617         if (fb->format->is_yuv && !icl_is_hdr_plane(dev_priv, plane->id)) {
4618                 if (plane_state->hw.color_encoding == DRM_COLOR_YCBCR_BT709)
4619                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709;
4620                 else
4621                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV601_TO_RGB709;
4622
4623                 if (plane_state->hw.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
4624                         plane_color_ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE;
4625         } else if (fb->format->is_yuv) {
4626                 plane_color_ctl |= PLANE_COLOR_INPUT_CSC_ENABLE;
4627         }
4628
4629         return plane_color_ctl;
4630 }
4631
4632 static int
4633 __intel_display_resume(struct drm_device *dev,
4634                        struct drm_atomic_state *state,
4635                        struct drm_modeset_acquire_ctx *ctx)
4636 {
4637         struct drm_crtc_state *crtc_state;
4638         struct drm_crtc *crtc;
4639         int i, ret;
4640
4641         intel_modeset_setup_hw_state(dev, ctx);
4642         intel_vga_redisable(to_i915(dev));
4643
4644         if (!state)
4645                 return 0;
4646
4647         /*
4648          * We've duplicated the state, pointers to the old state are invalid.
4649          *
4650          * Don't attempt to use the old state until we commit the duplicated state.
4651          */
4652         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
4653                 /*
4654                  * Force recalculation even if we restore
4655                  * current state. With fast modeset this may not result
4656                  * in a modeset when the state is compatible.
4657                  */
4658                 crtc_state->mode_changed = true;
4659         }
4660
4661         /* ignore any reset values/BIOS leftovers in the WM registers */
4662         if (!HAS_GMCH(to_i915(dev)))
4663                 to_intel_atomic_state(state)->skip_intermediate_wm = true;
4664
4665         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
4666
4667         WARN_ON(ret == -EDEADLK);
4668         return ret;
4669 }
4670
4671 static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv)
4672 {
4673         return (INTEL_INFO(dev_priv)->gpu_reset_clobbers_display &&
4674                 intel_has_gpu_reset(&dev_priv->gt));
4675 }
4676
4677 void intel_prepare_reset(struct drm_i915_private *dev_priv)
4678 {
4679         struct drm_device *dev = &dev_priv->drm;
4680         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
4681         struct drm_atomic_state *state;
4682         int ret;
4683
4684         /* reset doesn't touch the display */
4685         if (!i915_modparams.force_reset_modeset_test &&
4686             !gpu_reset_clobbers_display(dev_priv))
4687                 return;
4688
4689         /* We have a modeset vs reset deadlock, defensively unbreak it. */
4690         set_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags);
4691         smp_mb__after_atomic();
4692         wake_up_bit(&dev_priv->gt.reset.flags, I915_RESET_MODESET);
4693
4694         if (atomic_read(&dev_priv->gpu_error.pending_fb_pin)) {
4695                 DRM_DEBUG_KMS("Modeset potentially stuck, unbreaking through wedging\n");
4696                 intel_gt_set_wedged(&dev_priv->gt);
4697         }
4698
4699         /*
4700          * Need mode_config.mutex so that we don't
4701          * trample ongoing ->detect() and whatnot.
4702          */
4703         mutex_lock(&dev->mode_config.mutex);
4704         drm_modeset_acquire_init(ctx, 0);
4705         while (1) {
4706                 ret = drm_modeset_lock_all_ctx(dev, ctx);
4707                 if (ret != -EDEADLK)
4708                         break;
4709
4710                 drm_modeset_backoff(ctx);
4711         }
4712         /*
4713          * Disabling the crtcs gracefully seems nicer. Also the
4714          * g33 docs say we should at least disable all the planes.
4715          */
4716         state = drm_atomic_helper_duplicate_state(dev, ctx);
4717         if (IS_ERR(state)) {
4718                 ret = PTR_ERR(state);
4719                 DRM_ERROR("Duplicating state failed with %i\n", ret);
4720                 return;
4721         }
4722
4723         ret = drm_atomic_helper_disable_all(dev, ctx);
4724         if (ret) {
4725                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
4726                 drm_atomic_state_put(state);
4727                 return;
4728         }
4729
4730         dev_priv->modeset_restore_state = state;
4731         state->acquire_ctx = ctx;
4732 }
4733
4734 void intel_finish_reset(struct drm_i915_private *dev_priv)
4735 {
4736         struct drm_device *dev = &dev_priv->drm;
4737         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
4738         struct drm_atomic_state *state;
4739         int ret;
4740
4741         /* reset doesn't touch the display */
4742         if (!test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags))
4743                 return;
4744
4745         state = fetch_and_zero(&dev_priv->modeset_restore_state);
4746         if (!state)
4747                 goto unlock;
4748
4749         /* reset doesn't touch the display */
4750         if (!gpu_reset_clobbers_display(dev_priv)) {
4751                 /* for testing only restore the display */
4752                 ret = __intel_display_resume(dev, state, ctx);
4753                 if (ret)
4754                         DRM_ERROR("Restoring old state failed with %i\n", ret);
4755         } else {
4756                 /*
4757                  * The display has been reset as well,
4758                  * so need a full re-initialization.
4759                  */
4760                 intel_pps_unlock_regs_wa(dev_priv);
4761                 intel_modeset_init_hw(dev_priv);
4762                 intel_init_clock_gating(dev_priv);
4763
4764                 spin_lock_irq(&dev_priv->irq_lock);
4765                 if (dev_priv->display.hpd_irq_setup)
4766                         dev_priv->display.hpd_irq_setup(dev_priv);
4767                 spin_unlock_irq(&dev_priv->irq_lock);
4768
4769                 ret = __intel_display_resume(dev, state, ctx);
4770                 if (ret)
4771                         DRM_ERROR("Restoring old state failed with %i\n", ret);
4772
4773                 intel_hpd_init(dev_priv);
4774         }
4775
4776         drm_atomic_state_put(state);
4777 unlock:
4778         drm_modeset_drop_locks(ctx);
4779         drm_modeset_acquire_fini(ctx);
4780         mutex_unlock(&dev->mode_config.mutex);
4781
4782         clear_bit_unlock(I915_RESET_MODESET, &dev_priv->gt.reset.flags);
4783 }
4784
4785 static void icl_set_pipe_chicken(struct intel_crtc *crtc)
4786 {
4787         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4788         enum pipe pipe = crtc->pipe;
4789         u32 tmp;
4790
4791         tmp = I915_READ(PIPE_CHICKEN(pipe));
4792
4793         /*
4794          * Display WA #1153: icl
4795          * enable hardware to bypass the alpha math
4796          * and rounding for per-pixel values 00 and 0xff
4797          */
4798         tmp |= PER_PIXEL_ALPHA_BYPASS_EN;
4799         /*
4800          * Display WA # 1605353570: icl
4801          * Set the pixel rounding bit to 1 for allowing
4802          * passthrough of Frame buffer pixels unmodified
4803          * across pipe
4804          */
4805         tmp |= PIXEL_ROUNDING_TRUNC_FB_PASSTHRU;
4806         I915_WRITE(PIPE_CHICKEN(pipe), tmp);
4807 }
4808
4809 static void icl_enable_trans_port_sync(const struct intel_crtc_state *crtc_state)
4810 {
4811         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4812         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4813         u32 trans_ddi_func_ctl2_val;
4814         u8 master_select;
4815
4816         /*
4817          * Configure the master select and enable Transcoder Port Sync for
4818          * Slave CRTCs transcoder.
4819          */
4820         if (crtc_state->master_transcoder == INVALID_TRANSCODER)
4821                 return;
4822
4823         if (crtc_state->master_transcoder == TRANSCODER_EDP)
4824                 master_select = 0;
4825         else
4826                 master_select = crtc_state->master_transcoder + 1;
4827
4828         /* Set the master select bits for Tranascoder Port Sync */
4829         trans_ddi_func_ctl2_val = (PORT_SYNC_MODE_MASTER_SELECT(master_select) &
4830                                    PORT_SYNC_MODE_MASTER_SELECT_MASK) <<
4831                 PORT_SYNC_MODE_MASTER_SELECT_SHIFT;
4832         /* Enable Transcoder Port Sync */
4833         trans_ddi_func_ctl2_val |= PORT_SYNC_MODE_ENABLE;
4834
4835         I915_WRITE(TRANS_DDI_FUNC_CTL2(crtc_state->cpu_transcoder),
4836                    trans_ddi_func_ctl2_val);
4837 }
4838
4839 static void intel_fdi_normal_train(struct intel_crtc *crtc)
4840 {
4841         struct drm_device *dev = crtc->base.dev;
4842         struct drm_i915_private *dev_priv = to_i915(dev);
4843         enum pipe pipe = crtc->pipe;
4844         i915_reg_t reg;
4845         u32 temp;
4846
4847         /* enable normal train */
4848         reg = FDI_TX_CTL(pipe);
4849         temp = I915_READ(reg);
4850         if (IS_IVYBRIDGE(dev_priv)) {
4851                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
4852                 temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
4853         } else {
4854                 temp &= ~FDI_LINK_TRAIN_NONE;
4855                 temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
4856         }
4857         I915_WRITE(reg, temp);
4858
4859         reg = FDI_RX_CTL(pipe);
4860         temp = I915_READ(reg);
4861         if (HAS_PCH_CPT(dev_priv)) {
4862                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4863                 temp |= FDI_LINK_TRAIN_NORMAL_CPT;
4864         } else {
4865                 temp &= ~FDI_LINK_TRAIN_NONE;
4866                 temp |= FDI_LINK_TRAIN_NONE;
4867         }
4868         I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
4869
4870         /* wait one idle pattern time */
4871         POSTING_READ(reg);
4872         udelay(1000);
4873
4874         /* IVB wants error correction enabled */
4875         if (IS_IVYBRIDGE(dev_priv))
4876                 I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
4877                            FDI_FE_ERRC_ENABLE);
4878 }
4879
4880 /* The FDI link training functions for ILK/Ibexpeak. */
4881 static void ilk_fdi_link_train(struct intel_crtc *crtc,
4882                                const struct intel_crtc_state *crtc_state)
4883 {
4884         struct drm_device *dev = crtc->base.dev;
4885         struct drm_i915_private *dev_priv = to_i915(dev);
4886         enum pipe pipe = crtc->pipe;
4887         i915_reg_t reg;
4888         u32 temp, tries;
4889
4890         /* FDI needs bits from pipe first */
4891         assert_pipe_enabled(dev_priv, crtc_state->cpu_transcoder);
4892
4893         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4894            for train result */
4895         reg = FDI_RX_IMR(pipe);
4896         temp = I915_READ(reg);
4897         temp &= ~FDI_RX_SYMBOL_LOCK;
4898         temp &= ~FDI_RX_BIT_LOCK;
4899         I915_WRITE(reg, temp);
4900         I915_READ(reg);
4901         udelay(150);
4902
4903         /* enable CPU FDI TX and PCH FDI RX */
4904         reg = FDI_TX_CTL(pipe);
4905         temp = I915_READ(reg);
4906         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4907         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4908         temp &= ~FDI_LINK_TRAIN_NONE;
4909         temp |= FDI_LINK_TRAIN_PATTERN_1;
4910         I915_WRITE(reg, temp | FDI_TX_ENABLE);
4911
4912         reg = FDI_RX_CTL(pipe);
4913         temp = I915_READ(reg);
4914         temp &= ~FDI_LINK_TRAIN_NONE;
4915         temp |= FDI_LINK_TRAIN_PATTERN_1;
4916         I915_WRITE(reg, temp | FDI_RX_ENABLE);
4917
4918         POSTING_READ(reg);
4919         udelay(150);
4920
4921         /* Ironlake workaround, enable clock pointer after FDI enable*/
4922         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
4923         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR |
4924                    FDI_RX_PHASE_SYNC_POINTER_EN);
4925
4926         reg = FDI_RX_IIR(pipe);
4927         for (tries = 0; tries < 5; tries++) {
4928                 temp = I915_READ(reg);
4929                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4930
4931                 if ((temp & FDI_RX_BIT_LOCK)) {
4932                         DRM_DEBUG_KMS("FDI train 1 done.\n");
4933                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4934                         break;
4935                 }
4936         }
4937         if (tries == 5)
4938                 DRM_ERROR("FDI train 1 fail!\n");
4939
4940         /* Train 2 */
4941         reg = FDI_TX_CTL(pipe);
4942         temp = I915_READ(reg);
4943         temp &= ~FDI_LINK_TRAIN_NONE;
4944         temp |= FDI_LINK_TRAIN_PATTERN_2;
4945         I915_WRITE(reg, temp);
4946
4947         reg = FDI_RX_CTL(pipe);
4948         temp = I915_READ(reg);
4949         temp &= ~FDI_LINK_TRAIN_NONE;
4950         temp |= FDI_LINK_TRAIN_PATTERN_2;
4951         I915_WRITE(reg, temp);
4952
4953         POSTING_READ(reg);
4954         udelay(150);
4955
4956         reg = FDI_RX_IIR(pipe);
4957         for (tries = 0; tries < 5; tries++) {
4958                 temp = I915_READ(reg);
4959                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4960
4961                 if (temp & FDI_RX_SYMBOL_LOCK) {
4962                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4963                         DRM_DEBUG_KMS("FDI train 2 done.\n");
4964                         break;
4965                 }
4966         }
4967         if (tries == 5)
4968                 DRM_ERROR("FDI train 2 fail!\n");
4969
4970         DRM_DEBUG_KMS("FDI train done\n");
4971
4972 }
4973
4974 static const int snb_b_fdi_train_param[] = {
4975         FDI_LINK_TRAIN_400MV_0DB_SNB_B,
4976         FDI_LINK_TRAIN_400MV_6DB_SNB_B,
4977         FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
4978         FDI_LINK_TRAIN_800MV_0DB_SNB_B,
4979 };
4980
4981 /* The FDI link training functions for SNB/Cougarpoint. */
4982 static void gen6_fdi_link_train(struct intel_crtc *crtc,
4983                                 const struct intel_crtc_state *crtc_state)
4984 {
4985         struct drm_device *dev = crtc->base.dev;
4986         struct drm_i915_private *dev_priv = to_i915(dev);
4987         enum pipe pipe = crtc->pipe;
4988         i915_reg_t reg;
4989         u32 temp, i, retry;
4990
4991         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4992            for train result */
4993         reg = FDI_RX_IMR(pipe);
4994         temp = I915_READ(reg);
4995         temp &= ~FDI_RX_SYMBOL_LOCK;
4996         temp &= ~FDI_RX_BIT_LOCK;
4997         I915_WRITE(reg, temp);
4998
4999         POSTING_READ(reg);
5000         udelay(150);
5001
5002         /* enable CPU FDI TX and PCH FDI RX */
5003         reg = FDI_TX_CTL(pipe);
5004         temp = I915_READ(reg);
5005         temp &= ~FDI_DP_PORT_WIDTH_MASK;
5006         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
5007         temp &= ~FDI_LINK_TRAIN_NONE;
5008         temp |= FDI_LINK_TRAIN_PATTERN_1;
5009         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5010         /* SNB-B */
5011         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
5012         I915_WRITE(reg, temp | FDI_TX_ENABLE);
5013
5014         I915_WRITE(FDI_RX_MISC(pipe),
5015                    FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
5016
5017         reg = FDI_RX_CTL(pipe);
5018         temp = I915_READ(reg);
5019         if (HAS_PCH_CPT(dev_priv)) {
5020                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5021                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
5022         } else {
5023                 temp &= ~FDI_LINK_TRAIN_NONE;
5024                 temp |= FDI_LINK_TRAIN_PATTERN_1;
5025         }
5026         I915_WRITE(reg, temp | FDI_RX_ENABLE);
5027
5028         POSTING_READ(reg);
5029         udelay(150);
5030
5031         for (i = 0; i < 4; i++) {
5032                 reg = FDI_TX_CTL(pipe);
5033                 temp = I915_READ(reg);
5034                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5035                 temp |= snb_b_fdi_train_param[i];
5036                 I915_WRITE(reg, temp);
5037
5038                 POSTING_READ(reg);
5039                 udelay(500);
5040
5041                 for (retry = 0; retry < 5; retry++) {
5042                         reg = FDI_RX_IIR(pipe);
5043                         temp = I915_READ(reg);
5044                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
5045                         if (temp & FDI_RX_BIT_LOCK) {
5046                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
5047                                 DRM_DEBUG_KMS("FDI train 1 done.\n");
5048                                 break;
5049                         }
5050                         udelay(50);
5051                 }
5052                 if (retry < 5)
5053                         break;
5054         }
5055         if (i == 4)
5056                 DRM_ERROR("FDI train 1 fail!\n");
5057
5058         /* Train 2 */
5059         reg = FDI_TX_CTL(pipe);
5060         temp = I915_READ(reg);
5061         temp &= ~FDI_LINK_TRAIN_NONE;
5062         temp |= FDI_LINK_TRAIN_PATTERN_2;
5063         if (IS_GEN(dev_priv, 6)) {
5064                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5065                 /* SNB-B */
5066                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
5067         }
5068         I915_WRITE(reg, temp);
5069
5070         reg = FDI_RX_CTL(pipe);
5071         temp = I915_READ(reg);
5072         if (HAS_PCH_CPT(dev_priv)) {
5073                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5074                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
5075         } else {
5076                 temp &= ~FDI_LINK_TRAIN_NONE;
5077                 temp |= FDI_LINK_TRAIN_PATTERN_2;
5078         }
5079         I915_WRITE(reg, temp);
5080
5081         POSTING_READ(reg);
5082         udelay(150);
5083
5084         for (i = 0; i < 4; i++) {
5085                 reg = FDI_TX_CTL(pipe);
5086                 temp = I915_READ(reg);
5087                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5088                 temp |= snb_b_fdi_train_param[i];
5089                 I915_WRITE(reg, temp);
5090
5091                 POSTING_READ(reg);
5092                 udelay(500);
5093
5094                 for (retry = 0; retry < 5; retry++) {
5095                         reg = FDI_RX_IIR(pipe);
5096                         temp = I915_READ(reg);
5097                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
5098                         if (temp & FDI_RX_SYMBOL_LOCK) {
5099                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
5100                                 DRM_DEBUG_KMS("FDI train 2 done.\n");
5101                                 break;
5102                         }
5103                         udelay(50);
5104                 }
5105                 if (retry < 5)
5106                         break;
5107         }
5108         if (i == 4)
5109                 DRM_ERROR("FDI train 2 fail!\n");
5110
5111         DRM_DEBUG_KMS("FDI train done.\n");
5112 }
5113
5114 /* Manual link training for Ivy Bridge A0 parts */
5115 static void ivb_manual_fdi_link_train(struct intel_crtc *crtc,
5116                                       const struct intel_crtc_state *crtc_state)
5117 {
5118         struct drm_device *dev = crtc->base.dev;
5119         struct drm_i915_private *dev_priv = to_i915(dev);
5120         enum pipe pipe = crtc->pipe;
5121         i915_reg_t reg;
5122         u32 temp, i, j;
5123
5124         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
5125            for train result */
5126         reg = FDI_RX_IMR(pipe);
5127         temp = I915_READ(reg);
5128         temp &= ~FDI_RX_SYMBOL_LOCK;
5129         temp &= ~FDI_RX_BIT_LOCK;
5130         I915_WRITE(reg, temp);
5131
5132         POSTING_READ(reg);
5133         udelay(150);
5134
5135         DRM_DEBUG_KMS("FDI_RX_IIR before link train 0x%x\n",
5136                       I915_READ(FDI_RX_IIR(pipe)));
5137
5138         /* Try each vswing and preemphasis setting twice before moving on */
5139         for (j = 0; j < ARRAY_SIZE(snb_b_fdi_train_param) * 2; j++) {
5140                 /* disable first in case we need to retry */
5141                 reg = FDI_TX_CTL(pipe);
5142                 temp = I915_READ(reg);
5143                 temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
5144                 temp &= ~FDI_TX_ENABLE;
5145                 I915_WRITE(reg, temp);
5146
5147                 reg = FDI_RX_CTL(pipe);
5148                 temp = I915_READ(reg);
5149                 temp &= ~FDI_LINK_TRAIN_AUTO;
5150                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5151                 temp &= ~FDI_RX_ENABLE;
5152                 I915_WRITE(reg, temp);
5153
5154                 /* enable CPU FDI TX and PCH FDI RX */
5155                 reg = FDI_TX_CTL(pipe);
5156                 temp = I915_READ(reg);
5157                 temp &= ~FDI_DP_PORT_WIDTH_MASK;
5158                 temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
5159                 temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
5160                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5161                 temp |= snb_b_fdi_train_param[j/2];
5162                 temp |= FDI_COMPOSITE_SYNC;
5163                 I915_WRITE(reg, temp | FDI_TX_ENABLE);
5164
5165                 I915_WRITE(FDI_RX_MISC(pipe),
5166                            FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
5167
5168                 reg = FDI_RX_CTL(pipe);
5169                 temp = I915_READ(reg);
5170                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
5171                 temp |= FDI_COMPOSITE_SYNC;
5172                 I915_WRITE(reg, temp | FDI_RX_ENABLE);
5173
5174                 POSTING_READ(reg);
5175                 udelay(1); /* should be 0.5us */
5176
5177                 for (i = 0; i < 4; i++) {
5178                         reg = FDI_RX_IIR(pipe);
5179                         temp = I915_READ(reg);
5180                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
5181
5182                         if (temp & FDI_RX_BIT_LOCK ||
5183                             (I915_READ(reg) & FDI_RX_BIT_LOCK)) {
5184                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
5185                                 DRM_DEBUG_KMS("FDI train 1 done, level %i.\n",
5186                                               i);
5187                                 break;
5188                         }
5189                         udelay(1); /* should be 0.5us */
5190                 }
5191                 if (i == 4) {
5192                         DRM_DEBUG_KMS("FDI train 1 fail on vswing %d\n", j / 2);
5193                         continue;
5194                 }
5195
5196                 /* Train 2 */
5197                 reg = FDI_TX_CTL(pipe);
5198                 temp = I915_READ(reg);
5199                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
5200                 temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
5201                 I915_WRITE(reg, temp);
5202
5203                 reg = FDI_RX_CTL(pipe);
5204                 temp = I915_READ(reg);
5205                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5206                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
5207                 I915_WRITE(reg, temp);
5208
5209                 POSTING_READ(reg);
5210                 udelay(2); /* should be 1.5us */
5211
5212                 for (i = 0; i < 4; i++) {
5213                         reg = FDI_RX_IIR(pipe);
5214                         temp = I915_READ(reg);
5215                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
5216
5217                         if (temp & FDI_RX_SYMBOL_LOCK ||
5218                             (I915_READ(reg) & FDI_RX_SYMBOL_LOCK)) {
5219                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
5220                                 DRM_DEBUG_KMS("FDI train 2 done, level %i.\n",
5221                                               i);
5222                                 goto train_done;
5223                         }
5224                         udelay(2); /* should be 1.5us */
5225                 }
5226                 if (i == 4)
5227                         DRM_DEBUG_KMS("FDI train 2 fail on vswing %d\n", j / 2);
5228         }
5229
5230 train_done:
5231         DRM_DEBUG_KMS("FDI train done.\n");
5232 }
5233
5234 static void ilk_fdi_pll_enable(const struct intel_crtc_state *crtc_state)
5235 {
5236         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->uapi.crtc);
5237         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
5238         enum pipe pipe = intel_crtc->pipe;
5239         i915_reg_t reg;
5240         u32 temp;
5241
5242         /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
5243         reg = FDI_RX_CTL(pipe);
5244         temp = I915_READ(reg);
5245         temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16));
5246         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
5247         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
5248         I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
5249
5250         POSTING_READ(reg);
5251         udelay(200);
5252
5253         /* Switch from Rawclk to PCDclk */
5254         temp = I915_READ(reg);
5255         I915_WRITE(reg, temp | FDI_PCDCLK);
5256
5257         POSTING_READ(reg);
5258         udelay(200);
5259
5260         /* Enable CPU FDI TX PLL, always on for Ironlake */
5261         reg = FDI_TX_CTL(pipe);
5262         temp = I915_READ(reg);
5263         if ((temp & FDI_TX_PLL_ENABLE) == 0) {
5264                 I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
5265
5266                 POSTING_READ(reg);
5267                 udelay(100);
5268         }
5269 }
5270
5271 static void ilk_fdi_pll_disable(struct intel_crtc *intel_crtc)
5272 {
5273         struct drm_device *dev = intel_crtc->base.dev;
5274         struct drm_i915_private *dev_priv = to_i915(dev);
5275         enum pipe pipe = intel_crtc->pipe;
5276         i915_reg_t reg;
5277         u32 temp;
5278
5279         /* Switch from PCDclk to Rawclk */
5280         reg = FDI_RX_CTL(pipe);
5281         temp = I915_READ(reg);
5282         I915_WRITE(reg, temp & ~FDI_PCDCLK);
5283
5284         /* Disable CPU FDI TX PLL */
5285         reg = FDI_TX_CTL(pipe);
5286         temp = I915_READ(reg);
5287         I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
5288
5289         POSTING_READ(reg);
5290         udelay(100);
5291
5292         reg = FDI_RX_CTL(pipe);
5293         temp = I915_READ(reg);
5294         I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
5295
5296         /* Wait for the clocks to turn off. */
5297         POSTING_READ(reg);
5298         udelay(100);
5299 }
5300
5301 static void ilk_fdi_disable(struct intel_crtc *crtc)
5302 {
5303         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5304         enum pipe pipe = crtc->pipe;
5305         i915_reg_t reg;
5306         u32 temp;
5307
5308         /* disable CPU FDI tx and PCH FDI rx */
5309         reg = FDI_TX_CTL(pipe);
5310         temp = I915_READ(reg);
5311         I915_WRITE(reg, temp & ~FDI_TX_ENABLE);
5312         POSTING_READ(reg);
5313
5314         reg = FDI_RX_CTL(pipe);
5315         temp = I915_READ(reg);
5316         temp &= ~(0x7 << 16);
5317         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
5318         I915_WRITE(reg, temp & ~FDI_RX_ENABLE);
5319
5320         POSTING_READ(reg);
5321         udelay(100);
5322
5323         /* Ironlake workaround, disable clock pointer after downing FDI */
5324         if (HAS_PCH_IBX(dev_priv))
5325                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
5326
5327         /* still set train pattern 1 */
5328         reg = FDI_TX_CTL(pipe);
5329         temp = I915_READ(reg);
5330         temp &= ~FDI_LINK_TRAIN_NONE;
5331         temp |= FDI_LINK_TRAIN_PATTERN_1;
5332         I915_WRITE(reg, temp);
5333
5334         reg = FDI_RX_CTL(pipe);
5335         temp = I915_READ(reg);
5336         if (HAS_PCH_CPT(dev_priv)) {
5337                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5338                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
5339         } else {
5340                 temp &= ~FDI_LINK_TRAIN_NONE;
5341                 temp |= FDI_LINK_TRAIN_PATTERN_1;
5342         }
5343         /* BPC in FDI rx is consistent with that in PIPECONF */
5344         temp &= ~(0x07 << 16);
5345         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
5346         I915_WRITE(reg, temp);
5347
5348         POSTING_READ(reg);
5349         udelay(100);
5350 }
5351
5352 bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv)
5353 {
5354         struct drm_crtc *crtc;
5355         bool cleanup_done;
5356
5357         drm_for_each_crtc(crtc, &dev_priv->drm) {
5358                 struct drm_crtc_commit *commit;
5359                 spin_lock(&crtc->commit_lock);
5360                 commit = list_first_entry_or_null(&crtc->commit_list,
5361                                                   struct drm_crtc_commit, commit_entry);
5362                 cleanup_done = commit ?
5363                         try_wait_for_completion(&commit->cleanup_done) : true;
5364                 spin_unlock(&crtc->commit_lock);
5365
5366                 if (cleanup_done)
5367                         continue;
5368
5369                 drm_crtc_wait_one_vblank(crtc);
5370
5371                 return true;
5372         }
5373
5374         return false;
5375 }
5376
5377 void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
5378 {
5379         u32 temp;
5380
5381         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
5382
5383         mutex_lock(&dev_priv->sb_lock);
5384
5385         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5386         temp |= SBI_SSCCTL_DISABLE;
5387         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
5388
5389         mutex_unlock(&dev_priv->sb_lock);
5390 }
5391
5392 /* Program iCLKIP clock to the desired frequency */
5393 static void lpt_program_iclkip(const struct intel_crtc_state *crtc_state)
5394 {
5395         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5396         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5397         int clock = crtc_state->hw.adjusted_mode.crtc_clock;
5398         u32 divsel, phaseinc, auxdiv, phasedir = 0;
5399         u32 temp;
5400
5401         lpt_disable_iclkip(dev_priv);
5402
5403         /* The iCLK virtual clock root frequency is in MHz,
5404          * but the adjusted_mode->crtc_clock in in KHz. To get the
5405          * divisors, it is necessary to divide one by another, so we
5406          * convert the virtual clock precision to KHz here for higher
5407          * precision.
5408          */
5409         for (auxdiv = 0; auxdiv < 2; auxdiv++) {
5410                 u32 iclk_virtual_root_freq = 172800 * 1000;
5411                 u32 iclk_pi_range = 64;
5412                 u32 desired_divisor;
5413
5414                 desired_divisor = DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
5415                                                     clock << auxdiv);
5416                 divsel = (desired_divisor / iclk_pi_range) - 2;
5417                 phaseinc = desired_divisor % iclk_pi_range;
5418
5419                 /*
5420                  * Near 20MHz is a corner case which is
5421                  * out of range for the 7-bit divisor
5422                  */
5423                 if (divsel <= 0x7f)
5424                         break;
5425         }
5426
5427         /* This should not happen with any sane values */
5428         WARN_ON(SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
5429                 ~SBI_SSCDIVINTPHASE_DIVSEL_MASK);
5430         WARN_ON(SBI_SSCDIVINTPHASE_DIR(phasedir) &
5431                 ~SBI_SSCDIVINTPHASE_INCVAL_MASK);
5432
5433         DRM_DEBUG_KMS("iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
5434                         clock,
5435                         auxdiv,
5436                         divsel,
5437                         phasedir,
5438                         phaseinc);
5439
5440         mutex_lock(&dev_priv->sb_lock);
5441
5442         /* Program SSCDIVINTPHASE6 */
5443         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
5444         temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
5445         temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
5446         temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
5447         temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
5448         temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
5449         temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
5450         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK);
5451
5452         /* Program SSCAUXDIV */
5453         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
5454         temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
5455         temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
5456         intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK);
5457
5458         /* Enable modulator and associated divider */
5459         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5460         temp &= ~SBI_SSCCTL_DISABLE;
5461         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
5462
5463         mutex_unlock(&dev_priv->sb_lock);
5464
5465         /* Wait for initialization time */
5466         udelay(24);
5467
5468         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE);
5469 }
5470
5471 int lpt_get_iclkip(struct drm_i915_private *dev_priv)
5472 {
5473         u32 divsel, phaseinc, auxdiv;
5474         u32 iclk_virtual_root_freq = 172800 * 1000;
5475         u32 iclk_pi_range = 64;
5476         u32 desired_divisor;
5477         u32 temp;
5478
5479         if ((I915_READ(PIXCLK_GATE) & PIXCLK_GATE_UNGATE) == 0)
5480                 return 0;
5481
5482         mutex_lock(&dev_priv->sb_lock);
5483
5484         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5485         if (temp & SBI_SSCCTL_DISABLE) {
5486                 mutex_unlock(&dev_priv->sb_lock);
5487                 return 0;
5488         }
5489
5490         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
5491         divsel = (temp & SBI_SSCDIVINTPHASE_DIVSEL_MASK) >>
5492                 SBI_SSCDIVINTPHASE_DIVSEL_SHIFT;
5493         phaseinc = (temp & SBI_SSCDIVINTPHASE_INCVAL_MASK) >>
5494                 SBI_SSCDIVINTPHASE_INCVAL_SHIFT;
5495
5496         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
5497         auxdiv = (temp & SBI_SSCAUXDIV_FINALDIV2SEL_MASK) >>
5498                 SBI_SSCAUXDIV_FINALDIV2SEL_SHIFT;
5499
5500         mutex_unlock(&dev_priv->sb_lock);
5501
5502         desired_divisor = (divsel + 2) * iclk_pi_range + phaseinc;
5503
5504         return DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
5505                                  desired_divisor << auxdiv);
5506 }
5507
5508 static void ilk_pch_transcoder_set_timings(const struct intel_crtc_state *crtc_state,
5509                                            enum pipe pch_transcoder)
5510 {
5511         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5512         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5513         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
5514
5515         I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder),
5516                    I915_READ(HTOTAL(cpu_transcoder)));
5517         I915_WRITE(PCH_TRANS_HBLANK(pch_transcoder),
5518                    I915_READ(HBLANK(cpu_transcoder)));
5519         I915_WRITE(PCH_TRANS_HSYNC(pch_transcoder),
5520                    I915_READ(HSYNC(cpu_transcoder)));
5521
5522         I915_WRITE(PCH_TRANS_VTOTAL(pch_transcoder),
5523                    I915_READ(VTOTAL(cpu_transcoder)));
5524         I915_WRITE(PCH_TRANS_VBLANK(pch_transcoder),
5525                    I915_READ(VBLANK(cpu_transcoder)));
5526         I915_WRITE(PCH_TRANS_VSYNC(pch_transcoder),
5527                    I915_READ(VSYNC(cpu_transcoder)));
5528         I915_WRITE(PCH_TRANS_VSYNCSHIFT(pch_transcoder),
5529                    I915_READ(VSYNCSHIFT(cpu_transcoder)));
5530 }
5531
5532 static void cpt_set_fdi_bc_bifurcation(struct drm_i915_private *dev_priv, bool enable)
5533 {
5534         u32 temp;
5535
5536         temp = I915_READ(SOUTH_CHICKEN1);
5537         if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable)
5538                 return;
5539
5540         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_B)) & FDI_RX_ENABLE);
5541         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_C)) & FDI_RX_ENABLE);
5542
5543         temp &= ~FDI_BC_BIFURCATION_SELECT;
5544         if (enable)
5545                 temp |= FDI_BC_BIFURCATION_SELECT;
5546
5547         DRM_DEBUG_KMS("%sabling fdi C rx\n", enable ? "en" : "dis");
5548         I915_WRITE(SOUTH_CHICKEN1, temp);
5549         POSTING_READ(SOUTH_CHICKEN1);
5550 }
5551
5552 static void ivb_update_fdi_bc_bifurcation(const struct intel_crtc_state *crtc_state)
5553 {
5554         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5555         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5556
5557         switch (crtc->pipe) {
5558         case PIPE_A:
5559                 break;
5560         case PIPE_B:
5561                 if (crtc_state->fdi_lanes > 2)
5562                         cpt_set_fdi_bc_bifurcation(dev_priv, false);
5563                 else
5564                         cpt_set_fdi_bc_bifurcation(dev_priv, true);
5565
5566                 break;
5567         case PIPE_C:
5568                 cpt_set_fdi_bc_bifurcation(dev_priv, true);
5569
5570                 break;
5571         default:
5572                 BUG();
5573         }
5574 }
5575
5576 /*
5577  * Finds the encoder associated with the given CRTC. This can only be
5578  * used when we know that the CRTC isn't feeding multiple encoders!
5579  */
5580 static struct intel_encoder *
5581 intel_get_crtc_new_encoder(const struct intel_atomic_state *state,
5582                            const struct intel_crtc_state *crtc_state)
5583 {
5584         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5585         const struct drm_connector_state *connector_state;
5586         const struct drm_connector *connector;
5587         struct intel_encoder *encoder = NULL;
5588         int num_encoders = 0;
5589         int i;
5590
5591         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
5592                 if (connector_state->crtc != &crtc->base)
5593                         continue;
5594
5595                 encoder = to_intel_encoder(connector_state->best_encoder);
5596                 num_encoders++;
5597         }
5598
5599         WARN(num_encoders != 1, "%d encoders for pipe %c\n",
5600              num_encoders, pipe_name(crtc->pipe));
5601
5602         return encoder;
5603 }
5604
5605 /*
5606  * Enable PCH resources required for PCH ports:
5607  *   - PCH PLLs
5608  *   - FDI training & RX/TX
5609  *   - update transcoder timings
5610  *   - DP transcoding bits
5611  *   - transcoder
5612  */
5613 static void ilk_pch_enable(const struct intel_atomic_state *state,
5614                            const struct intel_crtc_state *crtc_state)
5615 {
5616         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5617         struct drm_device *dev = crtc->base.dev;
5618         struct drm_i915_private *dev_priv = to_i915(dev);
5619         enum pipe pipe = crtc->pipe;
5620         u32 temp;
5621
5622         assert_pch_transcoder_disabled(dev_priv, pipe);
5623
5624         if (IS_IVYBRIDGE(dev_priv))
5625                 ivb_update_fdi_bc_bifurcation(crtc_state);
5626
5627         /* Write the TU size bits before fdi link training, so that error
5628          * detection works. */
5629         I915_WRITE(FDI_RX_TUSIZE1(pipe),
5630                    I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
5631
5632         /* For PCH output, training FDI link */
5633         dev_priv->display.fdi_link_train(crtc, crtc_state);
5634
5635         /* We need to program the right clock selection before writing the pixel
5636          * mutliplier into the DPLL. */
5637         if (HAS_PCH_CPT(dev_priv)) {
5638                 u32 sel;
5639
5640                 temp = I915_READ(PCH_DPLL_SEL);
5641                 temp |= TRANS_DPLL_ENABLE(pipe);
5642                 sel = TRANS_DPLLB_SEL(pipe);
5643                 if (crtc_state->shared_dpll ==
5644                     intel_get_shared_dpll_by_id(dev_priv, DPLL_ID_PCH_PLL_B))
5645                         temp |= sel;
5646                 else
5647                         temp &= ~sel;
5648                 I915_WRITE(PCH_DPLL_SEL, temp);
5649         }
5650
5651         /* XXX: pch pll's can be enabled any time before we enable the PCH
5652          * transcoder, and we actually should do this to not upset any PCH
5653          * transcoder that already use the clock when we share it.
5654          *
5655          * Note that enable_shared_dpll tries to do the right thing, but
5656          * get_shared_dpll unconditionally resets the pll - we need that to have
5657          * the right LVDS enable sequence. */
5658         intel_enable_shared_dpll(crtc_state);
5659
5660         /* set transcoder timing, panel must allow it */
5661         assert_panel_unlocked(dev_priv, pipe);
5662         ilk_pch_transcoder_set_timings(crtc_state, pipe);
5663
5664         intel_fdi_normal_train(crtc);
5665
5666         /* For PCH DP, enable TRANS_DP_CTL */
5667         if (HAS_PCH_CPT(dev_priv) &&
5668             intel_crtc_has_dp_encoder(crtc_state)) {
5669                 const struct drm_display_mode *adjusted_mode =
5670                         &crtc_state->hw.adjusted_mode;
5671                 u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
5672                 i915_reg_t reg = TRANS_DP_CTL(pipe);
5673                 enum port port;
5674
5675                 temp = I915_READ(reg);
5676                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
5677                           TRANS_DP_SYNC_MASK |
5678                           TRANS_DP_BPC_MASK);
5679                 temp |= TRANS_DP_OUTPUT_ENABLE;
5680                 temp |= bpc << 9; /* same format but at 11:9 */
5681
5682                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
5683                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
5684                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
5685                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
5686
5687                 port = intel_get_crtc_new_encoder(state, crtc_state)->port;
5688                 WARN_ON(port < PORT_B || port > PORT_D);
5689                 temp |= TRANS_DP_PORT_SEL(port);
5690
5691                 I915_WRITE(reg, temp);
5692         }
5693
5694         ilk_enable_pch_transcoder(crtc_state);
5695 }
5696
5697 static void lpt_pch_enable(const struct intel_atomic_state *state,
5698                            const struct intel_crtc_state *crtc_state)
5699 {
5700         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5701         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5702         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
5703
5704         assert_pch_transcoder_disabled(dev_priv, PIPE_A);
5705
5706         lpt_program_iclkip(crtc_state);
5707
5708         /* Set transcoder timing. */
5709         ilk_pch_transcoder_set_timings(crtc_state, PIPE_A);
5710
5711         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
5712 }
5713
5714 static void cpt_verify_modeset(struct drm_i915_private *dev_priv,
5715                                enum pipe pipe)
5716 {
5717         i915_reg_t dslreg = PIPEDSL(pipe);
5718         u32 temp;
5719
5720         temp = I915_READ(dslreg);
5721         udelay(500);
5722         if (wait_for(I915_READ(dslreg) != temp, 5)) {
5723                 if (wait_for(I915_READ(dslreg) != temp, 5))
5724                         DRM_ERROR("mode set failed: pipe %c stuck\n", pipe_name(pipe));
5725         }
5726 }
5727
5728 /*
5729  * The hardware phase 0.0 refers to the center of the pixel.
5730  * We want to start from the top/left edge which is phase
5731  * -0.5. That matches how the hardware calculates the scaling
5732  * factors (from top-left of the first pixel to bottom-right
5733  * of the last pixel, as opposed to the pixel centers).
5734  *
5735  * For 4:2:0 subsampled chroma planes we obviously have to
5736  * adjust that so that the chroma sample position lands in
5737  * the right spot.
5738  *
5739  * Note that for packed YCbCr 4:2:2 formats there is no way to
5740  * control chroma siting. The hardware simply replicates the
5741  * chroma samples for both of the luma samples, and thus we don't
5742  * actually get the expected MPEG2 chroma siting convention :(
5743  * The same behaviour is observed on pre-SKL platforms as well.
5744  *
5745  * Theory behind the formula (note that we ignore sub-pixel
5746  * source coordinates):
5747  * s = source sample position
5748  * d = destination sample position
5749  *
5750  * Downscaling 4:1:
5751  * -0.5
5752  * | 0.0
5753  * | |     1.5 (initial phase)
5754  * | |     |
5755  * v v     v
5756  * | s | s | s | s |
5757  * |       d       |
5758  *
5759  * Upscaling 1:4:
5760  * -0.5
5761  * | -0.375 (initial phase)
5762  * | |     0.0
5763  * | |     |
5764  * v v     v
5765  * |       s       |
5766  * | d | d | d | d |
5767  */
5768 u16 skl_scaler_calc_phase(int sub, int scale, bool chroma_cosited)
5769 {
5770         int phase = -0x8000;
5771         u16 trip = 0;
5772
5773         if (chroma_cosited)
5774                 phase += (sub - 1) * 0x8000 / sub;
5775
5776         phase += scale / (2 * sub);
5777
5778         /*
5779          * Hardware initial phase limited to [-0.5:1.5].
5780          * Since the max hardware scale factor is 3.0, we
5781          * should never actually excdeed 1.0 here.
5782          */
5783         WARN_ON(phase < -0x8000 || phase > 0x18000);
5784
5785         if (phase < 0)
5786                 phase = 0x10000 + phase;
5787         else
5788                 trip = PS_PHASE_TRIP;
5789
5790         return ((phase >> 2) & PS_PHASE_MASK) | trip;
5791 }
5792
5793 #define SKL_MIN_SRC_W 8
5794 #define SKL_MAX_SRC_W 4096
5795 #define SKL_MIN_SRC_H 8
5796 #define SKL_MAX_SRC_H 4096
5797 #define SKL_MIN_DST_W 8
5798 #define SKL_MAX_DST_W 4096
5799 #define SKL_MIN_DST_H 8
5800 #define SKL_MAX_DST_H 4096
5801 #define ICL_MAX_SRC_W 5120
5802 #define ICL_MAX_SRC_H 4096
5803 #define ICL_MAX_DST_W 5120
5804 #define ICL_MAX_DST_H 4096
5805 #define SKL_MIN_YUV_420_SRC_W 16
5806 #define SKL_MIN_YUV_420_SRC_H 16
5807
5808 static int
5809 skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
5810                   unsigned int scaler_user, int *scaler_id,
5811                   int src_w, int src_h, int dst_w, int dst_h,
5812                   const struct drm_format_info *format,
5813                   u64 modifier, bool need_scaler)
5814 {
5815         struct intel_crtc_scaler_state *scaler_state =
5816                 &crtc_state->scaler_state;
5817         struct intel_crtc *intel_crtc =
5818                 to_intel_crtc(crtc_state->uapi.crtc);
5819         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
5820         const struct drm_display_mode *adjusted_mode =
5821                 &crtc_state->hw.adjusted_mode;
5822
5823         /*
5824          * Src coordinates are already rotated by 270 degrees for
5825          * the 90/270 degree plane rotation cases (to match the
5826          * GTT mapping), hence no need to account for rotation here.
5827          */
5828         if (src_w != dst_w || src_h != dst_h)
5829                 need_scaler = true;
5830
5831         /*
5832          * Scaling/fitting not supported in IF-ID mode in GEN9+
5833          * TODO: Interlace fetch mode doesn't support YUV420 planar formats.
5834          * Once NV12 is enabled, handle it here while allocating scaler
5835          * for NV12.
5836          */
5837         if (INTEL_GEN(dev_priv) >= 9 && crtc_state->hw.enable &&
5838             need_scaler && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
5839                 DRM_DEBUG_KMS("Pipe/Plane scaling not supported with IF-ID mode\n");
5840                 return -EINVAL;
5841         }
5842
5843         /*
5844          * if plane is being disabled or scaler is no more required or force detach
5845          *  - free scaler binded to this plane/crtc
5846          *  - in order to do this, update crtc->scaler_usage
5847          *
5848          * Here scaler state in crtc_state is set free so that
5849          * scaler can be assigned to other user. Actual register
5850          * update to free the scaler is done in plane/panel-fit programming.
5851          * For this purpose crtc/plane_state->scaler_id isn't reset here.
5852          */
5853         if (force_detach || !need_scaler) {
5854                 if (*scaler_id >= 0) {
5855                         scaler_state->scaler_users &= ~(1 << scaler_user);
5856                         scaler_state->scalers[*scaler_id].in_use = 0;
5857
5858                         DRM_DEBUG_KMS("scaler_user index %u.%u: "
5859                                 "Staged freeing scaler id %d scaler_users = 0x%x\n",
5860                                 intel_crtc->pipe, scaler_user, *scaler_id,
5861                                 scaler_state->scaler_users);
5862                         *scaler_id = -1;
5863                 }
5864                 return 0;
5865         }
5866
5867         if (format && intel_format_info_is_yuv_semiplanar(format, modifier) &&
5868             (src_h < SKL_MIN_YUV_420_SRC_H || src_w < SKL_MIN_YUV_420_SRC_W)) {
5869                 DRM_DEBUG_KMS("Planar YUV: src dimensions not met\n");
5870                 return -EINVAL;
5871         }
5872
5873         /* range checks */
5874         if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H ||
5875             dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
5876             (INTEL_GEN(dev_priv) >= 11 &&
5877              (src_w > ICL_MAX_SRC_W || src_h > ICL_MAX_SRC_H ||
5878               dst_w > ICL_MAX_DST_W || dst_h > ICL_MAX_DST_H)) ||
5879             (INTEL_GEN(dev_priv) < 11 &&
5880              (src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
5881               dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H))) {
5882                 DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u "
5883                         "size is out of scaler range\n",
5884                         intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h);
5885                 return -EINVAL;
5886         }
5887
5888         /* mark this plane as a scaler user in crtc_state */
5889         scaler_state->scaler_users |= (1 << scaler_user);
5890         DRM_DEBUG_KMS("scaler_user index %u.%u: "
5891                 "staged scaling request for %ux%u->%ux%u scaler_users = 0x%x\n",
5892                 intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h,
5893                 scaler_state->scaler_users);
5894
5895         return 0;
5896 }
5897
5898 /**
5899  * skl_update_scaler_crtc - Stages update to scaler state for a given crtc.
5900  *
5901  * @state: crtc's scaler state
5902  *
5903  * Return
5904  *     0 - scaler_usage updated successfully
5905  *    error - requested scaling cannot be supported or other error condition
5906  */
5907 int skl_update_scaler_crtc(struct intel_crtc_state *state)
5908 {
5909         const struct drm_display_mode *adjusted_mode = &state->hw.adjusted_mode;
5910         bool need_scaler = false;
5911
5912         if (state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
5913                 need_scaler = true;
5914
5915         return skl_update_scaler(state, !state->hw.active, SKL_CRTC_INDEX,
5916                                  &state->scaler_state.scaler_id,
5917                                  state->pipe_src_w, state->pipe_src_h,
5918                                  adjusted_mode->crtc_hdisplay,
5919                                  adjusted_mode->crtc_vdisplay, NULL, 0,
5920                                  need_scaler);
5921 }
5922
5923 /**
5924  * skl_update_scaler_plane - Stages update to scaler state for a given plane.
5925  * @crtc_state: crtc's scaler state
5926  * @plane_state: atomic plane state to update
5927  *
5928  * Return
5929  *     0 - scaler_usage updated successfully
5930  *    error - requested scaling cannot be supported or other error condition
5931  */
5932 static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
5933                                    struct intel_plane_state *plane_state)
5934 {
5935         struct intel_plane *intel_plane =
5936                 to_intel_plane(plane_state->uapi.plane);
5937         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
5938         struct drm_framebuffer *fb = plane_state->hw.fb;
5939         int ret;
5940         bool force_detach = !fb || !plane_state->uapi.visible;
5941         bool need_scaler = false;
5942
5943         /* Pre-gen11 and SDR planes always need a scaler for planar formats. */
5944         if (!icl_is_hdr_plane(dev_priv, intel_plane->id) &&
5945             fb && intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
5946                 need_scaler = true;
5947
5948         ret = skl_update_scaler(crtc_state, force_detach,
5949                                 drm_plane_index(&intel_plane->base),
5950                                 &plane_state->scaler_id,
5951                                 drm_rect_width(&plane_state->uapi.src) >> 16,
5952                                 drm_rect_height(&plane_state->uapi.src) >> 16,
5953                                 drm_rect_width(&plane_state->uapi.dst),
5954                                 drm_rect_height(&plane_state->uapi.dst),
5955                                 fb ? fb->format : NULL,
5956                                 fb ? fb->modifier : 0,
5957                                 need_scaler);
5958
5959         if (ret || plane_state->scaler_id < 0)
5960                 return ret;
5961
5962         /* check colorkey */
5963         if (plane_state->ckey.flags) {
5964                 DRM_DEBUG_KMS("[PLANE:%d:%s] scaling with color key not allowed",
5965                               intel_plane->base.base.id,
5966                               intel_plane->base.name);
5967                 return -EINVAL;
5968         }
5969
5970         /* Check src format */
5971         switch (fb->format->format) {
5972         case DRM_FORMAT_RGB565:
5973         case DRM_FORMAT_XBGR8888:
5974         case DRM_FORMAT_XRGB8888:
5975         case DRM_FORMAT_ABGR8888:
5976         case DRM_FORMAT_ARGB8888:
5977         case DRM_FORMAT_XRGB2101010:
5978         case DRM_FORMAT_XBGR2101010:
5979         case DRM_FORMAT_ARGB2101010:
5980         case DRM_FORMAT_ABGR2101010:
5981         case DRM_FORMAT_YUYV:
5982         case DRM_FORMAT_YVYU:
5983         case DRM_FORMAT_UYVY:
5984         case DRM_FORMAT_VYUY:
5985         case DRM_FORMAT_NV12:
5986         case DRM_FORMAT_P010:
5987         case DRM_FORMAT_P012:
5988         case DRM_FORMAT_P016:
5989         case DRM_FORMAT_Y210:
5990         case DRM_FORMAT_Y212:
5991         case DRM_FORMAT_Y216:
5992         case DRM_FORMAT_XVYU2101010:
5993         case DRM_FORMAT_XVYU12_16161616:
5994         case DRM_FORMAT_XVYU16161616:
5995                 break;
5996         case DRM_FORMAT_XBGR16161616F:
5997         case DRM_FORMAT_ABGR16161616F:
5998         case DRM_FORMAT_XRGB16161616F:
5999         case DRM_FORMAT_ARGB16161616F:
6000                 if (INTEL_GEN(dev_priv) >= 11)
6001                         break;
6002                 /* fall through */
6003         default:
6004                 DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n",
6005                               intel_plane->base.base.id, intel_plane->base.name,
6006                               fb->base.id, fb->format->format);
6007                 return -EINVAL;
6008         }
6009
6010         return 0;
6011 }
6012
6013 void skl_scaler_disable(const struct intel_crtc_state *old_crtc_state)
6014 {
6015         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
6016         int i;
6017
6018         for (i = 0; i < crtc->num_scalers; i++)
6019                 skl_detach_scaler(crtc, i);
6020 }
6021
6022 static void skl_pfit_enable(const struct intel_crtc_state *crtc_state)
6023 {
6024         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6025         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6026         enum pipe pipe = crtc->pipe;
6027         const struct intel_crtc_scaler_state *scaler_state =
6028                 &crtc_state->scaler_state;
6029
6030         if (crtc_state->pch_pfit.enabled) {
6031                 u16 uv_rgb_hphase, uv_rgb_vphase;
6032                 int pfit_w, pfit_h, hscale, vscale;
6033                 int id;
6034
6035                 if (WARN_ON(crtc_state->scaler_state.scaler_id < 0))
6036                         return;
6037
6038                 pfit_w = (crtc_state->pch_pfit.size >> 16) & 0xFFFF;
6039                 pfit_h = crtc_state->pch_pfit.size & 0xFFFF;
6040
6041                 hscale = (crtc_state->pipe_src_w << 16) / pfit_w;
6042                 vscale = (crtc_state->pipe_src_h << 16) / pfit_h;
6043
6044                 uv_rgb_hphase = skl_scaler_calc_phase(1, hscale, false);
6045                 uv_rgb_vphase = skl_scaler_calc_phase(1, vscale, false);
6046
6047                 id = scaler_state->scaler_id;
6048                 I915_WRITE(SKL_PS_CTRL(pipe, id), PS_SCALER_EN |
6049                         PS_FILTER_MEDIUM | scaler_state->scalers[id].mode);
6050                 I915_WRITE_FW(SKL_PS_VPHASE(pipe, id),
6051                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_vphase));
6052                 I915_WRITE_FW(SKL_PS_HPHASE(pipe, id),
6053                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_hphase));
6054                 I915_WRITE(SKL_PS_WIN_POS(pipe, id), crtc_state->pch_pfit.pos);
6055                 I915_WRITE(SKL_PS_WIN_SZ(pipe, id), crtc_state->pch_pfit.size);
6056         }
6057 }
6058
6059 static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state)
6060 {
6061         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6062         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6063         enum pipe pipe = crtc->pipe;
6064
6065         if (crtc_state->pch_pfit.enabled) {
6066                 /* Force use of hard-coded filter coefficients
6067                  * as some pre-programmed values are broken,
6068                  * e.g. x201.
6069                  */
6070                 if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv))
6071                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
6072                                                  PF_PIPE_SEL_IVB(pipe));
6073                 else
6074                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
6075                 I915_WRITE(PF_WIN_POS(pipe), crtc_state->pch_pfit.pos);
6076                 I915_WRITE(PF_WIN_SZ(pipe), crtc_state->pch_pfit.size);
6077         }
6078 }
6079
6080 void hsw_enable_ips(const struct intel_crtc_state *crtc_state)
6081 {
6082         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6083         struct drm_device *dev = crtc->base.dev;
6084         struct drm_i915_private *dev_priv = to_i915(dev);
6085
6086         if (!crtc_state->ips_enabled)
6087                 return;
6088
6089         /*
6090          * We can only enable IPS after we enable a plane and wait for a vblank
6091          * This function is called from post_plane_update, which is run after
6092          * a vblank wait.
6093          */
6094         WARN_ON(!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)));
6095
6096         if (IS_BROADWELL(dev_priv)) {
6097                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL,
6098                                                 IPS_ENABLE | IPS_PCODE_CONTROL));
6099                 /* Quoting Art Runyan: "its not safe to expect any particular
6100                  * value in IPS_CTL bit 31 after enabling IPS through the
6101                  * mailbox." Moreover, the mailbox may return a bogus state,
6102                  * so we need to just enable it and continue on.
6103                  */
6104         } else {
6105                 I915_WRITE(IPS_CTL, IPS_ENABLE);
6106                 /* The bit only becomes 1 in the next vblank, so this wait here
6107                  * is essentially intel_wait_for_vblank. If we don't have this
6108                  * and don't wait for vblanks until the end of crtc_enable, then
6109                  * the HW state readout code will complain that the expected
6110                  * IPS_CTL value is not the one we read. */
6111                 if (intel_de_wait_for_set(dev_priv, IPS_CTL, IPS_ENABLE, 50))
6112                         DRM_ERROR("Timed out waiting for IPS enable\n");
6113         }
6114 }
6115
6116 void hsw_disable_ips(const struct intel_crtc_state *crtc_state)
6117 {
6118         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6119         struct drm_device *dev = crtc->base.dev;
6120         struct drm_i915_private *dev_priv = to_i915(dev);
6121
6122         if (!crtc_state->ips_enabled)
6123                 return;
6124
6125         if (IS_BROADWELL(dev_priv)) {
6126                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
6127                 /*
6128                  * Wait for PCODE to finish disabling IPS. The BSpec specified
6129                  * 42ms timeout value leads to occasional timeouts so use 100ms
6130                  * instead.
6131                  */
6132                 if (intel_de_wait_for_clear(dev_priv, IPS_CTL, IPS_ENABLE, 100))
6133                         DRM_ERROR("Timed out waiting for IPS disable\n");
6134         } else {
6135                 I915_WRITE(IPS_CTL, 0);
6136                 POSTING_READ(IPS_CTL);
6137         }
6138
6139         /* We need to wait for a vblank before we can disable the plane. */
6140         intel_wait_for_vblank(dev_priv, crtc->pipe);
6141 }
6142
6143 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
6144 {
6145         if (intel_crtc->overlay)
6146                 (void) intel_overlay_switch_off(intel_crtc->overlay);
6147
6148         /* Let userspace switch the overlay on again. In most cases userspace
6149          * has to recompute where to put it anyway.
6150          */
6151 }
6152
6153 static bool hsw_pre_update_disable_ips(const struct intel_crtc_state *old_crtc_state,
6154                                        const struct intel_crtc_state *new_crtc_state)
6155 {
6156         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6157         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6158
6159         if (!old_crtc_state->ips_enabled)
6160                 return false;
6161
6162         if (needs_modeset(new_crtc_state))
6163                 return true;
6164
6165         /*
6166          * Workaround : Do not read or write the pipe palette/gamma data while
6167          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
6168          *
6169          * Disable IPS before we program the LUT.
6170          */
6171         if (IS_HASWELL(dev_priv) &&
6172             (new_crtc_state->uapi.color_mgmt_changed ||
6173              new_crtc_state->update_pipe) &&
6174             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
6175                 return true;
6176
6177         return !new_crtc_state->ips_enabled;
6178 }
6179
6180 static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_state,
6181                                        const struct intel_crtc_state *new_crtc_state)
6182 {
6183         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6184         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6185
6186         if (!new_crtc_state->ips_enabled)
6187                 return false;
6188
6189         if (needs_modeset(new_crtc_state))
6190                 return true;
6191
6192         /*
6193          * Workaround : Do not read or write the pipe palette/gamma data while
6194          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
6195          *
6196          * Re-enable IPS after the LUT has been programmed.
6197          */
6198         if (IS_HASWELL(dev_priv) &&
6199             (new_crtc_state->uapi.color_mgmt_changed ||
6200              new_crtc_state->update_pipe) &&
6201             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
6202                 return true;
6203
6204         /*
6205          * We can't read out IPS on broadwell, assume the worst and
6206          * forcibly enable IPS on the first fastset.
6207          */
6208         if (new_crtc_state->update_pipe &&
6209             old_crtc_state->hw.adjusted_mode.private_flags & I915_MODE_FLAG_INHERITED)
6210                 return true;
6211
6212         return !old_crtc_state->ips_enabled;
6213 }
6214
6215 static bool needs_nv12_wa(const struct intel_crtc_state *crtc_state)
6216 {
6217         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
6218
6219         if (!crtc_state->nv12_planes)
6220                 return false;
6221
6222         /* WA Display #0827: Gen9:all */
6223         if (IS_GEN(dev_priv, 9) && !IS_GEMINILAKE(dev_priv))
6224                 return true;
6225
6226         return false;
6227 }
6228
6229 static bool needs_scalerclk_wa(const struct intel_crtc_state *crtc_state)
6230 {
6231         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
6232
6233         /* Wa_2006604312:icl */
6234         if (crtc_state->scaler_state.scaler_users > 0 && IS_ICELAKE(dev_priv))
6235                 return true;
6236
6237         return false;
6238 }
6239
6240 static bool planes_enabling(const struct intel_crtc_state *old_crtc_state,
6241                             const struct intel_crtc_state *new_crtc_state)
6242 {
6243         return (!old_crtc_state->active_planes || needs_modeset(new_crtc_state)) &&
6244                 new_crtc_state->active_planes;
6245 }
6246
6247 static bool planes_disabling(const struct intel_crtc_state *old_crtc_state,
6248                              const struct intel_crtc_state *new_crtc_state)
6249 {
6250         return old_crtc_state->active_planes &&
6251                 (!new_crtc_state->active_planes || needs_modeset(new_crtc_state));
6252 }
6253
6254 static void intel_post_plane_update(struct intel_atomic_state *state,
6255                                     struct intel_crtc *crtc)
6256 {
6257         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6258         struct intel_plane *primary = to_intel_plane(crtc->base.primary);
6259         const struct intel_crtc_state *old_crtc_state =
6260                 intel_atomic_get_old_crtc_state(state, crtc);
6261         const struct intel_crtc_state *new_crtc_state =
6262                 intel_atomic_get_new_crtc_state(state, crtc);
6263         const struct intel_plane_state *new_primary_state =
6264                 intel_atomic_get_new_plane_state(state, primary);
6265         enum pipe pipe = crtc->pipe;
6266
6267         intel_frontbuffer_flip(dev_priv, new_crtc_state->fb_bits);
6268
6269         if (new_crtc_state->update_wm_post && new_crtc_state->hw.active)
6270                 intel_update_watermarks(crtc);
6271
6272         if (hsw_post_update_enable_ips(old_crtc_state, new_crtc_state))
6273                 hsw_enable_ips(new_crtc_state);
6274
6275         if (new_primary_state)
6276                 intel_fbc_post_update(crtc);
6277
6278         if (needs_nv12_wa(old_crtc_state) &&
6279             !needs_nv12_wa(new_crtc_state))
6280                 skl_wa_827(dev_priv, pipe, false);
6281
6282         if (needs_scalerclk_wa(old_crtc_state) &&
6283             !needs_scalerclk_wa(new_crtc_state))
6284                 icl_wa_scalerclkgating(dev_priv, pipe, false);
6285 }
6286
6287 static void intel_pre_plane_update(struct intel_atomic_state *state,
6288                                    struct intel_crtc *crtc)
6289 {
6290         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6291         struct intel_plane *primary = to_intel_plane(crtc->base.primary);
6292         const struct intel_crtc_state *old_crtc_state =
6293                 intel_atomic_get_old_crtc_state(state, crtc);
6294         const struct intel_crtc_state *new_crtc_state =
6295                 intel_atomic_get_new_crtc_state(state, crtc);
6296         const struct intel_plane_state *new_primary_state =
6297                 intel_atomic_get_new_plane_state(state, primary);
6298         enum pipe pipe = crtc->pipe;
6299
6300         if (hsw_pre_update_disable_ips(old_crtc_state, new_crtc_state))
6301                 hsw_disable_ips(old_crtc_state);
6302
6303         if (new_primary_state &&
6304             intel_fbc_pre_update(crtc, new_crtc_state, new_primary_state))
6305                 intel_wait_for_vblank(dev_priv, pipe);
6306
6307         /* Display WA 827 */
6308         if (!needs_nv12_wa(old_crtc_state) &&
6309             needs_nv12_wa(new_crtc_state))
6310                 skl_wa_827(dev_priv, pipe, true);
6311
6312         /* Wa_2006604312:icl */
6313         if (!needs_scalerclk_wa(old_crtc_state) &&
6314             needs_scalerclk_wa(new_crtc_state))
6315                 icl_wa_scalerclkgating(dev_priv, pipe, true);
6316
6317         /*
6318          * Vblank time updates from the shadow to live plane control register
6319          * are blocked if the memory self-refresh mode is active at that
6320          * moment. So to make sure the plane gets truly disabled, disable
6321          * first the self-refresh mode. The self-refresh enable bit in turn
6322          * will be checked/applied by the HW only at the next frame start
6323          * event which is after the vblank start event, so we need to have a
6324          * wait-for-vblank between disabling the plane and the pipe.
6325          */
6326         if (HAS_GMCH(dev_priv) && old_crtc_state->hw.active &&
6327             new_crtc_state->disable_cxsr && intel_set_memory_cxsr(dev_priv, false))
6328                 intel_wait_for_vblank(dev_priv, pipe);
6329
6330         /*
6331          * IVB workaround: must disable low power watermarks for at least
6332          * one frame before enabling scaling.  LP watermarks can be re-enabled
6333          * when scaling is disabled.
6334          *
6335          * WaCxSRDisabledForSpriteScaling:ivb
6336          */
6337         if (old_crtc_state->hw.active &&
6338             new_crtc_state->disable_lp_wm && ilk_disable_lp_wm(dev_priv))
6339                 intel_wait_for_vblank(dev_priv, pipe);
6340
6341         /*
6342          * If we're doing a modeset we don't need to do any
6343          * pre-vblank watermark programming here.
6344          */
6345         if (!needs_modeset(new_crtc_state)) {
6346                 /*
6347                  * For platforms that support atomic watermarks, program the
6348                  * 'intermediate' watermarks immediately.  On pre-gen9 platforms, these
6349                  * will be the intermediate values that are safe for both pre- and
6350                  * post- vblank; when vblank happens, the 'active' values will be set
6351                  * to the final 'target' values and we'll do this again to get the
6352                  * optimal watermarks.  For gen9+ platforms, the values we program here
6353                  * will be the final target values which will get automatically latched
6354                  * at vblank time; no further programming will be necessary.
6355                  *
6356                  * If a platform hasn't been transitioned to atomic watermarks yet,
6357                  * we'll continue to update watermarks the old way, if flags tell
6358                  * us to.
6359                  */
6360                 if (dev_priv->display.initial_watermarks)
6361                         dev_priv->display.initial_watermarks(state, crtc);
6362                 else if (new_crtc_state->update_wm_pre)
6363                         intel_update_watermarks(crtc);
6364         }
6365
6366         /*
6367          * Gen2 reports pipe underruns whenever all planes are disabled.
6368          * So disable underrun reporting before all the planes get disabled.
6369          *
6370          * We do this after .initial_watermarks() so that we have a
6371          * chance of catching underruns with the intermediate watermarks
6372          * vs. the old plane configuration.
6373          */
6374         if (IS_GEN(dev_priv, 2) && planes_disabling(old_crtc_state, new_crtc_state))
6375                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6376 }
6377
6378 static void intel_crtc_disable_planes(struct intel_atomic_state *state,
6379                                       struct intel_crtc *crtc)
6380 {
6381         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6382         const struct intel_crtc_state *new_crtc_state =
6383                 intel_atomic_get_new_crtc_state(state, crtc);
6384         unsigned int update_mask = new_crtc_state->update_planes;
6385         const struct intel_plane_state *old_plane_state;
6386         struct intel_plane *plane;
6387         unsigned fb_bits = 0;
6388         int i;
6389
6390         intel_crtc_dpms_overlay_disable(crtc);
6391
6392         for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
6393                 if (crtc->pipe != plane->pipe ||
6394                     !(update_mask & BIT(plane->id)))
6395                         continue;
6396
6397                 intel_disable_plane(plane, new_crtc_state);
6398
6399                 if (old_plane_state->uapi.visible)
6400                         fb_bits |= plane->frontbuffer_bit;
6401         }
6402
6403         intel_frontbuffer_flip(dev_priv, fb_bits);
6404 }
6405
6406 /*
6407  * intel_connector_primary_encoder - get the primary encoder for a connector
6408  * @connector: connector for which to return the encoder
6409  *
6410  * Returns the primary encoder for a connector. There is a 1:1 mapping from
6411  * all connectors to their encoder, except for DP-MST connectors which have
6412  * both a virtual and a primary encoder. These DP-MST primary encoders can be
6413  * pointed to by as many DP-MST connectors as there are pipes.
6414  */
6415 static struct intel_encoder *
6416 intel_connector_primary_encoder(struct intel_connector *connector)
6417 {
6418         struct intel_encoder *encoder;
6419
6420         if (connector->mst_port)
6421                 return &dp_to_dig_port(connector->mst_port)->base;
6422
6423         encoder = intel_attached_encoder(&connector->base);
6424         WARN_ON(!encoder);
6425
6426         return encoder;
6427 }
6428
6429 static void intel_encoders_update_prepare(struct intel_atomic_state *state)
6430 {
6431         struct drm_connector_state *new_conn_state;
6432         struct drm_connector *connector;
6433         int i;
6434
6435         for_each_new_connector_in_state(&state->base, connector, new_conn_state,
6436                                         i) {
6437                 struct intel_connector *intel_connector;
6438                 struct intel_encoder *encoder;
6439                 struct intel_crtc *crtc;
6440
6441                 if (!intel_connector_needs_modeset(state, connector))
6442                         continue;
6443
6444                 intel_connector = to_intel_connector(connector);
6445                 encoder = intel_connector_primary_encoder(intel_connector);
6446                 if (!encoder->update_prepare)
6447                         continue;
6448
6449                 crtc = new_conn_state->crtc ?
6450                         to_intel_crtc(new_conn_state->crtc) : NULL;
6451                 encoder->update_prepare(state, encoder, crtc);
6452         }
6453 }
6454
6455 static void intel_encoders_update_complete(struct intel_atomic_state *state)
6456 {
6457         struct drm_connector_state *new_conn_state;
6458         struct drm_connector *connector;
6459         int i;
6460
6461         for_each_new_connector_in_state(&state->base, connector, new_conn_state,
6462                                         i) {
6463                 struct intel_connector *intel_connector;
6464                 struct intel_encoder *encoder;
6465                 struct intel_crtc *crtc;
6466
6467                 if (!intel_connector_needs_modeset(state, connector))
6468                         continue;
6469
6470                 intel_connector = to_intel_connector(connector);
6471                 encoder = intel_connector_primary_encoder(intel_connector);
6472                 if (!encoder->update_complete)
6473                         continue;
6474
6475                 crtc = new_conn_state->crtc ?
6476                         to_intel_crtc(new_conn_state->crtc) : NULL;
6477                 encoder->update_complete(state, encoder, crtc);
6478         }
6479 }
6480
6481 static void intel_encoders_pre_pll_enable(struct intel_atomic_state *state,
6482                                           struct intel_crtc *crtc)
6483 {
6484         const struct intel_crtc_state *crtc_state =
6485                 intel_atomic_get_new_crtc_state(state, crtc);
6486         const struct drm_connector_state *conn_state;
6487         struct drm_connector *conn;
6488         int i;
6489
6490         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6491                 struct intel_encoder *encoder =
6492                         to_intel_encoder(conn_state->best_encoder);
6493
6494                 if (conn_state->crtc != &crtc->base)
6495                         continue;
6496
6497                 if (encoder->pre_pll_enable)
6498                         encoder->pre_pll_enable(encoder, crtc_state, conn_state);
6499         }
6500 }
6501
6502 static void intel_encoders_pre_enable(struct intel_atomic_state *state,
6503                                       struct intel_crtc *crtc)
6504 {
6505         const struct intel_crtc_state *crtc_state =
6506                 intel_atomic_get_new_crtc_state(state, crtc);
6507         const struct drm_connector_state *conn_state;
6508         struct drm_connector *conn;
6509         int i;
6510
6511         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6512                 struct intel_encoder *encoder =
6513                         to_intel_encoder(conn_state->best_encoder);
6514
6515                 if (conn_state->crtc != &crtc->base)
6516                         continue;
6517
6518                 if (encoder->pre_enable)
6519                         encoder->pre_enable(encoder, crtc_state, conn_state);
6520         }
6521 }
6522
6523 static void intel_encoders_enable(struct intel_atomic_state *state,
6524                                   struct intel_crtc *crtc)
6525 {
6526         const struct intel_crtc_state *crtc_state =
6527                 intel_atomic_get_new_crtc_state(state, crtc);
6528         const struct drm_connector_state *conn_state;
6529         struct drm_connector *conn;
6530         int i;
6531
6532         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6533                 struct intel_encoder *encoder =
6534                         to_intel_encoder(conn_state->best_encoder);
6535
6536                 if (conn_state->crtc != &crtc->base)
6537                         continue;
6538
6539                 if (encoder->enable)
6540                         encoder->enable(encoder, crtc_state, conn_state);
6541                 intel_opregion_notify_encoder(encoder, true);
6542         }
6543 }
6544
6545 static void intel_encoders_disable(struct intel_atomic_state *state,
6546                                    struct intel_crtc *crtc)
6547 {
6548         const struct intel_crtc_state *old_crtc_state =
6549                 intel_atomic_get_old_crtc_state(state, crtc);
6550         const struct drm_connector_state *old_conn_state;
6551         struct drm_connector *conn;
6552         int i;
6553
6554         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6555                 struct intel_encoder *encoder =
6556                         to_intel_encoder(old_conn_state->best_encoder);
6557
6558                 if (old_conn_state->crtc != &crtc->base)
6559                         continue;
6560
6561                 intel_opregion_notify_encoder(encoder, false);
6562                 if (encoder->disable)
6563                         encoder->disable(encoder, old_crtc_state, old_conn_state);
6564         }
6565 }
6566
6567 static void intel_encoders_post_disable(struct intel_atomic_state *state,
6568                                         struct intel_crtc *crtc)
6569 {
6570         const struct intel_crtc_state *old_crtc_state =
6571                 intel_atomic_get_old_crtc_state(state, crtc);
6572         const struct drm_connector_state *old_conn_state;
6573         struct drm_connector *conn;
6574         int i;
6575
6576         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6577                 struct intel_encoder *encoder =
6578                         to_intel_encoder(old_conn_state->best_encoder);
6579
6580                 if (old_conn_state->crtc != &crtc->base)
6581                         continue;
6582
6583                 if (encoder->post_disable)
6584                         encoder->post_disable(encoder, old_crtc_state, old_conn_state);
6585         }
6586 }
6587
6588 static void intel_encoders_post_pll_disable(struct intel_atomic_state *state,
6589                                             struct intel_crtc *crtc)
6590 {
6591         const struct intel_crtc_state *old_crtc_state =
6592                 intel_atomic_get_old_crtc_state(state, crtc);
6593         const struct drm_connector_state *old_conn_state;
6594         struct drm_connector *conn;
6595         int i;
6596
6597         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6598                 struct intel_encoder *encoder =
6599                         to_intel_encoder(old_conn_state->best_encoder);
6600
6601                 if (old_conn_state->crtc != &crtc->base)
6602                         continue;
6603
6604                 if (encoder->post_pll_disable)
6605                         encoder->post_pll_disable(encoder, old_crtc_state, old_conn_state);
6606         }
6607 }
6608
6609 static void intel_encoders_update_pipe(struct intel_atomic_state *state,
6610                                        struct intel_crtc *crtc)
6611 {
6612         const struct intel_crtc_state *crtc_state =
6613                 intel_atomic_get_new_crtc_state(state, crtc);
6614         const struct drm_connector_state *conn_state;
6615         struct drm_connector *conn;
6616         int i;
6617
6618         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6619                 struct intel_encoder *encoder =
6620                         to_intel_encoder(conn_state->best_encoder);
6621
6622                 if (conn_state->crtc != &crtc->base)
6623                         continue;
6624
6625                 if (encoder->update_pipe)
6626                         encoder->update_pipe(encoder, crtc_state, conn_state);
6627         }
6628 }
6629
6630 static void intel_disable_primary_plane(const struct intel_crtc_state *crtc_state)
6631 {
6632         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6633         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
6634
6635         plane->disable_plane(plane, crtc_state);
6636 }
6637
6638 static void ilk_crtc_enable(struct intel_atomic_state *state,
6639                             struct intel_crtc *crtc)
6640 {
6641         const struct intel_crtc_state *new_crtc_state =
6642                 intel_atomic_get_new_crtc_state(state, crtc);
6643         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6644         enum pipe pipe = crtc->pipe;
6645
6646         if (WARN_ON(crtc->active))
6647                 return;
6648
6649         /*
6650          * Sometimes spurious CPU pipe underruns happen during FDI
6651          * training, at least with VGA+HDMI cloning. Suppress them.
6652          *
6653          * On ILK we get an occasional spurious CPU pipe underruns
6654          * between eDP port A enable and vdd enable. Also PCH port
6655          * enable seems to result in the occasional CPU pipe underrun.
6656          *
6657          * Spurious PCH underruns also occur during PCH enabling.
6658          */
6659         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6660         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
6661
6662         if (new_crtc_state->has_pch_encoder)
6663                 intel_prepare_shared_dpll(new_crtc_state);
6664
6665         if (intel_crtc_has_dp_encoder(new_crtc_state))
6666                 intel_dp_set_m_n(new_crtc_state, M1_N1);
6667
6668         intel_set_pipe_timings(new_crtc_state);
6669         intel_set_pipe_src_size(new_crtc_state);
6670
6671         if (new_crtc_state->has_pch_encoder)
6672                 intel_cpu_transcoder_set_m_n(new_crtc_state,
6673                                              &new_crtc_state->fdi_m_n, NULL);
6674
6675         ilk_set_pipeconf(new_crtc_state);
6676
6677         crtc->active = true;
6678
6679         intel_encoders_pre_enable(state, crtc);
6680
6681         if (new_crtc_state->has_pch_encoder) {
6682                 /* Note: FDI PLL enabling _must_ be done before we enable the
6683                  * cpu pipes, hence this is separate from all the other fdi/pch
6684                  * enabling. */
6685                 ilk_fdi_pll_enable(new_crtc_state);
6686         } else {
6687                 assert_fdi_tx_disabled(dev_priv, pipe);
6688                 assert_fdi_rx_disabled(dev_priv, pipe);
6689         }
6690
6691         ilk_pfit_enable(new_crtc_state);
6692
6693         /*
6694          * On ILK+ LUT must be loaded before the pipe is running but with
6695          * clocks enabled
6696          */
6697         intel_color_load_luts(new_crtc_state);
6698         intel_color_commit(new_crtc_state);
6699         /* update DSPCNTR to configure gamma for pipe bottom color */
6700         intel_disable_primary_plane(new_crtc_state);
6701
6702         if (dev_priv->display.initial_watermarks)
6703                 dev_priv->display.initial_watermarks(state, crtc);
6704         intel_enable_pipe(new_crtc_state);
6705
6706         if (new_crtc_state->has_pch_encoder)
6707                 ilk_pch_enable(state, new_crtc_state);
6708
6709         intel_crtc_vblank_on(new_crtc_state);
6710
6711         intel_encoders_enable(state, crtc);
6712
6713         if (HAS_PCH_CPT(dev_priv))
6714                 cpt_verify_modeset(dev_priv, pipe);
6715
6716         /*
6717          * Must wait for vblank to avoid spurious PCH FIFO underruns.
6718          * And a second vblank wait is needed at least on ILK with
6719          * some interlaced HDMI modes. Let's do the double wait always
6720          * in case there are more corner cases we don't know about.
6721          */
6722         if (new_crtc_state->has_pch_encoder) {
6723                 intel_wait_for_vblank(dev_priv, pipe);
6724                 intel_wait_for_vblank(dev_priv, pipe);
6725         }
6726         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6727         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
6728 }
6729
6730 /* IPS only exists on ULT machines and is tied to pipe A. */
6731 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
6732 {
6733         return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A;
6734 }
6735
6736 static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv,
6737                                             enum pipe pipe, bool apply)
6738 {
6739         u32 val = I915_READ(CLKGATE_DIS_PSL(pipe));
6740         u32 mask = DPF_GATING_DIS | DPF_RAM_GATING_DIS | DPFR_GATING_DIS;
6741
6742         if (apply)
6743                 val |= mask;
6744         else
6745                 val &= ~mask;
6746
6747         I915_WRITE(CLKGATE_DIS_PSL(pipe), val);
6748 }
6749
6750 static void icl_pipe_mbus_enable(struct intel_crtc *crtc)
6751 {
6752         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6753         enum pipe pipe = crtc->pipe;
6754         u32 val;
6755
6756         val = MBUS_DBOX_A_CREDIT(2);
6757
6758         if (INTEL_GEN(dev_priv) >= 12) {
6759                 val |= MBUS_DBOX_BW_CREDIT(2);
6760                 val |= MBUS_DBOX_B_CREDIT(12);
6761         } else {
6762                 val |= MBUS_DBOX_BW_CREDIT(1);
6763                 val |= MBUS_DBOX_B_CREDIT(8);
6764         }
6765
6766         I915_WRITE(PIPE_MBUS_DBOX_CTL(pipe), val);
6767 }
6768
6769 static void hsw_set_frame_start_delay(const struct intel_crtc_state *crtc_state)
6770 {
6771         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6772         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6773         i915_reg_t reg = CHICKEN_TRANS(crtc_state->cpu_transcoder);
6774         u32 val;
6775
6776         val = I915_READ(reg);
6777         val &= ~HSW_FRAME_START_DELAY_MASK;
6778         val |= HSW_FRAME_START_DELAY(0);
6779         I915_WRITE(reg, val);
6780 }
6781
6782 static void hsw_crtc_enable(struct intel_atomic_state *state,
6783                             struct intel_crtc *crtc)
6784 {
6785         const struct intel_crtc_state *new_crtc_state =
6786                 intel_atomic_get_new_crtc_state(state, crtc);
6787         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6788         enum pipe pipe = crtc->pipe, hsw_workaround_pipe;
6789         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
6790         bool psl_clkgate_wa;
6791
6792         if (WARN_ON(crtc->active))
6793                 return;
6794
6795         intel_encoders_pre_pll_enable(state, crtc);
6796
6797         if (new_crtc_state->shared_dpll)
6798                 intel_enable_shared_dpll(new_crtc_state);
6799
6800         intel_encoders_pre_enable(state, crtc);
6801
6802         if (intel_crtc_has_dp_encoder(new_crtc_state))
6803                 intel_dp_set_m_n(new_crtc_state, M1_N1);
6804
6805         if (!transcoder_is_dsi(cpu_transcoder))
6806                 intel_set_pipe_timings(new_crtc_state);
6807
6808         if (INTEL_GEN(dev_priv) >= 11)
6809                 icl_enable_trans_port_sync(new_crtc_state);
6810
6811         intel_set_pipe_src_size(new_crtc_state);
6812
6813         if (cpu_transcoder != TRANSCODER_EDP &&
6814             !transcoder_is_dsi(cpu_transcoder))
6815                 I915_WRITE(PIPE_MULT(cpu_transcoder),
6816                            new_crtc_state->pixel_multiplier - 1);
6817
6818         if (new_crtc_state->has_pch_encoder)
6819                 intel_cpu_transcoder_set_m_n(new_crtc_state,
6820                                              &new_crtc_state->fdi_m_n, NULL);
6821
6822         if (!transcoder_is_dsi(cpu_transcoder)) {
6823                 hsw_set_frame_start_delay(new_crtc_state);
6824                 hsw_set_pipeconf(new_crtc_state);
6825         }
6826
6827         if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
6828                 bdw_set_pipemisc(new_crtc_state);
6829
6830         crtc->active = true;
6831
6832         /* Display WA #1180: WaDisableScalarClockGating: glk, cnl */
6833         psl_clkgate_wa = (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) &&
6834                 new_crtc_state->pch_pfit.enabled;
6835         if (psl_clkgate_wa)
6836                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, true);
6837
6838         if (INTEL_GEN(dev_priv) >= 9)
6839                 skl_pfit_enable(new_crtc_state);
6840         else
6841                 ilk_pfit_enable(new_crtc_state);
6842
6843         /*
6844          * On ILK+ LUT must be loaded before the pipe is running but with
6845          * clocks enabled
6846          */
6847         intel_color_load_luts(new_crtc_state);
6848         intel_color_commit(new_crtc_state);
6849         /* update DSPCNTR to configure gamma/csc for pipe bottom color */
6850         if (INTEL_GEN(dev_priv) < 9)
6851                 intel_disable_primary_plane(new_crtc_state);
6852
6853         if (INTEL_GEN(dev_priv) >= 11)
6854                 icl_set_pipe_chicken(crtc);
6855
6856         if (!transcoder_is_dsi(cpu_transcoder))
6857                 intel_ddi_enable_transcoder_func(new_crtc_state);
6858
6859         if (dev_priv->display.initial_watermarks)
6860                 dev_priv->display.initial_watermarks(state, crtc);
6861
6862         if (INTEL_GEN(dev_priv) >= 11)
6863                 icl_pipe_mbus_enable(crtc);
6864
6865         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
6866         if (!transcoder_is_dsi(cpu_transcoder))
6867                 intel_enable_pipe(new_crtc_state);
6868
6869         if (new_crtc_state->has_pch_encoder)
6870                 lpt_pch_enable(state, new_crtc_state);
6871
6872         intel_crtc_vblank_on(new_crtc_state);
6873
6874         intel_encoders_enable(state, crtc);
6875
6876         if (psl_clkgate_wa) {
6877                 intel_wait_for_vblank(dev_priv, pipe);
6878                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, false);
6879         }
6880
6881         /* If we change the relative order between pipe/planes enabling, we need
6882          * to change the workaround. */
6883         hsw_workaround_pipe = new_crtc_state->hsw_workaround_pipe;
6884         if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
6885                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
6886                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
6887         }
6888 }
6889
6890 void ilk_pfit_disable(const struct intel_crtc_state *old_crtc_state)
6891 {
6892         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
6893         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6894         enum pipe pipe = crtc->pipe;
6895
6896         /* To avoid upsetting the power well on haswell only disable the pfit if
6897          * it's in use. The hw state code will make sure we get this right. */
6898         if (old_crtc_state->pch_pfit.enabled) {
6899                 I915_WRITE(PF_CTL(pipe), 0);
6900                 I915_WRITE(PF_WIN_POS(pipe), 0);
6901                 I915_WRITE(PF_WIN_SZ(pipe), 0);
6902         }
6903 }
6904
6905 static void ilk_crtc_disable(struct intel_atomic_state *state,
6906                              struct intel_crtc *crtc)
6907 {
6908         const struct intel_crtc_state *old_crtc_state =
6909                 intel_atomic_get_old_crtc_state(state, crtc);
6910         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6911         enum pipe pipe = crtc->pipe;
6912
6913         /*
6914          * Sometimes spurious CPU pipe underruns happen when the
6915          * pipe is already disabled, but FDI RX/TX is still enabled.
6916          * Happens at least with VGA+HDMI cloning. Suppress them.
6917          */
6918         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6919         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
6920
6921         intel_encoders_disable(state, crtc);
6922
6923         intel_crtc_vblank_off(old_crtc_state);
6924
6925         intel_disable_pipe(old_crtc_state);
6926
6927         ilk_pfit_disable(old_crtc_state);
6928
6929         if (old_crtc_state->has_pch_encoder)
6930                 ilk_fdi_disable(crtc);
6931
6932         intel_encoders_post_disable(state, crtc);
6933
6934         if (old_crtc_state->has_pch_encoder) {
6935                 ilk_disable_pch_transcoder(dev_priv, pipe);
6936
6937                 if (HAS_PCH_CPT(dev_priv)) {
6938                         i915_reg_t reg;
6939                         u32 temp;
6940
6941                         /* disable TRANS_DP_CTL */
6942                         reg = TRANS_DP_CTL(pipe);
6943                         temp = I915_READ(reg);
6944                         temp &= ~(TRANS_DP_OUTPUT_ENABLE |
6945                                   TRANS_DP_PORT_SEL_MASK);
6946                         temp |= TRANS_DP_PORT_SEL_NONE;
6947                         I915_WRITE(reg, temp);
6948
6949                         /* disable DPLL_SEL */
6950                         temp = I915_READ(PCH_DPLL_SEL);
6951                         temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
6952                         I915_WRITE(PCH_DPLL_SEL, temp);
6953                 }
6954
6955                 ilk_fdi_pll_disable(crtc);
6956         }
6957
6958         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6959         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
6960 }
6961
6962 static void hsw_crtc_disable(struct intel_atomic_state *state,
6963                              struct intel_crtc *crtc)
6964 {
6965         /*
6966          * FIXME collapse everything to one hook.
6967          * Need care with mst->ddi interactions.
6968          */
6969         intel_encoders_disable(state, crtc);
6970         intel_encoders_post_disable(state, crtc);
6971 }
6972
6973 static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state)
6974 {
6975         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6976         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6977
6978         if (!crtc_state->gmch_pfit.control)
6979                 return;
6980
6981         /*
6982          * The panel fitter should only be adjusted whilst the pipe is disabled,
6983          * according to register description and PRM.
6984          */
6985         WARN_ON(I915_READ(PFIT_CONTROL) & PFIT_ENABLE);
6986         assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder);
6987
6988         I915_WRITE(PFIT_PGM_RATIOS, crtc_state->gmch_pfit.pgm_ratios);
6989         I915_WRITE(PFIT_CONTROL, crtc_state->gmch_pfit.control);
6990
6991         /* Border color in case we don't scale up to the full screen. Black by
6992          * default, change to something else for debugging. */
6993         I915_WRITE(BCLRPAT(crtc->pipe), 0);
6994 }
6995
6996 bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy)
6997 {
6998         if (phy == PHY_NONE)
6999                 return false;
7000
7001         if (IS_ELKHARTLAKE(dev_priv))
7002                 return phy <= PHY_C;
7003
7004         if (INTEL_GEN(dev_priv) >= 11)
7005                 return phy <= PHY_B;
7006
7007         return false;
7008 }
7009
7010 bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy)
7011 {
7012         if (INTEL_GEN(dev_priv) >= 12)
7013                 return phy >= PHY_D && phy <= PHY_I;
7014
7015         if (INTEL_GEN(dev_priv) >= 11 && !IS_ELKHARTLAKE(dev_priv))
7016                 return phy >= PHY_C && phy <= PHY_F;
7017
7018         return false;
7019 }
7020
7021 enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port)
7022 {
7023         if (IS_ELKHARTLAKE(i915) && port == PORT_D)
7024                 return PHY_A;
7025
7026         return (enum phy)port;
7027 }
7028
7029 enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port)
7030 {
7031         if (!intel_phy_is_tc(dev_priv, intel_port_to_phy(dev_priv, port)))
7032                 return PORT_TC_NONE;
7033
7034         if (INTEL_GEN(dev_priv) >= 12)
7035                 return port - PORT_D;
7036
7037         return port - PORT_C;
7038 }
7039
7040 enum intel_display_power_domain intel_port_to_power_domain(enum port port)
7041 {
7042         switch (port) {
7043         case PORT_A:
7044                 return POWER_DOMAIN_PORT_DDI_A_LANES;
7045         case PORT_B:
7046                 return POWER_DOMAIN_PORT_DDI_B_LANES;
7047         case PORT_C:
7048                 return POWER_DOMAIN_PORT_DDI_C_LANES;
7049         case PORT_D:
7050                 return POWER_DOMAIN_PORT_DDI_D_LANES;
7051         case PORT_E:
7052                 return POWER_DOMAIN_PORT_DDI_E_LANES;
7053         case PORT_F:
7054                 return POWER_DOMAIN_PORT_DDI_F_LANES;
7055         case PORT_G:
7056                 return POWER_DOMAIN_PORT_DDI_G_LANES;
7057         default:
7058                 MISSING_CASE(port);
7059                 return POWER_DOMAIN_PORT_OTHER;
7060         }
7061 }
7062
7063 enum intel_display_power_domain
7064 intel_aux_power_domain(struct intel_digital_port *dig_port)
7065 {
7066         struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
7067         enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port);
7068
7069         if (intel_phy_is_tc(dev_priv, phy) &&
7070             dig_port->tc_mode == TC_PORT_TBT_ALT) {
7071                 switch (dig_port->aux_ch) {
7072                 case AUX_CH_C:
7073                         return POWER_DOMAIN_AUX_C_TBT;
7074                 case AUX_CH_D:
7075                         return POWER_DOMAIN_AUX_D_TBT;
7076                 case AUX_CH_E:
7077                         return POWER_DOMAIN_AUX_E_TBT;
7078                 case AUX_CH_F:
7079                         return POWER_DOMAIN_AUX_F_TBT;
7080                 case AUX_CH_G:
7081                         return POWER_DOMAIN_AUX_G_TBT;
7082                 default:
7083                         MISSING_CASE(dig_port->aux_ch);
7084                         return POWER_DOMAIN_AUX_C_TBT;
7085                 }
7086         }
7087
7088         switch (dig_port->aux_ch) {
7089         case AUX_CH_A:
7090                 return POWER_DOMAIN_AUX_A;
7091         case AUX_CH_B:
7092                 return POWER_DOMAIN_AUX_B;
7093         case AUX_CH_C:
7094                 return POWER_DOMAIN_AUX_C;
7095         case AUX_CH_D:
7096                 return POWER_DOMAIN_AUX_D;
7097         case AUX_CH_E:
7098                 return POWER_DOMAIN_AUX_E;
7099         case AUX_CH_F:
7100                 return POWER_DOMAIN_AUX_F;
7101         case AUX_CH_G:
7102                 return POWER_DOMAIN_AUX_G;
7103         default:
7104                 MISSING_CASE(dig_port->aux_ch);
7105                 return POWER_DOMAIN_AUX_A;
7106         }
7107 }
7108
7109 static u64 get_crtc_power_domains(struct intel_crtc_state *crtc_state)
7110 {
7111         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7112         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7113         struct drm_encoder *encoder;
7114         enum pipe pipe = crtc->pipe;
7115         u64 mask;
7116         enum transcoder transcoder = crtc_state->cpu_transcoder;
7117
7118         if (!crtc_state->hw.active)
7119                 return 0;
7120
7121         mask = BIT_ULL(POWER_DOMAIN_PIPE(pipe));
7122         mask |= BIT_ULL(POWER_DOMAIN_TRANSCODER(transcoder));
7123         if (crtc_state->pch_pfit.enabled ||
7124             crtc_state->pch_pfit.force_thru)
7125                 mask |= BIT_ULL(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
7126
7127         drm_for_each_encoder_mask(encoder, &dev_priv->drm,
7128                                   crtc_state->uapi.encoder_mask) {
7129                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
7130
7131                 mask |= BIT_ULL(intel_encoder->power_domain);
7132         }
7133
7134         if (HAS_DDI(dev_priv) && crtc_state->has_audio)
7135                 mask |= BIT_ULL(POWER_DOMAIN_AUDIO);
7136
7137         if (crtc_state->shared_dpll)
7138                 mask |= BIT_ULL(POWER_DOMAIN_DISPLAY_CORE);
7139
7140         return mask;
7141 }
7142
7143 static u64
7144 modeset_get_crtc_power_domains(struct intel_crtc_state *crtc_state)
7145 {
7146         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7147         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7148         enum intel_display_power_domain domain;
7149         u64 domains, new_domains, old_domains;
7150
7151         old_domains = crtc->enabled_power_domains;
7152         crtc->enabled_power_domains = new_domains =
7153                 get_crtc_power_domains(crtc_state);
7154
7155         domains = new_domains & ~old_domains;
7156
7157         for_each_power_domain(domain, domains)
7158                 intel_display_power_get(dev_priv, domain);
7159
7160         return old_domains & ~new_domains;
7161 }
7162
7163 static void modeset_put_power_domains(struct drm_i915_private *dev_priv,
7164                                       u64 domains)
7165 {
7166         enum intel_display_power_domain domain;
7167
7168         for_each_power_domain(domain, domains)
7169                 intel_display_power_put_unchecked(dev_priv, domain);
7170 }
7171
7172 static void valleyview_crtc_enable(struct intel_atomic_state *state,
7173                                    struct intel_crtc *crtc)
7174 {
7175         const struct intel_crtc_state *new_crtc_state =
7176                 intel_atomic_get_new_crtc_state(state, crtc);
7177         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7178         enum pipe pipe = crtc->pipe;
7179
7180         if (WARN_ON(crtc->active))
7181                 return;
7182
7183         if (intel_crtc_has_dp_encoder(new_crtc_state))
7184                 intel_dp_set_m_n(new_crtc_state, M1_N1);
7185
7186         intel_set_pipe_timings(new_crtc_state);
7187         intel_set_pipe_src_size(new_crtc_state);
7188
7189         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
7190                 I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY);
7191                 I915_WRITE(CHV_CANVAS(pipe), 0);
7192         }
7193
7194         i9xx_set_pipeconf(new_crtc_state);
7195
7196         crtc->active = true;
7197
7198         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
7199
7200         intel_encoders_pre_pll_enable(state, crtc);
7201
7202         if (IS_CHERRYVIEW(dev_priv)) {
7203                 chv_prepare_pll(crtc, new_crtc_state);
7204                 chv_enable_pll(crtc, new_crtc_state);
7205         } else {
7206                 vlv_prepare_pll(crtc, new_crtc_state);
7207                 vlv_enable_pll(crtc, new_crtc_state);
7208         }
7209
7210         intel_encoders_pre_enable(state, crtc);
7211
7212         i9xx_pfit_enable(new_crtc_state);
7213
7214         intel_color_load_luts(new_crtc_state);
7215         intel_color_commit(new_crtc_state);
7216         /* update DSPCNTR to configure gamma for pipe bottom color */
7217         intel_disable_primary_plane(new_crtc_state);
7218
7219         dev_priv->display.initial_watermarks(state, crtc);
7220         intel_enable_pipe(new_crtc_state);
7221
7222         intel_crtc_vblank_on(new_crtc_state);
7223
7224         intel_encoders_enable(state, crtc);
7225 }
7226
7227 static void i9xx_set_pll_dividers(const struct intel_crtc_state *crtc_state)
7228 {
7229         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7230         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7231
7232         I915_WRITE(FP0(crtc->pipe), crtc_state->dpll_hw_state.fp0);
7233         I915_WRITE(FP1(crtc->pipe), crtc_state->dpll_hw_state.fp1);
7234 }
7235
7236 static void i9xx_crtc_enable(struct intel_atomic_state *state,
7237                              struct intel_crtc *crtc)
7238 {
7239         const struct intel_crtc_state *new_crtc_state =
7240                 intel_atomic_get_new_crtc_state(state, crtc);
7241         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7242         enum pipe pipe = crtc->pipe;
7243
7244         if (WARN_ON(crtc->active))
7245                 return;
7246
7247         i9xx_set_pll_dividers(new_crtc_state);
7248
7249         if (intel_crtc_has_dp_encoder(new_crtc_state))
7250                 intel_dp_set_m_n(new_crtc_state, M1_N1);
7251
7252         intel_set_pipe_timings(new_crtc_state);
7253         intel_set_pipe_src_size(new_crtc_state);
7254
7255         i9xx_set_pipeconf(new_crtc_state);
7256
7257         crtc->active = true;
7258
7259         if (!IS_GEN(dev_priv, 2))
7260                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
7261
7262         intel_encoders_pre_enable(state, crtc);
7263
7264         i9xx_enable_pll(crtc, new_crtc_state);
7265
7266         i9xx_pfit_enable(new_crtc_state);
7267
7268         intel_color_load_luts(new_crtc_state);
7269         intel_color_commit(new_crtc_state);
7270         /* update DSPCNTR to configure gamma for pipe bottom color */
7271         intel_disable_primary_plane(new_crtc_state);
7272
7273         if (dev_priv->display.initial_watermarks)
7274                 dev_priv->display.initial_watermarks(state, crtc);
7275         else
7276                 intel_update_watermarks(crtc);
7277         intel_enable_pipe(new_crtc_state);
7278
7279         intel_crtc_vblank_on(new_crtc_state);
7280
7281         intel_encoders_enable(state, crtc);
7282 }
7283
7284 static void i9xx_pfit_disable(const struct intel_crtc_state *old_crtc_state)
7285 {
7286         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
7287         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7288
7289         if (!old_crtc_state->gmch_pfit.control)
7290                 return;
7291
7292         assert_pipe_disabled(dev_priv, old_crtc_state->cpu_transcoder);
7293
7294         DRM_DEBUG_KMS("disabling pfit, current: 0x%08x\n",
7295                       I915_READ(PFIT_CONTROL));
7296         I915_WRITE(PFIT_CONTROL, 0);
7297 }
7298
7299 static void i9xx_crtc_disable(struct intel_atomic_state *state,
7300                               struct intel_crtc *crtc)
7301 {
7302         struct intel_crtc_state *old_crtc_state =
7303                 intel_atomic_get_old_crtc_state(state, crtc);
7304         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7305         enum pipe pipe = crtc->pipe;
7306
7307         /*
7308          * On gen2 planes are double buffered but the pipe isn't, so we must
7309          * wait for planes to fully turn off before disabling the pipe.
7310          */
7311         if (IS_GEN(dev_priv, 2))
7312                 intel_wait_for_vblank(dev_priv, pipe);
7313
7314         intel_encoders_disable(state, crtc);
7315
7316         intel_crtc_vblank_off(old_crtc_state);
7317
7318         intel_disable_pipe(old_crtc_state);
7319
7320         i9xx_pfit_disable(old_crtc_state);
7321
7322         intel_encoders_post_disable(state, crtc);
7323
7324         if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DSI)) {
7325                 if (IS_CHERRYVIEW(dev_priv))
7326                         chv_disable_pll(dev_priv, pipe);
7327                 else if (IS_VALLEYVIEW(dev_priv))
7328                         vlv_disable_pll(dev_priv, pipe);
7329                 else
7330                         i9xx_disable_pll(old_crtc_state);
7331         }
7332
7333         intel_encoders_post_pll_disable(state, crtc);
7334
7335         if (!IS_GEN(dev_priv, 2))
7336                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
7337
7338         if (!dev_priv->display.initial_watermarks)
7339                 intel_update_watermarks(crtc);
7340
7341         /* clock the pipe down to 640x480@60 to potentially save power */
7342         if (IS_I830(dev_priv))
7343                 i830_enable_pipe(dev_priv, pipe);
7344 }
7345
7346 static void intel_crtc_disable_noatomic(struct intel_crtc *crtc,
7347                                         struct drm_modeset_acquire_ctx *ctx)
7348 {
7349         struct intel_encoder *encoder;
7350         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7351         struct intel_bw_state *bw_state =
7352                 to_intel_bw_state(dev_priv->bw_obj.state);
7353         struct intel_crtc_state *crtc_state =
7354                 to_intel_crtc_state(crtc->base.state);
7355         enum intel_display_power_domain domain;
7356         struct intel_plane *plane;
7357         struct drm_atomic_state *state;
7358         struct intel_crtc_state *temp_crtc_state;
7359         enum pipe pipe = crtc->pipe;
7360         u64 domains;
7361         int ret;
7362
7363         if (!crtc_state->hw.active)
7364                 return;
7365
7366         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
7367                 const struct intel_plane_state *plane_state =
7368                         to_intel_plane_state(plane->base.state);
7369
7370                 if (plane_state->uapi.visible)
7371                         intel_plane_disable_noatomic(crtc, plane);
7372         }
7373
7374         state = drm_atomic_state_alloc(&dev_priv->drm);
7375         if (!state) {
7376                 DRM_DEBUG_KMS("failed to disable [CRTC:%d:%s], out of memory",
7377                               crtc->base.base.id, crtc->base.name);
7378                 return;
7379         }
7380
7381         state->acquire_ctx = ctx;
7382
7383         /* Everything's already locked, -EDEADLK can't happen. */
7384         temp_crtc_state = intel_atomic_get_crtc_state(state, crtc);
7385         ret = drm_atomic_add_affected_connectors(state, &crtc->base);
7386
7387         WARN_ON(IS_ERR(temp_crtc_state) || ret);
7388
7389         dev_priv->display.crtc_disable(to_intel_atomic_state(state), crtc);
7390
7391         drm_atomic_state_put(state);
7392
7393         DRM_DEBUG_KMS("[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
7394                       crtc->base.base.id, crtc->base.name);
7395
7396         crtc->active = false;
7397         crtc->base.enabled = false;
7398
7399         WARN_ON(drm_atomic_set_mode_for_crtc(&crtc_state->uapi, NULL) < 0);
7400         crtc_state->uapi.active = false;
7401         crtc_state->uapi.connector_mask = 0;
7402         crtc_state->uapi.encoder_mask = 0;
7403         intel_crtc_free_hw_state(crtc_state);
7404         memset(&crtc_state->hw, 0, sizeof(crtc_state->hw));
7405
7406         for_each_encoder_on_crtc(&dev_priv->drm, &crtc->base, encoder)
7407                 encoder->base.crtc = NULL;
7408
7409         intel_fbc_disable(crtc);
7410         intel_update_watermarks(crtc);
7411         intel_disable_shared_dpll(crtc_state);
7412
7413         domains = crtc->enabled_power_domains;
7414         for_each_power_domain(domain, domains)
7415                 intel_display_power_put_unchecked(dev_priv, domain);
7416         crtc->enabled_power_domains = 0;
7417
7418         dev_priv->active_pipes &= ~BIT(pipe);
7419         dev_priv->min_cdclk[pipe] = 0;
7420         dev_priv->min_voltage_level[pipe] = 0;
7421
7422         bw_state->data_rate[pipe] = 0;
7423         bw_state->num_active_planes[pipe] = 0;
7424 }
7425
7426 /*
7427  * turn all crtc's off, but do not adjust state
7428  * This has to be paired with a call to intel_modeset_setup_hw_state.
7429  */
7430 int intel_display_suspend(struct drm_device *dev)
7431 {
7432         struct drm_i915_private *dev_priv = to_i915(dev);
7433         struct drm_atomic_state *state;
7434         int ret;
7435
7436         state = drm_atomic_helper_suspend(dev);
7437         ret = PTR_ERR_OR_ZERO(state);
7438         if (ret)
7439                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
7440         else
7441                 dev_priv->modeset_restore_state = state;
7442         return ret;
7443 }
7444
7445 void intel_encoder_destroy(struct drm_encoder *encoder)
7446 {
7447         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
7448
7449         drm_encoder_cleanup(encoder);
7450         kfree(intel_encoder);
7451 }
7452
7453 /* Cross check the actual hw state with our own modeset state tracking (and it's
7454  * internal consistency). */
7455 static void intel_connector_verify_state(struct intel_crtc_state *crtc_state,
7456                                          struct drm_connector_state *conn_state)
7457 {
7458         struct intel_connector *connector = to_intel_connector(conn_state->connector);
7459
7460         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
7461                       connector->base.base.id,
7462                       connector->base.name);
7463
7464         if (connector->get_hw_state(connector)) {
7465                 struct intel_encoder *encoder = connector->encoder;
7466
7467                 I915_STATE_WARN(!crtc_state,
7468                          "connector enabled without attached crtc\n");
7469
7470                 if (!crtc_state)
7471                         return;
7472
7473                 I915_STATE_WARN(!crtc_state->hw.active,
7474                                 "connector is active, but attached crtc isn't\n");
7475
7476                 if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST)
7477                         return;
7478
7479                 I915_STATE_WARN(conn_state->best_encoder != &encoder->base,
7480                         "atomic encoder doesn't match attached encoder\n");
7481
7482                 I915_STATE_WARN(conn_state->crtc != encoder->base.crtc,
7483                         "attached encoder crtc differs from connector crtc\n");
7484         } else {
7485                 I915_STATE_WARN(crtc_state && crtc_state->hw.active,
7486                                 "attached crtc is active, but connector isn't\n");
7487                 I915_STATE_WARN(!crtc_state && conn_state->best_encoder,
7488                         "best encoder set without crtc!\n");
7489         }
7490 }
7491
7492 static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state)
7493 {
7494         if (crtc_state->hw.enable && crtc_state->has_pch_encoder)
7495                 return crtc_state->fdi_lanes;
7496
7497         return 0;
7498 }
7499
7500 static int ilk_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
7501                                struct intel_crtc_state *pipe_config)
7502 {
7503         struct drm_i915_private *dev_priv = to_i915(dev);
7504         struct drm_atomic_state *state = pipe_config->uapi.state;
7505         struct intel_crtc *other_crtc;
7506         struct intel_crtc_state *other_crtc_state;
7507
7508         DRM_DEBUG_KMS("checking fdi config on pipe %c, lanes %i\n",
7509                       pipe_name(pipe), pipe_config->fdi_lanes);
7510         if (pipe_config->fdi_lanes > 4) {
7511                 DRM_DEBUG_KMS("invalid fdi lane config on pipe %c: %i lanes\n",
7512                               pipe_name(pipe), pipe_config->fdi_lanes);
7513                 return -EINVAL;
7514         }
7515
7516         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
7517                 if (pipe_config->fdi_lanes > 2) {
7518                         DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n",
7519                                       pipe_config->fdi_lanes);
7520                         return -EINVAL;
7521                 } else {
7522                         return 0;
7523                 }
7524         }
7525
7526         if (INTEL_NUM_PIPES(dev_priv) == 2)
7527                 return 0;
7528
7529         /* Ivybridge 3 pipe is really complicated */
7530         switch (pipe) {
7531         case PIPE_A:
7532                 return 0;
7533         case PIPE_B:
7534                 if (pipe_config->fdi_lanes <= 2)
7535                         return 0;
7536
7537                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_C);
7538                 other_crtc_state =
7539                         intel_atomic_get_crtc_state(state, other_crtc);
7540                 if (IS_ERR(other_crtc_state))
7541                         return PTR_ERR(other_crtc_state);
7542
7543                 if (pipe_required_fdi_lanes(other_crtc_state) > 0) {
7544                         DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %c: %i lanes\n",
7545                                       pipe_name(pipe), pipe_config->fdi_lanes);
7546                         return -EINVAL;
7547                 }
7548                 return 0;
7549         case PIPE_C:
7550                 if (pipe_config->fdi_lanes > 2) {
7551                         DRM_DEBUG_KMS("only 2 lanes on pipe %c: required %i lanes\n",
7552                                       pipe_name(pipe), pipe_config->fdi_lanes);
7553                         return -EINVAL;
7554                 }
7555
7556                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_B);
7557                 other_crtc_state =
7558                         intel_atomic_get_crtc_state(state, other_crtc);
7559                 if (IS_ERR(other_crtc_state))
7560                         return PTR_ERR(other_crtc_state);
7561
7562                 if (pipe_required_fdi_lanes(other_crtc_state) > 2) {
7563                         DRM_DEBUG_KMS("fdi link B uses too many lanes to enable link C\n");
7564                         return -EINVAL;
7565                 }
7566                 return 0;
7567         default:
7568                 BUG();
7569         }
7570 }
7571
7572 #define RETRY 1
7573 static int ilk_fdi_compute_config(struct intel_crtc *intel_crtc,
7574                                   struct intel_crtc_state *pipe_config)
7575 {
7576         struct drm_device *dev = intel_crtc->base.dev;
7577         const struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
7578         int lane, link_bw, fdi_dotclock, ret;
7579         bool needs_recompute = false;
7580
7581 retry:
7582         /* FDI is a binary signal running at ~2.7GHz, encoding
7583          * each output octet as 10 bits. The actual frequency
7584          * is stored as a divider into a 100MHz clock, and the
7585          * mode pixel clock is stored in units of 1KHz.
7586          * Hence the bw of each lane in terms of the mode signal
7587          * is:
7588          */
7589         link_bw = intel_fdi_link_freq(to_i915(dev), pipe_config);
7590
7591         fdi_dotclock = adjusted_mode->crtc_clock;
7592
7593         lane = ilk_get_lanes_required(fdi_dotclock, link_bw,
7594                                       pipe_config->pipe_bpp);
7595
7596         pipe_config->fdi_lanes = lane;
7597
7598         intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock,
7599                                link_bw, &pipe_config->fdi_m_n, false, false);
7600
7601         ret = ilk_check_fdi_lanes(dev, intel_crtc->pipe, pipe_config);
7602         if (ret == -EDEADLK)
7603                 return ret;
7604
7605         if (ret == -EINVAL && pipe_config->pipe_bpp > 6*3) {
7606                 pipe_config->pipe_bpp -= 2*3;
7607                 DRM_DEBUG_KMS("fdi link bw constraint, reducing pipe bpp to %i\n",
7608                               pipe_config->pipe_bpp);
7609                 needs_recompute = true;
7610                 pipe_config->bw_constrained = true;
7611
7612                 goto retry;
7613         }
7614
7615         if (needs_recompute)
7616                 return RETRY;
7617
7618         return ret;
7619 }
7620
7621 bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state)
7622 {
7623         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7624         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7625
7626         /* IPS only exists on ULT machines and is tied to pipe A. */
7627         if (!hsw_crtc_supports_ips(crtc))
7628                 return false;
7629
7630         if (!i915_modparams.enable_ips)
7631                 return false;
7632
7633         if (crtc_state->pipe_bpp > 24)
7634                 return false;
7635
7636         /*
7637          * We compare against max which means we must take
7638          * the increased cdclk requirement into account when
7639          * calculating the new cdclk.
7640          *
7641          * Should measure whether using a lower cdclk w/o IPS
7642          */
7643         if (IS_BROADWELL(dev_priv) &&
7644             crtc_state->pixel_rate > dev_priv->max_cdclk_freq * 95 / 100)
7645                 return false;
7646
7647         return true;
7648 }
7649
7650 static bool hsw_compute_ips_config(struct intel_crtc_state *crtc_state)
7651 {
7652         struct drm_i915_private *dev_priv =
7653                 to_i915(crtc_state->uapi.crtc->dev);
7654         struct intel_atomic_state *intel_state =
7655                 to_intel_atomic_state(crtc_state->uapi.state);
7656
7657         if (!hsw_crtc_state_ips_capable(crtc_state))
7658                 return false;
7659
7660         /*
7661          * When IPS gets enabled, the pipe CRC changes. Since IPS gets
7662          * enabled and disabled dynamically based on package C states,
7663          * user space can't make reliable use of the CRCs, so let's just
7664          * completely disable it.
7665          */
7666         if (crtc_state->crc_enabled)
7667                 return false;
7668
7669         /* IPS should be fine as long as at least one plane is enabled. */
7670         if (!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)))
7671                 return false;
7672
7673         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
7674         if (IS_BROADWELL(dev_priv) &&
7675             crtc_state->pixel_rate > intel_state->cdclk.logical.cdclk * 95 / 100)
7676                 return false;
7677
7678         return true;
7679 }
7680
7681 static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc)
7682 {
7683         const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7684
7685         /* GDG double wide on either pipe, otherwise pipe A only */
7686         return INTEL_GEN(dev_priv) < 4 &&
7687                 (crtc->pipe == PIPE_A || IS_I915G(dev_priv));
7688 }
7689
7690 static u32 ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config)
7691 {
7692         u32 pixel_rate;
7693
7694         pixel_rate = pipe_config->hw.adjusted_mode.crtc_clock;
7695
7696         /*
7697          * We only use IF-ID interlacing. If we ever use
7698          * PF-ID we'll need to adjust the pixel_rate here.
7699          */
7700
7701         if (pipe_config->pch_pfit.enabled) {
7702                 u64 pipe_w, pipe_h, pfit_w, pfit_h;
7703                 u32 pfit_size = pipe_config->pch_pfit.size;
7704
7705                 pipe_w = pipe_config->pipe_src_w;
7706                 pipe_h = pipe_config->pipe_src_h;
7707
7708                 pfit_w = (pfit_size >> 16) & 0xFFFF;
7709                 pfit_h = pfit_size & 0xFFFF;
7710                 if (pipe_w < pfit_w)
7711                         pipe_w = pfit_w;
7712                 if (pipe_h < pfit_h)
7713                         pipe_h = pfit_h;
7714
7715                 if (WARN_ON(!pfit_w || !pfit_h))
7716                         return pixel_rate;
7717
7718                 pixel_rate = div_u64(mul_u32_u32(pixel_rate, pipe_w * pipe_h),
7719                                      pfit_w * pfit_h);
7720         }
7721
7722         return pixel_rate;
7723 }
7724
7725 static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state)
7726 {
7727         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
7728
7729         if (HAS_GMCH(dev_priv))
7730                 /* FIXME calculate proper pipe pixel rate for GMCH pfit */
7731                 crtc_state->pixel_rate =
7732                         crtc_state->hw.adjusted_mode.crtc_clock;
7733         else
7734                 crtc_state->pixel_rate =
7735                         ilk_pipe_pixel_rate(crtc_state);
7736 }
7737
7738 static int intel_crtc_compute_config(struct intel_crtc *crtc,
7739                                      struct intel_crtc_state *pipe_config)
7740 {
7741         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7742         const struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
7743         int clock_limit = dev_priv->max_dotclk_freq;
7744
7745         if (INTEL_GEN(dev_priv) < 4) {
7746                 clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
7747
7748                 /*
7749                  * Enable double wide mode when the dot clock
7750                  * is > 90% of the (display) core speed.
7751                  */
7752                 if (intel_crtc_supports_double_wide(crtc) &&
7753                     adjusted_mode->crtc_clock > clock_limit) {
7754                         clock_limit = dev_priv->max_dotclk_freq;
7755                         pipe_config->double_wide = true;
7756                 }
7757         }
7758
7759         if (adjusted_mode->crtc_clock > clock_limit) {
7760                 DRM_DEBUG_KMS("requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
7761                               adjusted_mode->crtc_clock, clock_limit,
7762                               yesno(pipe_config->double_wide));
7763                 return -EINVAL;
7764         }
7765
7766         if ((pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
7767              pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) &&
7768              pipe_config->hw.ctm) {
7769                 /*
7770                  * There is only one pipe CSC unit per pipe, and we need that
7771                  * for output conversion from RGB->YCBCR. So if CTM is already
7772                  * applied we can't support YCBCR420 output.
7773                  */
7774                 DRM_DEBUG_KMS("YCBCR420 and CTM together are not possible\n");
7775                 return -EINVAL;
7776         }
7777
7778         /*
7779          * Pipe horizontal size must be even in:
7780          * - DVO ganged mode
7781          * - LVDS dual channel mode
7782          * - Double wide pipe
7783          */
7784         if (pipe_config->pipe_src_w & 1) {
7785                 if (pipe_config->double_wide) {
7786                         DRM_DEBUG_KMS("Odd pipe source width not supported with double wide pipe\n");
7787                         return -EINVAL;
7788                 }
7789
7790                 if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_LVDS) &&
7791                     intel_is_dual_link_lvds(dev_priv)) {
7792                         DRM_DEBUG_KMS("Odd pipe source width not supported with dual link LVDS\n");
7793                         return -EINVAL;
7794                 }
7795         }
7796
7797         /* Cantiga+ cannot handle modes with a hsync front porch of 0.
7798          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
7799          */
7800         if ((INTEL_GEN(dev_priv) > 4 || IS_G4X(dev_priv)) &&
7801                 adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay)
7802                 return -EINVAL;
7803
7804         intel_crtc_compute_pixel_rate(pipe_config);
7805
7806         if (pipe_config->has_pch_encoder)
7807                 return ilk_fdi_compute_config(crtc, pipe_config);
7808
7809         return 0;
7810 }
7811
7812 static void
7813 intel_reduce_m_n_ratio(u32 *num, u32 *den)
7814 {
7815         while (*num > DATA_LINK_M_N_MASK ||
7816                *den > DATA_LINK_M_N_MASK) {
7817                 *num >>= 1;
7818                 *den >>= 1;
7819         }
7820 }
7821
7822 static void compute_m_n(unsigned int m, unsigned int n,
7823                         u32 *ret_m, u32 *ret_n,
7824                         bool constant_n)
7825 {
7826         /*
7827          * Several DP dongles in particular seem to be fussy about
7828          * too large link M/N values. Give N value as 0x8000 that
7829          * should be acceptable by specific devices. 0x8000 is the
7830          * specified fixed N value for asynchronous clock mode,
7831          * which the devices expect also in synchronous clock mode.
7832          */
7833         if (constant_n)
7834                 *ret_n = 0x8000;
7835         else
7836                 *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
7837
7838         *ret_m = div_u64(mul_u32_u32(m, *ret_n), n);
7839         intel_reduce_m_n_ratio(ret_m, ret_n);
7840 }
7841
7842 void
7843 intel_link_compute_m_n(u16 bits_per_pixel, int nlanes,
7844                        int pixel_clock, int link_clock,
7845                        struct intel_link_m_n *m_n,
7846                        bool constant_n, bool fec_enable)
7847 {
7848         u32 data_clock = bits_per_pixel * pixel_clock;
7849
7850         if (fec_enable)
7851                 data_clock = intel_dp_mode_to_fec_clock(data_clock);
7852
7853         m_n->tu = 64;
7854         compute_m_n(data_clock,
7855                     link_clock * nlanes * 8,
7856                     &m_n->gmch_m, &m_n->gmch_n,
7857                     constant_n);
7858
7859         compute_m_n(pixel_clock, link_clock,
7860                     &m_n->link_m, &m_n->link_n,
7861                     constant_n);
7862 }
7863
7864 static void intel_panel_sanitize_ssc(struct drm_i915_private *dev_priv)
7865 {
7866         /*
7867          * There may be no VBT; and if the BIOS enabled SSC we can
7868          * just keep using it to avoid unnecessary flicker.  Whereas if the
7869          * BIOS isn't using it, don't assume it will work even if the VBT
7870          * indicates as much.
7871          */
7872         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
7873                 bool bios_lvds_use_ssc = I915_READ(PCH_DREF_CONTROL) &
7874                         DREF_SSC1_ENABLE;
7875
7876                 if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) {
7877                         DRM_DEBUG_KMS("SSC %s by BIOS, overriding VBT which says %s\n",
7878                                       enableddisabled(bios_lvds_use_ssc),
7879                                       enableddisabled(dev_priv->vbt.lvds_use_ssc));
7880                         dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc;
7881                 }
7882         }
7883 }
7884
7885 static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
7886 {
7887         if (i915_modparams.panel_use_ssc >= 0)
7888                 return i915_modparams.panel_use_ssc != 0;
7889         return dev_priv->vbt.lvds_use_ssc
7890                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
7891 }
7892
7893 static u32 pnv_dpll_compute_fp(struct dpll *dpll)
7894 {
7895         return (1 << dpll->n) << 16 | dpll->m2;
7896 }
7897
7898 static u32 i9xx_dpll_compute_fp(struct dpll *dpll)
7899 {
7900         return dpll->n << 16 | dpll->m1 << 8 | dpll->m2;
7901 }
7902
7903 static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
7904                                      struct intel_crtc_state *crtc_state,
7905                                      struct dpll *reduced_clock)
7906 {
7907         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7908         u32 fp, fp2 = 0;
7909
7910         if (IS_PINEVIEW(dev_priv)) {
7911                 fp = pnv_dpll_compute_fp(&crtc_state->dpll);
7912                 if (reduced_clock)
7913                         fp2 = pnv_dpll_compute_fp(reduced_clock);
7914         } else {
7915                 fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
7916                 if (reduced_clock)
7917                         fp2 = i9xx_dpll_compute_fp(reduced_clock);
7918         }
7919
7920         crtc_state->dpll_hw_state.fp0 = fp;
7921
7922         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7923             reduced_clock) {
7924                 crtc_state->dpll_hw_state.fp1 = fp2;
7925         } else {
7926                 crtc_state->dpll_hw_state.fp1 = fp;
7927         }
7928 }
7929
7930 static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe
7931                 pipe)
7932 {
7933         u32 reg_val;
7934
7935         /*
7936          * PLLB opamp always calibrates to max value of 0x3f, force enable it
7937          * and set it to a reasonable value instead.
7938          */
7939         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7940         reg_val &= 0xffffff00;
7941         reg_val |= 0x00000030;
7942         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7943
7944         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7945         reg_val &= 0x00ffffff;
7946         reg_val |= 0x8c000000;
7947         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7948
7949         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7950         reg_val &= 0xffffff00;
7951         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7952
7953         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7954         reg_val &= 0x00ffffff;
7955         reg_val |= 0xb0000000;
7956         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7957 }
7958
7959 static void intel_pch_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
7960                                          const struct intel_link_m_n *m_n)
7961 {
7962         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7963         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7964         enum pipe pipe = crtc->pipe;
7965
7966         I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
7967         I915_WRITE(PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
7968         I915_WRITE(PCH_TRANS_LINK_M1(pipe), m_n->link_m);
7969         I915_WRITE(PCH_TRANS_LINK_N1(pipe), m_n->link_n);
7970 }
7971
7972 static bool transcoder_has_m2_n2(struct drm_i915_private *dev_priv,
7973                                  enum transcoder transcoder)
7974 {
7975         if (IS_HASWELL(dev_priv))
7976                 return transcoder == TRANSCODER_EDP;
7977
7978         /*
7979          * Strictly speaking some registers are available before
7980          * gen7, but we only support DRRS on gen7+
7981          */
7982         return IS_GEN(dev_priv, 7) || IS_CHERRYVIEW(dev_priv);
7983 }
7984
7985 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
7986                                          const struct intel_link_m_n *m_n,
7987                                          const struct intel_link_m_n *m2_n2)
7988 {
7989         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7990         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7991         enum pipe pipe = crtc->pipe;
7992         enum transcoder transcoder = crtc_state->cpu_transcoder;
7993
7994         if (INTEL_GEN(dev_priv) >= 5) {
7995                 I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m);
7996                 I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
7997                 I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
7998                 I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n);
7999                 /*
8000                  *  M2_N2 registers are set only if DRRS is supported
8001                  * (to make sure the registers are not unnecessarily accessed).
8002                  */
8003                 if (m2_n2 && crtc_state->has_drrs &&
8004                     transcoder_has_m2_n2(dev_priv, transcoder)) {
8005                         I915_WRITE(PIPE_DATA_M2(transcoder),
8006                                         TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
8007                         I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n);
8008                         I915_WRITE(PIPE_LINK_M2(transcoder), m2_n2->link_m);
8009                         I915_WRITE(PIPE_LINK_N2(transcoder), m2_n2->link_n);
8010                 }
8011         } else {
8012                 I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
8013                 I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
8014                 I915_WRITE(PIPE_LINK_M_G4X(pipe), m_n->link_m);
8015                 I915_WRITE(PIPE_LINK_N_G4X(pipe), m_n->link_n);
8016         }
8017 }
8018
8019 void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, enum link_m_n_set m_n)
8020 {
8021         const struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL;
8022
8023         if (m_n == M1_N1) {
8024                 dp_m_n = &crtc_state->dp_m_n;
8025                 dp_m2_n2 = &crtc_state->dp_m2_n2;
8026         } else if (m_n == M2_N2) {
8027
8028                 /*
8029                  * M2_N2 registers are not supported. Hence m2_n2 divider value
8030                  * needs to be programmed into M1_N1.
8031                  */
8032                 dp_m_n = &crtc_state->dp_m2_n2;
8033         } else {
8034                 DRM_ERROR("Unsupported divider value\n");
8035                 return;
8036         }
8037
8038         if (crtc_state->has_pch_encoder)
8039                 intel_pch_transcoder_set_m_n(crtc_state, &crtc_state->dp_m_n);
8040         else
8041                 intel_cpu_transcoder_set_m_n(crtc_state, dp_m_n, dp_m2_n2);
8042 }
8043
8044 static void vlv_compute_dpll(struct intel_crtc *crtc,
8045                              struct intel_crtc_state *pipe_config)
8046 {
8047         pipe_config->dpll_hw_state.dpll = DPLL_INTEGRATED_REF_CLK_VLV |
8048                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
8049         if (crtc->pipe != PIPE_A)
8050                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
8051
8052         /* DPLL not used with DSI, but still need the rest set up */
8053         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
8054                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE |
8055                         DPLL_EXT_BUFFER_ENABLE_VLV;
8056
8057         pipe_config->dpll_hw_state.dpll_md =
8058                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
8059 }
8060
8061 static void chv_compute_dpll(struct intel_crtc *crtc,
8062                              struct intel_crtc_state *pipe_config)
8063 {
8064         pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV |
8065                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
8066         if (crtc->pipe != PIPE_A)
8067                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
8068
8069         /* DPLL not used with DSI, but still need the rest set up */
8070         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
8071                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE;
8072
8073         pipe_config->dpll_hw_state.dpll_md =
8074                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
8075 }
8076
8077 static void vlv_prepare_pll(struct intel_crtc *crtc,
8078                             const struct intel_crtc_state *pipe_config)
8079 {
8080         struct drm_device *dev = crtc->base.dev;
8081         struct drm_i915_private *dev_priv = to_i915(dev);
8082         enum pipe pipe = crtc->pipe;
8083         u32 mdiv;
8084         u32 bestn, bestm1, bestm2, bestp1, bestp2;
8085         u32 coreclk, reg_val;
8086
8087         /* Enable Refclk */
8088         I915_WRITE(DPLL(pipe),
8089                    pipe_config->dpll_hw_state.dpll &
8090                    ~(DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV));
8091
8092         /* No need to actually set up the DPLL with DSI */
8093         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8094                 return;
8095
8096         vlv_dpio_get(dev_priv);
8097
8098         bestn = pipe_config->dpll.n;
8099         bestm1 = pipe_config->dpll.m1;
8100         bestm2 = pipe_config->dpll.m2;
8101         bestp1 = pipe_config->dpll.p1;
8102         bestp2 = pipe_config->dpll.p2;
8103
8104         /* See eDP HDMI DPIO driver vbios notes doc */
8105
8106         /* PLL B needs special handling */
8107         if (pipe == PIPE_B)
8108                 vlv_pllb_recal_opamp(dev_priv, pipe);
8109
8110         /* Set up Tx target for periodic Rcomp update */
8111         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9_BCAST, 0x0100000f);
8112
8113         /* Disable target IRef on PLL */
8114         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW8(pipe));
8115         reg_val &= 0x00ffffff;
8116         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW8(pipe), reg_val);
8117
8118         /* Disable fast lock */
8119         vlv_dpio_write(dev_priv, pipe, VLV_CMN_DW0, 0x610);
8120
8121         /* Set idtafcrecal before PLL is enabled */
8122         mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
8123         mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT));
8124         mdiv |= ((bestn << DPIO_N_SHIFT));
8125         mdiv |= (1 << DPIO_K_SHIFT);
8126
8127         /*
8128          * Post divider depends on pixel clock rate, DAC vs digital (and LVDS,
8129          * but we don't support that).
8130          * Note: don't use the DAC post divider as it seems unstable.
8131          */
8132         mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
8133         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
8134
8135         mdiv |= DPIO_ENABLE_CALIBRATION;
8136         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
8137
8138         /* Set HBR and RBR LPF coefficients */
8139         if (pipe_config->port_clock == 162000 ||
8140             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_ANALOG) ||
8141             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI))
8142                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
8143                                  0x009f0003);
8144         else
8145                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
8146                                  0x00d0000f);
8147
8148         if (intel_crtc_has_dp_encoder(pipe_config)) {
8149                 /* Use SSC source */
8150                 if (pipe == PIPE_A)
8151                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
8152                                          0x0df40000);
8153                 else
8154                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
8155                                          0x0df70000);
8156         } else { /* HDMI or VGA */
8157                 /* Use bend source */
8158                 if (pipe == PIPE_A)
8159                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
8160                                          0x0df70000);
8161                 else
8162                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
8163                                          0x0df40000);
8164         }
8165
8166         coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe));
8167         coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
8168         if (intel_crtc_has_dp_encoder(pipe_config))
8169                 coreclk |= 0x01000000;
8170         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk);
8171
8172         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000);
8173
8174         vlv_dpio_put(dev_priv);
8175 }
8176
8177 static void chv_prepare_pll(struct intel_crtc *crtc,
8178                             const struct intel_crtc_state *pipe_config)
8179 {
8180         struct drm_device *dev = crtc->base.dev;
8181         struct drm_i915_private *dev_priv = to_i915(dev);
8182         enum pipe pipe = crtc->pipe;
8183         enum dpio_channel port = vlv_pipe_to_channel(pipe);
8184         u32 loopfilter, tribuf_calcntr;
8185         u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
8186         u32 dpio_val;
8187         int vco;
8188
8189         /* Enable Refclk and SSC */
8190         I915_WRITE(DPLL(pipe),
8191                    pipe_config->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE);
8192
8193         /* No need to actually set up the DPLL with DSI */
8194         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8195                 return;
8196
8197         bestn = pipe_config->dpll.n;
8198         bestm2_frac = pipe_config->dpll.m2 & 0x3fffff;
8199         bestm1 = pipe_config->dpll.m1;
8200         bestm2 = pipe_config->dpll.m2 >> 22;
8201         bestp1 = pipe_config->dpll.p1;
8202         bestp2 = pipe_config->dpll.p2;
8203         vco = pipe_config->dpll.vco;
8204         dpio_val = 0;
8205         loopfilter = 0;
8206
8207         vlv_dpio_get(dev_priv);
8208
8209         /* p1 and p2 divider */
8210         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port),
8211                         5 << DPIO_CHV_S1_DIV_SHIFT |
8212                         bestp1 << DPIO_CHV_P1_DIV_SHIFT |
8213                         bestp2 << DPIO_CHV_P2_DIV_SHIFT |
8214                         1 << DPIO_CHV_K_DIV_SHIFT);
8215
8216         /* Feedback post-divider - m2 */
8217         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2);
8218
8219         /* Feedback refclk divider - n and m1 */
8220         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port),
8221                         DPIO_CHV_M1_DIV_BY_2 |
8222                         1 << DPIO_CHV_N_DIV_SHIFT);
8223
8224         /* M2 fraction division */
8225         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
8226
8227         /* M2 fraction division enable */
8228         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
8229         dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
8230         dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT);
8231         if (bestm2_frac)
8232                 dpio_val |= DPIO_CHV_FRAC_DIV_EN;
8233         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), dpio_val);
8234
8235         /* Program digital lock detect threshold */
8236         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW9(port));
8237         dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
8238                                         DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
8239         dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT);
8240         if (!bestm2_frac)
8241                 dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
8242         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW9(port), dpio_val);
8243
8244         /* Loop filter */
8245         if (vco == 5400000) {
8246                 loopfilter |= (0x3 << DPIO_CHV_PROP_COEFF_SHIFT);
8247                 loopfilter |= (0x8 << DPIO_CHV_INT_COEFF_SHIFT);
8248                 loopfilter |= (0x1 << DPIO_CHV_GAIN_CTRL_SHIFT);
8249                 tribuf_calcntr = 0x9;
8250         } else if (vco <= 6200000) {
8251                 loopfilter |= (0x5 << DPIO_CHV_PROP_COEFF_SHIFT);
8252                 loopfilter |= (0xB << DPIO_CHV_INT_COEFF_SHIFT);
8253                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
8254                 tribuf_calcntr = 0x9;
8255         } else if (vco <= 6480000) {
8256                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
8257                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
8258                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
8259                 tribuf_calcntr = 0x8;
8260         } else {
8261                 /* Not supported. Apply the same limits as in the max case */
8262                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
8263                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
8264                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
8265                 tribuf_calcntr = 0;
8266         }
8267         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter);
8268
8269         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW8(port));
8270         dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
8271         dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT);
8272         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW8(port), dpio_val);
8273
8274         /* AFC Recal */
8275         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port),
8276                         vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) |
8277                         DPIO_AFC_RECAL);
8278
8279         vlv_dpio_put(dev_priv);
8280 }
8281
8282 /**
8283  * vlv_force_pll_on - forcibly enable just the PLL
8284  * @dev_priv: i915 private structure
8285  * @pipe: pipe PLL to enable
8286  * @dpll: PLL configuration
8287  *
8288  * Enable the PLL for @pipe using the supplied @dpll config. To be used
8289  * in cases where we need the PLL enabled even when @pipe is not going to
8290  * be enabled.
8291  */
8292 int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe,
8293                      const struct dpll *dpll)
8294 {
8295         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
8296         struct intel_crtc_state *pipe_config;
8297
8298         pipe_config = intel_crtc_state_alloc(crtc);
8299         if (!pipe_config)
8300                 return -ENOMEM;
8301
8302         pipe_config->cpu_transcoder = (enum transcoder)pipe;
8303         pipe_config->pixel_multiplier = 1;
8304         pipe_config->dpll = *dpll;
8305
8306         if (IS_CHERRYVIEW(dev_priv)) {
8307                 chv_compute_dpll(crtc, pipe_config);
8308                 chv_prepare_pll(crtc, pipe_config);
8309                 chv_enable_pll(crtc, pipe_config);
8310         } else {
8311                 vlv_compute_dpll(crtc, pipe_config);
8312                 vlv_prepare_pll(crtc, pipe_config);
8313                 vlv_enable_pll(crtc, pipe_config);
8314         }
8315
8316         kfree(pipe_config);
8317
8318         return 0;
8319 }
8320
8321 /**
8322  * vlv_force_pll_off - forcibly disable just the PLL
8323  * @dev_priv: i915 private structure
8324  * @pipe: pipe PLL to disable
8325  *
8326  * Disable the PLL for @pipe. To be used in cases where we need
8327  * the PLL enabled even when @pipe is not going to be enabled.
8328  */
8329 void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe)
8330 {
8331         if (IS_CHERRYVIEW(dev_priv))
8332                 chv_disable_pll(dev_priv, pipe);
8333         else
8334                 vlv_disable_pll(dev_priv, pipe);
8335 }
8336
8337 static void i9xx_compute_dpll(struct intel_crtc *crtc,
8338                               struct intel_crtc_state *crtc_state,
8339                               struct dpll *reduced_clock)
8340 {
8341         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8342         u32 dpll;
8343         struct dpll *clock = &crtc_state->dpll;
8344
8345         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
8346
8347         dpll = DPLL_VGA_MODE_DIS;
8348
8349         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
8350                 dpll |= DPLLB_MODE_LVDS;
8351         else
8352                 dpll |= DPLLB_MODE_DAC_SERIAL;
8353
8354         if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
8355             IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
8356                 dpll |= (crtc_state->pixel_multiplier - 1)
8357                         << SDVO_MULTIPLIER_SHIFT_HIRES;
8358         }
8359
8360         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
8361             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
8362                 dpll |= DPLL_SDVO_HIGH_SPEED;
8363
8364         if (intel_crtc_has_dp_encoder(crtc_state))
8365                 dpll |= DPLL_SDVO_HIGH_SPEED;
8366
8367         /* compute bitmask from p1 value */
8368         if (IS_PINEVIEW(dev_priv))
8369                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
8370         else {
8371                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8372                 if (IS_G4X(dev_priv) && reduced_clock)
8373                         dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
8374         }
8375         switch (clock->p2) {
8376         case 5:
8377                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
8378                 break;
8379         case 7:
8380                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
8381                 break;
8382         case 10:
8383                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
8384                 break;
8385         case 14:
8386                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
8387                 break;
8388         }
8389         if (INTEL_GEN(dev_priv) >= 4)
8390                 dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
8391
8392         if (crtc_state->sdvo_tv_clock)
8393                 dpll |= PLL_REF_INPUT_TVCLKINBC;
8394         else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8395                  intel_panel_use_ssc(dev_priv))
8396                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8397         else
8398                 dpll |= PLL_REF_INPUT_DREFCLK;
8399
8400         dpll |= DPLL_VCO_ENABLE;
8401         crtc_state->dpll_hw_state.dpll = dpll;
8402
8403         if (INTEL_GEN(dev_priv) >= 4) {
8404                 u32 dpll_md = (crtc_state->pixel_multiplier - 1)
8405                         << DPLL_MD_UDI_MULTIPLIER_SHIFT;
8406                 crtc_state->dpll_hw_state.dpll_md = dpll_md;
8407         }
8408 }
8409
8410 static void i8xx_compute_dpll(struct intel_crtc *crtc,
8411                               struct intel_crtc_state *crtc_state,
8412                               struct dpll *reduced_clock)
8413 {
8414         struct drm_device *dev = crtc->base.dev;
8415         struct drm_i915_private *dev_priv = to_i915(dev);
8416         u32 dpll;
8417         struct dpll *clock = &crtc_state->dpll;
8418
8419         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
8420
8421         dpll = DPLL_VGA_MODE_DIS;
8422
8423         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8424                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8425         } else {
8426                 if (clock->p1 == 2)
8427                         dpll |= PLL_P1_DIVIDE_BY_TWO;
8428                 else
8429                         dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8430                 if (clock->p2 == 4)
8431                         dpll |= PLL_P2_DIVIDE_BY_4;
8432         }
8433
8434         /*
8435          * Bspec:
8436          * "[Almador Errata}: For the correct operation of the muxed DVO pins
8437          *  (GDEVSELB/I2Cdata, GIRDBY/I2CClk) and (GFRAMEB/DVI_Data,
8438          *  GTRDYB/DVI_Clk): Bit 31 (DPLL VCO Enable) and Bit 30 (2X Clock
8439          *  Enable) must be set to “1” in both the DPLL A Control Register
8440          *  (06014h-06017h) and DPLL B Control Register (06018h-0601Bh)."
8441          *
8442          * For simplicity We simply keep both bits always enabled in
8443          * both DPLLS. The spec says we should disable the DVO 2X clock
8444          * when not needed, but this seems to work fine in practice.
8445          */
8446         if (IS_I830(dev_priv) ||
8447             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO))
8448                 dpll |= DPLL_DVO_2X_MODE;
8449
8450         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8451             intel_panel_use_ssc(dev_priv))
8452                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8453         else
8454                 dpll |= PLL_REF_INPUT_DREFCLK;
8455
8456         dpll |= DPLL_VCO_ENABLE;
8457         crtc_state->dpll_hw_state.dpll = dpll;
8458 }
8459
8460 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state)
8461 {
8462         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8463         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8464         enum pipe pipe = crtc->pipe;
8465         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
8466         const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
8467         u32 crtc_vtotal, crtc_vblank_end;
8468         int vsyncshift = 0;
8469
8470         /* We need to be careful not to changed the adjusted mode, for otherwise
8471          * the hw state checker will get angry at the mismatch. */
8472         crtc_vtotal = adjusted_mode->crtc_vtotal;
8473         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
8474
8475         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
8476                 /* the chip adds 2 halflines automatically */
8477                 crtc_vtotal -= 1;
8478                 crtc_vblank_end -= 1;
8479
8480                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
8481                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
8482                 else
8483                         vsyncshift = adjusted_mode->crtc_hsync_start -
8484                                 adjusted_mode->crtc_htotal / 2;
8485                 if (vsyncshift < 0)
8486                         vsyncshift += adjusted_mode->crtc_htotal;
8487         }
8488
8489         if (INTEL_GEN(dev_priv) > 3)
8490                 I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift);
8491
8492         I915_WRITE(HTOTAL(cpu_transcoder),
8493                    (adjusted_mode->crtc_hdisplay - 1) |
8494                    ((adjusted_mode->crtc_htotal - 1) << 16));
8495         I915_WRITE(HBLANK(cpu_transcoder),
8496                    (adjusted_mode->crtc_hblank_start - 1) |
8497                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
8498         I915_WRITE(HSYNC(cpu_transcoder),
8499                    (adjusted_mode->crtc_hsync_start - 1) |
8500                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
8501
8502         I915_WRITE(VTOTAL(cpu_transcoder),
8503                    (adjusted_mode->crtc_vdisplay - 1) |
8504                    ((crtc_vtotal - 1) << 16));
8505         I915_WRITE(VBLANK(cpu_transcoder),
8506                    (adjusted_mode->crtc_vblank_start - 1) |
8507                    ((crtc_vblank_end - 1) << 16));
8508         I915_WRITE(VSYNC(cpu_transcoder),
8509                    (adjusted_mode->crtc_vsync_start - 1) |
8510                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
8511
8512         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
8513          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
8514          * documented on the DDI_FUNC_CTL register description, EDP Input Select
8515          * bits. */
8516         if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP &&
8517             (pipe == PIPE_B || pipe == PIPE_C))
8518                 I915_WRITE(VTOTAL(pipe), I915_READ(VTOTAL(cpu_transcoder)));
8519
8520 }
8521
8522 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state)
8523 {
8524         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8525         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8526         enum pipe pipe = crtc->pipe;
8527
8528         /* pipesrc controls the size that is scaled from, which should
8529          * always be the user's requested size.
8530          */
8531         I915_WRITE(PIPESRC(pipe),
8532                    ((crtc_state->pipe_src_w - 1) << 16) |
8533                    (crtc_state->pipe_src_h - 1));
8534 }
8535
8536 static bool intel_pipe_is_interlaced(const struct intel_crtc_state *crtc_state)
8537 {
8538         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
8539         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
8540
8541         if (IS_GEN(dev_priv, 2))
8542                 return false;
8543
8544         if (INTEL_GEN(dev_priv) >= 9 ||
8545             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
8546                 return I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK_HSW;
8547         else
8548                 return I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK;
8549 }
8550
8551 static void intel_get_pipe_timings(struct intel_crtc *crtc,
8552                                    struct intel_crtc_state *pipe_config)
8553 {
8554         struct drm_device *dev = crtc->base.dev;
8555         struct drm_i915_private *dev_priv = to_i915(dev);
8556         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
8557         u32 tmp;
8558
8559         tmp = I915_READ(HTOTAL(cpu_transcoder));
8560         pipe_config->hw.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
8561         pipe_config->hw.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
8562
8563         if (!transcoder_is_dsi(cpu_transcoder)) {
8564                 tmp = I915_READ(HBLANK(cpu_transcoder));
8565                 pipe_config->hw.adjusted_mode.crtc_hblank_start =
8566                                                         (tmp & 0xffff) + 1;
8567                 pipe_config->hw.adjusted_mode.crtc_hblank_end =
8568                                                 ((tmp >> 16) & 0xffff) + 1;
8569         }
8570         tmp = I915_READ(HSYNC(cpu_transcoder));
8571         pipe_config->hw.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
8572         pipe_config->hw.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
8573
8574         tmp = I915_READ(VTOTAL(cpu_transcoder));
8575         pipe_config->hw.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
8576         pipe_config->hw.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
8577
8578         if (!transcoder_is_dsi(cpu_transcoder)) {
8579                 tmp = I915_READ(VBLANK(cpu_transcoder));
8580                 pipe_config->hw.adjusted_mode.crtc_vblank_start =
8581                                                         (tmp & 0xffff) + 1;
8582                 pipe_config->hw.adjusted_mode.crtc_vblank_end =
8583                                                 ((tmp >> 16) & 0xffff) + 1;
8584         }
8585         tmp = I915_READ(VSYNC(cpu_transcoder));
8586         pipe_config->hw.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
8587         pipe_config->hw.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
8588
8589         if (intel_pipe_is_interlaced(pipe_config)) {
8590                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
8591                 pipe_config->hw.adjusted_mode.crtc_vtotal += 1;
8592                 pipe_config->hw.adjusted_mode.crtc_vblank_end += 1;
8593         }
8594 }
8595
8596 static void intel_get_pipe_src_size(struct intel_crtc *crtc,
8597                                     struct intel_crtc_state *pipe_config)
8598 {
8599         struct drm_device *dev = crtc->base.dev;
8600         struct drm_i915_private *dev_priv = to_i915(dev);
8601         u32 tmp;
8602
8603         tmp = I915_READ(PIPESRC(crtc->pipe));
8604         pipe_config->pipe_src_h = (tmp & 0xffff) + 1;
8605         pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1;
8606
8607         pipe_config->hw.mode.vdisplay = pipe_config->pipe_src_h;
8608         pipe_config->hw.mode.hdisplay = pipe_config->pipe_src_w;
8609 }
8610
8611 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
8612                                  struct intel_crtc_state *pipe_config)
8613 {
8614         mode->hdisplay = pipe_config->hw.adjusted_mode.crtc_hdisplay;
8615         mode->htotal = pipe_config->hw.adjusted_mode.crtc_htotal;
8616         mode->hsync_start = pipe_config->hw.adjusted_mode.crtc_hsync_start;
8617         mode->hsync_end = pipe_config->hw.adjusted_mode.crtc_hsync_end;
8618
8619         mode->vdisplay = pipe_config->hw.adjusted_mode.crtc_vdisplay;
8620         mode->vtotal = pipe_config->hw.adjusted_mode.crtc_vtotal;
8621         mode->vsync_start = pipe_config->hw.adjusted_mode.crtc_vsync_start;
8622         mode->vsync_end = pipe_config->hw.adjusted_mode.crtc_vsync_end;
8623
8624         mode->flags = pipe_config->hw.adjusted_mode.flags;
8625         mode->type = DRM_MODE_TYPE_DRIVER;
8626
8627         mode->clock = pipe_config->hw.adjusted_mode.crtc_clock;
8628
8629         mode->hsync = drm_mode_hsync(mode);
8630         mode->vrefresh = drm_mode_vrefresh(mode);
8631         drm_mode_set_name(mode);
8632 }
8633
8634 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
8635 {
8636         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8637         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8638         u32 pipeconf;
8639
8640         pipeconf = 0;
8641
8642         /* we keep both pipes enabled on 830 */
8643         if (IS_I830(dev_priv))
8644                 pipeconf |= I915_READ(PIPECONF(crtc->pipe)) & PIPECONF_ENABLE;
8645
8646         if (crtc_state->double_wide)
8647                 pipeconf |= PIPECONF_DOUBLE_WIDE;
8648
8649         /* only g4x and later have fancy bpc/dither controls */
8650         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
8651             IS_CHERRYVIEW(dev_priv)) {
8652                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
8653                 if (crtc_state->dither && crtc_state->pipe_bpp != 30)
8654                         pipeconf |= PIPECONF_DITHER_EN |
8655                                     PIPECONF_DITHER_TYPE_SP;
8656
8657                 switch (crtc_state->pipe_bpp) {
8658                 case 18:
8659                         pipeconf |= PIPECONF_6BPC;
8660                         break;
8661                 case 24:
8662                         pipeconf |= PIPECONF_8BPC;
8663                         break;
8664                 case 30:
8665                         pipeconf |= PIPECONF_10BPC;
8666                         break;
8667                 default:
8668                         /* Case prevented by intel_choose_pipe_bpp_dither. */
8669                         BUG();
8670                 }
8671         }
8672
8673         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
8674                 if (INTEL_GEN(dev_priv) < 4 ||
8675                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
8676                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
8677                 else
8678                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
8679         } else {
8680                 pipeconf |= PIPECONF_PROGRESSIVE;
8681         }
8682
8683         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
8684              crtc_state->limited_color_range)
8685                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
8686
8687         pipeconf |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
8688
8689         pipeconf |= PIPECONF_FRAME_START_DELAY(0);
8690
8691         I915_WRITE(PIPECONF(crtc->pipe), pipeconf);
8692         POSTING_READ(PIPECONF(crtc->pipe));
8693 }
8694
8695 static int i8xx_crtc_compute_clock(struct intel_crtc *crtc,
8696                                    struct intel_crtc_state *crtc_state)
8697 {
8698         struct drm_device *dev = crtc->base.dev;
8699         struct drm_i915_private *dev_priv = to_i915(dev);
8700         const struct intel_limit *limit;
8701         int refclk = 48000;
8702
8703         memset(&crtc_state->dpll_hw_state, 0,
8704                sizeof(crtc_state->dpll_hw_state));
8705
8706         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8707                 if (intel_panel_use_ssc(dev_priv)) {
8708                         refclk = dev_priv->vbt.lvds_ssc_freq;
8709                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8710                 }
8711
8712                 limit = &intel_limits_i8xx_lvds;
8713         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO)) {
8714                 limit = &intel_limits_i8xx_dvo;
8715         } else {
8716                 limit = &intel_limits_i8xx_dac;
8717         }
8718
8719         if (!crtc_state->clock_set &&
8720             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8721                                  refclk, NULL, &crtc_state->dpll)) {
8722                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8723                 return -EINVAL;
8724         }
8725
8726         i8xx_compute_dpll(crtc, crtc_state, NULL);
8727
8728         return 0;
8729 }
8730
8731 static int g4x_crtc_compute_clock(struct intel_crtc *crtc,
8732                                   struct intel_crtc_state *crtc_state)
8733 {
8734         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8735         const struct intel_limit *limit;
8736         int refclk = 96000;
8737
8738         memset(&crtc_state->dpll_hw_state, 0,
8739                sizeof(crtc_state->dpll_hw_state));
8740
8741         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8742                 if (intel_panel_use_ssc(dev_priv)) {
8743                         refclk = dev_priv->vbt.lvds_ssc_freq;
8744                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8745                 }
8746
8747                 if (intel_is_dual_link_lvds(dev_priv))
8748                         limit = &intel_limits_g4x_dual_channel_lvds;
8749                 else
8750                         limit = &intel_limits_g4x_single_channel_lvds;
8751         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) ||
8752                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
8753                 limit = &intel_limits_g4x_hdmi;
8754         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) {
8755                 limit = &intel_limits_g4x_sdvo;
8756         } else {
8757                 /* The option is for other outputs */
8758                 limit = &intel_limits_i9xx_sdvo;
8759         }
8760
8761         if (!crtc_state->clock_set &&
8762             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8763                                 refclk, NULL, &crtc_state->dpll)) {
8764                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8765                 return -EINVAL;
8766         }
8767
8768         i9xx_compute_dpll(crtc, crtc_state, NULL);
8769
8770         return 0;
8771 }
8772
8773 static int pnv_crtc_compute_clock(struct intel_crtc *crtc,
8774                                   struct intel_crtc_state *crtc_state)
8775 {
8776         struct drm_device *dev = crtc->base.dev;
8777         struct drm_i915_private *dev_priv = to_i915(dev);
8778         const struct intel_limit *limit;
8779         int refclk = 96000;
8780
8781         memset(&crtc_state->dpll_hw_state, 0,
8782                sizeof(crtc_state->dpll_hw_state));
8783
8784         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8785                 if (intel_panel_use_ssc(dev_priv)) {
8786                         refclk = dev_priv->vbt.lvds_ssc_freq;
8787                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8788                 }
8789
8790                 limit = &pnv_limits_lvds;
8791         } else {
8792                 limit = &pnv_limits_sdvo;
8793         }
8794
8795         if (!crtc_state->clock_set &&
8796             !pnv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8797                                 refclk, NULL, &crtc_state->dpll)) {
8798                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8799                 return -EINVAL;
8800         }
8801
8802         i9xx_compute_dpll(crtc, crtc_state, NULL);
8803
8804         return 0;
8805 }
8806
8807 static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
8808                                    struct intel_crtc_state *crtc_state)
8809 {
8810         struct drm_device *dev = crtc->base.dev;
8811         struct drm_i915_private *dev_priv = to_i915(dev);
8812         const struct intel_limit *limit;
8813         int refclk = 96000;
8814
8815         memset(&crtc_state->dpll_hw_state, 0,
8816                sizeof(crtc_state->dpll_hw_state));
8817
8818         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8819                 if (intel_panel_use_ssc(dev_priv)) {
8820                         refclk = dev_priv->vbt.lvds_ssc_freq;
8821                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8822                 }
8823
8824                 limit = &intel_limits_i9xx_lvds;
8825         } else {
8826                 limit = &intel_limits_i9xx_sdvo;
8827         }
8828
8829         if (!crtc_state->clock_set &&
8830             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8831                                  refclk, NULL, &crtc_state->dpll)) {
8832                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8833                 return -EINVAL;
8834         }
8835
8836         i9xx_compute_dpll(crtc, crtc_state, NULL);
8837
8838         return 0;
8839 }
8840
8841 static int chv_crtc_compute_clock(struct intel_crtc *crtc,
8842                                   struct intel_crtc_state *crtc_state)
8843 {
8844         int refclk = 100000;
8845         const struct intel_limit *limit = &intel_limits_chv;
8846
8847         memset(&crtc_state->dpll_hw_state, 0,
8848                sizeof(crtc_state->dpll_hw_state));
8849
8850         if (!crtc_state->clock_set &&
8851             !chv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8852                                 refclk, NULL, &crtc_state->dpll)) {
8853                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8854                 return -EINVAL;
8855         }
8856
8857         chv_compute_dpll(crtc, crtc_state);
8858
8859         return 0;
8860 }
8861
8862 static int vlv_crtc_compute_clock(struct intel_crtc *crtc,
8863                                   struct intel_crtc_state *crtc_state)
8864 {
8865         int refclk = 100000;
8866         const struct intel_limit *limit = &intel_limits_vlv;
8867
8868         memset(&crtc_state->dpll_hw_state, 0,
8869                sizeof(crtc_state->dpll_hw_state));
8870
8871         if (!crtc_state->clock_set &&
8872             !vlv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8873                                 refclk, NULL, &crtc_state->dpll)) {
8874                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8875                 return -EINVAL;
8876         }
8877
8878         vlv_compute_dpll(crtc, crtc_state);
8879
8880         return 0;
8881 }
8882
8883 static bool i9xx_has_pfit(struct drm_i915_private *dev_priv)
8884 {
8885         if (IS_I830(dev_priv))
8886                 return false;
8887
8888         return INTEL_GEN(dev_priv) >= 4 ||
8889                 IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
8890 }
8891
8892 static void i9xx_get_pfit_config(struct intel_crtc *crtc,
8893                                  struct intel_crtc_state *pipe_config)
8894 {
8895         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8896         u32 tmp;
8897
8898         if (!i9xx_has_pfit(dev_priv))
8899                 return;
8900
8901         tmp = I915_READ(PFIT_CONTROL);
8902         if (!(tmp & PFIT_ENABLE))
8903                 return;
8904
8905         /* Check whether the pfit is attached to our pipe. */
8906         if (INTEL_GEN(dev_priv) < 4) {
8907                 if (crtc->pipe != PIPE_B)
8908                         return;
8909         } else {
8910                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
8911                         return;
8912         }
8913
8914         pipe_config->gmch_pfit.control = tmp;
8915         pipe_config->gmch_pfit.pgm_ratios = I915_READ(PFIT_PGM_RATIOS);
8916 }
8917
8918 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
8919                                struct intel_crtc_state *pipe_config)
8920 {
8921         struct drm_device *dev = crtc->base.dev;
8922         struct drm_i915_private *dev_priv = to_i915(dev);
8923         enum pipe pipe = crtc->pipe;
8924         struct dpll clock;
8925         u32 mdiv;
8926         int refclk = 100000;
8927
8928         /* In case of DSI, DPLL will not be used */
8929         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8930                 return;
8931
8932         vlv_dpio_get(dev_priv);
8933         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
8934         vlv_dpio_put(dev_priv);
8935
8936         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
8937         clock.m2 = mdiv & DPIO_M2DIV_MASK;
8938         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
8939         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
8940         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
8941
8942         pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
8943 }
8944
8945 static void
8946 i9xx_get_initial_plane_config(struct intel_crtc *crtc,
8947                               struct intel_initial_plane_config *plane_config)
8948 {
8949         struct drm_device *dev = crtc->base.dev;
8950         struct drm_i915_private *dev_priv = to_i915(dev);
8951         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
8952         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
8953         enum pipe pipe;
8954         u32 val, base, offset;
8955         int fourcc, pixel_format;
8956         unsigned int aligned_height;
8957         struct drm_framebuffer *fb;
8958         struct intel_framebuffer *intel_fb;
8959
8960         if (!plane->get_hw_state(plane, &pipe))
8961                 return;
8962
8963         WARN_ON(pipe != crtc->pipe);
8964
8965         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
8966         if (!intel_fb) {
8967                 DRM_DEBUG_KMS("failed to alloc fb\n");
8968                 return;
8969         }
8970
8971         fb = &intel_fb->base;
8972
8973         fb->dev = dev;
8974
8975         val = I915_READ(DSPCNTR(i9xx_plane));
8976
8977         if (INTEL_GEN(dev_priv) >= 4) {
8978                 if (val & DISPPLANE_TILED) {
8979                         plane_config->tiling = I915_TILING_X;
8980                         fb->modifier = I915_FORMAT_MOD_X_TILED;
8981                 }
8982
8983                 if (val & DISPPLANE_ROTATE_180)
8984                         plane_config->rotation = DRM_MODE_ROTATE_180;
8985         }
8986
8987         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B &&
8988             val & DISPPLANE_MIRROR)
8989                 plane_config->rotation |= DRM_MODE_REFLECT_X;
8990
8991         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
8992         fourcc = i9xx_format_to_fourcc(pixel_format);
8993         fb->format = drm_format_info(fourcc);
8994
8995         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
8996                 offset = I915_READ(DSPOFFSET(i9xx_plane));
8997                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
8998         } else if (INTEL_GEN(dev_priv) >= 4) {
8999                 if (plane_config->tiling)
9000                         offset = I915_READ(DSPTILEOFF(i9xx_plane));
9001                 else
9002                         offset = I915_READ(DSPLINOFF(i9xx_plane));
9003                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
9004         } else {
9005                 base = I915_READ(DSPADDR(i9xx_plane));
9006         }
9007         plane_config->base = base;
9008
9009         val = I915_READ(PIPESRC(pipe));
9010         fb->width = ((val >> 16) & 0xfff) + 1;
9011         fb->height = ((val >> 0) & 0xfff) + 1;
9012
9013         val = I915_READ(DSPSTRIDE(i9xx_plane));
9014         fb->pitches[0] = val & 0xffffffc0;
9015
9016         aligned_height = intel_fb_align_height(fb, 0, fb->height);
9017
9018         plane_config->size = fb->pitches[0] * aligned_height;
9019
9020         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
9021                       crtc->base.name, plane->base.name, fb->width, fb->height,
9022                       fb->format->cpp[0] * 8, base, fb->pitches[0],
9023                       plane_config->size);
9024
9025         plane_config->fb = intel_fb;
9026 }
9027
9028 static void chv_crtc_clock_get(struct intel_crtc *crtc,
9029                                struct intel_crtc_state *pipe_config)
9030 {
9031         struct drm_device *dev = crtc->base.dev;
9032         struct drm_i915_private *dev_priv = to_i915(dev);
9033         enum pipe pipe = crtc->pipe;
9034         enum dpio_channel port = vlv_pipe_to_channel(pipe);
9035         struct dpll clock;
9036         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
9037         int refclk = 100000;
9038
9039         /* In case of DSI, DPLL will not be used */
9040         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
9041                 return;
9042
9043         vlv_dpio_get(dev_priv);
9044         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
9045         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
9046         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
9047         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
9048         pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
9049         vlv_dpio_put(dev_priv);
9050
9051         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
9052         clock.m2 = (pll_dw0 & 0xff) << 22;
9053         if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
9054                 clock.m2 |= pll_dw2 & 0x3fffff;
9055         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
9056         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
9057         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
9058
9059         pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
9060 }
9061
9062 static enum intel_output_format
9063 bdw_get_pipemisc_output_format(struct intel_crtc *crtc)
9064 {
9065         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9066         u32 tmp;
9067
9068         tmp = I915_READ(PIPEMISC(crtc->pipe));
9069
9070         if (tmp & PIPEMISC_YUV420_ENABLE) {
9071                 /* We support 4:2:0 in full blend mode only */
9072                 WARN_ON((tmp & PIPEMISC_YUV420_MODE_FULL_BLEND) == 0);
9073
9074                 return INTEL_OUTPUT_FORMAT_YCBCR420;
9075         } else if (tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV) {
9076                 return INTEL_OUTPUT_FORMAT_YCBCR444;
9077         } else {
9078                 return INTEL_OUTPUT_FORMAT_RGB;
9079         }
9080 }
9081
9082 static void i9xx_get_pipe_color_config(struct intel_crtc_state *crtc_state)
9083 {
9084         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9085         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
9086         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9087         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
9088         u32 tmp;
9089
9090         tmp = I915_READ(DSPCNTR(i9xx_plane));
9091
9092         if (tmp & DISPPLANE_GAMMA_ENABLE)
9093                 crtc_state->gamma_enable = true;
9094
9095         if (!HAS_GMCH(dev_priv) &&
9096             tmp & DISPPLANE_PIPE_CSC_ENABLE)
9097                 crtc_state->csc_enable = true;
9098 }
9099
9100 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
9101                                  struct intel_crtc_state *pipe_config)
9102 {
9103         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9104         enum intel_display_power_domain power_domain;
9105         intel_wakeref_t wakeref;
9106         u32 tmp;
9107         bool ret;
9108
9109         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
9110         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
9111         if (!wakeref)
9112                 return false;
9113
9114         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
9115         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9116         pipe_config->shared_dpll = NULL;
9117         pipe_config->master_transcoder = INVALID_TRANSCODER;
9118
9119         ret = false;
9120
9121         tmp = I915_READ(PIPECONF(crtc->pipe));
9122         if (!(tmp & PIPECONF_ENABLE))
9123                 goto out;
9124
9125         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
9126             IS_CHERRYVIEW(dev_priv)) {
9127                 switch (tmp & PIPECONF_BPC_MASK) {
9128                 case PIPECONF_6BPC:
9129                         pipe_config->pipe_bpp = 18;
9130                         break;
9131                 case PIPECONF_8BPC:
9132                         pipe_config->pipe_bpp = 24;
9133                         break;
9134                 case PIPECONF_10BPC:
9135                         pipe_config->pipe_bpp = 30;
9136                         break;
9137                 default:
9138                         break;
9139                 }
9140         }
9141
9142         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
9143             (tmp & PIPECONF_COLOR_RANGE_SELECT))
9144                 pipe_config->limited_color_range = true;
9145
9146         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_I9XX) >>
9147                 PIPECONF_GAMMA_MODE_SHIFT;
9148
9149         if (IS_CHERRYVIEW(dev_priv))
9150                 pipe_config->cgm_mode = I915_READ(CGM_PIPE_MODE(crtc->pipe));
9151
9152         i9xx_get_pipe_color_config(pipe_config);
9153         intel_color_get_config(pipe_config);
9154
9155         if (INTEL_GEN(dev_priv) < 4)
9156                 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
9157
9158         intel_get_pipe_timings(crtc, pipe_config);
9159         intel_get_pipe_src_size(crtc, pipe_config);
9160
9161         i9xx_get_pfit_config(crtc, pipe_config);
9162
9163         if (INTEL_GEN(dev_priv) >= 4) {
9164                 /* No way to read it out on pipes B and C */
9165                 if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
9166                         tmp = dev_priv->chv_dpll_md[crtc->pipe];
9167                 else
9168                         tmp = I915_READ(DPLL_MD(crtc->pipe));
9169                 pipe_config->pixel_multiplier =
9170                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
9171                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
9172                 pipe_config->dpll_hw_state.dpll_md = tmp;
9173         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
9174                    IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
9175                 tmp = I915_READ(DPLL(crtc->pipe));
9176                 pipe_config->pixel_multiplier =
9177                         ((tmp & SDVO_MULTIPLIER_MASK)
9178                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
9179         } else {
9180                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
9181                  * port and will be fixed up in the encoder->get_config
9182                  * function. */
9183                 pipe_config->pixel_multiplier = 1;
9184         }
9185         pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
9186         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
9187                 pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
9188                 pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe));
9189         } else {
9190                 /* Mask out read-only status bits. */
9191                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
9192                                                      DPLL_PORTC_READY_MASK |
9193                                                      DPLL_PORTB_READY_MASK);
9194         }
9195
9196         if (IS_CHERRYVIEW(dev_priv))
9197                 chv_crtc_clock_get(crtc, pipe_config);
9198         else if (IS_VALLEYVIEW(dev_priv))
9199                 vlv_crtc_clock_get(crtc, pipe_config);
9200         else
9201                 i9xx_crtc_clock_get(crtc, pipe_config);
9202
9203         /*
9204          * Normally the dotclock is filled in by the encoder .get_config()
9205          * but in case the pipe is enabled w/o any ports we need a sane
9206          * default.
9207          */
9208         pipe_config->hw.adjusted_mode.crtc_clock =
9209                 pipe_config->port_clock / pipe_config->pixel_multiplier;
9210
9211         ret = true;
9212
9213 out:
9214         intel_display_power_put(dev_priv, power_domain, wakeref);
9215
9216         return ret;
9217 }
9218
9219 static void ilk_init_pch_refclk(struct drm_i915_private *dev_priv)
9220 {
9221         struct intel_encoder *encoder;
9222         int i;
9223         u32 val, final;
9224         bool has_lvds = false;
9225         bool has_cpu_edp = false;
9226         bool has_panel = false;
9227         bool has_ck505 = false;
9228         bool can_ssc = false;
9229         bool using_ssc_source = false;
9230
9231         /* We need to take the global config into account */
9232         for_each_intel_encoder(&dev_priv->drm, encoder) {
9233                 switch (encoder->type) {
9234                 case INTEL_OUTPUT_LVDS:
9235                         has_panel = true;
9236                         has_lvds = true;
9237                         break;
9238                 case INTEL_OUTPUT_EDP:
9239                         has_panel = true;
9240                         if (encoder->port == PORT_A)
9241                                 has_cpu_edp = true;
9242                         break;
9243                 default:
9244                         break;
9245                 }
9246         }
9247
9248         if (HAS_PCH_IBX(dev_priv)) {
9249                 has_ck505 = dev_priv->vbt.display_clock_mode;
9250                 can_ssc = has_ck505;
9251         } else {
9252                 has_ck505 = false;
9253                 can_ssc = true;
9254         }
9255
9256         /* Check if any DPLLs are using the SSC source */
9257         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
9258                 u32 temp = I915_READ(PCH_DPLL(i));
9259
9260                 if (!(temp & DPLL_VCO_ENABLE))
9261                         continue;
9262
9263                 if ((temp & PLL_REF_INPUT_MASK) ==
9264                     PLLB_REF_INPUT_SPREADSPECTRUMIN) {
9265                         using_ssc_source = true;
9266                         break;
9267                 }
9268         }
9269
9270         DRM_DEBUG_KMS("has_panel %d has_lvds %d has_ck505 %d using_ssc_source %d\n",
9271                       has_panel, has_lvds, has_ck505, using_ssc_source);
9272
9273         /* Ironlake: try to setup display ref clock before DPLL
9274          * enabling. This is only under driver's control after
9275          * PCH B stepping, previous chipset stepping should be
9276          * ignoring this setting.
9277          */
9278         val = I915_READ(PCH_DREF_CONTROL);
9279
9280         /* As we must carefully and slowly disable/enable each source in turn,
9281          * compute the final state we want first and check if we need to
9282          * make any changes at all.
9283          */
9284         final = val;
9285         final &= ~DREF_NONSPREAD_SOURCE_MASK;
9286         if (has_ck505)
9287                 final |= DREF_NONSPREAD_CK505_ENABLE;
9288         else
9289                 final |= DREF_NONSPREAD_SOURCE_ENABLE;
9290
9291         final &= ~DREF_SSC_SOURCE_MASK;
9292         final &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9293         final &= ~DREF_SSC1_ENABLE;
9294
9295         if (has_panel) {
9296                 final |= DREF_SSC_SOURCE_ENABLE;
9297
9298                 if (intel_panel_use_ssc(dev_priv) && can_ssc)
9299                         final |= DREF_SSC1_ENABLE;
9300
9301                 if (has_cpu_edp) {
9302                         if (intel_panel_use_ssc(dev_priv) && can_ssc)
9303                                 final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
9304                         else
9305                                 final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
9306                 } else
9307                         final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9308         } else if (using_ssc_source) {
9309                 final |= DREF_SSC_SOURCE_ENABLE;
9310                 final |= DREF_SSC1_ENABLE;
9311         }
9312
9313         if (final == val)
9314                 return;
9315
9316         /* Always enable nonspread source */
9317         val &= ~DREF_NONSPREAD_SOURCE_MASK;
9318
9319         if (has_ck505)
9320                 val |= DREF_NONSPREAD_CK505_ENABLE;
9321         else
9322                 val |= DREF_NONSPREAD_SOURCE_ENABLE;
9323
9324         if (has_panel) {
9325                 val &= ~DREF_SSC_SOURCE_MASK;
9326                 val |= DREF_SSC_SOURCE_ENABLE;
9327
9328                 /* SSC must be turned on before enabling the CPU output  */
9329                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
9330                         DRM_DEBUG_KMS("Using SSC on panel\n");
9331                         val |= DREF_SSC1_ENABLE;
9332                 } else
9333                         val &= ~DREF_SSC1_ENABLE;
9334
9335                 /* Get SSC going before enabling the outputs */
9336                 I915_WRITE(PCH_DREF_CONTROL, val);
9337                 POSTING_READ(PCH_DREF_CONTROL);
9338                 udelay(200);
9339
9340                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9341
9342                 /* Enable CPU source on CPU attached eDP */
9343                 if (has_cpu_edp) {
9344                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
9345                                 DRM_DEBUG_KMS("Using SSC on eDP\n");
9346                                 val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
9347                         } else
9348                                 val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
9349                 } else
9350                         val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9351
9352                 I915_WRITE(PCH_DREF_CONTROL, val);
9353                 POSTING_READ(PCH_DREF_CONTROL);
9354                 udelay(200);
9355         } else {
9356                 DRM_DEBUG_KMS("Disabling CPU source output\n");
9357
9358                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9359
9360                 /* Turn off CPU output */
9361                 val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9362
9363                 I915_WRITE(PCH_DREF_CONTROL, val);
9364                 POSTING_READ(PCH_DREF_CONTROL);
9365                 udelay(200);
9366
9367                 if (!using_ssc_source) {
9368                         DRM_DEBUG_KMS("Disabling SSC source\n");
9369
9370                         /* Turn off the SSC source */
9371                         val &= ~DREF_SSC_SOURCE_MASK;
9372                         val |= DREF_SSC_SOURCE_DISABLE;
9373
9374                         /* Turn off SSC1 */
9375                         val &= ~DREF_SSC1_ENABLE;
9376
9377                         I915_WRITE(PCH_DREF_CONTROL, val);
9378                         POSTING_READ(PCH_DREF_CONTROL);
9379                         udelay(200);
9380                 }
9381         }
9382
9383         BUG_ON(val != final);
9384 }
9385
9386 static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv)
9387 {
9388         u32 tmp;
9389
9390         tmp = I915_READ(SOUTH_CHICKEN2);
9391         tmp |= FDI_MPHY_IOSFSB_RESET_CTL;
9392         I915_WRITE(SOUTH_CHICKEN2, tmp);
9393
9394         if (wait_for_us(I915_READ(SOUTH_CHICKEN2) &
9395                         FDI_MPHY_IOSFSB_RESET_STATUS, 100))
9396                 DRM_ERROR("FDI mPHY reset assert timeout\n");
9397
9398         tmp = I915_READ(SOUTH_CHICKEN2);
9399         tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL;
9400         I915_WRITE(SOUTH_CHICKEN2, tmp);
9401
9402         if (wait_for_us((I915_READ(SOUTH_CHICKEN2) &
9403                          FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100))
9404                 DRM_ERROR("FDI mPHY reset de-assert timeout\n");
9405 }
9406
9407 /* WaMPhyProgramming:hsw */
9408 static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv)
9409 {
9410         u32 tmp;
9411
9412         tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY);
9413         tmp &= ~(0xFF << 24);
9414         tmp |= (0x12 << 24);
9415         intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY);
9416
9417         tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY);
9418         tmp |= (1 << 11);
9419         intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY);
9420
9421         tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY);
9422         tmp |= (1 << 11);
9423         intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY);
9424
9425         tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY);
9426         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
9427         intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY);
9428
9429         tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY);
9430         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
9431         intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY);
9432
9433         tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY);
9434         tmp &= ~(7 << 13);
9435         tmp |= (5 << 13);
9436         intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY);
9437
9438         tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY);
9439         tmp &= ~(7 << 13);
9440         tmp |= (5 << 13);
9441         intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY);
9442
9443         tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY);
9444         tmp &= ~0xFF;
9445         tmp |= 0x1C;
9446         intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY);
9447
9448         tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY);
9449         tmp &= ~0xFF;
9450         tmp |= 0x1C;
9451         intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY);
9452
9453         tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY);
9454         tmp &= ~(0xFF << 16);
9455         tmp |= (0x1C << 16);
9456         intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY);
9457
9458         tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY);
9459         tmp &= ~(0xFF << 16);
9460         tmp |= (0x1C << 16);
9461         intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY);
9462
9463         tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY);
9464         tmp |= (1 << 27);
9465         intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY);
9466
9467         tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY);
9468         tmp |= (1 << 27);
9469         intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY);
9470
9471         tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY);
9472         tmp &= ~(0xF << 28);
9473         tmp |= (4 << 28);
9474         intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY);
9475
9476         tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY);
9477         tmp &= ~(0xF << 28);
9478         tmp |= (4 << 28);
9479         intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY);
9480 }
9481
9482 /* Implements 3 different sequences from BSpec chapter "Display iCLK
9483  * Programming" based on the parameters passed:
9484  * - Sequence to enable CLKOUT_DP
9485  * - Sequence to enable CLKOUT_DP without spread
9486  * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O
9487  */
9488 static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv,
9489                                  bool with_spread, bool with_fdi)
9490 {
9491         u32 reg, tmp;
9492
9493         if (WARN(with_fdi && !with_spread, "FDI requires downspread\n"))
9494                 with_spread = true;
9495         if (WARN(HAS_PCH_LPT_LP(dev_priv) &&
9496             with_fdi, "LP PCH doesn't have FDI\n"))
9497                 with_fdi = false;
9498
9499         mutex_lock(&dev_priv->sb_lock);
9500
9501         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9502         tmp &= ~SBI_SSCCTL_DISABLE;
9503         tmp |= SBI_SSCCTL_PATHALT;
9504         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9505
9506         udelay(24);
9507
9508         if (with_spread) {
9509                 tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9510                 tmp &= ~SBI_SSCCTL_PATHALT;
9511                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9512
9513                 if (with_fdi) {
9514                         lpt_reset_fdi_mphy(dev_priv);
9515                         lpt_program_fdi_mphy(dev_priv);
9516                 }
9517         }
9518
9519         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
9520         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
9521         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
9522         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
9523
9524         mutex_unlock(&dev_priv->sb_lock);
9525 }
9526
9527 /* Sequence to disable CLKOUT_DP */
9528 void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv)
9529 {
9530         u32 reg, tmp;
9531
9532         mutex_lock(&dev_priv->sb_lock);
9533
9534         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
9535         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
9536         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
9537         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
9538
9539         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9540         if (!(tmp & SBI_SSCCTL_DISABLE)) {
9541                 if (!(tmp & SBI_SSCCTL_PATHALT)) {
9542                         tmp |= SBI_SSCCTL_PATHALT;
9543                         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9544                         udelay(32);
9545                 }
9546                 tmp |= SBI_SSCCTL_DISABLE;
9547                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9548         }
9549
9550         mutex_unlock(&dev_priv->sb_lock);
9551 }
9552
9553 #define BEND_IDX(steps) ((50 + (steps)) / 5)
9554
9555 static const u16 sscdivintphase[] = {
9556         [BEND_IDX( 50)] = 0x3B23,
9557         [BEND_IDX( 45)] = 0x3B23,
9558         [BEND_IDX( 40)] = 0x3C23,
9559         [BEND_IDX( 35)] = 0x3C23,
9560         [BEND_IDX( 30)] = 0x3D23,
9561         [BEND_IDX( 25)] = 0x3D23,
9562         [BEND_IDX( 20)] = 0x3E23,
9563         [BEND_IDX( 15)] = 0x3E23,
9564         [BEND_IDX( 10)] = 0x3F23,
9565         [BEND_IDX(  5)] = 0x3F23,
9566         [BEND_IDX(  0)] = 0x0025,
9567         [BEND_IDX( -5)] = 0x0025,
9568         [BEND_IDX(-10)] = 0x0125,
9569         [BEND_IDX(-15)] = 0x0125,
9570         [BEND_IDX(-20)] = 0x0225,
9571         [BEND_IDX(-25)] = 0x0225,
9572         [BEND_IDX(-30)] = 0x0325,
9573         [BEND_IDX(-35)] = 0x0325,
9574         [BEND_IDX(-40)] = 0x0425,
9575         [BEND_IDX(-45)] = 0x0425,
9576         [BEND_IDX(-50)] = 0x0525,
9577 };
9578
9579 /*
9580  * Bend CLKOUT_DP
9581  * steps -50 to 50 inclusive, in steps of 5
9582  * < 0 slow down the clock, > 0 speed up the clock, 0 == no bend (135MHz)
9583  * change in clock period = -(steps / 10) * 5.787 ps
9584  */
9585 static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps)
9586 {
9587         u32 tmp;
9588         int idx = BEND_IDX(steps);
9589
9590         if (WARN_ON(steps % 5 != 0))
9591                 return;
9592
9593         if (WARN_ON(idx >= ARRAY_SIZE(sscdivintphase)))
9594                 return;
9595
9596         mutex_lock(&dev_priv->sb_lock);
9597
9598         if (steps % 10 != 0)
9599                 tmp = 0xAAAAAAAB;
9600         else
9601                 tmp = 0x00000000;
9602         intel_sbi_write(dev_priv, SBI_SSCDITHPHASE, tmp, SBI_ICLK);
9603
9604         tmp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE, SBI_ICLK);
9605         tmp &= 0xffff0000;
9606         tmp |= sscdivintphase[idx];
9607         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE, tmp, SBI_ICLK);
9608
9609         mutex_unlock(&dev_priv->sb_lock);
9610 }
9611
9612 #undef BEND_IDX
9613
9614 static bool spll_uses_pch_ssc(struct drm_i915_private *dev_priv)
9615 {
9616         u32 fuse_strap = I915_READ(FUSE_STRAP);
9617         u32 ctl = I915_READ(SPLL_CTL);
9618
9619         if ((ctl & SPLL_PLL_ENABLE) == 0)
9620                 return false;
9621
9622         if ((ctl & SPLL_REF_MASK) == SPLL_REF_MUXED_SSC &&
9623             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
9624                 return true;
9625
9626         if (IS_BROADWELL(dev_priv) &&
9627             (ctl & SPLL_REF_MASK) == SPLL_REF_PCH_SSC_BDW)
9628                 return true;
9629
9630         return false;
9631 }
9632
9633 static bool wrpll_uses_pch_ssc(struct drm_i915_private *dev_priv,
9634                                enum intel_dpll_id id)
9635 {
9636         u32 fuse_strap = I915_READ(FUSE_STRAP);
9637         u32 ctl = I915_READ(WRPLL_CTL(id));
9638
9639         if ((ctl & WRPLL_PLL_ENABLE) == 0)
9640                 return false;
9641
9642         if ((ctl & WRPLL_REF_MASK) == WRPLL_REF_PCH_SSC)
9643                 return true;
9644
9645         if ((IS_BROADWELL(dev_priv) || IS_HSW_ULT(dev_priv)) &&
9646             (ctl & WRPLL_REF_MASK) == WRPLL_REF_MUXED_SSC_BDW &&
9647             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
9648                 return true;
9649
9650         return false;
9651 }
9652
9653 static void lpt_init_pch_refclk(struct drm_i915_private *dev_priv)
9654 {
9655         struct intel_encoder *encoder;
9656         bool has_fdi = false;
9657
9658         for_each_intel_encoder(&dev_priv->drm, encoder) {
9659                 switch (encoder->type) {
9660                 case INTEL_OUTPUT_ANALOG:
9661                         has_fdi = true;
9662                         break;
9663                 default:
9664                         break;
9665                 }
9666         }
9667
9668         /*
9669          * The BIOS may have decided to use the PCH SSC
9670          * reference so we must not disable it until the
9671          * relevant PLLs have stopped relying on it. We'll
9672          * just leave the PCH SSC reference enabled in case
9673          * any active PLL is using it. It will get disabled
9674          * after runtime suspend if we don't have FDI.
9675          *
9676          * TODO: Move the whole reference clock handling
9677          * to the modeset sequence proper so that we can
9678          * actually enable/disable/reconfigure these things
9679          * safely. To do that we need to introduce a real
9680          * clock hierarchy. That would also allow us to do
9681          * clock bending finally.
9682          */
9683         dev_priv->pch_ssc_use = 0;
9684
9685         if (spll_uses_pch_ssc(dev_priv)) {
9686                 DRM_DEBUG_KMS("SPLL using PCH SSC\n");
9687                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_SPLL);
9688         }
9689
9690         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL1)) {
9691                 DRM_DEBUG_KMS("WRPLL1 using PCH SSC\n");
9692                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_WRPLL1);
9693         }
9694
9695         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL2)) {
9696                 DRM_DEBUG_KMS("WRPLL2 using PCH SSC\n");
9697                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_WRPLL2);
9698         }
9699
9700         if (dev_priv->pch_ssc_use)
9701                 return;
9702
9703         if (has_fdi) {
9704                 lpt_bend_clkout_dp(dev_priv, 0);
9705                 lpt_enable_clkout_dp(dev_priv, true, true);
9706         } else {
9707                 lpt_disable_clkout_dp(dev_priv);
9708         }
9709 }
9710
9711 /*
9712  * Initialize reference clocks when the driver loads
9713  */
9714 void intel_init_pch_refclk(struct drm_i915_private *dev_priv)
9715 {
9716         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
9717                 ilk_init_pch_refclk(dev_priv);
9718         else if (HAS_PCH_LPT(dev_priv))
9719                 lpt_init_pch_refclk(dev_priv);
9720 }
9721
9722 static void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state)
9723 {
9724         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9725         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9726         enum pipe pipe = crtc->pipe;
9727         u32 val;
9728
9729         val = 0;
9730
9731         switch (crtc_state->pipe_bpp) {
9732         case 18:
9733                 val |= PIPECONF_6BPC;
9734                 break;
9735         case 24:
9736                 val |= PIPECONF_8BPC;
9737                 break;
9738         case 30:
9739                 val |= PIPECONF_10BPC;
9740                 break;
9741         case 36:
9742                 val |= PIPECONF_12BPC;
9743                 break;
9744         default:
9745                 /* Case prevented by intel_choose_pipe_bpp_dither. */
9746                 BUG();
9747         }
9748
9749         if (crtc_state->dither)
9750                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
9751
9752         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
9753                 val |= PIPECONF_INTERLACED_ILK;
9754         else
9755                 val |= PIPECONF_PROGRESSIVE;
9756
9757         /*
9758          * This would end up with an odd purple hue over
9759          * the entire display. Make sure we don't do it.
9760          */
9761         WARN_ON(crtc_state->limited_color_range &&
9762                 crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB);
9763
9764         if (crtc_state->limited_color_range)
9765                 val |= PIPECONF_COLOR_RANGE_SELECT;
9766
9767         if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
9768                 val |= PIPECONF_OUTPUT_COLORSPACE_YUV709;
9769
9770         val |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
9771
9772         val |= PIPECONF_FRAME_START_DELAY(0);
9773
9774         I915_WRITE(PIPECONF(pipe), val);
9775         POSTING_READ(PIPECONF(pipe));
9776 }
9777
9778 static void hsw_set_pipeconf(const struct intel_crtc_state *crtc_state)
9779 {
9780         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9781         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9782         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
9783         u32 val = 0;
9784
9785         if (IS_HASWELL(dev_priv) && crtc_state->dither)
9786                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
9787
9788         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
9789                 val |= PIPECONF_INTERLACED_ILK;
9790         else
9791                 val |= PIPECONF_PROGRESSIVE;
9792
9793         if (IS_HASWELL(dev_priv) &&
9794             crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
9795                 val |= PIPECONF_OUTPUT_COLORSPACE_YUV_HSW;
9796
9797         I915_WRITE(PIPECONF(cpu_transcoder), val);
9798         POSTING_READ(PIPECONF(cpu_transcoder));
9799 }
9800
9801 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state)
9802 {
9803         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9804         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9805         u32 val = 0;
9806
9807         switch (crtc_state->pipe_bpp) {
9808         case 18:
9809                 val |= PIPEMISC_DITHER_6_BPC;
9810                 break;
9811         case 24:
9812                 val |= PIPEMISC_DITHER_8_BPC;
9813                 break;
9814         case 30:
9815                 val |= PIPEMISC_DITHER_10_BPC;
9816                 break;
9817         case 36:
9818                 val |= PIPEMISC_DITHER_12_BPC;
9819                 break;
9820         default:
9821                 MISSING_CASE(crtc_state->pipe_bpp);
9822                 break;
9823         }
9824
9825         if (crtc_state->dither)
9826                 val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
9827
9828         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
9829             crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444)
9830                 val |= PIPEMISC_OUTPUT_COLORSPACE_YUV;
9831
9832         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
9833                 val |= PIPEMISC_YUV420_ENABLE |
9834                         PIPEMISC_YUV420_MODE_FULL_BLEND;
9835
9836         if (INTEL_GEN(dev_priv) >= 11 &&
9837             (crtc_state->active_planes & ~(icl_hdr_plane_mask() |
9838                                            BIT(PLANE_CURSOR))) == 0)
9839                 val |= PIPEMISC_HDR_MODE_PRECISION;
9840
9841         I915_WRITE(PIPEMISC(crtc->pipe), val);
9842 }
9843
9844 int bdw_get_pipemisc_bpp(struct intel_crtc *crtc)
9845 {
9846         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9847         u32 tmp;
9848
9849         tmp = I915_READ(PIPEMISC(crtc->pipe));
9850
9851         switch (tmp & PIPEMISC_DITHER_BPC_MASK) {
9852         case PIPEMISC_DITHER_6_BPC:
9853                 return 18;
9854         case PIPEMISC_DITHER_8_BPC:
9855                 return 24;
9856         case PIPEMISC_DITHER_10_BPC:
9857                 return 30;
9858         case PIPEMISC_DITHER_12_BPC:
9859                 return 36;
9860         default:
9861                 MISSING_CASE(tmp);
9862                 return 0;
9863         }
9864 }
9865
9866 int ilk_get_lanes_required(int target_clock, int link_bw, int bpp)
9867 {
9868         /*
9869          * Account for spread spectrum to avoid
9870          * oversubscribing the link. Max center spread
9871          * is 2.5%; use 5% for safety's sake.
9872          */
9873         u32 bps = target_clock * bpp * 21 / 20;
9874         return DIV_ROUND_UP(bps, link_bw * 8);
9875 }
9876
9877 static bool ilk_needs_fb_cb_tune(struct dpll *dpll, int factor)
9878 {
9879         return i9xx_dpll_compute_m(dpll) < factor * dpll->n;
9880 }
9881
9882 static void ilk_compute_dpll(struct intel_crtc *crtc,
9883                              struct intel_crtc_state *crtc_state,
9884                              struct dpll *reduced_clock)
9885 {
9886         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9887         u32 dpll, fp, fp2;
9888         int factor;
9889
9890         /* Enable autotuning of the PLL clock (if permissible) */
9891         factor = 21;
9892         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
9893                 if ((intel_panel_use_ssc(dev_priv) &&
9894                      dev_priv->vbt.lvds_ssc_freq == 100000) ||
9895                     (HAS_PCH_IBX(dev_priv) &&
9896                      intel_is_dual_link_lvds(dev_priv)))
9897                         factor = 25;
9898         } else if (crtc_state->sdvo_tv_clock) {
9899                 factor = 20;
9900         }
9901
9902         fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
9903
9904         if (ilk_needs_fb_cb_tune(&crtc_state->dpll, factor))
9905                 fp |= FP_CB_TUNE;
9906
9907         if (reduced_clock) {
9908                 fp2 = i9xx_dpll_compute_fp(reduced_clock);
9909
9910                 if (reduced_clock->m < factor * reduced_clock->n)
9911                         fp2 |= FP_CB_TUNE;
9912         } else {
9913                 fp2 = fp;
9914         }
9915
9916         dpll = 0;
9917
9918         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
9919                 dpll |= DPLLB_MODE_LVDS;
9920         else
9921                 dpll |= DPLLB_MODE_DAC_SERIAL;
9922
9923         dpll |= (crtc_state->pixel_multiplier - 1)
9924                 << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
9925
9926         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
9927             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
9928                 dpll |= DPLL_SDVO_HIGH_SPEED;
9929
9930         if (intel_crtc_has_dp_encoder(crtc_state))
9931                 dpll |= DPLL_SDVO_HIGH_SPEED;
9932
9933         /*
9934          * The high speed IO clock is only really required for
9935          * SDVO/HDMI/DP, but we also enable it for CRT to make it
9936          * possible to share the DPLL between CRT and HDMI. Enabling
9937          * the clock needlessly does no real harm, except use up a
9938          * bit of power potentially.
9939          *
9940          * We'll limit this to IVB with 3 pipes, since it has only two
9941          * DPLLs and so DPLL sharing is the only way to get three pipes
9942          * driving PCH ports at the same time. On SNB we could do this,
9943          * and potentially avoid enabling the second DPLL, but it's not
9944          * clear if it''s a win or loss power wise. No point in doing
9945          * this on ILK at all since it has a fixed DPLL<->pipe mapping.
9946          */
9947         if (INTEL_NUM_PIPES(dev_priv) == 3 &&
9948             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG))
9949                 dpll |= DPLL_SDVO_HIGH_SPEED;
9950
9951         /* compute bitmask from p1 value */
9952         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
9953         /* also FPA1 */
9954         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
9955
9956         switch (crtc_state->dpll.p2) {
9957         case 5:
9958                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
9959                 break;
9960         case 7:
9961                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
9962                 break;
9963         case 10:
9964                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
9965                 break;
9966         case 14:
9967                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
9968                 break;
9969         }
9970
9971         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
9972             intel_panel_use_ssc(dev_priv))
9973                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
9974         else
9975                 dpll |= PLL_REF_INPUT_DREFCLK;
9976
9977         dpll |= DPLL_VCO_ENABLE;
9978
9979         crtc_state->dpll_hw_state.dpll = dpll;
9980         crtc_state->dpll_hw_state.fp0 = fp;
9981         crtc_state->dpll_hw_state.fp1 = fp2;
9982 }
9983
9984 static int ilk_crtc_compute_clock(struct intel_crtc *crtc,
9985                                   struct intel_crtc_state *crtc_state)
9986 {
9987         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9988         struct intel_atomic_state *state =
9989                 to_intel_atomic_state(crtc_state->uapi.state);
9990         const struct intel_limit *limit;
9991         int refclk = 120000;
9992
9993         memset(&crtc_state->dpll_hw_state, 0,
9994                sizeof(crtc_state->dpll_hw_state));
9995
9996         /* CPU eDP is the only output that doesn't need a PCH PLL of its own. */
9997         if (!crtc_state->has_pch_encoder)
9998                 return 0;
9999
10000         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
10001                 if (intel_panel_use_ssc(dev_priv)) {
10002                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n",
10003                                       dev_priv->vbt.lvds_ssc_freq);
10004                         refclk = dev_priv->vbt.lvds_ssc_freq;
10005                 }
10006
10007                 if (intel_is_dual_link_lvds(dev_priv)) {
10008                         if (refclk == 100000)
10009                                 limit = &ilk_limits_dual_lvds_100m;
10010                         else
10011                                 limit = &ilk_limits_dual_lvds;
10012                 } else {
10013                         if (refclk == 100000)
10014                                 limit = &ilk_limits_single_lvds_100m;
10015                         else
10016                                 limit = &ilk_limits_single_lvds;
10017                 }
10018         } else {
10019                 limit = &ilk_limits_dac;
10020         }
10021
10022         if (!crtc_state->clock_set &&
10023             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
10024                                 refclk, NULL, &crtc_state->dpll)) {
10025                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
10026                 return -EINVAL;
10027         }
10028
10029         ilk_compute_dpll(crtc, crtc_state, NULL);
10030
10031         if (!intel_reserve_shared_dplls(state, crtc, NULL)) {
10032                 DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
10033                               pipe_name(crtc->pipe));
10034                 return -EINVAL;
10035         }
10036
10037         return 0;
10038 }
10039
10040 static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
10041                                          struct intel_link_m_n *m_n)
10042 {
10043         struct drm_device *dev = crtc->base.dev;
10044         struct drm_i915_private *dev_priv = to_i915(dev);
10045         enum pipe pipe = crtc->pipe;
10046
10047         m_n->link_m = I915_READ(PCH_TRANS_LINK_M1(pipe));
10048         m_n->link_n = I915_READ(PCH_TRANS_LINK_N1(pipe));
10049         m_n->gmch_m = I915_READ(PCH_TRANS_DATA_M1(pipe))
10050                 & ~TU_SIZE_MASK;
10051         m_n->gmch_n = I915_READ(PCH_TRANS_DATA_N1(pipe));
10052         m_n->tu = ((I915_READ(PCH_TRANS_DATA_M1(pipe))
10053                     & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
10054 }
10055
10056 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
10057                                          enum transcoder transcoder,
10058                                          struct intel_link_m_n *m_n,
10059                                          struct intel_link_m_n *m2_n2)
10060 {
10061         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10062         enum pipe pipe = crtc->pipe;
10063
10064         if (INTEL_GEN(dev_priv) >= 5) {
10065                 m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder));
10066                 m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder));
10067                 m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder))
10068                         & ~TU_SIZE_MASK;
10069                 m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder));
10070                 m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder))
10071                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
10072
10073                 if (m2_n2 && transcoder_has_m2_n2(dev_priv, transcoder)) {
10074                         m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder));
10075                         m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder));
10076                         m2_n2->gmch_m = I915_READ(PIPE_DATA_M2(transcoder))
10077                                         & ~TU_SIZE_MASK;
10078                         m2_n2->gmch_n = I915_READ(PIPE_DATA_N2(transcoder));
10079                         m2_n2->tu = ((I915_READ(PIPE_DATA_M2(transcoder))
10080                                         & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
10081                 }
10082         } else {
10083                 m_n->link_m = I915_READ(PIPE_LINK_M_G4X(pipe));
10084                 m_n->link_n = I915_READ(PIPE_LINK_N_G4X(pipe));
10085                 m_n->gmch_m = I915_READ(PIPE_DATA_M_G4X(pipe))
10086                         & ~TU_SIZE_MASK;
10087                 m_n->gmch_n = I915_READ(PIPE_DATA_N_G4X(pipe));
10088                 m_n->tu = ((I915_READ(PIPE_DATA_M_G4X(pipe))
10089                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
10090         }
10091 }
10092
10093 void intel_dp_get_m_n(struct intel_crtc *crtc,
10094                       struct intel_crtc_state *pipe_config)
10095 {
10096         if (pipe_config->has_pch_encoder)
10097                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
10098         else
10099                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
10100                                              &pipe_config->dp_m_n,
10101                                              &pipe_config->dp_m2_n2);
10102 }
10103
10104 static void ilk_get_fdi_m_n_config(struct intel_crtc *crtc,
10105                                    struct intel_crtc_state *pipe_config)
10106 {
10107         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
10108                                      &pipe_config->fdi_m_n, NULL);
10109 }
10110
10111 static void skl_get_pfit_config(struct intel_crtc *crtc,
10112                                 struct intel_crtc_state *pipe_config)
10113 {
10114         struct drm_device *dev = crtc->base.dev;
10115         struct drm_i915_private *dev_priv = to_i915(dev);
10116         struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state;
10117         u32 ps_ctrl = 0;
10118         int id = -1;
10119         int i;
10120
10121         /* find scaler attached to this pipe */
10122         for (i = 0; i < crtc->num_scalers; i++) {
10123                 ps_ctrl = I915_READ(SKL_PS_CTRL(crtc->pipe, i));
10124                 if (ps_ctrl & PS_SCALER_EN && !(ps_ctrl & PS_PLANE_SEL_MASK)) {
10125                         id = i;
10126                         pipe_config->pch_pfit.enabled = true;
10127                         pipe_config->pch_pfit.pos = I915_READ(SKL_PS_WIN_POS(crtc->pipe, i));
10128                         pipe_config->pch_pfit.size = I915_READ(SKL_PS_WIN_SZ(crtc->pipe, i));
10129                         scaler_state->scalers[i].in_use = true;
10130                         break;
10131                 }
10132         }
10133
10134         scaler_state->scaler_id = id;
10135         if (id >= 0) {
10136                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
10137         } else {
10138                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
10139         }
10140 }
10141
10142 static void
10143 skl_get_initial_plane_config(struct intel_crtc *crtc,
10144                              struct intel_initial_plane_config *plane_config)
10145 {
10146         struct drm_device *dev = crtc->base.dev;
10147         struct drm_i915_private *dev_priv = to_i915(dev);
10148         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
10149         enum plane_id plane_id = plane->id;
10150         enum pipe pipe;
10151         u32 val, base, offset, stride_mult, tiling, alpha;
10152         int fourcc, pixel_format;
10153         unsigned int aligned_height;
10154         struct drm_framebuffer *fb;
10155         struct intel_framebuffer *intel_fb;
10156
10157         if (!plane->get_hw_state(plane, &pipe))
10158                 return;
10159
10160         WARN_ON(pipe != crtc->pipe);
10161
10162         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
10163         if (!intel_fb) {
10164                 DRM_DEBUG_KMS("failed to alloc fb\n");
10165                 return;
10166         }
10167
10168         fb = &intel_fb->base;
10169
10170         fb->dev = dev;
10171
10172         val = I915_READ(PLANE_CTL(pipe, plane_id));
10173
10174         if (INTEL_GEN(dev_priv) >= 11)
10175                 pixel_format = val & ICL_PLANE_CTL_FORMAT_MASK;
10176         else
10177                 pixel_format = val & PLANE_CTL_FORMAT_MASK;
10178
10179         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
10180                 alpha = I915_READ(PLANE_COLOR_CTL(pipe, plane_id));
10181                 alpha &= PLANE_COLOR_ALPHA_MASK;
10182         } else {
10183                 alpha = val & PLANE_CTL_ALPHA_MASK;
10184         }
10185
10186         fourcc = skl_format_to_fourcc(pixel_format,
10187                                       val & PLANE_CTL_ORDER_RGBX, alpha);
10188         fb->format = drm_format_info(fourcc);
10189
10190         tiling = val & PLANE_CTL_TILED_MASK;
10191         switch (tiling) {
10192         case PLANE_CTL_TILED_LINEAR:
10193                 fb->modifier = DRM_FORMAT_MOD_LINEAR;
10194                 break;
10195         case PLANE_CTL_TILED_X:
10196                 plane_config->tiling = I915_TILING_X;
10197                 fb->modifier = I915_FORMAT_MOD_X_TILED;
10198                 break;
10199         case PLANE_CTL_TILED_Y:
10200                 plane_config->tiling = I915_TILING_Y;
10201                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
10202                         fb->modifier = INTEL_GEN(dev_priv) >= 12 ?
10203                                 I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS :
10204                                 I915_FORMAT_MOD_Y_TILED_CCS;
10205                 else
10206                         fb->modifier = I915_FORMAT_MOD_Y_TILED;
10207                 break;
10208         case PLANE_CTL_TILED_YF:
10209                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
10210                         fb->modifier = I915_FORMAT_MOD_Yf_TILED_CCS;
10211                 else
10212                         fb->modifier = I915_FORMAT_MOD_Yf_TILED;
10213                 break;
10214         default:
10215                 MISSING_CASE(tiling);
10216                 goto error;
10217         }
10218
10219         /*
10220          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
10221          * while i915 HW rotation is clockwise, thats why this swapping.
10222          */
10223         switch (val & PLANE_CTL_ROTATE_MASK) {
10224         case PLANE_CTL_ROTATE_0:
10225                 plane_config->rotation = DRM_MODE_ROTATE_0;
10226                 break;
10227         case PLANE_CTL_ROTATE_90:
10228                 plane_config->rotation = DRM_MODE_ROTATE_270;
10229                 break;
10230         case PLANE_CTL_ROTATE_180:
10231                 plane_config->rotation = DRM_MODE_ROTATE_180;
10232                 break;
10233         case PLANE_CTL_ROTATE_270:
10234                 plane_config->rotation = DRM_MODE_ROTATE_90;
10235                 break;
10236         }
10237
10238         if (INTEL_GEN(dev_priv) >= 10 &&
10239             val & PLANE_CTL_FLIP_HORIZONTAL)
10240                 plane_config->rotation |= DRM_MODE_REFLECT_X;
10241
10242         base = I915_READ(PLANE_SURF(pipe, plane_id)) & 0xfffff000;
10243         plane_config->base = base;
10244
10245         offset = I915_READ(PLANE_OFFSET(pipe, plane_id));
10246
10247         val = I915_READ(PLANE_SIZE(pipe, plane_id));
10248         fb->height = ((val >> 16) & 0xffff) + 1;
10249         fb->width = ((val >> 0) & 0xffff) + 1;
10250
10251         val = I915_READ(PLANE_STRIDE(pipe, plane_id));
10252         stride_mult = skl_plane_stride_mult(fb, 0, DRM_MODE_ROTATE_0);
10253         fb->pitches[0] = (val & 0x3ff) * stride_mult;
10254
10255         aligned_height = intel_fb_align_height(fb, 0, fb->height);
10256
10257         plane_config->size = fb->pitches[0] * aligned_height;
10258
10259         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
10260                       crtc->base.name, plane->base.name, fb->width, fb->height,
10261                       fb->format->cpp[0] * 8, base, fb->pitches[0],
10262                       plane_config->size);
10263
10264         plane_config->fb = intel_fb;
10265         return;
10266
10267 error:
10268         kfree(intel_fb);
10269 }
10270
10271 static void ilk_get_pfit_config(struct intel_crtc *crtc,
10272                                 struct intel_crtc_state *pipe_config)
10273 {
10274         struct drm_device *dev = crtc->base.dev;
10275         struct drm_i915_private *dev_priv = to_i915(dev);
10276         u32 tmp;
10277
10278         tmp = I915_READ(PF_CTL(crtc->pipe));
10279
10280         if (tmp & PF_ENABLE) {
10281                 pipe_config->pch_pfit.enabled = true;
10282                 pipe_config->pch_pfit.pos = I915_READ(PF_WIN_POS(crtc->pipe));
10283                 pipe_config->pch_pfit.size = I915_READ(PF_WIN_SZ(crtc->pipe));
10284
10285                 /* We currently do not free assignements of panel fitters on
10286                  * ivb/hsw (since we don't use the higher upscaling modes which
10287                  * differentiates them) so just WARN about this case for now. */
10288                 if (IS_GEN(dev_priv, 7)) {
10289                         WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) !=
10290                                 PF_PIPE_SEL_IVB(crtc->pipe));
10291                 }
10292         }
10293 }
10294
10295 static bool ilk_get_pipe_config(struct intel_crtc *crtc,
10296                                 struct intel_crtc_state *pipe_config)
10297 {
10298         struct drm_device *dev = crtc->base.dev;
10299         struct drm_i915_private *dev_priv = to_i915(dev);
10300         enum intel_display_power_domain power_domain;
10301         intel_wakeref_t wakeref;
10302         u32 tmp;
10303         bool ret;
10304
10305         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
10306         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
10307         if (!wakeref)
10308                 return false;
10309
10310         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
10311         pipe_config->shared_dpll = NULL;
10312         pipe_config->master_transcoder = INVALID_TRANSCODER;
10313
10314         ret = false;
10315         tmp = I915_READ(PIPECONF(crtc->pipe));
10316         if (!(tmp & PIPECONF_ENABLE))
10317                 goto out;
10318
10319         switch (tmp & PIPECONF_BPC_MASK) {
10320         case PIPECONF_6BPC:
10321                 pipe_config->pipe_bpp = 18;
10322                 break;
10323         case PIPECONF_8BPC:
10324                 pipe_config->pipe_bpp = 24;
10325                 break;
10326         case PIPECONF_10BPC:
10327                 pipe_config->pipe_bpp = 30;
10328                 break;
10329         case PIPECONF_12BPC:
10330                 pipe_config->pipe_bpp = 36;
10331                 break;
10332         default:
10333                 break;
10334         }
10335
10336         if (tmp & PIPECONF_COLOR_RANGE_SELECT)
10337                 pipe_config->limited_color_range = true;
10338
10339         switch (tmp & PIPECONF_OUTPUT_COLORSPACE_MASK) {
10340         case PIPECONF_OUTPUT_COLORSPACE_YUV601:
10341         case PIPECONF_OUTPUT_COLORSPACE_YUV709:
10342                 pipe_config->output_format = INTEL_OUTPUT_FORMAT_YCBCR444;
10343                 break;
10344         default:
10345                 pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
10346                 break;
10347         }
10348
10349         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_ILK) >>
10350                 PIPECONF_GAMMA_MODE_SHIFT;
10351
10352         pipe_config->csc_mode = I915_READ(PIPE_CSC_MODE(crtc->pipe));
10353
10354         i9xx_get_pipe_color_config(pipe_config);
10355         intel_color_get_config(pipe_config);
10356
10357         if (I915_READ(PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
10358                 struct intel_shared_dpll *pll;
10359                 enum intel_dpll_id pll_id;
10360
10361                 pipe_config->has_pch_encoder = true;
10362
10363                 tmp = I915_READ(FDI_RX_CTL(crtc->pipe));
10364                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
10365                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
10366
10367                 ilk_get_fdi_m_n_config(crtc, pipe_config);
10368
10369                 if (HAS_PCH_IBX(dev_priv)) {
10370                         /*
10371                          * The pipe->pch transcoder and pch transcoder->pll
10372                          * mapping is fixed.
10373                          */
10374                         pll_id = (enum intel_dpll_id) crtc->pipe;
10375                 } else {
10376                         tmp = I915_READ(PCH_DPLL_SEL);
10377                         if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
10378                                 pll_id = DPLL_ID_PCH_PLL_B;
10379                         else
10380                                 pll_id= DPLL_ID_PCH_PLL_A;
10381                 }
10382
10383                 pipe_config->shared_dpll =
10384                         intel_get_shared_dpll_by_id(dev_priv, pll_id);
10385                 pll = pipe_config->shared_dpll;
10386
10387                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
10388                                                 &pipe_config->dpll_hw_state));
10389
10390                 tmp = pipe_config->dpll_hw_state.dpll;
10391                 pipe_config->pixel_multiplier =
10392                         ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
10393                          >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
10394
10395                 ilk_pch_clock_get(crtc, pipe_config);
10396         } else {
10397                 pipe_config->pixel_multiplier = 1;
10398         }
10399
10400         intel_get_pipe_timings(crtc, pipe_config);
10401         intel_get_pipe_src_size(crtc, pipe_config);
10402
10403         ilk_get_pfit_config(crtc, pipe_config);
10404
10405         ret = true;
10406
10407 out:
10408         intel_display_power_put(dev_priv, power_domain, wakeref);
10409
10410         return ret;
10411 }
10412
10413 static int hsw_crtc_compute_clock(struct intel_crtc *crtc,
10414                                   struct intel_crtc_state *crtc_state)
10415 {
10416         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10417         struct intel_atomic_state *state =
10418                 to_intel_atomic_state(crtc_state->uapi.state);
10419
10420         if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) ||
10421             INTEL_GEN(dev_priv) >= 11) {
10422                 struct intel_encoder *encoder =
10423                         intel_get_crtc_new_encoder(state, crtc_state);
10424
10425                 if (!intel_reserve_shared_dplls(state, crtc, encoder)) {
10426                         DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
10427                                       pipe_name(crtc->pipe));
10428                         return -EINVAL;
10429                 }
10430         }
10431
10432         return 0;
10433 }
10434
10435 static void cnl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port,
10436                             struct intel_crtc_state *pipe_config)
10437 {
10438         enum intel_dpll_id id;
10439         u32 temp;
10440
10441         temp = I915_READ(DPCLKA_CFGCR0) & DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
10442         id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port);
10443
10444         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL2))
10445                 return;
10446
10447         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10448 }
10449
10450 static void icl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port,
10451                             struct intel_crtc_state *pipe_config)
10452 {
10453         enum phy phy = intel_port_to_phy(dev_priv, port);
10454         enum icl_port_dpll_id port_dpll_id;
10455         enum intel_dpll_id id;
10456         u32 temp;
10457
10458         if (intel_phy_is_combo(dev_priv, phy)) {
10459                 temp = I915_READ(ICL_DPCLKA_CFGCR0) &
10460                         ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy);
10461                 id = temp >> ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy);
10462                 port_dpll_id = ICL_PORT_DPLL_DEFAULT;
10463         } else if (intel_phy_is_tc(dev_priv, phy)) {
10464                 u32 clk_sel = I915_READ(DDI_CLK_SEL(port)) & DDI_CLK_SEL_MASK;
10465
10466                 if (clk_sel == DDI_CLK_SEL_MG) {
10467                         id = icl_tc_port_to_pll_id(intel_port_to_tc(dev_priv,
10468                                                                     port));
10469                         port_dpll_id = ICL_PORT_DPLL_MG_PHY;
10470                 } else {
10471                         WARN_ON(clk_sel < DDI_CLK_SEL_TBT_162);
10472                         id = DPLL_ID_ICL_TBTPLL;
10473                         port_dpll_id = ICL_PORT_DPLL_DEFAULT;
10474                 }
10475         } else {
10476                 WARN(1, "Invalid port %x\n", port);
10477                 return;
10478         }
10479
10480         pipe_config->icl_port_dplls[port_dpll_id].pll =
10481                 intel_get_shared_dpll_by_id(dev_priv, id);
10482
10483         icl_set_active_port_dpll(pipe_config, port_dpll_id);
10484 }
10485
10486 static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv,
10487                                 enum port port,
10488                                 struct intel_crtc_state *pipe_config)
10489 {
10490         enum intel_dpll_id id;
10491
10492         switch (port) {
10493         case PORT_A:
10494                 id = DPLL_ID_SKL_DPLL0;
10495                 break;
10496         case PORT_B:
10497                 id = DPLL_ID_SKL_DPLL1;
10498                 break;
10499         case PORT_C:
10500                 id = DPLL_ID_SKL_DPLL2;
10501                 break;
10502         default:
10503                 DRM_ERROR("Incorrect port type\n");
10504                 return;
10505         }
10506
10507         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10508 }
10509
10510 static void skl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port,
10511                             struct intel_crtc_state *pipe_config)
10512 {
10513         enum intel_dpll_id id;
10514         u32 temp;
10515
10516         temp = I915_READ(DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port);
10517         id = temp >> (port * 3 + 1);
10518
10519         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL3))
10520                 return;
10521
10522         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10523 }
10524
10525 static void hsw_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port,
10526                             struct intel_crtc_state *pipe_config)
10527 {
10528         enum intel_dpll_id id;
10529         u32 ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
10530
10531         switch (ddi_pll_sel) {
10532         case PORT_CLK_SEL_WRPLL1:
10533                 id = DPLL_ID_WRPLL1;
10534                 break;
10535         case PORT_CLK_SEL_WRPLL2:
10536                 id = DPLL_ID_WRPLL2;
10537                 break;
10538         case PORT_CLK_SEL_SPLL:
10539                 id = DPLL_ID_SPLL;
10540                 break;
10541         case PORT_CLK_SEL_LCPLL_810:
10542                 id = DPLL_ID_LCPLL_810;
10543                 break;
10544         case PORT_CLK_SEL_LCPLL_1350:
10545                 id = DPLL_ID_LCPLL_1350;
10546                 break;
10547         case PORT_CLK_SEL_LCPLL_2700:
10548                 id = DPLL_ID_LCPLL_2700;
10549                 break;
10550         default:
10551                 MISSING_CASE(ddi_pll_sel);
10552                 /* fall through */
10553         case PORT_CLK_SEL_NONE:
10554                 return;
10555         }
10556
10557         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10558 }
10559
10560 static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
10561                                      struct intel_crtc_state *pipe_config,
10562                                      u64 *power_domain_mask,
10563                                      intel_wakeref_t *wakerefs)
10564 {
10565         struct drm_device *dev = crtc->base.dev;
10566         struct drm_i915_private *dev_priv = to_i915(dev);
10567         enum intel_display_power_domain power_domain;
10568         unsigned long panel_transcoder_mask = 0;
10569         unsigned long enabled_panel_transcoders = 0;
10570         enum transcoder panel_transcoder;
10571         intel_wakeref_t wf;
10572         u32 tmp;
10573
10574         if (INTEL_GEN(dev_priv) >= 11)
10575                 panel_transcoder_mask |=
10576                         BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1);
10577
10578         if (HAS_TRANSCODER_EDP(dev_priv))
10579                 panel_transcoder_mask |= BIT(TRANSCODER_EDP);
10580
10581         /*
10582          * The pipe->transcoder mapping is fixed with the exception of the eDP
10583          * and DSI transcoders handled below.
10584          */
10585         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
10586
10587         /*
10588          * XXX: Do intel_display_power_get_if_enabled before reading this (for
10589          * consistency and less surprising code; it's in always on power).
10590          */
10591         for_each_set_bit(panel_transcoder,
10592                          &panel_transcoder_mask,
10593                          ARRAY_SIZE(INTEL_INFO(dev_priv)->trans_offsets)) {
10594                 bool force_thru = false;
10595                 enum pipe trans_pipe;
10596
10597                 tmp = I915_READ(TRANS_DDI_FUNC_CTL(panel_transcoder));
10598                 if (!(tmp & TRANS_DDI_FUNC_ENABLE))
10599                         continue;
10600
10601                 /*
10602                  * Log all enabled ones, only use the first one.
10603                  *
10604                  * FIXME: This won't work for two separate DSI displays.
10605                  */
10606                 enabled_panel_transcoders |= BIT(panel_transcoder);
10607                 if (enabled_panel_transcoders != BIT(panel_transcoder))
10608                         continue;
10609
10610                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
10611                 default:
10612                         WARN(1, "unknown pipe linked to transcoder %s\n",
10613                              transcoder_name(panel_transcoder));
10614                         /* fall through */
10615                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
10616                         force_thru = true;
10617                         /* fall through */
10618                 case TRANS_DDI_EDP_INPUT_A_ON:
10619                         trans_pipe = PIPE_A;
10620                         break;
10621                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
10622                         trans_pipe = PIPE_B;
10623                         break;
10624                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
10625                         trans_pipe = PIPE_C;
10626                         break;
10627                 case TRANS_DDI_EDP_INPUT_D_ONOFF:
10628                         trans_pipe = PIPE_D;
10629                         break;
10630                 }
10631
10632                 if (trans_pipe == crtc->pipe) {
10633                         pipe_config->cpu_transcoder = panel_transcoder;
10634                         pipe_config->pch_pfit.force_thru = force_thru;
10635                 }
10636         }
10637
10638         /*
10639          * Valid combos: none, eDP, DSI0, DSI1, DSI0+DSI1
10640          */
10641         WARN_ON((enabled_panel_transcoders & BIT(TRANSCODER_EDP)) &&
10642                 enabled_panel_transcoders != BIT(TRANSCODER_EDP));
10643
10644         power_domain = POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder);
10645         WARN_ON(*power_domain_mask & BIT_ULL(power_domain));
10646
10647         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10648         if (!wf)
10649                 return false;
10650
10651         wakerefs[power_domain] = wf;
10652         *power_domain_mask |= BIT_ULL(power_domain);
10653
10654         tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
10655
10656         return tmp & PIPECONF_ENABLE;
10657 }
10658
10659 static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
10660                                          struct intel_crtc_state *pipe_config,
10661                                          u64 *power_domain_mask,
10662                                          intel_wakeref_t *wakerefs)
10663 {
10664         struct drm_device *dev = crtc->base.dev;
10665         struct drm_i915_private *dev_priv = to_i915(dev);
10666         enum intel_display_power_domain power_domain;
10667         enum transcoder cpu_transcoder;
10668         intel_wakeref_t wf;
10669         enum port port;
10670         u32 tmp;
10671
10672         for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) {
10673                 if (port == PORT_A)
10674                         cpu_transcoder = TRANSCODER_DSI_A;
10675                 else
10676                         cpu_transcoder = TRANSCODER_DSI_C;
10677
10678                 power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
10679                 WARN_ON(*power_domain_mask & BIT_ULL(power_domain));
10680
10681                 wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10682                 if (!wf)
10683                         continue;
10684
10685                 wakerefs[power_domain] = wf;
10686                 *power_domain_mask |= BIT_ULL(power_domain);
10687
10688                 /*
10689                  * The PLL needs to be enabled with a valid divider
10690                  * configuration, otherwise accessing DSI registers will hang
10691                  * the machine. See BSpec North Display Engine
10692                  * registers/MIPI[BXT]. We can break out here early, since we
10693                  * need the same DSI PLL to be enabled for both DSI ports.
10694                  */
10695                 if (!bxt_dsi_pll_is_enabled(dev_priv))
10696                         break;
10697
10698                 /* XXX: this works for video mode only */
10699                 tmp = I915_READ(BXT_MIPI_PORT_CTRL(port));
10700                 if (!(tmp & DPI_ENABLE))
10701                         continue;
10702
10703                 tmp = I915_READ(MIPI_CTRL(port));
10704                 if ((tmp & BXT_PIPE_SELECT_MASK) != BXT_PIPE_SELECT(crtc->pipe))
10705                         continue;
10706
10707                 pipe_config->cpu_transcoder = cpu_transcoder;
10708                 break;
10709         }
10710
10711         return transcoder_is_dsi(pipe_config->cpu_transcoder);
10712 }
10713
10714 static void hsw_get_ddi_port_state(struct intel_crtc *crtc,
10715                                    struct intel_crtc_state *pipe_config)
10716 {
10717         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10718         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
10719         struct intel_shared_dpll *pll;
10720         enum port port;
10721         u32 tmp;
10722
10723         if (transcoder_is_dsi(cpu_transcoder)) {
10724                 port = (cpu_transcoder == TRANSCODER_DSI_A) ?
10725                                                 PORT_A : PORT_B;
10726         } else {
10727                 tmp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
10728                 if (INTEL_GEN(dev_priv) >= 12)
10729                         port = TGL_TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp);
10730                 else
10731                         port = TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp);
10732         }
10733
10734         if (INTEL_GEN(dev_priv) >= 11)
10735                 icl_get_ddi_pll(dev_priv, port, pipe_config);
10736         else if (IS_CANNONLAKE(dev_priv))
10737                 cnl_get_ddi_pll(dev_priv, port, pipe_config);
10738         else if (IS_GEN9_BC(dev_priv))
10739                 skl_get_ddi_pll(dev_priv, port, pipe_config);
10740         else if (IS_GEN9_LP(dev_priv))
10741                 bxt_get_ddi_pll(dev_priv, port, pipe_config);
10742         else
10743                 hsw_get_ddi_pll(dev_priv, port, pipe_config);
10744
10745         pll = pipe_config->shared_dpll;
10746         if (pll) {
10747                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
10748                                                 &pipe_config->dpll_hw_state));
10749         }
10750
10751         /*
10752          * Haswell has only FDI/PCH transcoder A. It is which is connected to
10753          * DDI E. So just check whether this pipe is wired to DDI E and whether
10754          * the PCH transcoder is on.
10755          */
10756         if (INTEL_GEN(dev_priv) < 9 &&
10757             (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) {
10758                 pipe_config->has_pch_encoder = true;
10759
10760                 tmp = I915_READ(FDI_RX_CTL(PIPE_A));
10761                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
10762                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
10763
10764                 ilk_get_fdi_m_n_config(crtc, pipe_config);
10765         }
10766 }
10767
10768 static enum transcoder transcoder_master_readout(struct drm_i915_private *dev_priv,
10769                                                  enum transcoder cpu_transcoder)
10770 {
10771         u32 trans_port_sync, master_select;
10772
10773         trans_port_sync = I915_READ(TRANS_DDI_FUNC_CTL2(cpu_transcoder));
10774
10775         if ((trans_port_sync & PORT_SYNC_MODE_ENABLE) == 0)
10776                 return INVALID_TRANSCODER;
10777
10778         master_select = trans_port_sync &
10779                         PORT_SYNC_MODE_MASTER_SELECT_MASK;
10780         if (master_select == 0)
10781                 return TRANSCODER_EDP;
10782         else
10783                 return master_select - 1;
10784 }
10785
10786 static void icl_get_trans_port_sync_config(struct intel_crtc_state *crtc_state)
10787 {
10788         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
10789         u32 transcoders;
10790         enum transcoder cpu_transcoder;
10791
10792         crtc_state->master_transcoder = transcoder_master_readout(dev_priv,
10793                                                                   crtc_state->cpu_transcoder);
10794
10795         transcoders = BIT(TRANSCODER_A) |
10796                 BIT(TRANSCODER_B) |
10797                 BIT(TRANSCODER_C) |
10798                 BIT(TRANSCODER_D);
10799         for_each_cpu_transcoder_masked(dev_priv, cpu_transcoder, transcoders) {
10800                 enum intel_display_power_domain power_domain;
10801                 intel_wakeref_t trans_wakeref;
10802
10803                 power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
10804                 trans_wakeref = intel_display_power_get_if_enabled(dev_priv,
10805                                                                    power_domain);
10806
10807                 if (!trans_wakeref)
10808                         continue;
10809
10810                 if (transcoder_master_readout(dev_priv, cpu_transcoder) ==
10811                     crtc_state->cpu_transcoder)
10812                         crtc_state->sync_mode_slaves_mask |= BIT(cpu_transcoder);
10813
10814                 intel_display_power_put(dev_priv, power_domain, trans_wakeref);
10815         }
10816
10817         WARN_ON(crtc_state->master_transcoder != INVALID_TRANSCODER &&
10818                 crtc_state->sync_mode_slaves_mask);
10819 }
10820
10821 static bool hsw_get_pipe_config(struct intel_crtc *crtc,
10822                                 struct intel_crtc_state *pipe_config)
10823 {
10824         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10825         intel_wakeref_t wakerefs[POWER_DOMAIN_NUM], wf;
10826         enum intel_display_power_domain power_domain;
10827         u64 power_domain_mask;
10828         bool active;
10829
10830         pipe_config->master_transcoder = INVALID_TRANSCODER;
10831
10832         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
10833         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10834         if (!wf)
10835                 return false;
10836
10837         wakerefs[power_domain] = wf;
10838         power_domain_mask = BIT_ULL(power_domain);
10839
10840         pipe_config->shared_dpll = NULL;
10841
10842         active = hsw_get_transcoder_state(crtc, pipe_config,
10843                                           &power_domain_mask, wakerefs);
10844
10845         if (IS_GEN9_LP(dev_priv) &&
10846             bxt_get_dsi_transcoder_state(crtc, pipe_config,
10847                                          &power_domain_mask, wakerefs)) {
10848                 WARN_ON(active);
10849                 active = true;
10850         }
10851
10852         if (!active)
10853                 goto out;
10854
10855         if (!transcoder_is_dsi(pipe_config->cpu_transcoder) ||
10856             INTEL_GEN(dev_priv) >= 11) {
10857                 hsw_get_ddi_port_state(crtc, pipe_config);
10858                 intel_get_pipe_timings(crtc, pipe_config);
10859         }
10860
10861         intel_get_pipe_src_size(crtc, pipe_config);
10862
10863         if (IS_HASWELL(dev_priv)) {
10864                 u32 tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
10865
10866                 if (tmp & PIPECONF_OUTPUT_COLORSPACE_YUV_HSW)
10867                         pipe_config->output_format = INTEL_OUTPUT_FORMAT_YCBCR444;
10868                 else
10869                         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
10870         } else {
10871                 pipe_config->output_format =
10872                         bdw_get_pipemisc_output_format(crtc);
10873
10874                 /*
10875                  * Currently there is no interface defined to
10876                  * check user preference between RGB/YCBCR444
10877                  * or YCBCR420. So the only possible case for
10878                  * YCBCR444 usage is driving YCBCR420 output
10879                  * with LSPCON, when pipe is configured for
10880                  * YCBCR444 output and LSPCON takes care of
10881                  * downsampling it.
10882                  */
10883                 pipe_config->lspcon_downsampling =
10884                         pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR444;
10885         }
10886
10887         pipe_config->gamma_mode = I915_READ(GAMMA_MODE(crtc->pipe));
10888
10889         pipe_config->csc_mode = I915_READ(PIPE_CSC_MODE(crtc->pipe));
10890
10891         if (INTEL_GEN(dev_priv) >= 9) {
10892                 u32 tmp = I915_READ(SKL_BOTTOM_COLOR(crtc->pipe));
10893
10894                 if (tmp & SKL_BOTTOM_COLOR_GAMMA_ENABLE)
10895                         pipe_config->gamma_enable = true;
10896
10897                 if (tmp & SKL_BOTTOM_COLOR_CSC_ENABLE)
10898                         pipe_config->csc_enable = true;
10899         } else {
10900                 i9xx_get_pipe_color_config(pipe_config);
10901         }
10902
10903         intel_color_get_config(pipe_config);
10904
10905         power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
10906         WARN_ON(power_domain_mask & BIT_ULL(power_domain));
10907
10908         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10909         if (wf) {
10910                 wakerefs[power_domain] = wf;
10911                 power_domain_mask |= BIT_ULL(power_domain);
10912
10913                 if (INTEL_GEN(dev_priv) >= 9)
10914                         skl_get_pfit_config(crtc, pipe_config);
10915                 else
10916                         ilk_get_pfit_config(crtc, pipe_config);
10917         }
10918
10919         if (hsw_crtc_supports_ips(crtc)) {
10920                 if (IS_HASWELL(dev_priv))
10921                         pipe_config->ips_enabled = I915_READ(IPS_CTL) & IPS_ENABLE;
10922                 else {
10923                         /*
10924                          * We cannot readout IPS state on broadwell, set to
10925                          * true so we can set it to a defined state on first
10926                          * commit.
10927                          */
10928                         pipe_config->ips_enabled = true;
10929                 }
10930         }
10931
10932         if (pipe_config->cpu_transcoder != TRANSCODER_EDP &&
10933             !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
10934                 pipe_config->pixel_multiplier =
10935                         I915_READ(PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
10936         } else {
10937                 pipe_config->pixel_multiplier = 1;
10938         }
10939
10940         if (INTEL_GEN(dev_priv) >= 11 &&
10941             !transcoder_is_dsi(pipe_config->cpu_transcoder))
10942                 icl_get_trans_port_sync_config(pipe_config);
10943
10944 out:
10945         for_each_power_domain(power_domain, power_domain_mask)
10946                 intel_display_power_put(dev_priv,
10947                                         power_domain, wakerefs[power_domain]);
10948
10949         return active;
10950 }
10951
10952 static u32 intel_cursor_base(const struct intel_plane_state *plane_state)
10953 {
10954         struct drm_i915_private *dev_priv =
10955                 to_i915(plane_state->uapi.plane->dev);
10956         const struct drm_framebuffer *fb = plane_state->hw.fb;
10957         const struct drm_i915_gem_object *obj = intel_fb_obj(fb);
10958         u32 base;
10959
10960         if (INTEL_INFO(dev_priv)->display.cursor_needs_physical)
10961                 base = obj->phys_handle->busaddr;
10962         else
10963                 base = intel_plane_ggtt_offset(plane_state);
10964
10965         return base + plane_state->color_plane[0].offset;
10966 }
10967
10968 static u32 intel_cursor_position(const struct intel_plane_state *plane_state)
10969 {
10970         int x = plane_state->uapi.dst.x1;
10971         int y = plane_state->uapi.dst.y1;
10972         u32 pos = 0;
10973
10974         if (x < 0) {
10975                 pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
10976                 x = -x;
10977         }
10978         pos |= x << CURSOR_X_SHIFT;
10979
10980         if (y < 0) {
10981                 pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
10982                 y = -y;
10983         }
10984         pos |= y << CURSOR_Y_SHIFT;
10985
10986         return pos;
10987 }
10988
10989 static bool intel_cursor_size_ok(const struct intel_plane_state *plane_state)
10990 {
10991         const struct drm_mode_config *config =
10992                 &plane_state->uapi.plane->dev->mode_config;
10993         int width = drm_rect_width(&plane_state->uapi.dst);
10994         int height = drm_rect_height(&plane_state->uapi.dst);
10995
10996         return width > 0 && width <= config->cursor_width &&
10997                 height > 0 && height <= config->cursor_height;
10998 }
10999
11000 static int intel_cursor_check_surface(struct intel_plane_state *plane_state)
11001 {
11002         struct drm_i915_private *dev_priv =
11003                 to_i915(plane_state->uapi.plane->dev);
11004         unsigned int rotation = plane_state->hw.rotation;
11005         int src_x, src_y;
11006         u32 offset;
11007         int ret;
11008
11009         ret = intel_plane_compute_gtt(plane_state);
11010         if (ret)
11011                 return ret;
11012
11013         if (!plane_state->uapi.visible)
11014                 return 0;
11015
11016         src_x = plane_state->uapi.src.x1 >> 16;
11017         src_y = plane_state->uapi.src.y1 >> 16;
11018
11019         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
11020         offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
11021                                                     plane_state, 0);
11022
11023         if (src_x != 0 || src_y != 0) {
11024                 DRM_DEBUG_KMS("Arbitrary cursor panning not supported\n");
11025                 return -EINVAL;
11026         }
11027
11028         /*
11029          * Put the final coordinates back so that the src
11030          * coordinate checks will see the right values.
11031          */
11032         drm_rect_translate_to(&plane_state->uapi.src,
11033                               src_x << 16, src_y << 16);
11034
11035         /* ILK+ do this automagically in hardware */
11036         if (HAS_GMCH(dev_priv) && rotation & DRM_MODE_ROTATE_180) {
11037                 const struct drm_framebuffer *fb = plane_state->hw.fb;
11038                 int src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
11039                 int src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
11040
11041                 offset += (src_h * src_w - 1) * fb->format->cpp[0];
11042         }
11043
11044         plane_state->color_plane[0].offset = offset;
11045         plane_state->color_plane[0].x = src_x;
11046         plane_state->color_plane[0].y = src_y;
11047
11048         return 0;
11049 }
11050
11051 static int intel_check_cursor(struct intel_crtc_state *crtc_state,
11052                               struct intel_plane_state *plane_state)
11053 {
11054         const struct drm_framebuffer *fb = plane_state->hw.fb;
11055         int ret;
11056
11057         if (fb && fb->modifier != DRM_FORMAT_MOD_LINEAR) {
11058                 DRM_DEBUG_KMS("cursor cannot be tiled\n");
11059                 return -EINVAL;
11060         }
11061
11062         ret = drm_atomic_helper_check_plane_state(&plane_state->uapi,
11063                                                   &crtc_state->uapi,
11064                                                   DRM_PLANE_HELPER_NO_SCALING,
11065                                                   DRM_PLANE_HELPER_NO_SCALING,
11066                                                   true, true);
11067         if (ret)
11068                 return ret;
11069
11070         /* Use the unclipped src/dst rectangles, which we program to hw */
11071         plane_state->uapi.src = drm_plane_state_src(&plane_state->uapi);
11072         plane_state->uapi.dst = drm_plane_state_dest(&plane_state->uapi);
11073
11074         ret = intel_cursor_check_surface(plane_state);
11075         if (ret)
11076                 return ret;
11077
11078         if (!plane_state->uapi.visible)
11079                 return 0;
11080
11081         ret = intel_plane_check_src_coordinates(plane_state);
11082         if (ret)
11083                 return ret;
11084
11085         return 0;
11086 }
11087
11088 static unsigned int
11089 i845_cursor_max_stride(struct intel_plane *plane,
11090                        u32 pixel_format, u64 modifier,
11091                        unsigned int rotation)
11092 {
11093         return 2048;
11094 }
11095
11096 static u32 i845_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state)
11097 {
11098         u32 cntl = 0;
11099
11100         if (crtc_state->gamma_enable)
11101                 cntl |= CURSOR_GAMMA_ENABLE;
11102
11103         return cntl;
11104 }
11105
11106 static u32 i845_cursor_ctl(const struct intel_crtc_state *crtc_state,
11107                            const struct intel_plane_state *plane_state)
11108 {
11109         return CURSOR_ENABLE |
11110                 CURSOR_FORMAT_ARGB |
11111                 CURSOR_STRIDE(plane_state->color_plane[0].stride);
11112 }
11113
11114 static bool i845_cursor_size_ok(const struct intel_plane_state *plane_state)
11115 {
11116         int width = drm_rect_width(&plane_state->uapi.dst);
11117
11118         /*
11119          * 845g/865g are only limited by the width of their cursors,
11120          * the height is arbitrary up to the precision of the register.
11121          */
11122         return intel_cursor_size_ok(plane_state) && IS_ALIGNED(width, 64);
11123 }
11124
11125 static int i845_check_cursor(struct intel_crtc_state *crtc_state,
11126                              struct intel_plane_state *plane_state)
11127 {
11128         const struct drm_framebuffer *fb = plane_state->hw.fb;
11129         int ret;
11130
11131         ret = intel_check_cursor(crtc_state, plane_state);
11132         if (ret)
11133                 return ret;
11134
11135         /* if we want to turn off the cursor ignore width and height */
11136         if (!fb)
11137                 return 0;
11138
11139         /* Check for which cursor types we support */
11140         if (!i845_cursor_size_ok(plane_state)) {
11141                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
11142                           drm_rect_width(&plane_state->uapi.dst),
11143                           drm_rect_height(&plane_state->uapi.dst));
11144                 return -EINVAL;
11145         }
11146
11147         WARN_ON(plane_state->uapi.visible &&
11148                 plane_state->color_plane[0].stride != fb->pitches[0]);
11149
11150         switch (fb->pitches[0]) {
11151         case 256:
11152         case 512:
11153         case 1024:
11154         case 2048:
11155                 break;
11156         default:
11157                 DRM_DEBUG_KMS("Invalid cursor stride (%u)\n",
11158                               fb->pitches[0]);
11159                 return -EINVAL;
11160         }
11161
11162         plane_state->ctl = i845_cursor_ctl(crtc_state, plane_state);
11163
11164         return 0;
11165 }
11166
11167 static void i845_update_cursor(struct intel_plane *plane,
11168                                const struct intel_crtc_state *crtc_state,
11169                                const struct intel_plane_state *plane_state)
11170 {
11171         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11172         u32 cntl = 0, base = 0, pos = 0, size = 0;
11173         unsigned long irqflags;
11174
11175         if (plane_state && plane_state->uapi.visible) {
11176                 unsigned int width = drm_rect_width(&plane_state->uapi.dst);
11177                 unsigned int height = drm_rect_height(&plane_state->uapi.dst);
11178
11179                 cntl = plane_state->ctl |
11180                         i845_cursor_ctl_crtc(crtc_state);
11181
11182                 size = (height << 12) | width;
11183
11184                 base = intel_cursor_base(plane_state);
11185                 pos = intel_cursor_position(plane_state);
11186         }
11187
11188         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
11189
11190         /* On these chipsets we can only modify the base/size/stride
11191          * whilst the cursor is disabled.
11192          */
11193         if (plane->cursor.base != base ||
11194             plane->cursor.size != size ||
11195             plane->cursor.cntl != cntl) {
11196                 I915_WRITE_FW(CURCNTR(PIPE_A), 0);
11197                 I915_WRITE_FW(CURBASE(PIPE_A), base);
11198                 I915_WRITE_FW(CURSIZE, size);
11199                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
11200                 I915_WRITE_FW(CURCNTR(PIPE_A), cntl);
11201
11202                 plane->cursor.base = base;
11203                 plane->cursor.size = size;
11204                 plane->cursor.cntl = cntl;
11205         } else {
11206                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
11207         }
11208
11209         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
11210 }
11211
11212 static void i845_disable_cursor(struct intel_plane *plane,
11213                                 const struct intel_crtc_state *crtc_state)
11214 {
11215         i845_update_cursor(plane, crtc_state, NULL);
11216 }
11217
11218 static bool i845_cursor_get_hw_state(struct intel_plane *plane,
11219                                      enum pipe *pipe)
11220 {
11221         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11222         enum intel_display_power_domain power_domain;
11223         intel_wakeref_t wakeref;
11224         bool ret;
11225
11226         power_domain = POWER_DOMAIN_PIPE(PIPE_A);
11227         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
11228         if (!wakeref)
11229                 return false;
11230
11231         ret = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
11232
11233         *pipe = PIPE_A;
11234
11235         intel_display_power_put(dev_priv, power_domain, wakeref);
11236
11237         return ret;
11238 }
11239
11240 static unsigned int
11241 i9xx_cursor_max_stride(struct intel_plane *plane,
11242                        u32 pixel_format, u64 modifier,
11243                        unsigned int rotation)
11244 {
11245         return plane->base.dev->mode_config.cursor_width * 4;
11246 }
11247
11248 static u32 i9xx_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state)
11249 {
11250         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
11251         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11252         u32 cntl = 0;
11253
11254         if (INTEL_GEN(dev_priv) >= 11)
11255                 return cntl;
11256
11257         if (crtc_state->gamma_enable)
11258                 cntl = MCURSOR_GAMMA_ENABLE;
11259
11260         if (crtc_state->csc_enable)
11261                 cntl |= MCURSOR_PIPE_CSC_ENABLE;
11262
11263         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
11264                 cntl |= MCURSOR_PIPE_SELECT(crtc->pipe);
11265
11266         return cntl;
11267 }
11268
11269 static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state,
11270                            const struct intel_plane_state *plane_state)
11271 {
11272         struct drm_i915_private *dev_priv =
11273                 to_i915(plane_state->uapi.plane->dev);
11274         u32 cntl = 0;
11275
11276         if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
11277                 cntl |= MCURSOR_TRICKLE_FEED_DISABLE;
11278
11279         switch (drm_rect_width(&plane_state->uapi.dst)) {
11280         case 64:
11281                 cntl |= MCURSOR_MODE_64_ARGB_AX;
11282                 break;
11283         case 128:
11284                 cntl |= MCURSOR_MODE_128_ARGB_AX;
11285                 break;
11286         case 256:
11287                 cntl |= MCURSOR_MODE_256_ARGB_AX;
11288                 break;
11289         default:
11290                 MISSING_CASE(drm_rect_width(&plane_state->uapi.dst));
11291                 return 0;
11292         }
11293
11294         if (plane_state->hw.rotation & DRM_MODE_ROTATE_180)
11295                 cntl |= MCURSOR_ROTATE_180;
11296
11297         return cntl;
11298 }
11299
11300 static bool i9xx_cursor_size_ok(const struct intel_plane_state *plane_state)
11301 {
11302         struct drm_i915_private *dev_priv =
11303                 to_i915(plane_state->uapi.plane->dev);
11304         int width = drm_rect_width(&plane_state->uapi.dst);
11305         int height = drm_rect_height(&plane_state->uapi.dst);
11306
11307         if (!intel_cursor_size_ok(plane_state))
11308                 return false;
11309
11310         /* Cursor width is limited to a few power-of-two sizes */
11311         switch (width) {
11312         case 256:
11313         case 128:
11314         case 64:
11315                 break;
11316         default:
11317                 return false;
11318         }
11319
11320         /*
11321          * IVB+ have CUR_FBC_CTL which allows an arbitrary cursor
11322          * height from 8 lines up to the cursor width, when the
11323          * cursor is not rotated. Everything else requires square
11324          * cursors.
11325          */
11326         if (HAS_CUR_FBC(dev_priv) &&
11327             plane_state->hw.rotation & DRM_MODE_ROTATE_0) {
11328                 if (height < 8 || height > width)
11329                         return false;
11330         } else {
11331                 if (height != width)
11332                         return false;
11333         }
11334
11335         return true;
11336 }
11337
11338 static int i9xx_check_cursor(struct intel_crtc_state *crtc_state,
11339                              struct intel_plane_state *plane_state)
11340 {
11341         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
11342         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11343         const struct drm_framebuffer *fb = plane_state->hw.fb;
11344         enum pipe pipe = plane->pipe;
11345         int ret;
11346
11347         ret = intel_check_cursor(crtc_state, plane_state);
11348         if (ret)
11349                 return ret;
11350
11351         /* if we want to turn off the cursor ignore width and height */
11352         if (!fb)
11353                 return 0;
11354
11355         /* Check for which cursor types we support */
11356         if (!i9xx_cursor_size_ok(plane_state)) {
11357                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
11358                           drm_rect_width(&plane_state->uapi.dst),
11359                           drm_rect_height(&plane_state->uapi.dst));
11360                 return -EINVAL;
11361         }
11362
11363         WARN_ON(plane_state->uapi.visible &&
11364                 plane_state->color_plane[0].stride != fb->pitches[0]);
11365
11366         if (fb->pitches[0] !=
11367             drm_rect_width(&plane_state->uapi.dst) * fb->format->cpp[0]) {
11368                 DRM_DEBUG_KMS("Invalid cursor stride (%u) (cursor width %d)\n",
11369                               fb->pitches[0],
11370                               drm_rect_width(&plane_state->uapi.dst));
11371                 return -EINVAL;
11372         }
11373
11374         /*
11375          * There's something wrong with the cursor on CHV pipe C.
11376          * If it straddles the left edge of the screen then
11377          * moving it away from the edge or disabling it often
11378          * results in a pipe underrun, and often that can lead to
11379          * dead pipe (constant underrun reported, and it scans
11380          * out just a solid color). To recover from that, the
11381          * display power well must be turned off and on again.
11382          * Refuse the put the cursor into that compromised position.
11383          */
11384         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_C &&
11385             plane_state->uapi.visible && plane_state->uapi.dst.x1 < 0) {
11386                 DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
11387                 return -EINVAL;
11388         }
11389
11390         plane_state->ctl = i9xx_cursor_ctl(crtc_state, plane_state);
11391
11392         return 0;
11393 }
11394
11395 static void i9xx_update_cursor(struct intel_plane *plane,
11396                                const struct intel_crtc_state *crtc_state,
11397                                const struct intel_plane_state *plane_state)
11398 {
11399         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11400         enum pipe pipe = plane->pipe;
11401         u32 cntl = 0, base = 0, pos = 0, fbc_ctl = 0;
11402         unsigned long irqflags;
11403
11404         if (plane_state && plane_state->uapi.visible) {
11405                 unsigned width = drm_rect_width(&plane_state->uapi.dst);
11406                 unsigned height = drm_rect_height(&plane_state->uapi.dst);
11407
11408                 cntl = plane_state->ctl |
11409                         i9xx_cursor_ctl_crtc(crtc_state);
11410
11411                 if (width != height)
11412                         fbc_ctl = CUR_FBC_CTL_EN | (height - 1);
11413
11414                 base = intel_cursor_base(plane_state);
11415                 pos = intel_cursor_position(plane_state);
11416         }
11417
11418         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
11419
11420         /*
11421          * On some platforms writing CURCNTR first will also
11422          * cause CURPOS to be armed by the CURBASE write.
11423          * Without the CURCNTR write the CURPOS write would
11424          * arm itself. Thus we always update CURCNTR before
11425          * CURPOS.
11426          *
11427          * On other platforms CURPOS always requires the
11428          * CURBASE write to arm the update. Additonally
11429          * a write to any of the cursor register will cancel
11430          * an already armed cursor update. Thus leaving out
11431          * the CURBASE write after CURPOS could lead to a
11432          * cursor that doesn't appear to move, or even change
11433          * shape. Thus we always write CURBASE.
11434          *
11435          * The other registers are armed by by the CURBASE write
11436          * except when the plane is getting enabled at which time
11437          * the CURCNTR write arms the update.
11438          */
11439
11440         if (INTEL_GEN(dev_priv) >= 9)
11441                 skl_write_cursor_wm(plane, crtc_state);
11442
11443         if (plane->cursor.base != base ||
11444             plane->cursor.size != fbc_ctl ||
11445             plane->cursor.cntl != cntl) {
11446                 if (HAS_CUR_FBC(dev_priv))
11447                         I915_WRITE_FW(CUR_FBC_CTL(pipe), fbc_ctl);
11448                 I915_WRITE_FW(CURCNTR(pipe), cntl);
11449                 I915_WRITE_FW(CURPOS(pipe), pos);
11450                 I915_WRITE_FW(CURBASE(pipe), base);
11451
11452                 plane->cursor.base = base;
11453                 plane->cursor.size = fbc_ctl;
11454                 plane->cursor.cntl = cntl;
11455         } else {
11456                 I915_WRITE_FW(CURPOS(pipe), pos);
11457                 I915_WRITE_FW(CURBASE(pipe), base);
11458         }
11459
11460         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
11461 }
11462
11463 static void i9xx_disable_cursor(struct intel_plane *plane,
11464                                 const struct intel_crtc_state *crtc_state)
11465 {
11466         i9xx_update_cursor(plane, crtc_state, NULL);
11467 }
11468
11469 static bool i9xx_cursor_get_hw_state(struct intel_plane *plane,
11470                                      enum pipe *pipe)
11471 {
11472         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11473         enum intel_display_power_domain power_domain;
11474         intel_wakeref_t wakeref;
11475         bool ret;
11476         u32 val;
11477
11478         /*
11479          * Not 100% correct for planes that can move between pipes,
11480          * but that's only the case for gen2-3 which don't have any
11481          * display power wells.
11482          */
11483         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
11484         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
11485         if (!wakeref)
11486                 return false;
11487
11488         val = I915_READ(CURCNTR(plane->pipe));
11489
11490         ret = val & MCURSOR_MODE;
11491
11492         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
11493                 *pipe = plane->pipe;
11494         else
11495                 *pipe = (val & MCURSOR_PIPE_SELECT_MASK) >>
11496                         MCURSOR_PIPE_SELECT_SHIFT;
11497
11498         intel_display_power_put(dev_priv, power_domain, wakeref);
11499
11500         return ret;
11501 }
11502
11503 /* VESA 640x480x72Hz mode to set on the pipe */
11504 static const struct drm_display_mode load_detect_mode = {
11505         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
11506                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
11507 };
11508
11509 struct drm_framebuffer *
11510 intel_framebuffer_create(struct drm_i915_gem_object *obj,
11511                          struct drm_mode_fb_cmd2 *mode_cmd)
11512 {
11513         struct intel_framebuffer *intel_fb;
11514         int ret;
11515
11516         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
11517         if (!intel_fb)
11518                 return ERR_PTR(-ENOMEM);
11519
11520         ret = intel_framebuffer_init(intel_fb, obj, mode_cmd);
11521         if (ret)
11522                 goto err;
11523
11524         return &intel_fb->base;
11525
11526 err:
11527         kfree(intel_fb);
11528         return ERR_PTR(ret);
11529 }
11530
11531 static int intel_modeset_disable_planes(struct drm_atomic_state *state,
11532                                         struct drm_crtc *crtc)
11533 {
11534         struct drm_plane *plane;
11535         struct drm_plane_state *plane_state;
11536         int ret, i;
11537
11538         ret = drm_atomic_add_affected_planes(state, crtc);
11539         if (ret)
11540                 return ret;
11541
11542         for_each_new_plane_in_state(state, plane, plane_state, i) {
11543                 if (plane_state->crtc != crtc)
11544                         continue;
11545
11546                 ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
11547                 if (ret)
11548                         return ret;
11549
11550                 drm_atomic_set_fb_for_plane(plane_state, NULL);
11551         }
11552
11553         return 0;
11554 }
11555
11556 int intel_get_load_detect_pipe(struct drm_connector *connector,
11557                                struct intel_load_detect_pipe *old,
11558                                struct drm_modeset_acquire_ctx *ctx)
11559 {
11560         struct intel_crtc *intel_crtc;
11561         struct intel_encoder *intel_encoder =
11562                 intel_attached_encoder(connector);
11563         struct drm_crtc *possible_crtc;
11564         struct drm_encoder *encoder = &intel_encoder->base;
11565         struct drm_crtc *crtc = NULL;
11566         struct drm_device *dev = encoder->dev;
11567         struct drm_i915_private *dev_priv = to_i915(dev);
11568         struct drm_mode_config *config = &dev->mode_config;
11569         struct drm_atomic_state *state = NULL, *restore_state = NULL;
11570         struct drm_connector_state *connector_state;
11571         struct intel_crtc_state *crtc_state;
11572         int ret, i = -1;
11573
11574         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
11575                       connector->base.id, connector->name,
11576                       encoder->base.id, encoder->name);
11577
11578         old->restore_state = NULL;
11579
11580         WARN_ON(!drm_modeset_is_locked(&config->connection_mutex));
11581
11582         /*
11583          * Algorithm gets a little messy:
11584          *
11585          *   - if the connector already has an assigned crtc, use it (but make
11586          *     sure it's on first)
11587          *
11588          *   - try to find the first unused crtc that can drive this connector,
11589          *     and use that if we find one
11590          */
11591
11592         /* See if we already have a CRTC for this connector */
11593         if (connector->state->crtc) {
11594                 crtc = connector->state->crtc;
11595
11596                 ret = drm_modeset_lock(&crtc->mutex, ctx);
11597                 if (ret)
11598                         goto fail;
11599
11600                 /* Make sure the crtc and connector are running */
11601                 goto found;
11602         }
11603
11604         /* Find an unused one (if possible) */
11605         for_each_crtc(dev, possible_crtc) {
11606                 i++;
11607                 if (!(encoder->possible_crtcs & (1 << i)))
11608                         continue;
11609
11610                 ret = drm_modeset_lock(&possible_crtc->mutex, ctx);
11611                 if (ret)
11612                         goto fail;
11613
11614                 if (possible_crtc->state->enable) {
11615                         drm_modeset_unlock(&possible_crtc->mutex);
11616                         continue;
11617                 }
11618
11619                 crtc = possible_crtc;
11620                 break;
11621         }
11622
11623         /*
11624          * If we didn't find an unused CRTC, don't use any.
11625          */
11626         if (!crtc) {
11627                 DRM_DEBUG_KMS("no pipe available for load-detect\n");
11628                 ret = -ENODEV;
11629                 goto fail;
11630         }
11631
11632 found:
11633         intel_crtc = to_intel_crtc(crtc);
11634
11635         state = drm_atomic_state_alloc(dev);
11636         restore_state = drm_atomic_state_alloc(dev);
11637         if (!state || !restore_state) {
11638                 ret = -ENOMEM;
11639                 goto fail;
11640         }
11641
11642         state->acquire_ctx = ctx;
11643         restore_state->acquire_ctx = ctx;
11644
11645         connector_state = drm_atomic_get_connector_state(state, connector);
11646         if (IS_ERR(connector_state)) {
11647                 ret = PTR_ERR(connector_state);
11648                 goto fail;
11649         }
11650
11651         ret = drm_atomic_set_crtc_for_connector(connector_state, crtc);
11652         if (ret)
11653                 goto fail;
11654
11655         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
11656         if (IS_ERR(crtc_state)) {
11657                 ret = PTR_ERR(crtc_state);
11658                 goto fail;
11659         }
11660
11661         crtc_state->uapi.active = true;
11662
11663         ret = drm_atomic_set_mode_for_crtc(&crtc_state->uapi,
11664                                            &load_detect_mode);
11665         if (ret)
11666                 goto fail;
11667
11668         ret = intel_modeset_disable_planes(state, crtc);
11669         if (ret)
11670                 goto fail;
11671
11672         ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector));
11673         if (!ret)
11674                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, crtc));
11675         if (!ret)
11676                 ret = drm_atomic_add_affected_planes(restore_state, crtc);
11677         if (ret) {
11678                 DRM_DEBUG_KMS("Failed to create a copy of old state to restore: %i\n", ret);
11679                 goto fail;
11680         }
11681
11682         ret = drm_atomic_commit(state);
11683         if (ret) {
11684                 DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
11685                 goto fail;
11686         }
11687
11688         old->restore_state = restore_state;
11689         drm_atomic_state_put(state);
11690
11691         /* let the connector get through one full cycle before testing */
11692         intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
11693         return true;
11694
11695 fail:
11696         if (state) {
11697                 drm_atomic_state_put(state);
11698                 state = NULL;
11699         }
11700         if (restore_state) {
11701                 drm_atomic_state_put(restore_state);
11702                 restore_state = NULL;
11703         }
11704
11705         if (ret == -EDEADLK)
11706                 return ret;
11707
11708         return false;
11709 }
11710
11711 void intel_release_load_detect_pipe(struct drm_connector *connector,
11712                                     struct intel_load_detect_pipe *old,
11713                                     struct drm_modeset_acquire_ctx *ctx)
11714 {
11715         struct intel_encoder *intel_encoder =
11716                 intel_attached_encoder(connector);
11717         struct drm_encoder *encoder = &intel_encoder->base;
11718         struct drm_atomic_state *state = old->restore_state;
11719         int ret;
11720
11721         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
11722                       connector->base.id, connector->name,
11723                       encoder->base.id, encoder->name);
11724
11725         if (!state)
11726                 return;
11727
11728         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
11729         if (ret)
11730                 DRM_DEBUG_KMS("Couldn't release load detect pipe: %i\n", ret);
11731         drm_atomic_state_put(state);
11732 }
11733
11734 static int i9xx_pll_refclk(struct drm_device *dev,
11735                            const struct intel_crtc_state *pipe_config)
11736 {
11737         struct drm_i915_private *dev_priv = to_i915(dev);
11738         u32 dpll = pipe_config->dpll_hw_state.dpll;
11739
11740         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
11741                 return dev_priv->vbt.lvds_ssc_freq;
11742         else if (HAS_PCH_SPLIT(dev_priv))
11743                 return 120000;
11744         else if (!IS_GEN(dev_priv, 2))
11745                 return 96000;
11746         else
11747                 return 48000;
11748 }
11749
11750 /* Returns the clock of the currently programmed mode of the given pipe. */
11751 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
11752                                 struct intel_crtc_state *pipe_config)
11753 {
11754         struct drm_device *dev = crtc->base.dev;
11755         struct drm_i915_private *dev_priv = to_i915(dev);
11756         enum pipe pipe = crtc->pipe;
11757         u32 dpll = pipe_config->dpll_hw_state.dpll;
11758         u32 fp;
11759         struct dpll clock;
11760         int port_clock;
11761         int refclk = i9xx_pll_refclk(dev, pipe_config);
11762
11763         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
11764                 fp = pipe_config->dpll_hw_state.fp0;
11765         else
11766                 fp = pipe_config->dpll_hw_state.fp1;
11767
11768         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
11769         if (IS_PINEVIEW(dev_priv)) {
11770                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
11771                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
11772         } else {
11773                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
11774                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
11775         }
11776
11777         if (!IS_GEN(dev_priv, 2)) {
11778                 if (IS_PINEVIEW(dev_priv))
11779                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
11780                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
11781                 else
11782                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
11783                                DPLL_FPA01_P1_POST_DIV_SHIFT);
11784
11785                 switch (dpll & DPLL_MODE_MASK) {
11786                 case DPLLB_MODE_DAC_SERIAL:
11787                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
11788                                 5 : 10;
11789                         break;
11790                 case DPLLB_MODE_LVDS:
11791                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
11792                                 7 : 14;
11793                         break;
11794                 default:
11795                         DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed "
11796                                   "mode\n", (int)(dpll & DPLL_MODE_MASK));
11797                         return;
11798                 }
11799
11800                 if (IS_PINEVIEW(dev_priv))
11801                         port_clock = pnv_calc_dpll_params(refclk, &clock);
11802                 else
11803                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
11804         } else {
11805                 u32 lvds = IS_I830(dev_priv) ? 0 : I915_READ(LVDS);
11806                 bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN);
11807
11808                 if (is_lvds) {
11809                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
11810                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
11811
11812                         if (lvds & LVDS_CLKB_POWER_UP)
11813                                 clock.p2 = 7;
11814                         else
11815                                 clock.p2 = 14;
11816                 } else {
11817                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
11818                                 clock.p1 = 2;
11819                         else {
11820                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
11821                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
11822                         }
11823                         if (dpll & PLL_P2_DIVIDE_BY_4)
11824                                 clock.p2 = 4;
11825                         else
11826                                 clock.p2 = 2;
11827                 }
11828
11829                 port_clock = i9xx_calc_dpll_params(refclk, &clock);
11830         }
11831
11832         /*
11833          * This value includes pixel_multiplier. We will use
11834          * port_clock to compute adjusted_mode.crtc_clock in the
11835          * encoder's get_config() function.
11836          */
11837         pipe_config->port_clock = port_clock;
11838 }
11839
11840 int intel_dotclock_calculate(int link_freq,
11841                              const struct intel_link_m_n *m_n)
11842 {
11843         /*
11844          * The calculation for the data clock is:
11845          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
11846          * But we want to avoid losing precison if possible, so:
11847          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
11848          *
11849          * and the link clock is simpler:
11850          * link_clock = (m * link_clock) / n
11851          */
11852
11853         if (!m_n->link_n)
11854                 return 0;
11855
11856         return div_u64(mul_u32_u32(m_n->link_m, link_freq), m_n->link_n);
11857 }
11858
11859 static void ilk_pch_clock_get(struct intel_crtc *crtc,
11860                               struct intel_crtc_state *pipe_config)
11861 {
11862         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11863
11864         /* read out port_clock from the DPLL */
11865         i9xx_crtc_clock_get(crtc, pipe_config);
11866
11867         /*
11868          * In case there is an active pipe without active ports,
11869          * we may need some idea for the dotclock anyway.
11870          * Calculate one based on the FDI configuration.
11871          */
11872         pipe_config->hw.adjusted_mode.crtc_clock =
11873                 intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
11874                                          &pipe_config->fdi_m_n);
11875 }
11876
11877 static void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
11878                                    struct intel_crtc *crtc)
11879 {
11880         memset(crtc_state, 0, sizeof(*crtc_state));
11881
11882         __drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
11883
11884         crtc_state->cpu_transcoder = INVALID_TRANSCODER;
11885         crtc_state->master_transcoder = INVALID_TRANSCODER;
11886         crtc_state->hsw_workaround_pipe = INVALID_PIPE;
11887         crtc_state->output_format = INTEL_OUTPUT_FORMAT_INVALID;
11888         crtc_state->scaler_state.scaler_id = -1;
11889         crtc_state->mst_master_transcoder = INVALID_TRANSCODER;
11890 }
11891
11892 static struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
11893 {
11894         struct intel_crtc_state *crtc_state;
11895
11896         crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
11897
11898         if (crtc_state)
11899                 intel_crtc_state_reset(crtc_state, crtc);
11900
11901         return crtc_state;
11902 }
11903
11904 /* Returns the currently programmed mode of the given encoder. */
11905 struct drm_display_mode *
11906 intel_encoder_current_mode(struct intel_encoder *encoder)
11907 {
11908         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
11909         struct intel_crtc_state *crtc_state;
11910         struct drm_display_mode *mode;
11911         struct intel_crtc *crtc;
11912         enum pipe pipe;
11913
11914         if (!encoder->get_hw_state(encoder, &pipe))
11915                 return NULL;
11916
11917         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
11918
11919         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
11920         if (!mode)
11921                 return NULL;
11922
11923         crtc_state = intel_crtc_state_alloc(crtc);
11924         if (!crtc_state) {
11925                 kfree(mode);
11926                 return NULL;
11927         }
11928
11929         if (!dev_priv->display.get_pipe_config(crtc, crtc_state)) {
11930                 kfree(crtc_state);
11931                 kfree(mode);
11932                 return NULL;
11933         }
11934
11935         encoder->get_config(encoder, crtc_state);
11936
11937         intel_mode_from_pipe_config(mode, crtc_state);
11938
11939         kfree(crtc_state);
11940
11941         return mode;
11942 }
11943
11944 static void intel_crtc_destroy(struct drm_crtc *crtc)
11945 {
11946         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11947
11948         drm_crtc_cleanup(crtc);
11949         kfree(intel_crtc);
11950 }
11951
11952 /**
11953  * intel_wm_need_update - Check whether watermarks need updating
11954  * @cur: current plane state
11955  * @new: new plane state
11956  *
11957  * Check current plane state versus the new one to determine whether
11958  * watermarks need to be recalculated.
11959  *
11960  * Returns true or false.
11961  */
11962 static bool intel_wm_need_update(const struct intel_plane_state *cur,
11963                                  struct intel_plane_state *new)
11964 {
11965         /* Update watermarks on tiling or size changes. */
11966         if (new->uapi.visible != cur->uapi.visible)
11967                 return true;
11968
11969         if (!cur->hw.fb || !new->hw.fb)
11970                 return false;
11971
11972         if (cur->hw.fb->modifier != new->hw.fb->modifier ||
11973             cur->hw.rotation != new->hw.rotation ||
11974             drm_rect_width(&new->uapi.src) != drm_rect_width(&cur->uapi.src) ||
11975             drm_rect_height(&new->uapi.src) != drm_rect_height(&cur->uapi.src) ||
11976             drm_rect_width(&new->uapi.dst) != drm_rect_width(&cur->uapi.dst) ||
11977             drm_rect_height(&new->uapi.dst) != drm_rect_height(&cur->uapi.dst))
11978                 return true;
11979
11980         return false;
11981 }
11982
11983 static bool needs_scaling(const struct intel_plane_state *state)
11984 {
11985         int src_w = drm_rect_width(&state->uapi.src) >> 16;
11986         int src_h = drm_rect_height(&state->uapi.src) >> 16;
11987         int dst_w = drm_rect_width(&state->uapi.dst);
11988         int dst_h = drm_rect_height(&state->uapi.dst);
11989
11990         return (src_w != dst_w || src_h != dst_h);
11991 }
11992
11993 int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state,
11994                                     struct intel_crtc_state *crtc_state,
11995                                     const struct intel_plane_state *old_plane_state,
11996                                     struct intel_plane_state *plane_state)
11997 {
11998         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
11999         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
12000         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12001         bool mode_changed = needs_modeset(crtc_state);
12002         bool was_crtc_enabled = old_crtc_state->hw.active;
12003         bool is_crtc_enabled = crtc_state->hw.active;
12004         bool turn_off, turn_on, visible, was_visible;
12005         int ret;
12006
12007         if (INTEL_GEN(dev_priv) >= 9 && plane->id != PLANE_CURSOR) {
12008                 ret = skl_update_scaler_plane(crtc_state, plane_state);
12009                 if (ret)
12010                         return ret;
12011         }
12012
12013         was_visible = old_plane_state->uapi.visible;
12014         visible = plane_state->uapi.visible;
12015
12016         if (!was_crtc_enabled && WARN_ON(was_visible))
12017                 was_visible = false;
12018
12019         /*
12020          * Visibility is calculated as if the crtc was on, but
12021          * after scaler setup everything depends on it being off
12022          * when the crtc isn't active.
12023          *
12024          * FIXME this is wrong for watermarks. Watermarks should also
12025          * be computed as if the pipe would be active. Perhaps move
12026          * per-plane wm computation to the .check_plane() hook, and
12027          * only combine the results from all planes in the current place?
12028          */
12029         if (!is_crtc_enabled) {
12030                 plane_state->uapi.visible = visible = false;
12031                 crtc_state->active_planes &= ~BIT(plane->id);
12032                 crtc_state->data_rate[plane->id] = 0;
12033                 crtc_state->min_cdclk[plane->id] = 0;
12034         }
12035
12036         if (!was_visible && !visible)
12037                 return 0;
12038
12039         turn_off = was_visible && (!visible || mode_changed);
12040         turn_on = visible && (!was_visible || mode_changed);
12041
12042         DRM_DEBUG_ATOMIC("[CRTC:%d:%s] with [PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n",
12043                          crtc->base.base.id, crtc->base.name,
12044                          plane->base.base.id, plane->base.name,
12045                          was_visible, visible,
12046                          turn_off, turn_on, mode_changed);
12047
12048         if (turn_on) {
12049                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
12050                         crtc_state->update_wm_pre = true;
12051
12052                 /* must disable cxsr around plane enable/disable */
12053                 if (plane->id != PLANE_CURSOR)
12054                         crtc_state->disable_cxsr = true;
12055         } else if (turn_off) {
12056                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
12057                         crtc_state->update_wm_post = true;
12058
12059                 /* must disable cxsr around plane enable/disable */
12060                 if (plane->id != PLANE_CURSOR)
12061                         crtc_state->disable_cxsr = true;
12062         } else if (intel_wm_need_update(old_plane_state, plane_state)) {
12063                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) {
12064                         /* FIXME bollocks */
12065                         crtc_state->update_wm_pre = true;
12066                         crtc_state->update_wm_post = true;
12067                 }
12068         }
12069
12070         if (visible || was_visible)
12071                 crtc_state->fb_bits |= plane->frontbuffer_bit;
12072
12073         /*
12074          * ILK/SNB DVSACNTR/Sprite Enable
12075          * IVB SPR_CTL/Sprite Enable
12076          * "When in Self Refresh Big FIFO mode, a write to enable the
12077          *  plane will be internally buffered and delayed while Big FIFO
12078          *  mode is exiting."
12079          *
12080          * Which means that enabling the sprite can take an extra frame
12081          * when we start in big FIFO mode (LP1+). Thus we need to drop
12082          * down to LP0 and wait for vblank in order to make sure the
12083          * sprite gets enabled on the next vblank after the register write.
12084          * Doing otherwise would risk enabling the sprite one frame after
12085          * we've already signalled flip completion. We can resume LP1+
12086          * once the sprite has been enabled.
12087          *
12088          *
12089          * WaCxSRDisabledForSpriteScaling:ivb
12090          * IVB SPR_SCALE/Scaling Enable
12091          * "Low Power watermarks must be disabled for at least one
12092          *  frame before enabling sprite scaling, and kept disabled
12093          *  until sprite scaling is disabled."
12094          *
12095          * ILK/SNB DVSASCALE/Scaling Enable
12096          * "When in Self Refresh Big FIFO mode, scaling enable will be
12097          *  masked off while Big FIFO mode is exiting."
12098          *
12099          * Despite the w/a only being listed for IVB we assume that
12100          * the ILK/SNB note has similar ramifications, hence we apply
12101          * the w/a on all three platforms.
12102          *
12103          * With experimental results seems this is needed also for primary
12104          * plane, not only sprite plane.
12105          */
12106         if (plane->id != PLANE_CURSOR &&
12107             (IS_GEN_RANGE(dev_priv, 5, 6) ||
12108              IS_IVYBRIDGE(dev_priv)) &&
12109             (turn_on || (!needs_scaling(old_plane_state) &&
12110                          needs_scaling(plane_state))))
12111                 crtc_state->disable_lp_wm = true;
12112
12113         return 0;
12114 }
12115
12116 static bool encoders_cloneable(const struct intel_encoder *a,
12117                                const struct intel_encoder *b)
12118 {
12119         /* masks could be asymmetric, so check both ways */
12120         return a == b || (a->cloneable & (1 << b->type) &&
12121                           b->cloneable & (1 << a->type));
12122 }
12123
12124 static bool check_single_encoder_cloning(struct drm_atomic_state *state,
12125                                          struct intel_crtc *crtc,
12126                                          struct intel_encoder *encoder)
12127 {
12128         struct intel_encoder *source_encoder;
12129         struct drm_connector *connector;
12130         struct drm_connector_state *connector_state;
12131         int i;
12132
12133         for_each_new_connector_in_state(state, connector, connector_state, i) {
12134                 if (connector_state->crtc != &crtc->base)
12135                         continue;
12136
12137                 source_encoder =
12138                         to_intel_encoder(connector_state->best_encoder);
12139                 if (!encoders_cloneable(encoder, source_encoder))
12140                         return false;
12141         }
12142
12143         return true;
12144 }
12145
12146 static int icl_add_linked_planes(struct intel_atomic_state *state)
12147 {
12148         struct intel_plane *plane, *linked;
12149         struct intel_plane_state *plane_state, *linked_plane_state;
12150         int i;
12151
12152         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12153                 linked = plane_state->planar_linked_plane;
12154
12155                 if (!linked)
12156                         continue;
12157
12158                 linked_plane_state = intel_atomic_get_plane_state(state, linked);
12159                 if (IS_ERR(linked_plane_state))
12160                         return PTR_ERR(linked_plane_state);
12161
12162                 WARN_ON(linked_plane_state->planar_linked_plane != plane);
12163                 WARN_ON(linked_plane_state->planar_slave == plane_state->planar_slave);
12164         }
12165
12166         return 0;
12167 }
12168
12169 static int icl_check_nv12_planes(struct intel_crtc_state *crtc_state)
12170 {
12171         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
12172         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12173         struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->uapi.state);
12174         struct intel_plane *plane, *linked;
12175         struct intel_plane_state *plane_state;
12176         int i;
12177
12178         if (INTEL_GEN(dev_priv) < 11)
12179                 return 0;
12180
12181         /*
12182          * Destroy all old plane links and make the slave plane invisible
12183          * in the crtc_state->active_planes mask.
12184          */
12185         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12186                 if (plane->pipe != crtc->pipe || !plane_state->planar_linked_plane)
12187                         continue;
12188
12189                 plane_state->planar_linked_plane = NULL;
12190                 if (plane_state->planar_slave && !plane_state->uapi.visible) {
12191                         crtc_state->active_planes &= ~BIT(plane->id);
12192                         crtc_state->update_planes |= BIT(plane->id);
12193                 }
12194
12195                 plane_state->planar_slave = false;
12196         }
12197
12198         if (!crtc_state->nv12_planes)
12199                 return 0;
12200
12201         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12202                 struct intel_plane_state *linked_state = NULL;
12203
12204                 if (plane->pipe != crtc->pipe ||
12205                     !(crtc_state->nv12_planes & BIT(plane->id)))
12206                         continue;
12207
12208                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, linked) {
12209                         if (!icl_is_nv12_y_plane(linked->id))
12210                                 continue;
12211
12212                         if (crtc_state->active_planes & BIT(linked->id))
12213                                 continue;
12214
12215                         linked_state = intel_atomic_get_plane_state(state, linked);
12216                         if (IS_ERR(linked_state))
12217                                 return PTR_ERR(linked_state);
12218
12219                         break;
12220                 }
12221
12222                 if (!linked_state) {
12223                         DRM_DEBUG_KMS("Need %d free Y planes for planar YUV\n",
12224                                       hweight8(crtc_state->nv12_planes));
12225
12226                         return -EINVAL;
12227                 }
12228
12229                 plane_state->planar_linked_plane = linked;
12230
12231                 linked_state->planar_slave = true;
12232                 linked_state->planar_linked_plane = plane;
12233                 crtc_state->active_planes |= BIT(linked->id);
12234                 crtc_state->update_planes |= BIT(linked->id);
12235                 DRM_DEBUG_KMS("Using %s as Y plane for %s\n", linked->base.name, plane->base.name);
12236
12237                 /* Copy parameters to slave plane */
12238                 linked_state->ctl = plane_state->ctl | PLANE_CTL_YUV420_Y_PLANE;
12239                 linked_state->color_ctl = plane_state->color_ctl;
12240                 memcpy(linked_state->color_plane, plane_state->color_plane,
12241                        sizeof(linked_state->color_plane));
12242
12243                 intel_plane_copy_uapi_to_hw_state(linked_state, plane_state);
12244                 linked_state->uapi.src = plane_state->uapi.src;
12245                 linked_state->uapi.dst = plane_state->uapi.dst;
12246
12247                 if (icl_is_hdr_plane(dev_priv, plane->id)) {
12248                         if (linked->id == PLANE_SPRITE5)
12249                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_7;
12250                         else if (linked->id == PLANE_SPRITE4)
12251                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_6;
12252                         else
12253                                 MISSING_CASE(linked->id);
12254                 }
12255         }
12256
12257         return 0;
12258 }
12259
12260 static bool c8_planes_changed(const struct intel_crtc_state *new_crtc_state)
12261 {
12262         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
12263         struct intel_atomic_state *state =
12264                 to_intel_atomic_state(new_crtc_state->uapi.state);
12265         const struct intel_crtc_state *old_crtc_state =
12266                 intel_atomic_get_old_crtc_state(state, crtc);
12267
12268         return !old_crtc_state->c8_planes != !new_crtc_state->c8_planes;
12269 }
12270
12271 static bool
12272 intel_atomic_is_master_connector(struct intel_crtc_state *crtc_state)
12273 {
12274         struct drm_crtc *crtc = crtc_state->uapi.crtc;
12275         struct drm_atomic_state *state = crtc_state->uapi.state;
12276         struct drm_connector *connector;
12277         struct drm_connector_state *connector_state;
12278         int i;
12279
12280         for_each_new_connector_in_state(state, connector, connector_state, i) {
12281                 if (connector_state->crtc != crtc)
12282                         continue;
12283                 if (connector->has_tile &&
12284                     connector->tile_h_loc == connector->num_h_tile - 1 &&
12285                     connector->tile_v_loc == connector->num_v_tile - 1)
12286                         return true;
12287         }
12288
12289         return false;
12290 }
12291
12292 static void reset_port_sync_mode_state(struct intel_crtc_state *crtc_state)
12293 {
12294         crtc_state->master_transcoder = INVALID_TRANSCODER;
12295         crtc_state->sync_mode_slaves_mask = 0;
12296 }
12297
12298 static int icl_compute_port_sync_crtc_state(struct drm_connector *connector,
12299                                             struct intel_crtc_state *crtc_state,
12300                                             int num_tiled_conns)
12301 {
12302         struct drm_crtc *crtc = crtc_state->uapi.crtc;
12303         struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->uapi.state);
12304         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
12305         struct drm_connector *master_connector;
12306         struct drm_connector_list_iter conn_iter;
12307         struct drm_crtc *master_crtc = NULL;
12308         struct drm_crtc_state *master_crtc_state;
12309         struct intel_crtc_state *master_pipe_config;
12310
12311         if (INTEL_GEN(dev_priv) < 11)
12312                 return 0;
12313
12314         if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP))
12315                 return 0;
12316
12317         /*
12318          * In case of tiled displays there could be one or more slaves but there is
12319          * only one master. Lets make the CRTC used by the connector corresponding
12320          * to the last horizonal and last vertical tile a master/genlock CRTC.
12321          * All the other CRTCs corresponding to other tiles of the same Tile group
12322          * are the slave CRTCs and hold a pointer to their genlock CRTC.
12323          * If all tiles not present do not make master slave assignments.
12324          */
12325         if (!connector->has_tile ||
12326             crtc_state->hw.mode.hdisplay != connector->tile_h_size ||
12327             crtc_state->hw.mode.vdisplay != connector->tile_v_size ||
12328             num_tiled_conns < connector->num_h_tile * connector->num_v_tile) {
12329                 reset_port_sync_mode_state(crtc_state);
12330                 return 0;
12331         }
12332         /* Last Horizontal and last vertical tile connector is a master
12333          * Master's crtc state is already populated in slave for port sync
12334          */
12335         if (connector->tile_h_loc == connector->num_h_tile - 1 &&
12336             connector->tile_v_loc == connector->num_v_tile - 1)
12337                 return 0;
12338
12339         /* Loop through all connectors and configure the Slave crtc_state
12340          * to point to the correct master.
12341          */
12342         drm_connector_list_iter_begin(&dev_priv->drm, &conn_iter);
12343         drm_for_each_connector_iter(master_connector, &conn_iter) {
12344                 struct drm_connector_state *master_conn_state = NULL;
12345
12346                 if (!(master_connector->has_tile &&
12347                       master_connector->tile_group->id == connector->tile_group->id))
12348                         continue;
12349                 if (master_connector->tile_h_loc != master_connector->num_h_tile - 1 ||
12350                     master_connector->tile_v_loc != master_connector->num_v_tile - 1)
12351                         continue;
12352
12353                 master_conn_state = drm_atomic_get_connector_state(&state->base,
12354                                                                    master_connector);
12355                 if (IS_ERR(master_conn_state)) {
12356                         drm_connector_list_iter_end(&conn_iter);
12357                         return PTR_ERR(master_conn_state);
12358                 }
12359                 if (master_conn_state->crtc) {
12360                         master_crtc = master_conn_state->crtc;
12361                         break;
12362                 }
12363         }
12364         drm_connector_list_iter_end(&conn_iter);
12365
12366         if (!master_crtc) {
12367                 DRM_DEBUG_KMS("Could not find Master CRTC for Slave CRTC %d\n",
12368                               crtc->base.id);
12369                 return -EINVAL;
12370         }
12371
12372         master_crtc_state = drm_atomic_get_crtc_state(&state->base,
12373                                                       master_crtc);
12374         if (IS_ERR(master_crtc_state))
12375                 return PTR_ERR(master_crtc_state);
12376
12377         master_pipe_config = to_intel_crtc_state(master_crtc_state);
12378         crtc_state->master_transcoder = master_pipe_config->cpu_transcoder;
12379         master_pipe_config->sync_mode_slaves_mask |=
12380                 BIT(crtc_state->cpu_transcoder);
12381         DRM_DEBUG_KMS("Master Transcoder = %s added for Slave CRTC = %d, slave transcoder bitmask = %d\n",
12382                       transcoder_name(crtc_state->master_transcoder),
12383                       crtc->base.id,
12384                       master_pipe_config->sync_mode_slaves_mask);
12385
12386         return 0;
12387 }
12388
12389 static int intel_crtc_atomic_check(struct intel_atomic_state *state,
12390                                    struct intel_crtc *crtc)
12391 {
12392         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12393         struct intel_crtc_state *crtc_state =
12394                 intel_atomic_get_new_crtc_state(state, crtc);
12395         bool mode_changed = needs_modeset(crtc_state);
12396         int ret;
12397
12398         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv) &&
12399             mode_changed && !crtc_state->hw.active)
12400                 crtc_state->update_wm_post = true;
12401
12402         if (mode_changed && crtc_state->hw.enable &&
12403             dev_priv->display.crtc_compute_clock &&
12404             !WARN_ON(crtc_state->shared_dpll)) {
12405                 ret = dev_priv->display.crtc_compute_clock(crtc, crtc_state);
12406                 if (ret)
12407                         return ret;
12408         }
12409
12410         /*
12411          * May need to update pipe gamma enable bits
12412          * when C8 planes are getting enabled/disabled.
12413          */
12414         if (c8_planes_changed(crtc_state))
12415                 crtc_state->uapi.color_mgmt_changed = true;
12416
12417         if (mode_changed || crtc_state->update_pipe ||
12418             crtc_state->uapi.color_mgmt_changed) {
12419                 ret = intel_color_check(crtc_state);
12420                 if (ret)
12421                         return ret;
12422         }
12423
12424         ret = 0;
12425         if (dev_priv->display.compute_pipe_wm) {
12426                 ret = dev_priv->display.compute_pipe_wm(crtc_state);
12427                 if (ret) {
12428                         DRM_DEBUG_KMS("Target pipe watermarks are invalid\n");
12429                         return ret;
12430                 }
12431         }
12432
12433         if (dev_priv->display.compute_intermediate_wm) {
12434                 if (WARN_ON(!dev_priv->display.compute_pipe_wm))
12435                         return 0;
12436
12437                 /*
12438                  * Calculate 'intermediate' watermarks that satisfy both the
12439                  * old state and the new state.  We can program these
12440                  * immediately.
12441                  */
12442                 ret = dev_priv->display.compute_intermediate_wm(crtc_state);
12443                 if (ret) {
12444                         DRM_DEBUG_KMS("No valid intermediate pipe watermarks are possible\n");
12445                         return ret;
12446                 }
12447         }
12448
12449         if (INTEL_GEN(dev_priv) >= 9) {
12450                 if (mode_changed || crtc_state->update_pipe)
12451                         ret = skl_update_scaler_crtc(crtc_state);
12452                 if (!ret)
12453                         ret = intel_atomic_setup_scalers(dev_priv, crtc,
12454                                                          crtc_state);
12455         }
12456
12457         if (HAS_IPS(dev_priv))
12458                 crtc_state->ips_enabled = hsw_compute_ips_config(crtc_state);
12459
12460         return ret;
12461 }
12462
12463 static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
12464 {
12465         struct intel_connector *connector;
12466         struct drm_connector_list_iter conn_iter;
12467
12468         drm_connector_list_iter_begin(dev, &conn_iter);
12469         for_each_intel_connector_iter(connector, &conn_iter) {
12470                 if (connector->base.state->crtc)
12471                         drm_connector_put(&connector->base);
12472
12473                 if (connector->base.encoder) {
12474                         connector->base.state->best_encoder =
12475                                 connector->base.encoder;
12476                         connector->base.state->crtc =
12477                                 connector->base.encoder->crtc;
12478
12479                         drm_connector_get(&connector->base);
12480                 } else {
12481                         connector->base.state->best_encoder = NULL;
12482                         connector->base.state->crtc = NULL;
12483                 }
12484         }
12485         drm_connector_list_iter_end(&conn_iter);
12486 }
12487
12488 static int
12489 compute_sink_pipe_bpp(const struct drm_connector_state *conn_state,
12490                       struct intel_crtc_state *pipe_config)
12491 {
12492         struct drm_connector *connector = conn_state->connector;
12493         const struct drm_display_info *info = &connector->display_info;
12494         int bpp;
12495
12496         switch (conn_state->max_bpc) {
12497         case 6 ... 7:
12498                 bpp = 6 * 3;
12499                 break;
12500         case 8 ... 9:
12501                 bpp = 8 * 3;
12502                 break;
12503         case 10 ... 11:
12504                 bpp = 10 * 3;
12505                 break;
12506         case 12:
12507                 bpp = 12 * 3;
12508                 break;
12509         default:
12510                 return -EINVAL;
12511         }
12512
12513         if (bpp < pipe_config->pipe_bpp) {
12514                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Limiting display bpp to %d instead of "
12515                               "EDID bpp %d, requested bpp %d, max platform bpp %d\n",
12516                               connector->base.id, connector->name,
12517                               bpp, 3 * info->bpc, 3 * conn_state->max_requested_bpc,
12518                               pipe_config->pipe_bpp);
12519
12520                 pipe_config->pipe_bpp = bpp;
12521         }
12522
12523         return 0;
12524 }
12525
12526 static int
12527 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
12528                           struct intel_crtc_state *pipe_config)
12529 {
12530         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12531         struct drm_atomic_state *state = pipe_config->uapi.state;
12532         struct drm_connector *connector;
12533         struct drm_connector_state *connector_state;
12534         int bpp, i;
12535
12536         if ((IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
12537             IS_CHERRYVIEW(dev_priv)))
12538                 bpp = 10*3;
12539         else if (INTEL_GEN(dev_priv) >= 5)
12540                 bpp = 12*3;
12541         else
12542                 bpp = 8*3;
12543
12544         pipe_config->pipe_bpp = bpp;
12545
12546         /* Clamp display bpp to connector max bpp */
12547         for_each_new_connector_in_state(state, connector, connector_state, i) {
12548                 int ret;
12549
12550                 if (connector_state->crtc != &crtc->base)
12551                         continue;
12552
12553                 ret = compute_sink_pipe_bpp(connector_state, pipe_config);
12554                 if (ret)
12555                         return ret;
12556         }
12557
12558         return 0;
12559 }
12560
12561 static void intel_dump_crtc_timings(const struct drm_display_mode *mode)
12562 {
12563         DRM_DEBUG_KMS("crtc timings: %d %d %d %d %d %d %d %d %d, "
12564                       "type: 0x%x flags: 0x%x\n",
12565                       mode->crtc_clock,
12566                       mode->crtc_hdisplay, mode->crtc_hsync_start,
12567                       mode->crtc_hsync_end, mode->crtc_htotal,
12568                       mode->crtc_vdisplay, mode->crtc_vsync_start,
12569                       mode->crtc_vsync_end, mode->crtc_vtotal,
12570                       mode->type, mode->flags);
12571 }
12572
12573 static inline void
12574 intel_dump_m_n_config(const struct intel_crtc_state *pipe_config,
12575                       const char *id, unsigned int lane_count,
12576                       const struct intel_link_m_n *m_n)
12577 {
12578         DRM_DEBUG_KMS("%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
12579                       id, lane_count,
12580                       m_n->gmch_m, m_n->gmch_n,
12581                       m_n->link_m, m_n->link_n, m_n->tu);
12582 }
12583
12584 static void
12585 intel_dump_infoframe(struct drm_i915_private *dev_priv,
12586                      const union hdmi_infoframe *frame)
12587 {
12588         if ((drm_debug & DRM_UT_KMS) == 0)
12589                 return;
12590
12591         hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, frame);
12592 }
12593
12594 #define OUTPUT_TYPE(x) [INTEL_OUTPUT_ ## x] = #x
12595
12596 static const char * const output_type_str[] = {
12597         OUTPUT_TYPE(UNUSED),
12598         OUTPUT_TYPE(ANALOG),
12599         OUTPUT_TYPE(DVO),
12600         OUTPUT_TYPE(SDVO),
12601         OUTPUT_TYPE(LVDS),
12602         OUTPUT_TYPE(TVOUT),
12603         OUTPUT_TYPE(HDMI),
12604         OUTPUT_TYPE(DP),
12605         OUTPUT_TYPE(EDP),
12606         OUTPUT_TYPE(DSI),
12607         OUTPUT_TYPE(DDI),
12608         OUTPUT_TYPE(DP_MST),
12609 };
12610
12611 #undef OUTPUT_TYPE
12612
12613 static void snprintf_output_types(char *buf, size_t len,
12614                                   unsigned int output_types)
12615 {
12616         char *str = buf;
12617         int i;
12618
12619         str[0] = '\0';
12620
12621         for (i = 0; i < ARRAY_SIZE(output_type_str); i++) {
12622                 int r;
12623
12624                 if ((output_types & BIT(i)) == 0)
12625                         continue;
12626
12627                 r = snprintf(str, len, "%s%s",
12628                              str != buf ? "," : "", output_type_str[i]);
12629                 if (r >= len)
12630                         break;
12631                 str += r;
12632                 len -= r;
12633
12634                 output_types &= ~BIT(i);
12635         }
12636
12637         WARN_ON_ONCE(output_types != 0);
12638 }
12639
12640 static const char * const output_format_str[] = {
12641         [INTEL_OUTPUT_FORMAT_INVALID] = "Invalid",
12642         [INTEL_OUTPUT_FORMAT_RGB] = "RGB",
12643         [INTEL_OUTPUT_FORMAT_YCBCR420] = "YCBCR4:2:0",
12644         [INTEL_OUTPUT_FORMAT_YCBCR444] = "YCBCR4:4:4",
12645 };
12646
12647 static const char *output_formats(enum intel_output_format format)
12648 {
12649         if (format >= ARRAY_SIZE(output_format_str))
12650                 format = INTEL_OUTPUT_FORMAT_INVALID;
12651         return output_format_str[format];
12652 }
12653
12654 static void intel_dump_plane_state(const struct intel_plane_state *plane_state)
12655 {
12656         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
12657         const struct drm_framebuffer *fb = plane_state->hw.fb;
12658         struct drm_format_name_buf format_name;
12659
12660         if (!fb) {
12661                 DRM_DEBUG_KMS("[PLANE:%d:%s] fb: [NOFB], visible: %s\n",
12662                               plane->base.base.id, plane->base.name,
12663                               yesno(plane_state->uapi.visible));
12664                 return;
12665         }
12666
12667         DRM_DEBUG_KMS("[PLANE:%d:%s] fb: [FB:%d] %ux%u format = %s, visible: %s\n",
12668                       plane->base.base.id, plane->base.name,
12669                       fb->base.id, fb->width, fb->height,
12670                       drm_get_format_name(fb->format->format, &format_name),
12671                       yesno(plane_state->uapi.visible));
12672         DRM_DEBUG_KMS("\trotation: 0x%x, scaler: %d\n",
12673                       plane_state->hw.rotation, plane_state->scaler_id);
12674         if (plane_state->uapi.visible)
12675                 DRM_DEBUG_KMS("\tsrc: " DRM_RECT_FP_FMT " dst: " DRM_RECT_FMT "\n",
12676                               DRM_RECT_FP_ARG(&plane_state->uapi.src),
12677                               DRM_RECT_ARG(&plane_state->uapi.dst));
12678 }
12679
12680 static void intel_dump_pipe_config(const struct intel_crtc_state *pipe_config,
12681                                    struct intel_atomic_state *state,
12682                                    const char *context)
12683 {
12684         struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
12685         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12686         const struct intel_plane_state *plane_state;
12687         struct intel_plane *plane;
12688         char buf[64];
12689         int i;
12690
12691         DRM_DEBUG_KMS("[CRTC:%d:%s] enable: %s %s\n",
12692                       crtc->base.base.id, crtc->base.name,
12693                       yesno(pipe_config->hw.enable), context);
12694
12695         if (!pipe_config->hw.enable)
12696                 goto dump_planes;
12697
12698         snprintf_output_types(buf, sizeof(buf), pipe_config->output_types);
12699         DRM_DEBUG_KMS("active: %s, output_types: %s (0x%x), output format: %s\n",
12700                       yesno(pipe_config->hw.active),
12701                       buf, pipe_config->output_types,
12702                       output_formats(pipe_config->output_format));
12703
12704         DRM_DEBUG_KMS("cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
12705                       transcoder_name(pipe_config->cpu_transcoder),
12706                       pipe_config->pipe_bpp, pipe_config->dither);
12707
12708         if (pipe_config->has_pch_encoder)
12709                 intel_dump_m_n_config(pipe_config, "fdi",
12710                                       pipe_config->fdi_lanes,
12711                                       &pipe_config->fdi_m_n);
12712
12713         if (intel_crtc_has_dp_encoder(pipe_config)) {
12714                 intel_dump_m_n_config(pipe_config, "dp m_n",
12715                                 pipe_config->lane_count, &pipe_config->dp_m_n);
12716                 if (pipe_config->has_drrs)
12717                         intel_dump_m_n_config(pipe_config, "dp m2_n2",
12718                                               pipe_config->lane_count,
12719                                               &pipe_config->dp_m2_n2);
12720         }
12721
12722         DRM_DEBUG_KMS("audio: %i, infoframes: %i, infoframes enabled: 0x%x\n",
12723                       pipe_config->has_audio, pipe_config->has_infoframe,
12724                       pipe_config->infoframes.enable);
12725
12726         if (pipe_config->infoframes.enable &
12727             intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GENERAL_CONTROL))
12728                 DRM_DEBUG_KMS("GCP: 0x%x\n", pipe_config->infoframes.gcp);
12729         if (pipe_config->infoframes.enable &
12730             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_AVI))
12731                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.avi);
12732         if (pipe_config->infoframes.enable &
12733             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_SPD))
12734                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.spd);
12735         if (pipe_config->infoframes.enable &
12736             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_VENDOR))
12737                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.hdmi);
12738
12739         DRM_DEBUG_KMS("requested mode:\n");
12740         drm_mode_debug_printmodeline(&pipe_config->hw.mode);
12741         DRM_DEBUG_KMS("adjusted mode:\n");
12742         drm_mode_debug_printmodeline(&pipe_config->hw.adjusted_mode);
12743         intel_dump_crtc_timings(&pipe_config->hw.adjusted_mode);
12744         DRM_DEBUG_KMS("port clock: %d, pipe src size: %dx%d, pixel rate %d\n",
12745                       pipe_config->port_clock,
12746                       pipe_config->pipe_src_w, pipe_config->pipe_src_h,
12747                       pipe_config->pixel_rate);
12748
12749         if (INTEL_GEN(dev_priv) >= 9)
12750                 DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
12751                               crtc->num_scalers,
12752                               pipe_config->scaler_state.scaler_users,
12753                               pipe_config->scaler_state.scaler_id);
12754
12755         if (HAS_GMCH(dev_priv))
12756                 DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
12757                               pipe_config->gmch_pfit.control,
12758                               pipe_config->gmch_pfit.pgm_ratios,
12759                               pipe_config->gmch_pfit.lvds_border_bits);
12760         else
12761                 DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s, force thru: %s\n",
12762                               pipe_config->pch_pfit.pos,
12763                               pipe_config->pch_pfit.size,
12764                               enableddisabled(pipe_config->pch_pfit.enabled),
12765                               yesno(pipe_config->pch_pfit.force_thru));
12766
12767         DRM_DEBUG_KMS("ips: %i, double wide: %i\n",
12768                       pipe_config->ips_enabled, pipe_config->double_wide);
12769
12770         intel_dpll_dump_hw_state(dev_priv, &pipe_config->dpll_hw_state);
12771
12772         if (IS_CHERRYVIEW(dev_priv))
12773                 DRM_DEBUG_KMS("cgm_mode: 0x%x gamma_mode: 0x%x gamma_enable: %d csc_enable: %d\n",
12774                               pipe_config->cgm_mode, pipe_config->gamma_mode,
12775                               pipe_config->gamma_enable, pipe_config->csc_enable);
12776         else
12777                 DRM_DEBUG_KMS("csc_mode: 0x%x gamma_mode: 0x%x gamma_enable: %d csc_enable: %d\n",
12778                               pipe_config->csc_mode, pipe_config->gamma_mode,
12779                               pipe_config->gamma_enable, pipe_config->csc_enable);
12780
12781         DRM_DEBUG_KMS("MST master transcoder: %s\n",
12782                       transcoder_name(pipe_config->mst_master_transcoder));
12783
12784 dump_planes:
12785         if (!state)
12786                 return;
12787
12788         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12789                 if (plane->pipe == crtc->pipe)
12790                         intel_dump_plane_state(plane_state);
12791         }
12792 }
12793
12794 static bool check_digital_port_conflicts(struct intel_atomic_state *state)
12795 {
12796         struct drm_device *dev = state->base.dev;
12797         struct drm_connector *connector;
12798         struct drm_connector_list_iter conn_iter;
12799         unsigned int used_ports = 0;
12800         unsigned int used_mst_ports = 0;
12801         bool ret = true;
12802
12803         /*
12804          * We're going to peek into connector->state,
12805          * hence connection_mutex must be held.
12806          */
12807         drm_modeset_lock_assert_held(&dev->mode_config.connection_mutex);
12808
12809         /*
12810          * Walk the connector list instead of the encoder
12811          * list to detect the problem on ddi platforms
12812          * where there's just one encoder per digital port.
12813          */
12814         drm_connector_list_iter_begin(dev, &conn_iter);
12815         drm_for_each_connector_iter(connector, &conn_iter) {
12816                 struct drm_connector_state *connector_state;
12817                 struct intel_encoder *encoder;
12818
12819                 connector_state =
12820                         drm_atomic_get_new_connector_state(&state->base,
12821                                                            connector);
12822                 if (!connector_state)
12823                         connector_state = connector->state;
12824
12825                 if (!connector_state->best_encoder)
12826                         continue;
12827
12828                 encoder = to_intel_encoder(connector_state->best_encoder);
12829
12830                 WARN_ON(!connector_state->crtc);
12831
12832                 switch (encoder->type) {
12833                         unsigned int port_mask;
12834                 case INTEL_OUTPUT_DDI:
12835                         if (WARN_ON(!HAS_DDI(to_i915(dev))))
12836                                 break;
12837                         /* else, fall through */
12838                 case INTEL_OUTPUT_DP:
12839                 case INTEL_OUTPUT_HDMI:
12840                 case INTEL_OUTPUT_EDP:
12841                         port_mask = 1 << encoder->port;
12842
12843                         /* the same port mustn't appear more than once */
12844                         if (used_ports & port_mask)
12845                                 ret = false;
12846
12847                         used_ports |= port_mask;
12848                         break;
12849                 case INTEL_OUTPUT_DP_MST:
12850                         used_mst_ports |=
12851                                 1 << encoder->port;
12852                         break;
12853                 default:
12854                         break;
12855                 }
12856         }
12857         drm_connector_list_iter_end(&conn_iter);
12858
12859         /* can't mix MST and SST/HDMI on the same port */
12860         if (used_ports & used_mst_ports)
12861                 return false;
12862
12863         return ret;
12864 }
12865
12866 static void
12867 intel_crtc_copy_uapi_to_hw_state_nomodeset(struct intel_crtc_state *crtc_state)
12868 {
12869         intel_crtc_copy_color_blobs(crtc_state);
12870 }
12871
12872 static void
12873 intel_crtc_copy_uapi_to_hw_state(struct intel_crtc_state *crtc_state)
12874 {
12875         crtc_state->hw.enable = crtc_state->uapi.enable;
12876         crtc_state->hw.active = crtc_state->uapi.active;
12877         crtc_state->hw.mode = crtc_state->uapi.mode;
12878         crtc_state->hw.adjusted_mode = crtc_state->uapi.adjusted_mode;
12879         intel_crtc_copy_uapi_to_hw_state_nomodeset(crtc_state);
12880 }
12881
12882 static void intel_crtc_copy_hw_to_uapi_state(struct intel_crtc_state *crtc_state)
12883 {
12884         crtc_state->uapi.enable = crtc_state->hw.enable;
12885         crtc_state->uapi.active = crtc_state->hw.active;
12886         WARN_ON(drm_atomic_set_mode_for_crtc(&crtc_state->uapi, &crtc_state->hw.mode) < 0);
12887
12888         crtc_state->uapi.adjusted_mode = crtc_state->hw.adjusted_mode;
12889
12890         /* copy color blobs to uapi */
12891         drm_property_replace_blob(&crtc_state->uapi.degamma_lut,
12892                                   crtc_state->hw.degamma_lut);
12893         drm_property_replace_blob(&crtc_state->uapi.gamma_lut,
12894                                   crtc_state->hw.gamma_lut);
12895         drm_property_replace_blob(&crtc_state->uapi.ctm,
12896                                   crtc_state->hw.ctm);
12897 }
12898
12899 static int
12900 intel_crtc_prepare_cleared_state(struct intel_crtc_state *crtc_state)
12901 {
12902         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
12903         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12904         struct intel_crtc_state *saved_state;
12905
12906         saved_state = intel_crtc_state_alloc(crtc);
12907         if (!saved_state)
12908                 return -ENOMEM;
12909
12910         /* free the old crtc_state->hw members */
12911         intel_crtc_free_hw_state(crtc_state);
12912
12913         /* FIXME: before the switch to atomic started, a new pipe_config was
12914          * kzalloc'd. Code that depends on any field being zero should be
12915          * fixed, so that the crtc_state can be safely duplicated. For now,
12916          * only fields that are know to not cause problems are preserved. */
12917
12918         saved_state->uapi = crtc_state->uapi;
12919         saved_state->scaler_state = crtc_state->scaler_state;
12920         saved_state->shared_dpll = crtc_state->shared_dpll;
12921         saved_state->dpll_hw_state = crtc_state->dpll_hw_state;
12922         memcpy(saved_state->icl_port_dplls, crtc_state->icl_port_dplls,
12923                sizeof(saved_state->icl_port_dplls));
12924         saved_state->crc_enabled = crtc_state->crc_enabled;
12925         if (IS_G4X(dev_priv) ||
12926             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
12927                 saved_state->wm = crtc_state->wm;
12928         /*
12929          * Save the slave bitmask which gets filled for master crtc state during
12930          * slave atomic check call. For all other CRTCs reset the port sync variables
12931          * crtc_state->master_transcoder needs to be set to INVALID
12932          */
12933         reset_port_sync_mode_state(saved_state);
12934         if (intel_atomic_is_master_connector(crtc_state))
12935                 saved_state->sync_mode_slaves_mask =
12936                         crtc_state->sync_mode_slaves_mask;
12937
12938         memcpy(crtc_state, saved_state, sizeof(*crtc_state));
12939         kfree(saved_state);
12940
12941         intel_crtc_copy_uapi_to_hw_state(crtc_state);
12942
12943         return 0;
12944 }
12945
12946 static int
12947 intel_modeset_pipe_config(struct intel_crtc_state *pipe_config)
12948 {
12949         struct drm_crtc *crtc = pipe_config->uapi.crtc;
12950         struct drm_atomic_state *state = pipe_config->uapi.state;
12951         struct intel_encoder *encoder;
12952         struct drm_connector *connector;
12953         struct drm_connector_state *connector_state;
12954         int base_bpp, ret;
12955         int i, tile_group_id = -1, num_tiled_conns = 0;
12956         bool retry = true;
12957
12958         pipe_config->cpu_transcoder =
12959                 (enum transcoder) to_intel_crtc(crtc)->pipe;
12960
12961         /*
12962          * Sanitize sync polarity flags based on requested ones. If neither
12963          * positive or negative polarity is requested, treat this as meaning
12964          * negative polarity.
12965          */
12966         if (!(pipe_config->hw.adjusted_mode.flags &
12967               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
12968                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
12969
12970         if (!(pipe_config->hw.adjusted_mode.flags &
12971               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
12972                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
12973
12974         ret = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
12975                                         pipe_config);
12976         if (ret)
12977                 return ret;
12978
12979         base_bpp = pipe_config->pipe_bpp;
12980
12981         /*
12982          * Determine the real pipe dimensions. Note that stereo modes can
12983          * increase the actual pipe size due to the frame doubling and
12984          * insertion of additional space for blanks between the frame. This
12985          * is stored in the crtc timings. We use the requested mode to do this
12986          * computation to clearly distinguish it from the adjusted mode, which
12987          * can be changed by the connectors in the below retry loop.
12988          */
12989         drm_mode_get_hv_timing(&pipe_config->hw.mode,
12990                                &pipe_config->pipe_src_w,
12991                                &pipe_config->pipe_src_h);
12992
12993         for_each_new_connector_in_state(state, connector, connector_state, i) {
12994                 if (connector_state->crtc != crtc)
12995                         continue;
12996
12997                 encoder = to_intel_encoder(connector_state->best_encoder);
12998
12999                 if (!check_single_encoder_cloning(state, to_intel_crtc(crtc), encoder)) {
13000                         DRM_DEBUG_KMS("rejecting invalid cloning configuration\n");
13001                         return -EINVAL;
13002                 }
13003
13004                 /*
13005                  * Determine output_types before calling the .compute_config()
13006                  * hooks so that the hooks can use this information safely.
13007                  */
13008                 if (encoder->compute_output_type)
13009                         pipe_config->output_types |=
13010                                 BIT(encoder->compute_output_type(encoder, pipe_config,
13011                                                                  connector_state));
13012                 else
13013                         pipe_config->output_types |= BIT(encoder->type);
13014         }
13015
13016 encoder_retry:
13017         /* Ensure the port clock defaults are reset when retrying. */
13018         pipe_config->port_clock = 0;
13019         pipe_config->pixel_multiplier = 1;
13020
13021         /* Fill in default crtc timings, allow encoders to overwrite them. */
13022         drm_mode_set_crtcinfo(&pipe_config->hw.adjusted_mode,
13023                               CRTC_STEREO_DOUBLE);
13024
13025         /* Get tile_group_id of tiled connector */
13026         for_each_new_connector_in_state(state, connector, connector_state, i) {
13027                 if (connector_state->crtc == crtc &&
13028                     connector->has_tile) {
13029                         tile_group_id = connector->tile_group->id;
13030                         break;
13031                 }
13032         }
13033
13034         /* Get total number of tiled connectors in state that belong to
13035          * this tile group.
13036          */
13037         for_each_new_connector_in_state(state, connector, connector_state, i) {
13038                 if (connector->has_tile &&
13039                     connector->tile_group->id == tile_group_id)
13040                         num_tiled_conns++;
13041         }
13042
13043         /* Pass our mode to the connectors and the CRTC to give them a chance to
13044          * adjust it according to limitations or connector properties, and also
13045          * a chance to reject the mode entirely.
13046          */
13047         for_each_new_connector_in_state(state, connector, connector_state, i) {
13048                 if (connector_state->crtc != crtc)
13049                         continue;
13050
13051                 ret = icl_compute_port_sync_crtc_state(connector, pipe_config,
13052                                                        num_tiled_conns);
13053                 if (ret) {
13054                         DRM_DEBUG_KMS("Cannot assign Sync Mode CRTCs: %d\n",
13055                                       ret);
13056                         return ret;
13057                 }
13058
13059                 encoder = to_intel_encoder(connector_state->best_encoder);
13060                 ret = encoder->compute_config(encoder, pipe_config,
13061                                               connector_state);
13062                 if (ret < 0) {
13063                         if (ret != -EDEADLK)
13064                                 DRM_DEBUG_KMS("Encoder config failure: %d\n",
13065                                               ret);
13066                         return ret;
13067                 }
13068         }
13069
13070         /* Set default port clock if not overwritten by the encoder. Needs to be
13071          * done afterwards in case the encoder adjusts the mode. */
13072         if (!pipe_config->port_clock)
13073                 pipe_config->port_clock = pipe_config->hw.adjusted_mode.crtc_clock
13074                         * pipe_config->pixel_multiplier;
13075
13076         ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
13077         if (ret == -EDEADLK)
13078                 return ret;
13079         if (ret < 0) {
13080                 DRM_DEBUG_KMS("CRTC fixup failed\n");
13081                 return ret;
13082         }
13083
13084         if (ret == RETRY) {
13085                 if (WARN(!retry, "loop in pipe configuration computation\n"))
13086                         return -EINVAL;
13087
13088                 DRM_DEBUG_KMS("CRTC bw constrained, retrying\n");
13089                 retry = false;
13090                 goto encoder_retry;
13091         }
13092
13093         /* Dithering seems to not pass-through bits correctly when it should, so
13094          * only enable it on 6bpc panels and when its not a compliance
13095          * test requesting 6bpc video pattern.
13096          */
13097         pipe_config->dither = (pipe_config->pipe_bpp == 6*3) &&
13098                 !pipe_config->dither_force_disable;
13099         DRM_DEBUG_KMS("hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
13100                       base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
13101
13102         /*
13103          * Make drm_calc_timestamping_constants in
13104          * drm_atomic_helper_update_legacy_modeset_state() happy
13105          */
13106         pipe_config->uapi.adjusted_mode = pipe_config->hw.adjusted_mode;
13107
13108         return 0;
13109 }
13110
13111 bool intel_fuzzy_clock_check(int clock1, int clock2)
13112 {
13113         int diff;
13114
13115         if (clock1 == clock2)
13116                 return true;
13117
13118         if (!clock1 || !clock2)
13119                 return false;
13120
13121         diff = abs(clock1 - clock2);
13122
13123         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
13124                 return true;
13125
13126         return false;
13127 }
13128
13129 static bool
13130 intel_compare_m_n(unsigned int m, unsigned int n,
13131                   unsigned int m2, unsigned int n2,
13132                   bool exact)
13133 {
13134         if (m == m2 && n == n2)
13135                 return true;
13136
13137         if (exact || !m || !n || !m2 || !n2)
13138                 return false;
13139
13140         BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
13141
13142         if (n > n2) {
13143                 while (n > n2) {
13144                         m2 <<= 1;
13145                         n2 <<= 1;
13146                 }
13147         } else if (n < n2) {
13148                 while (n < n2) {
13149                         m <<= 1;
13150                         n <<= 1;
13151                 }
13152         }
13153
13154         if (n != n2)
13155                 return false;
13156
13157         return intel_fuzzy_clock_check(m, m2);
13158 }
13159
13160 static bool
13161 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
13162                        const struct intel_link_m_n *m2_n2,
13163                        bool exact)
13164 {
13165         return m_n->tu == m2_n2->tu &&
13166                 intel_compare_m_n(m_n->gmch_m, m_n->gmch_n,
13167                                   m2_n2->gmch_m, m2_n2->gmch_n, exact) &&
13168                 intel_compare_m_n(m_n->link_m, m_n->link_n,
13169                                   m2_n2->link_m, m2_n2->link_n, exact);
13170 }
13171
13172 static bool
13173 intel_compare_infoframe(const union hdmi_infoframe *a,
13174                         const union hdmi_infoframe *b)
13175 {
13176         return memcmp(a, b, sizeof(*a)) == 0;
13177 }
13178
13179 static void
13180 pipe_config_infoframe_mismatch(struct drm_i915_private *dev_priv,
13181                                bool fastset, const char *name,
13182                                const union hdmi_infoframe *a,
13183                                const union hdmi_infoframe *b)
13184 {
13185         if (fastset) {
13186                 if ((drm_debug & DRM_UT_KMS) == 0)
13187                         return;
13188
13189                 DRM_DEBUG_KMS("fastset mismatch in %s infoframe\n", name);
13190                 DRM_DEBUG_KMS("expected:\n");
13191                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, a);
13192                 DRM_DEBUG_KMS("found:\n");
13193                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, b);
13194         } else {
13195                 DRM_ERROR("mismatch in %s infoframe\n", name);
13196                 DRM_ERROR("expected:\n");
13197                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, a);
13198                 DRM_ERROR("found:\n");
13199                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, b);
13200         }
13201 }
13202
13203 static void __printf(4, 5)
13204 pipe_config_mismatch(bool fastset, const struct intel_crtc *crtc,
13205                      const char *name, const char *format, ...)
13206 {
13207         struct va_format vaf;
13208         va_list args;
13209
13210         va_start(args, format);
13211         vaf.fmt = format;
13212         vaf.va = &args;
13213
13214         if (fastset)
13215                 DRM_DEBUG_KMS("[CRTC:%d:%s] fastset mismatch in %s %pV\n",
13216                               crtc->base.base.id, crtc->base.name, name, &vaf);
13217         else
13218                 DRM_ERROR("[CRTC:%d:%s] mismatch in %s %pV\n",
13219                           crtc->base.base.id, crtc->base.name, name, &vaf);
13220
13221         va_end(args);
13222 }
13223
13224 static bool fastboot_enabled(struct drm_i915_private *dev_priv)
13225 {
13226         if (i915_modparams.fastboot != -1)
13227                 return i915_modparams.fastboot;
13228
13229         /* Enable fastboot by default on Skylake and newer */
13230         if (INTEL_GEN(dev_priv) >= 9)
13231                 return true;
13232
13233         /* Enable fastboot by default on VLV and CHV */
13234         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
13235                 return true;
13236
13237         /* Disabled by default on all others */
13238         return false;
13239 }
13240
13241 static bool
13242 intel_pipe_config_compare(const struct intel_crtc_state *current_config,
13243                           const struct intel_crtc_state *pipe_config,
13244                           bool fastset)
13245 {
13246         struct drm_i915_private *dev_priv = to_i915(current_config->uapi.crtc->dev);
13247         struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
13248         bool ret = true;
13249         u32 bp_gamma = 0;
13250         bool fixup_inherited = fastset &&
13251                 (current_config->hw.mode.private_flags & I915_MODE_FLAG_INHERITED) &&
13252                 !(pipe_config->hw.mode.private_flags & I915_MODE_FLAG_INHERITED);
13253
13254         if (fixup_inherited && !fastboot_enabled(dev_priv)) {
13255                 DRM_DEBUG_KMS("initial modeset and fastboot not set\n");
13256                 ret = false;
13257         }
13258
13259 #define PIPE_CONF_CHECK_X(name) do { \
13260         if (current_config->name != pipe_config->name) { \
13261                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13262                                      "(expected 0x%08x, found 0x%08x)", \
13263                                      current_config->name, \
13264                                      pipe_config->name); \
13265                 ret = false; \
13266         } \
13267 } while (0)
13268
13269 #define PIPE_CONF_CHECK_I(name) do { \
13270         if (current_config->name != pipe_config->name) { \
13271                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13272                                      "(expected %i, found %i)", \
13273                                      current_config->name, \
13274                                      pipe_config->name); \
13275                 ret = false; \
13276         } \
13277 } while (0)
13278
13279 #define PIPE_CONF_CHECK_BOOL(name) do { \
13280         if (current_config->name != pipe_config->name) { \
13281                 pipe_config_mismatch(fastset, crtc,  __stringify(name), \
13282                                      "(expected %s, found %s)", \
13283                                      yesno(current_config->name), \
13284                                      yesno(pipe_config->name)); \
13285                 ret = false; \
13286         } \
13287 } while (0)
13288
13289 /*
13290  * Checks state where we only read out the enabling, but not the entire
13291  * state itself (like full infoframes or ELD for audio). These states
13292  * require a full modeset on bootup to fix up.
13293  */
13294 #define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) do { \
13295         if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \
13296                 PIPE_CONF_CHECK_BOOL(name); \
13297         } else { \
13298                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13299                                      "unable to verify whether state matches exactly, forcing modeset (expected %s, found %s)", \
13300                                      yesno(current_config->name), \
13301                                      yesno(pipe_config->name)); \
13302                 ret = false; \
13303         } \
13304 } while (0)
13305
13306 #define PIPE_CONF_CHECK_P(name) do { \
13307         if (current_config->name != pipe_config->name) { \
13308                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13309                                      "(expected %p, found %p)", \
13310                                      current_config->name, \
13311                                      pipe_config->name); \
13312                 ret = false; \
13313         } \
13314 } while (0)
13315
13316 #define PIPE_CONF_CHECK_M_N(name) do { \
13317         if (!intel_compare_link_m_n(&current_config->name, \
13318                                     &pipe_config->name,\
13319                                     !fastset)) { \
13320                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13321                                      "(expected tu %i gmch %i/%i link %i/%i, " \
13322                                      "found tu %i, gmch %i/%i link %i/%i)", \
13323                                      current_config->name.tu, \
13324                                      current_config->name.gmch_m, \
13325                                      current_config->name.gmch_n, \
13326                                      current_config->name.link_m, \
13327                                      current_config->name.link_n, \
13328                                      pipe_config->name.tu, \
13329                                      pipe_config->name.gmch_m, \
13330                                      pipe_config->name.gmch_n, \
13331                                      pipe_config->name.link_m, \
13332                                      pipe_config->name.link_n); \
13333                 ret = false; \
13334         } \
13335 } while (0)
13336
13337 /* This is required for BDW+ where there is only one set of registers for
13338  * switching between high and low RR.
13339  * This macro can be used whenever a comparison has to be made between one
13340  * hw state and multiple sw state variables.
13341  */
13342 #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) do { \
13343         if (!intel_compare_link_m_n(&current_config->name, \
13344                                     &pipe_config->name, !fastset) && \
13345             !intel_compare_link_m_n(&current_config->alt_name, \
13346                                     &pipe_config->name, !fastset)) { \
13347                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13348                                      "(expected tu %i gmch %i/%i link %i/%i, " \
13349                                      "or tu %i gmch %i/%i link %i/%i, " \
13350                                      "found tu %i, gmch %i/%i link %i/%i)", \
13351                                      current_config->name.tu, \
13352                                      current_config->name.gmch_m, \
13353                                      current_config->name.gmch_n, \
13354                                      current_config->name.link_m, \
13355                                      current_config->name.link_n, \
13356                                      current_config->alt_name.tu, \
13357                                      current_config->alt_name.gmch_m, \
13358                                      current_config->alt_name.gmch_n, \
13359                                      current_config->alt_name.link_m, \
13360                                      current_config->alt_name.link_n, \
13361                                      pipe_config->name.tu, \
13362                                      pipe_config->name.gmch_m, \
13363                                      pipe_config->name.gmch_n, \
13364                                      pipe_config->name.link_m, \
13365                                      pipe_config->name.link_n); \
13366                 ret = false; \
13367         } \
13368 } while (0)
13369
13370 #define PIPE_CONF_CHECK_FLAGS(name, mask) do { \
13371         if ((current_config->name ^ pipe_config->name) & (mask)) { \
13372                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13373                                      "(%x) (expected %i, found %i)", \
13374                                      (mask), \
13375                                      current_config->name & (mask), \
13376                                      pipe_config->name & (mask)); \
13377                 ret = false; \
13378         } \
13379 } while (0)
13380
13381 #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) do { \
13382         if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
13383                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13384                                      "(expected %i, found %i)", \
13385                                      current_config->name, \
13386                                      pipe_config->name); \
13387                 ret = false; \
13388         } \
13389 } while (0)
13390
13391 #define PIPE_CONF_CHECK_INFOFRAME(name) do { \
13392         if (!intel_compare_infoframe(&current_config->infoframes.name, \
13393                                      &pipe_config->infoframes.name)) { \
13394                 pipe_config_infoframe_mismatch(dev_priv, fastset, __stringify(name), \
13395                                                &current_config->infoframes.name, \
13396                                                &pipe_config->infoframes.name); \
13397                 ret = false; \
13398         } \
13399 } while (0)
13400
13401 #define PIPE_CONF_CHECK_COLOR_LUT(name1, name2, bit_precision) do { \
13402         if (current_config->name1 != pipe_config->name1) { \
13403                 pipe_config_mismatch(fastset, crtc, __stringify(name1), \
13404                                 "(expected %i, found %i, won't compare lut values)", \
13405                                 current_config->name1, \
13406                                 pipe_config->name1); \
13407                 ret = false;\
13408         } else { \
13409                 if (!intel_color_lut_equal(current_config->name2, \
13410                                         pipe_config->name2, pipe_config->name1, \
13411                                         bit_precision)) { \
13412                         pipe_config_mismatch(fastset, crtc, __stringify(name2), \
13413                                         "hw_state doesn't match sw_state"); \
13414                         ret = false; \
13415                 } \
13416         } \
13417 } while (0)
13418
13419 #define PIPE_CONF_QUIRK(quirk) \
13420         ((current_config->quirks | pipe_config->quirks) & (quirk))
13421
13422         PIPE_CONF_CHECK_I(cpu_transcoder);
13423
13424         PIPE_CONF_CHECK_BOOL(has_pch_encoder);
13425         PIPE_CONF_CHECK_I(fdi_lanes);
13426         PIPE_CONF_CHECK_M_N(fdi_m_n);
13427
13428         PIPE_CONF_CHECK_I(lane_count);
13429         PIPE_CONF_CHECK_X(lane_lat_optim_mask);
13430
13431         if (INTEL_GEN(dev_priv) < 8) {
13432                 PIPE_CONF_CHECK_M_N(dp_m_n);
13433
13434                 if (current_config->has_drrs)
13435                         PIPE_CONF_CHECK_M_N(dp_m2_n2);
13436         } else
13437                 PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
13438
13439         PIPE_CONF_CHECK_X(output_types);
13440
13441         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hdisplay);
13442         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_htotal);
13443         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hblank_start);
13444         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hblank_end);
13445         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hsync_start);
13446         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hsync_end);
13447
13448         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vdisplay);
13449         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vtotal);
13450         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vblank_start);
13451         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vblank_end);
13452         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vsync_start);
13453         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vsync_end);
13454
13455         PIPE_CONF_CHECK_I(pixel_multiplier);
13456         PIPE_CONF_CHECK_I(output_format);
13457         PIPE_CONF_CHECK_I(dc3co_exitline);
13458         PIPE_CONF_CHECK_BOOL(has_hdmi_sink);
13459         if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
13460             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
13461                 PIPE_CONF_CHECK_BOOL(limited_color_range);
13462
13463         PIPE_CONF_CHECK_BOOL(hdmi_scrambling);
13464         PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio);
13465         PIPE_CONF_CHECK_BOOL(has_infoframe);
13466         PIPE_CONF_CHECK_BOOL(fec_enable);
13467
13468         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio);
13469
13470         PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13471                               DRM_MODE_FLAG_INTERLACE);
13472
13473         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
13474                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13475                                       DRM_MODE_FLAG_PHSYNC);
13476                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13477                                       DRM_MODE_FLAG_NHSYNC);
13478                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13479                                       DRM_MODE_FLAG_PVSYNC);
13480                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13481                                       DRM_MODE_FLAG_NVSYNC);
13482         }
13483
13484         PIPE_CONF_CHECK_X(gmch_pfit.control);
13485         /* pfit ratios are autocomputed by the hw on gen4+ */
13486         if (INTEL_GEN(dev_priv) < 4)
13487                 PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios);
13488         PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
13489
13490         /*
13491          * Changing the EDP transcoder input mux
13492          * (A_ONOFF vs. A_ON) requires a full modeset.
13493          */
13494         PIPE_CONF_CHECK_BOOL(pch_pfit.force_thru);
13495
13496         if (!fastset) {
13497                 PIPE_CONF_CHECK_I(pipe_src_w);
13498                 PIPE_CONF_CHECK_I(pipe_src_h);
13499
13500                 PIPE_CONF_CHECK_BOOL(pch_pfit.enabled);
13501                 if (current_config->pch_pfit.enabled) {
13502                         PIPE_CONF_CHECK_X(pch_pfit.pos);
13503                         PIPE_CONF_CHECK_X(pch_pfit.size);
13504                 }
13505
13506                 PIPE_CONF_CHECK_I(scaler_state.scaler_id);
13507                 PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate);
13508
13509                 PIPE_CONF_CHECK_X(gamma_mode);
13510                 if (IS_CHERRYVIEW(dev_priv))
13511                         PIPE_CONF_CHECK_X(cgm_mode);
13512                 else
13513                         PIPE_CONF_CHECK_X(csc_mode);
13514                 PIPE_CONF_CHECK_BOOL(gamma_enable);
13515                 PIPE_CONF_CHECK_BOOL(csc_enable);
13516
13517                 bp_gamma = intel_color_get_gamma_bit_precision(pipe_config);
13518                 if (bp_gamma)
13519                         PIPE_CONF_CHECK_COLOR_LUT(gamma_mode, hw.gamma_lut, bp_gamma);
13520
13521         }
13522
13523         PIPE_CONF_CHECK_BOOL(double_wide);
13524
13525         PIPE_CONF_CHECK_P(shared_dpll);
13526         PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
13527         PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
13528         PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
13529         PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
13530         PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
13531         PIPE_CONF_CHECK_X(dpll_hw_state.spll);
13532         PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
13533         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
13534         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
13535         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0);
13536         PIPE_CONF_CHECK_X(dpll_hw_state.ebb0);
13537         PIPE_CONF_CHECK_X(dpll_hw_state.ebb4);
13538         PIPE_CONF_CHECK_X(dpll_hw_state.pll0);
13539         PIPE_CONF_CHECK_X(dpll_hw_state.pll1);
13540         PIPE_CONF_CHECK_X(dpll_hw_state.pll2);
13541         PIPE_CONF_CHECK_X(dpll_hw_state.pll3);
13542         PIPE_CONF_CHECK_X(dpll_hw_state.pll6);
13543         PIPE_CONF_CHECK_X(dpll_hw_state.pll8);
13544         PIPE_CONF_CHECK_X(dpll_hw_state.pll9);
13545         PIPE_CONF_CHECK_X(dpll_hw_state.pll10);
13546         PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12);
13547         PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl);
13548         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1);
13549         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl);
13550         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0);
13551         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1);
13552         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf);
13553         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock);
13554         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc);
13555         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias);
13556         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias);
13557
13558         PIPE_CONF_CHECK_X(dsi_pll.ctrl);
13559         PIPE_CONF_CHECK_X(dsi_pll.div);
13560
13561         if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5)
13562                 PIPE_CONF_CHECK_I(pipe_bpp);
13563
13564         PIPE_CONF_CHECK_CLOCK_FUZZY(hw.adjusted_mode.crtc_clock);
13565         PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
13566
13567         PIPE_CONF_CHECK_I(min_voltage_level);
13568
13569         PIPE_CONF_CHECK_X(infoframes.enable);
13570         PIPE_CONF_CHECK_X(infoframes.gcp);
13571         PIPE_CONF_CHECK_INFOFRAME(avi);
13572         PIPE_CONF_CHECK_INFOFRAME(spd);
13573         PIPE_CONF_CHECK_INFOFRAME(hdmi);
13574         PIPE_CONF_CHECK_INFOFRAME(drm);
13575
13576         PIPE_CONF_CHECK_I(sync_mode_slaves_mask);
13577         PIPE_CONF_CHECK_I(master_transcoder);
13578
13579         PIPE_CONF_CHECK_I(dsc.compression_enable);
13580         PIPE_CONF_CHECK_I(dsc.dsc_split);
13581         PIPE_CONF_CHECK_I(dsc.compressed_bpp);
13582
13583         PIPE_CONF_CHECK_I(mst_master_transcoder);
13584
13585 #undef PIPE_CONF_CHECK_X
13586 #undef PIPE_CONF_CHECK_I
13587 #undef PIPE_CONF_CHECK_BOOL
13588 #undef PIPE_CONF_CHECK_BOOL_INCOMPLETE
13589 #undef PIPE_CONF_CHECK_P
13590 #undef PIPE_CONF_CHECK_FLAGS
13591 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
13592 #undef PIPE_CONF_CHECK_COLOR_LUT
13593 #undef PIPE_CONF_QUIRK
13594
13595         return ret;
13596 }
13597
13598 static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv,
13599                                            const struct intel_crtc_state *pipe_config)
13600 {
13601         if (pipe_config->has_pch_encoder) {
13602                 int fdi_dotclock = intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
13603                                                             &pipe_config->fdi_m_n);
13604                 int dotclock = pipe_config->hw.adjusted_mode.crtc_clock;
13605
13606                 /*
13607                  * FDI already provided one idea for the dotclock.
13608                  * Yell if the encoder disagrees.
13609                  */
13610                 WARN(!intel_fuzzy_clock_check(fdi_dotclock, dotclock),
13611                      "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
13612                      fdi_dotclock, dotclock);
13613         }
13614 }
13615
13616 static void verify_wm_state(struct intel_crtc *crtc,
13617                             struct intel_crtc_state *new_crtc_state)
13618 {
13619         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13620         struct skl_hw_state {
13621                 struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
13622                 struct skl_ddb_entry ddb_uv[I915_MAX_PLANES];
13623                 struct skl_ddb_allocation ddb;
13624                 struct skl_pipe_wm wm;
13625         } *hw;
13626         struct skl_ddb_allocation *sw_ddb;
13627         struct skl_pipe_wm *sw_wm;
13628         struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
13629         const enum pipe pipe = crtc->pipe;
13630         int plane, level, max_level = ilk_wm_max_level(dev_priv);
13631
13632         if (INTEL_GEN(dev_priv) < 9 || !new_crtc_state->hw.active)
13633                 return;
13634
13635         hw = kzalloc(sizeof(*hw), GFP_KERNEL);
13636         if (!hw)
13637                 return;
13638
13639         skl_pipe_wm_get_hw_state(crtc, &hw->wm);
13640         sw_wm = &new_crtc_state->wm.skl.optimal;
13641
13642         skl_pipe_ddb_get_hw_state(crtc, hw->ddb_y, hw->ddb_uv);
13643
13644         skl_ddb_get_hw_state(dev_priv, &hw->ddb);
13645         sw_ddb = &dev_priv->wm.skl_hw.ddb;
13646
13647         if (INTEL_GEN(dev_priv) >= 11 &&
13648             hw->ddb.enabled_slices != sw_ddb->enabled_slices)
13649                 DRM_ERROR("mismatch in DBUF Slices (expected %u, got %u)\n",
13650                           sw_ddb->enabled_slices,
13651                           hw->ddb.enabled_slices);
13652
13653         /* planes */
13654         for_each_universal_plane(dev_priv, pipe, plane) {
13655                 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
13656
13657                 hw_plane_wm = &hw->wm.planes[plane];
13658                 sw_plane_wm = &sw_wm->planes[plane];
13659
13660                 /* Watermarks */
13661                 for (level = 0; level <= max_level; level++) {
13662                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
13663                                                 &sw_plane_wm->wm[level]))
13664                                 continue;
13665
13666                         DRM_ERROR("mismatch in WM pipe %c plane %d level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
13667                                   pipe_name(pipe), plane + 1, level,
13668                                   sw_plane_wm->wm[level].plane_en,
13669                                   sw_plane_wm->wm[level].plane_res_b,
13670                                   sw_plane_wm->wm[level].plane_res_l,
13671                                   hw_plane_wm->wm[level].plane_en,
13672                                   hw_plane_wm->wm[level].plane_res_b,
13673                                   hw_plane_wm->wm[level].plane_res_l);
13674                 }
13675
13676                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
13677                                          &sw_plane_wm->trans_wm)) {
13678                         DRM_ERROR("mismatch in trans WM pipe %c plane %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
13679                                   pipe_name(pipe), plane + 1,
13680                                   sw_plane_wm->trans_wm.plane_en,
13681                                   sw_plane_wm->trans_wm.plane_res_b,
13682                                   sw_plane_wm->trans_wm.plane_res_l,
13683                                   hw_plane_wm->trans_wm.plane_en,
13684                                   hw_plane_wm->trans_wm.plane_res_b,
13685                                   hw_plane_wm->trans_wm.plane_res_l);
13686                 }
13687
13688                 /* DDB */
13689                 hw_ddb_entry = &hw->ddb_y[plane];
13690                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[plane];
13691
13692                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
13693                         DRM_ERROR("mismatch in DDB state pipe %c plane %d (expected (%u,%u), found (%u,%u))\n",
13694                                   pipe_name(pipe), plane + 1,
13695                                   sw_ddb_entry->start, sw_ddb_entry->end,
13696                                   hw_ddb_entry->start, hw_ddb_entry->end);
13697                 }
13698         }
13699
13700         /*
13701          * cursor
13702          * If the cursor plane isn't active, we may not have updated it's ddb
13703          * allocation. In that case since the ddb allocation will be updated
13704          * once the plane becomes visible, we can skip this check
13705          */
13706         if (1) {
13707                 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
13708
13709                 hw_plane_wm = &hw->wm.planes[PLANE_CURSOR];
13710                 sw_plane_wm = &sw_wm->planes[PLANE_CURSOR];
13711
13712                 /* Watermarks */
13713                 for (level = 0; level <= max_level; level++) {
13714                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
13715                                                 &sw_plane_wm->wm[level]))
13716                                 continue;
13717
13718                         DRM_ERROR("mismatch in WM pipe %c cursor level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
13719                                   pipe_name(pipe), level,
13720                                   sw_plane_wm->wm[level].plane_en,
13721                                   sw_plane_wm->wm[level].plane_res_b,
13722                                   sw_plane_wm->wm[level].plane_res_l,
13723                                   hw_plane_wm->wm[level].plane_en,
13724                                   hw_plane_wm->wm[level].plane_res_b,
13725                                   hw_plane_wm->wm[level].plane_res_l);
13726                 }
13727
13728                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
13729                                          &sw_plane_wm->trans_wm)) {
13730                         DRM_ERROR("mismatch in trans WM pipe %c cursor (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
13731                                   pipe_name(pipe),
13732                                   sw_plane_wm->trans_wm.plane_en,
13733                                   sw_plane_wm->trans_wm.plane_res_b,
13734                                   sw_plane_wm->trans_wm.plane_res_l,
13735                                   hw_plane_wm->trans_wm.plane_en,
13736                                   hw_plane_wm->trans_wm.plane_res_b,
13737                                   hw_plane_wm->trans_wm.plane_res_l);
13738                 }
13739
13740                 /* DDB */
13741                 hw_ddb_entry = &hw->ddb_y[PLANE_CURSOR];
13742                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR];
13743
13744                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
13745                         DRM_ERROR("mismatch in DDB state pipe %c cursor (expected (%u,%u), found (%u,%u))\n",
13746                                   pipe_name(pipe),
13747                                   sw_ddb_entry->start, sw_ddb_entry->end,
13748                                   hw_ddb_entry->start, hw_ddb_entry->end);
13749                 }
13750         }
13751
13752         kfree(hw);
13753 }
13754
13755 static void
13756 verify_connector_state(struct intel_atomic_state *state,
13757                        struct intel_crtc *crtc)
13758 {
13759         struct drm_connector *connector;
13760         struct drm_connector_state *new_conn_state;
13761         int i;
13762
13763         for_each_new_connector_in_state(&state->base, connector, new_conn_state, i) {
13764                 struct drm_encoder *encoder = connector->encoder;
13765                 struct intel_crtc_state *crtc_state = NULL;
13766
13767                 if (new_conn_state->crtc != &crtc->base)
13768                         continue;
13769
13770                 if (crtc)
13771                         crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
13772
13773                 intel_connector_verify_state(crtc_state, new_conn_state);
13774
13775                 I915_STATE_WARN(new_conn_state->best_encoder != encoder,
13776                      "connector's atomic encoder doesn't match legacy encoder\n");
13777         }
13778 }
13779
13780 static void
13781 verify_encoder_state(struct drm_i915_private *dev_priv, struct intel_atomic_state *state)
13782 {
13783         struct intel_encoder *encoder;
13784         struct drm_connector *connector;
13785         struct drm_connector_state *old_conn_state, *new_conn_state;
13786         int i;
13787
13788         for_each_intel_encoder(&dev_priv->drm, encoder) {
13789                 bool enabled = false, found = false;
13790                 enum pipe pipe;
13791
13792                 DRM_DEBUG_KMS("[ENCODER:%d:%s]\n",
13793                               encoder->base.base.id,
13794                               encoder->base.name);
13795
13796                 for_each_oldnew_connector_in_state(&state->base, connector, old_conn_state,
13797                                                    new_conn_state, i) {
13798                         if (old_conn_state->best_encoder == &encoder->base)
13799                                 found = true;
13800
13801                         if (new_conn_state->best_encoder != &encoder->base)
13802                                 continue;
13803                         found = enabled = true;
13804
13805                         I915_STATE_WARN(new_conn_state->crtc !=
13806                                         encoder->base.crtc,
13807                              "connector's crtc doesn't match encoder crtc\n");
13808                 }
13809
13810                 if (!found)
13811                         continue;
13812
13813                 I915_STATE_WARN(!!encoder->base.crtc != enabled,
13814                      "encoder's enabled state mismatch "
13815                      "(expected %i, found %i)\n",
13816                      !!encoder->base.crtc, enabled);
13817
13818                 if (!encoder->base.crtc) {
13819                         bool active;
13820
13821                         active = encoder->get_hw_state(encoder, &pipe);
13822                         I915_STATE_WARN(active,
13823                              "encoder detached but still enabled on pipe %c.\n",
13824                              pipe_name(pipe));
13825                 }
13826         }
13827 }
13828
13829 static void
13830 verify_crtc_state(struct intel_crtc *crtc,
13831                   struct intel_crtc_state *old_crtc_state,
13832                   struct intel_crtc_state *new_crtc_state)
13833 {
13834         struct drm_device *dev = crtc->base.dev;
13835         struct drm_i915_private *dev_priv = to_i915(dev);
13836         struct intel_encoder *encoder;
13837         struct intel_crtc_state *pipe_config = old_crtc_state;
13838         struct drm_atomic_state *state = old_crtc_state->uapi.state;
13839         bool active;
13840
13841         __drm_atomic_helper_crtc_destroy_state(&old_crtc_state->uapi);
13842         intel_crtc_free_hw_state(old_crtc_state);
13843         intel_crtc_state_reset(old_crtc_state, crtc);
13844         old_crtc_state->uapi.state = state;
13845
13846         DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.base.id, crtc->base.name);
13847
13848         active = dev_priv->display.get_pipe_config(crtc, pipe_config);
13849
13850         /* we keep both pipes enabled on 830 */
13851         if (IS_I830(dev_priv))
13852                 active = new_crtc_state->hw.active;
13853
13854         I915_STATE_WARN(new_crtc_state->hw.active != active,
13855                         "crtc active state doesn't match with hw state "
13856                         "(expected %i, found %i)\n",
13857                         new_crtc_state->hw.active, active);
13858
13859         I915_STATE_WARN(crtc->active != new_crtc_state->hw.active,
13860                         "transitional active state does not match atomic hw state "
13861                         "(expected %i, found %i)\n",
13862                         new_crtc_state->hw.active, crtc->active);
13863
13864         for_each_encoder_on_crtc(dev, &crtc->base, encoder) {
13865                 enum pipe pipe;
13866
13867                 active = encoder->get_hw_state(encoder, &pipe);
13868                 I915_STATE_WARN(active != new_crtc_state->hw.active,
13869                                 "[ENCODER:%i] active %i with crtc active %i\n",
13870                                 encoder->base.base.id, active,
13871                                 new_crtc_state->hw.active);
13872
13873                 I915_STATE_WARN(active && crtc->pipe != pipe,
13874                                 "Encoder connected to wrong pipe %c\n",
13875                                 pipe_name(pipe));
13876
13877                 if (active)
13878                         encoder->get_config(encoder, pipe_config);
13879         }
13880
13881         intel_crtc_compute_pixel_rate(pipe_config);
13882
13883         if (!new_crtc_state->hw.active)
13884                 return;
13885
13886         intel_pipe_config_sanity_check(dev_priv, pipe_config);
13887
13888         if (!intel_pipe_config_compare(new_crtc_state,
13889                                        pipe_config, false)) {
13890                 I915_STATE_WARN(1, "pipe state doesn't match!\n");
13891                 intel_dump_pipe_config(pipe_config, NULL, "[hw state]");
13892                 intel_dump_pipe_config(new_crtc_state, NULL, "[sw state]");
13893         }
13894 }
13895
13896 static void
13897 intel_verify_planes(struct intel_atomic_state *state)
13898 {
13899         struct intel_plane *plane;
13900         const struct intel_plane_state *plane_state;
13901         int i;
13902
13903         for_each_new_intel_plane_in_state(state, plane,
13904                                           plane_state, i)
13905                 assert_plane(plane, plane_state->planar_slave ||
13906                              plane_state->uapi.visible);
13907 }
13908
13909 static void
13910 verify_single_dpll_state(struct drm_i915_private *dev_priv,
13911                          struct intel_shared_dpll *pll,
13912                          struct intel_crtc *crtc,
13913                          struct intel_crtc_state *new_crtc_state)
13914 {
13915         struct intel_dpll_hw_state dpll_hw_state;
13916         unsigned int crtc_mask;
13917         bool active;
13918
13919         memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
13920
13921         DRM_DEBUG_KMS("%s\n", pll->info->name);
13922
13923         active = pll->info->funcs->get_hw_state(dev_priv, pll, &dpll_hw_state);
13924
13925         if (!(pll->info->flags & INTEL_DPLL_ALWAYS_ON)) {
13926                 I915_STATE_WARN(!pll->on && pll->active_mask,
13927                      "pll in active use but not on in sw tracking\n");
13928                 I915_STATE_WARN(pll->on && !pll->active_mask,
13929                      "pll is on but not used by any active crtc\n");
13930                 I915_STATE_WARN(pll->on != active,
13931                      "pll on state mismatch (expected %i, found %i)\n",
13932                      pll->on, active);
13933         }
13934
13935         if (!crtc) {
13936                 I915_STATE_WARN(pll->active_mask & ~pll->state.crtc_mask,
13937                                 "more active pll users than references: %x vs %x\n",
13938                                 pll->active_mask, pll->state.crtc_mask);
13939
13940                 return;
13941         }
13942
13943         crtc_mask = drm_crtc_mask(&crtc->base);
13944
13945         if (new_crtc_state->hw.active)
13946                 I915_STATE_WARN(!(pll->active_mask & crtc_mask),
13947                                 "pll active mismatch (expected pipe %c in active mask 0x%02x)\n",
13948                                 pipe_name(drm_crtc_index(&crtc->base)), pll->active_mask);
13949         else
13950                 I915_STATE_WARN(pll->active_mask & crtc_mask,
13951                                 "pll active mismatch (didn't expect pipe %c in active mask 0x%02x)\n",
13952                                 pipe_name(drm_crtc_index(&crtc->base)), pll->active_mask);
13953
13954         I915_STATE_WARN(!(pll->state.crtc_mask & crtc_mask),
13955                         "pll enabled crtcs mismatch (expected 0x%x in 0x%02x)\n",
13956                         crtc_mask, pll->state.crtc_mask);
13957
13958         I915_STATE_WARN(pll->on && memcmp(&pll->state.hw_state,
13959                                           &dpll_hw_state,
13960                                           sizeof(dpll_hw_state)),
13961                         "pll hw state mismatch\n");
13962 }
13963
13964 static void
13965 verify_shared_dpll_state(struct intel_crtc *crtc,
13966                          struct intel_crtc_state *old_crtc_state,
13967                          struct intel_crtc_state *new_crtc_state)
13968 {
13969         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13970
13971         if (new_crtc_state->shared_dpll)
13972                 verify_single_dpll_state(dev_priv, new_crtc_state->shared_dpll, crtc, new_crtc_state);
13973
13974         if (old_crtc_state->shared_dpll &&
13975             old_crtc_state->shared_dpll != new_crtc_state->shared_dpll) {
13976                 unsigned int crtc_mask = drm_crtc_mask(&crtc->base);
13977                 struct intel_shared_dpll *pll = old_crtc_state->shared_dpll;
13978
13979                 I915_STATE_WARN(pll->active_mask & crtc_mask,
13980                                 "pll active mismatch (didn't expect pipe %c in active mask)\n",
13981                                 pipe_name(drm_crtc_index(&crtc->base)));
13982                 I915_STATE_WARN(pll->state.crtc_mask & crtc_mask,
13983                                 "pll enabled crtcs mismatch (found %x in enabled mask)\n",
13984                                 pipe_name(drm_crtc_index(&crtc->base)));
13985         }
13986 }
13987
13988 static void
13989 intel_modeset_verify_crtc(struct intel_crtc *crtc,
13990                           struct intel_atomic_state *state,
13991                           struct intel_crtc_state *old_crtc_state,
13992                           struct intel_crtc_state *new_crtc_state)
13993 {
13994         if (!needs_modeset(new_crtc_state) && !new_crtc_state->update_pipe)
13995                 return;
13996
13997         verify_wm_state(crtc, new_crtc_state);
13998         verify_connector_state(state, crtc);
13999         verify_crtc_state(crtc, old_crtc_state, new_crtc_state);
14000         verify_shared_dpll_state(crtc, old_crtc_state, new_crtc_state);
14001 }
14002
14003 static void
14004 verify_disabled_dpll_state(struct drm_i915_private *dev_priv)
14005 {
14006         int i;
14007
14008         for (i = 0; i < dev_priv->num_shared_dpll; i++)
14009                 verify_single_dpll_state(dev_priv, &dev_priv->shared_dplls[i], NULL, NULL);
14010 }
14011
14012 static void
14013 intel_modeset_verify_disabled(struct drm_i915_private *dev_priv,
14014                               struct intel_atomic_state *state)
14015 {
14016         verify_encoder_state(dev_priv, state);
14017         verify_connector_state(state, NULL);
14018         verify_disabled_dpll_state(dev_priv);
14019 }
14020
14021 static void
14022 intel_crtc_update_active_timings(const struct intel_crtc_state *crtc_state)
14023 {
14024         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
14025         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14026         const struct drm_display_mode *adjusted_mode =
14027                 &crtc_state->hw.adjusted_mode;
14028
14029         drm_calc_timestamping_constants(&crtc->base, adjusted_mode);
14030
14031         /*
14032          * The scanline counter increments at the leading edge of hsync.
14033          *
14034          * On most platforms it starts counting from vtotal-1 on the
14035          * first active line. That means the scanline counter value is
14036          * always one less than what we would expect. Ie. just after
14037          * start of vblank, which also occurs at start of hsync (on the
14038          * last active line), the scanline counter will read vblank_start-1.
14039          *
14040          * On gen2 the scanline counter starts counting from 1 instead
14041          * of vtotal-1, so we have to subtract one (or rather add vtotal-1
14042          * to keep the value positive), instead of adding one.
14043          *
14044          * On HSW+ the behaviour of the scanline counter depends on the output
14045          * type. For DP ports it behaves like most other platforms, but on HDMI
14046          * there's an extra 1 line difference. So we need to add two instead of
14047          * one to the value.
14048          *
14049          * On VLV/CHV DSI the scanline counter would appear to increment
14050          * approx. 1/3 of a scanline before start of vblank. Unfortunately
14051          * that means we can't tell whether we're in vblank or not while
14052          * we're on that particular line. We must still set scanline_offset
14053          * to 1 so that the vblank timestamps come out correct when we query
14054          * the scanline counter from within the vblank interrupt handler.
14055          * However if queried just before the start of vblank we'll get an
14056          * answer that's slightly in the future.
14057          */
14058         if (IS_GEN(dev_priv, 2)) {
14059                 int vtotal;
14060
14061                 vtotal = adjusted_mode->crtc_vtotal;
14062                 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
14063                         vtotal /= 2;
14064
14065                 crtc->scanline_offset = vtotal - 1;
14066         } else if (HAS_DDI(dev_priv) &&
14067                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
14068                 crtc->scanline_offset = 2;
14069         } else {
14070                 crtc->scanline_offset = 1;
14071         }
14072 }
14073
14074 static void intel_modeset_clear_plls(struct intel_atomic_state *state)
14075 {
14076         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14077         struct intel_crtc_state *new_crtc_state;
14078         struct intel_crtc *crtc;
14079         int i;
14080
14081         if (!dev_priv->display.crtc_compute_clock)
14082                 return;
14083
14084         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
14085                 if (!needs_modeset(new_crtc_state))
14086                         continue;
14087
14088                 intel_release_shared_dplls(state, crtc);
14089         }
14090 }
14091
14092 /*
14093  * This implements the workaround described in the "notes" section of the mode
14094  * set sequence documentation. When going from no pipes or single pipe to
14095  * multiple pipes, and planes are enabled after the pipe, we need to wait at
14096  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
14097  */
14098 static int hsw_mode_set_planes_workaround(struct intel_atomic_state *state)
14099 {
14100         struct intel_crtc_state *crtc_state;
14101         struct intel_crtc *crtc;
14102         struct intel_crtc_state *first_crtc_state = NULL;
14103         struct intel_crtc_state *other_crtc_state = NULL;
14104         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
14105         int i;
14106
14107         /* look at all crtc's that are going to be enabled in during modeset */
14108         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
14109                 if (!crtc_state->hw.active ||
14110                     !needs_modeset(crtc_state))
14111                         continue;
14112
14113                 if (first_crtc_state) {
14114                         other_crtc_state = crtc_state;
14115                         break;
14116                 } else {
14117                         first_crtc_state = crtc_state;
14118                         first_pipe = crtc->pipe;
14119                 }
14120         }
14121
14122         /* No workaround needed? */
14123         if (!first_crtc_state)
14124                 return 0;
14125
14126         /* w/a possibly needed, check how many crtc's are already enabled. */
14127         for_each_intel_crtc(state->base.dev, crtc) {
14128                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
14129                 if (IS_ERR(crtc_state))
14130                         return PTR_ERR(crtc_state);
14131
14132                 crtc_state->hsw_workaround_pipe = INVALID_PIPE;
14133
14134                 if (!crtc_state->hw.active ||
14135                     needs_modeset(crtc_state))
14136                         continue;
14137
14138                 /* 2 or more enabled crtcs means no need for w/a */
14139                 if (enabled_pipe != INVALID_PIPE)
14140                         return 0;
14141
14142                 enabled_pipe = crtc->pipe;
14143         }
14144
14145         if (enabled_pipe != INVALID_PIPE)
14146                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
14147         else if (other_crtc_state)
14148                 other_crtc_state->hsw_workaround_pipe = first_pipe;
14149
14150         return 0;
14151 }
14152
14153 static int intel_modeset_checks(struct intel_atomic_state *state)
14154 {
14155         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14156         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14157         struct intel_crtc *crtc;
14158         int ret, i;
14159
14160         /* keep the current setting */
14161         if (!state->cdclk.force_min_cdclk_changed)
14162                 state->cdclk.force_min_cdclk = dev_priv->cdclk.force_min_cdclk;
14163
14164         state->modeset = true;
14165         state->active_pipes = dev_priv->active_pipes;
14166         state->cdclk.logical = dev_priv->cdclk.logical;
14167         state->cdclk.actual = dev_priv->cdclk.actual;
14168
14169         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14170                                             new_crtc_state, i) {
14171                 if (new_crtc_state->hw.active)
14172                         state->active_pipes |= BIT(crtc->pipe);
14173                 else
14174                         state->active_pipes &= ~BIT(crtc->pipe);
14175
14176                 if (old_crtc_state->hw.active != new_crtc_state->hw.active)
14177                         state->active_pipe_changes |= BIT(crtc->pipe);
14178         }
14179
14180         if (state->active_pipe_changes) {
14181                 ret = intel_atomic_lock_global_state(state);
14182                 if (ret)
14183                         return ret;
14184         }
14185
14186         ret = intel_modeset_calc_cdclk(state);
14187         if (ret)
14188                 return ret;
14189
14190         intel_modeset_clear_plls(state);
14191
14192         if (IS_HASWELL(dev_priv))
14193                 return hsw_mode_set_planes_workaround(state);
14194
14195         return 0;
14196 }
14197
14198 /*
14199  * Handle calculation of various watermark data at the end of the atomic check
14200  * phase.  The code here should be run after the per-crtc and per-plane 'check'
14201  * handlers to ensure that all derived state has been updated.
14202  */
14203 static int calc_watermark_data(struct intel_atomic_state *state)
14204 {
14205         struct drm_device *dev = state->base.dev;
14206         struct drm_i915_private *dev_priv = to_i915(dev);
14207
14208         /* Is there platform-specific watermark information to calculate? */
14209         if (dev_priv->display.compute_global_watermarks)
14210                 return dev_priv->display.compute_global_watermarks(state);
14211
14212         return 0;
14213 }
14214
14215 static void intel_crtc_check_fastset(const struct intel_crtc_state *old_crtc_state,
14216                                      struct intel_crtc_state *new_crtc_state)
14217 {
14218         if (!intel_pipe_config_compare(old_crtc_state, new_crtc_state, true))
14219                 return;
14220
14221         new_crtc_state->uapi.mode_changed = false;
14222         new_crtc_state->update_pipe = true;
14223 }
14224
14225 static void intel_crtc_copy_fastset(const struct intel_crtc_state *old_crtc_state,
14226                                     struct intel_crtc_state *new_crtc_state)
14227 {
14228         /*
14229          * If we're not doing the full modeset we want to
14230          * keep the current M/N values as they may be
14231          * sufficiently different to the computed values
14232          * to cause problems.
14233          *
14234          * FIXME: should really copy more fuzzy state here
14235          */
14236         new_crtc_state->fdi_m_n = old_crtc_state->fdi_m_n;
14237         new_crtc_state->dp_m_n = old_crtc_state->dp_m_n;
14238         new_crtc_state->dp_m2_n2 = old_crtc_state->dp_m2_n2;
14239         new_crtc_state->has_drrs = old_crtc_state->has_drrs;
14240 }
14241
14242 static int intel_crtc_add_planes_to_state(struct intel_atomic_state *state,
14243                                           struct intel_crtc *crtc,
14244                                           u8 plane_ids_mask)
14245 {
14246         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14247         struct intel_plane *plane;
14248
14249         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
14250                 struct intel_plane_state *plane_state;
14251
14252                 if ((plane_ids_mask & BIT(plane->id)) == 0)
14253                         continue;
14254
14255                 plane_state = intel_atomic_get_plane_state(state, plane);
14256                 if (IS_ERR(plane_state))
14257                         return PTR_ERR(plane_state);
14258         }
14259
14260         return 0;
14261 }
14262
14263 static bool active_planes_affects_min_cdclk(struct drm_i915_private *dev_priv)
14264 {
14265         /* See {hsw,vlv,ivb}_plane_ratio() */
14266         return IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv) ||
14267                 IS_CHERRYVIEW(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
14268                 IS_IVYBRIDGE(dev_priv);
14269 }
14270
14271 static int intel_atomic_check_planes(struct intel_atomic_state *state,
14272                                      bool *need_modeset)
14273 {
14274         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14275         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14276         struct intel_plane_state *plane_state;
14277         struct intel_plane *plane;
14278         struct intel_crtc *crtc;
14279         int i, ret;
14280
14281         ret = icl_add_linked_planes(state);
14282         if (ret)
14283                 return ret;
14284
14285         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
14286                 ret = intel_plane_atomic_check(state, plane);
14287                 if (ret) {
14288                         DRM_DEBUG_ATOMIC("[PLANE:%d:%s] atomic driver check failed\n",
14289                                          plane->base.base.id, plane->base.name);
14290                         return ret;
14291                 }
14292         }
14293
14294         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14295                                             new_crtc_state, i) {
14296                 u8 old_active_planes, new_active_planes;
14297
14298                 ret = icl_check_nv12_planes(new_crtc_state);
14299                 if (ret)
14300                         return ret;
14301
14302                 /*
14303                  * On some platforms the number of active planes affects
14304                  * the planes' minimum cdclk calculation. Add such planes
14305                  * to the state before we compute the minimum cdclk.
14306                  */
14307                 if (!active_planes_affects_min_cdclk(dev_priv))
14308                         continue;
14309
14310                 old_active_planes = old_crtc_state->active_planes & ~BIT(PLANE_CURSOR);
14311                 new_active_planes = new_crtc_state->active_planes & ~BIT(PLANE_CURSOR);
14312
14313                 if (hweight8(old_active_planes) == hweight8(new_active_planes))
14314                         continue;
14315
14316                 ret = intel_crtc_add_planes_to_state(state, crtc, new_active_planes);
14317                 if (ret)
14318                         return ret;
14319         }
14320
14321         /*
14322          * active_planes bitmask has been updated, and potentially
14323          * affected planes are part of the state. We can now
14324          * compute the minimum cdclk for each plane.
14325          */
14326         for_each_new_intel_plane_in_state(state, plane, plane_state, i)
14327                 *need_modeset |= intel_plane_calc_min_cdclk(state, plane);
14328
14329         return 0;
14330 }
14331
14332 static int intel_atomic_check_crtcs(struct intel_atomic_state *state)
14333 {
14334         struct intel_crtc_state *crtc_state;
14335         struct intel_crtc *crtc;
14336         int i;
14337
14338         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
14339                 int ret = intel_crtc_atomic_check(state, crtc);
14340                 if (ret) {
14341                         DRM_DEBUG_ATOMIC("[CRTC:%d:%s] atomic driver check failed\n",
14342                                          crtc->base.base.id, crtc->base.name);
14343                         return ret;
14344                 }
14345         }
14346
14347         return 0;
14348 }
14349
14350 static bool intel_cpu_transcoder_needs_modeset(struct intel_atomic_state *state,
14351                                                enum transcoder transcoder)
14352 {
14353         struct intel_crtc_state *new_crtc_state;
14354         struct intel_crtc *crtc;
14355         int i;
14356
14357         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
14358                 if (new_crtc_state->cpu_transcoder == transcoder)
14359                         return needs_modeset(new_crtc_state);
14360
14361         return false;
14362 }
14363
14364 static void
14365 intel_modeset_synced_crtcs(struct intel_atomic_state *state,
14366                            u8 transcoders)
14367 {
14368         struct intel_crtc_state *new_crtc_state;
14369         struct intel_crtc *crtc;
14370         int i;
14371
14372         for_each_new_intel_crtc_in_state(state, crtc,
14373                                          new_crtc_state, i) {
14374                 if (transcoders & BIT(new_crtc_state->cpu_transcoder)) {
14375                         new_crtc_state->uapi.mode_changed = true;
14376                         new_crtc_state->update_pipe = false;
14377                 }
14378         }
14379 }
14380
14381 static void
14382 intel_atomic_check_synced_crtcs(struct intel_atomic_state *state)
14383 {
14384         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14385         struct intel_crtc_state *new_crtc_state;
14386         struct intel_crtc *crtc;
14387         int i;
14388
14389         if (INTEL_GEN(dev_priv) < 11)
14390                 return;
14391
14392         for_each_new_intel_crtc_in_state(state, crtc,
14393                                          new_crtc_state, i) {
14394                 if (is_trans_port_sync_master(new_crtc_state) &&
14395                     needs_modeset(new_crtc_state)) {
14396                         intel_modeset_synced_crtcs(state,
14397                                                    new_crtc_state->sync_mode_slaves_mask);
14398                 } else if (is_trans_port_sync_slave(new_crtc_state) &&
14399                            needs_modeset(new_crtc_state)) {
14400                         intel_modeset_synced_crtcs(state,
14401                                                    BIT(new_crtc_state->master_transcoder));
14402                 }
14403         }
14404 }
14405
14406 static int
14407 intel_modeset_all_tiles(struct intel_atomic_state *state, int tile_grp_id)
14408 {
14409         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14410         struct drm_connector *connector;
14411         struct drm_connector_list_iter conn_iter;
14412         int ret = 0;
14413
14414         drm_connector_list_iter_begin(&dev_priv->drm, &conn_iter);
14415         drm_for_each_connector_iter(connector, &conn_iter) {
14416                 struct drm_connector_state *conn_state;
14417                 struct drm_crtc_state *crtc_state;
14418
14419                 if (!connector->has_tile ||
14420                     connector->tile_group->id != tile_grp_id)
14421                         continue;
14422                 conn_state = drm_atomic_get_connector_state(&state->base,
14423                                                             connector);
14424                 if (IS_ERR(conn_state)) {
14425                         ret =  PTR_ERR(conn_state);
14426                         break;
14427                 }
14428
14429                 if (!conn_state->crtc)
14430                         continue;
14431
14432                 crtc_state = drm_atomic_get_crtc_state(&state->base,
14433                                                        conn_state->crtc);
14434                 if (IS_ERR(crtc_state)) {
14435                         ret = PTR_ERR(conn_state);
14436                         break;
14437                 }
14438                 crtc_state->mode_changed = true;
14439                 ret = drm_atomic_add_affected_connectors(&state->base,
14440                                                          conn_state->crtc);
14441                 if (ret)
14442                         break;
14443         }
14444         drm_connector_list_iter_end(&conn_iter);
14445
14446         return ret;
14447 }
14448
14449 static int
14450 intel_atomic_check_tiled_conns(struct intel_atomic_state *state)
14451 {
14452         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14453         struct drm_connector *connector;
14454         struct drm_connector_state *old_conn_state, *new_conn_state;
14455         int i, ret;
14456
14457         if (INTEL_GEN(dev_priv) < 11)
14458                 return 0;
14459
14460         /* Is tiled, mark all other tiled CRTCs as needing a modeset */
14461         for_each_oldnew_connector_in_state(&state->base, connector,
14462                                            old_conn_state, new_conn_state, i) {
14463                 if (!connector->has_tile)
14464                         continue;
14465                 if (!intel_connector_needs_modeset(state, connector))
14466                         continue;
14467
14468                 ret = intel_modeset_all_tiles(state, connector->tile_group->id);
14469                 if (ret)
14470                         return ret;
14471         }
14472
14473         return 0;
14474 }
14475
14476 /**
14477  * intel_atomic_check - validate state object
14478  * @dev: drm device
14479  * @_state: state to validate
14480  */
14481 static int intel_atomic_check(struct drm_device *dev,
14482                               struct drm_atomic_state *_state)
14483 {
14484         struct drm_i915_private *dev_priv = to_i915(dev);
14485         struct intel_atomic_state *state = to_intel_atomic_state(_state);
14486         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14487         struct intel_crtc *crtc;
14488         int ret, i;
14489         bool any_ms = false;
14490
14491         /* Catch I915_MODE_FLAG_INHERITED */
14492         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14493                                             new_crtc_state, i) {
14494                 if (new_crtc_state->hw.mode.private_flags !=
14495                     old_crtc_state->hw.mode.private_flags)
14496                         new_crtc_state->uapi.mode_changed = true;
14497         }
14498
14499         ret = drm_atomic_helper_check_modeset(dev, &state->base);
14500         if (ret)
14501                 goto fail;
14502
14503         /**
14504          * This check adds all the connectors in current state that belong to
14505          * the same tile group to a full modeset.
14506          * This function directly sets the mode_changed to true and we also call
14507          * drm_atomic_add_affected_connectors(). Hence we are not explicitly
14508          * calling drm_atomic_helper_check_modeset() after this.
14509          *
14510          * Fixme: Handle some corner cases where one of the
14511          * tiled connectors gets disconnected and tile info is lost but since it
14512          * was previously synced to other conn, we need to add that to the modeset.
14513          */
14514         ret = intel_atomic_check_tiled_conns(state);
14515         if (ret)
14516                 goto fail;
14517
14518         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14519                                             new_crtc_state, i) {
14520                 if (!needs_modeset(new_crtc_state)) {
14521                         /* Light copy */
14522                         intel_crtc_copy_uapi_to_hw_state_nomodeset(new_crtc_state);
14523
14524                         continue;
14525                 }
14526
14527                 if (!new_crtc_state->uapi.enable) {
14528                         intel_crtc_copy_uapi_to_hw_state(new_crtc_state);
14529                         continue;
14530                 }
14531
14532                 ret = intel_crtc_prepare_cleared_state(new_crtc_state);
14533                 if (ret)
14534                         goto fail;
14535
14536                 ret = intel_modeset_pipe_config(new_crtc_state);
14537                 if (ret)
14538                         goto fail;
14539
14540                 intel_crtc_check_fastset(old_crtc_state, new_crtc_state);
14541         }
14542
14543         /**
14544          * Check if fastset is allowed by external dependencies like other
14545          * pipes and transcoders.
14546          *
14547          * Right now it only forces a fullmodeset when the MST master
14548          * transcoder did not changed but the pipe of the master transcoder
14549          * needs a fullmodeset so all slaves also needs to do a fullmodeset.
14550          */
14551         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
14552                 enum transcoder master = new_crtc_state->mst_master_transcoder;
14553
14554                 if (!new_crtc_state->hw.enable ||
14555                     needs_modeset(new_crtc_state) ||
14556                     !intel_dp_mst_is_slave_trans(new_crtc_state))
14557                         continue;
14558
14559                 if (intel_cpu_transcoder_needs_modeset(state, master)) {
14560                         new_crtc_state->uapi.mode_changed = true;
14561                         new_crtc_state->update_pipe = false;
14562                 }
14563         }
14564
14565         /**
14566          * In case of port synced crtcs, if one of the synced crtcs
14567          * needs a full modeset, all other synced crtcs should be
14568          * forced a full modeset. This checks if fastset is allowed
14569          * by other dependencies like the synced crtcs.
14570          * Here we set the mode_changed to true directly to force full
14571          * modeset hence we do not explicitly call the function
14572          * drm_atomic_helper_check_modeset().
14573          */
14574         intel_atomic_check_synced_crtcs(state);
14575
14576         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14577                                             new_crtc_state, i) {
14578                 if (needs_modeset(new_crtc_state)) {
14579                         any_ms = true;
14580                         continue;
14581                 }
14582
14583                 if (!new_crtc_state->update_pipe)
14584                         continue;
14585
14586                 intel_crtc_copy_fastset(old_crtc_state, new_crtc_state);
14587         }
14588
14589         if (any_ms && !check_digital_port_conflicts(state)) {
14590                 DRM_DEBUG_KMS("rejecting conflicting digital port configuration\n");
14591                 ret = EINVAL;
14592                 goto fail;
14593         }
14594
14595         ret = drm_dp_mst_atomic_check(&state->base);
14596         if (ret)
14597                 goto fail;
14598
14599         any_ms |= state->cdclk.force_min_cdclk_changed;
14600
14601         ret = intel_atomic_check_planes(state, &any_ms);
14602         if (ret)
14603                 goto fail;
14604
14605         if (any_ms) {
14606                 ret = intel_modeset_checks(state);
14607                 if (ret)
14608                         goto fail;
14609         } else {
14610                 state->cdclk.logical = dev_priv->cdclk.logical;
14611         }
14612
14613         ret = intel_atomic_check_crtcs(state);
14614         if (ret)
14615                 goto fail;
14616
14617         intel_fbc_choose_crtc(dev_priv, state);
14618         ret = calc_watermark_data(state);
14619         if (ret)
14620                 goto fail;
14621
14622         ret = intel_bw_atomic_check(state);
14623         if (ret)
14624                 goto fail;
14625
14626         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14627                                             new_crtc_state, i) {
14628                 if (!needs_modeset(new_crtc_state) &&
14629                     !new_crtc_state->update_pipe)
14630                         continue;
14631
14632                 intel_dump_pipe_config(new_crtc_state, state,
14633                                        needs_modeset(new_crtc_state) ?
14634                                        "[modeset]" : "[fastset]");
14635         }
14636
14637         return 0;
14638
14639  fail:
14640         if (ret == -EDEADLK)
14641                 return ret;
14642
14643         /*
14644          * FIXME would probably be nice to know which crtc specifically
14645          * caused the failure, in cases where we can pinpoint it.
14646          */
14647         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14648                                             new_crtc_state, i)
14649                 intel_dump_pipe_config(new_crtc_state, state, "[failed]");
14650
14651         return ret;
14652 }
14653
14654 static int intel_atomic_prepare_commit(struct intel_atomic_state *state)
14655 {
14656         return drm_atomic_helper_prepare_planes(state->base.dev,
14657                                                 &state->base);
14658 }
14659
14660 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
14661 {
14662         struct drm_device *dev = crtc->base.dev;
14663         struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)];
14664
14665         if (!vblank->max_vblank_count)
14666                 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
14667
14668         return crtc->base.funcs->get_vblank_counter(&crtc->base);
14669 }
14670
14671 void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
14672                                   struct intel_crtc_state *crtc_state)
14673 {
14674         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14675
14676         if (!IS_GEN(dev_priv, 2) || crtc_state->active_planes)
14677                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
14678
14679         if (crtc_state->has_pch_encoder) {
14680                 enum pipe pch_transcoder =
14681                         intel_crtc_pch_transcoder(crtc);
14682
14683                 intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
14684         }
14685 }
14686
14687 static void intel_pipe_fastset(const struct intel_crtc_state *old_crtc_state,
14688                                const struct intel_crtc_state *new_crtc_state)
14689 {
14690         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
14691         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14692
14693         /*
14694          * Update pipe size and adjust fitter if needed: the reason for this is
14695          * that in compute_mode_changes we check the native mode (not the pfit
14696          * mode) to see if we can flip rather than do a full mode set. In the
14697          * fastboot case, we'll flip, but if we don't update the pipesrc and
14698          * pfit state, we'll end up with a big fb scanned out into the wrong
14699          * sized surface.
14700          */
14701         intel_set_pipe_src_size(new_crtc_state);
14702
14703         /* on skylake this is done by detaching scalers */
14704         if (INTEL_GEN(dev_priv) >= 9) {
14705                 skl_detach_scalers(new_crtc_state);
14706
14707                 if (new_crtc_state->pch_pfit.enabled)
14708                         skl_pfit_enable(new_crtc_state);
14709         } else if (HAS_PCH_SPLIT(dev_priv)) {
14710                 if (new_crtc_state->pch_pfit.enabled)
14711                         ilk_pfit_enable(new_crtc_state);
14712                 else if (old_crtc_state->pch_pfit.enabled)
14713                         ilk_pfit_disable(old_crtc_state);
14714         }
14715
14716         if (INTEL_GEN(dev_priv) >= 11)
14717                 icl_set_pipe_chicken(crtc);
14718 }
14719
14720 static void commit_pipe_config(struct intel_atomic_state *state,
14721                                struct intel_crtc_state *old_crtc_state,
14722                                struct intel_crtc_state *new_crtc_state)
14723 {
14724         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
14725         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14726         bool modeset = needs_modeset(new_crtc_state);
14727
14728         /*
14729          * During modesets pipe configuration was programmed as the
14730          * CRTC was enabled.
14731          */
14732         if (!modeset) {
14733                 if (new_crtc_state->uapi.color_mgmt_changed ||
14734                     new_crtc_state->update_pipe)
14735                         intel_color_commit(new_crtc_state);
14736
14737                 if (INTEL_GEN(dev_priv) >= 9)
14738                         skl_detach_scalers(new_crtc_state);
14739
14740                 if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
14741                         bdw_set_pipemisc(new_crtc_state);
14742
14743                 if (new_crtc_state->update_pipe)
14744                         intel_pipe_fastset(old_crtc_state, new_crtc_state);
14745         }
14746
14747         if (dev_priv->display.atomic_update_watermarks)
14748                 dev_priv->display.atomic_update_watermarks(state, crtc);
14749 }
14750
14751 static void intel_update_crtc(struct intel_crtc *crtc,
14752                               struct intel_atomic_state *state,
14753                               struct intel_crtc_state *old_crtc_state,
14754                               struct intel_crtc_state *new_crtc_state)
14755 {
14756         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14757         bool modeset = needs_modeset(new_crtc_state);
14758         struct intel_plane_state *new_plane_state =
14759                 intel_atomic_get_new_plane_state(state,
14760                                                  to_intel_plane(crtc->base.primary));
14761
14762         if (modeset) {
14763                 intel_crtc_update_active_timings(new_crtc_state);
14764
14765                 dev_priv->display.crtc_enable(state, crtc);
14766
14767                 /* vblanks work again, re-enable pipe CRC. */
14768                 intel_crtc_enable_pipe_crc(crtc);
14769         } else {
14770                 if (new_crtc_state->preload_luts &&
14771                     (new_crtc_state->uapi.color_mgmt_changed ||
14772                      new_crtc_state->update_pipe))
14773                         intel_color_load_luts(new_crtc_state);
14774
14775                 intel_pre_plane_update(state, crtc);
14776
14777                 if (new_crtc_state->update_pipe)
14778                         intel_encoders_update_pipe(state, crtc);
14779         }
14780
14781         if (new_crtc_state->update_pipe && !new_crtc_state->enable_fbc)
14782                 intel_fbc_disable(crtc);
14783         else if (new_plane_state)
14784                 intel_fbc_enable(crtc, new_crtc_state, new_plane_state);
14785
14786         /* Perform vblank evasion around commit operation */
14787         intel_pipe_update_start(new_crtc_state);
14788
14789         commit_pipe_config(state, old_crtc_state, new_crtc_state);
14790
14791         if (INTEL_GEN(dev_priv) >= 9)
14792                 skl_update_planes_on_crtc(state, crtc);
14793         else
14794                 i9xx_update_planes_on_crtc(state, crtc);
14795
14796         intel_pipe_update_end(new_crtc_state);
14797
14798         /*
14799          * We usually enable FIFO underrun interrupts as part of the
14800          * CRTC enable sequence during modesets.  But when we inherit a
14801          * valid pipe configuration from the BIOS we need to take care
14802          * of enabling them on the CRTC's first fastset.
14803          */
14804         if (new_crtc_state->update_pipe && !modeset &&
14805             old_crtc_state->hw.mode.private_flags & I915_MODE_FLAG_INHERITED)
14806                 intel_crtc_arm_fifo_underrun(crtc, new_crtc_state);
14807 }
14808
14809 static struct intel_crtc *intel_get_slave_crtc(const struct intel_crtc_state *new_crtc_state)
14810 {
14811         struct drm_i915_private *dev_priv = to_i915(new_crtc_state->uapi.crtc->dev);
14812         enum transcoder slave_transcoder;
14813
14814         WARN_ON(!is_power_of_2(new_crtc_state->sync_mode_slaves_mask));
14815
14816         slave_transcoder = ffs(new_crtc_state->sync_mode_slaves_mask) - 1;
14817         return intel_get_crtc_for_pipe(dev_priv,
14818                                        (enum pipe)slave_transcoder);
14819 }
14820
14821 static void intel_old_crtc_state_disables(struct intel_atomic_state *state,
14822                                           struct intel_crtc_state *old_crtc_state,
14823                                           struct intel_crtc_state *new_crtc_state,
14824                                           struct intel_crtc *crtc)
14825 {
14826         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14827
14828         intel_crtc_disable_planes(state, crtc);
14829
14830         /*
14831          * We need to disable pipe CRC before disabling the pipe,
14832          * or we race against vblank off.
14833          */
14834         intel_crtc_disable_pipe_crc(crtc);
14835
14836         dev_priv->display.crtc_disable(state, crtc);
14837         crtc->active = false;
14838         intel_fbc_disable(crtc);
14839         intel_disable_shared_dpll(old_crtc_state);
14840
14841         /* FIXME unify this for all platforms */
14842         if (!new_crtc_state->hw.active &&
14843             !HAS_GMCH(dev_priv) &&
14844             dev_priv->display.initial_watermarks)
14845                 dev_priv->display.initial_watermarks(state, crtc);
14846 }
14847
14848 static void intel_commit_modeset_disables(struct intel_atomic_state *state)
14849 {
14850         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
14851         struct intel_crtc *crtc;
14852         u32 handled = 0;
14853         int i;
14854
14855         /* Only disable port sync and MST slaves */
14856         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14857                                             new_crtc_state, i) {
14858                 if (!needs_modeset(new_crtc_state))
14859                         continue;
14860
14861                 if (!old_crtc_state->hw.active)
14862                         continue;
14863
14864                 /* In case of Transcoder port Sync master slave CRTCs can be
14865                  * assigned in any order and we need to make sure that
14866                  * slave CRTCs are disabled first and then master CRTC since
14867                  * Slave vblanks are masked till Master Vblanks.
14868                  */
14869                 if (!is_trans_port_sync_slave(old_crtc_state) &&
14870                     !intel_dp_mst_is_slave_trans(old_crtc_state))
14871                         continue;
14872
14873                 intel_pre_plane_update(state, crtc);
14874                 intel_old_crtc_state_disables(state, old_crtc_state,
14875                                               new_crtc_state, crtc);
14876                 handled |= BIT(crtc->pipe);
14877         }
14878
14879         /* Disable everything else left on */
14880         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14881                                             new_crtc_state, i) {
14882                 if (!needs_modeset(new_crtc_state) ||
14883                     (handled & BIT(crtc->pipe)))
14884                         continue;
14885
14886                 intel_pre_plane_update(state, crtc);
14887                 if (old_crtc_state->hw.active)
14888                         intel_old_crtc_state_disables(state, old_crtc_state,
14889                                                       new_crtc_state, crtc);
14890         }
14891 }
14892
14893 static void intel_commit_modeset_enables(struct intel_atomic_state *state)
14894 {
14895         struct intel_crtc *crtc;
14896         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14897         int i;
14898
14899         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
14900                 if (!new_crtc_state->hw.active)
14901                         continue;
14902
14903                 intel_update_crtc(crtc, state, old_crtc_state,
14904                                   new_crtc_state);
14905         }
14906 }
14907
14908 static void intel_crtc_enable_trans_port_sync(struct intel_crtc *crtc,
14909                                               struct intel_atomic_state *state,
14910                                               struct intel_crtc_state *new_crtc_state)
14911 {
14912         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14913
14914         intel_crtc_update_active_timings(new_crtc_state);
14915         dev_priv->display.crtc_enable(state, crtc);
14916         intel_crtc_enable_pipe_crc(crtc);
14917 }
14918
14919 static void intel_set_dp_tp_ctl_normal(struct intel_crtc *crtc,
14920                                        struct intel_atomic_state *state)
14921 {
14922         struct drm_connector *uninitialized_var(conn);
14923         struct drm_connector_state *conn_state;
14924         struct intel_dp *intel_dp;
14925         int i;
14926
14927         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
14928                 if (conn_state->crtc == &crtc->base)
14929                         break;
14930         }
14931         intel_dp = enc_to_intel_dp(&intel_attached_encoder(conn)->base);
14932         intel_dp_stop_link_train(intel_dp);
14933 }
14934
14935 /*
14936  * TODO: This is only called from port sync and it is identical to what will be
14937  * executed again in intel_update_crtc() over port sync pipes
14938  */
14939 static void intel_post_crtc_enable_updates(struct intel_crtc *crtc,
14940                                            struct intel_atomic_state *state)
14941 {
14942         struct intel_crtc_state *new_crtc_state =
14943                 intel_atomic_get_new_crtc_state(state, crtc);
14944         struct intel_crtc_state *old_crtc_state =
14945                 intel_atomic_get_old_crtc_state(state, crtc);
14946         struct intel_plane_state *new_plane_state =
14947                 intel_atomic_get_new_plane_state(state,
14948                                                  to_intel_plane(crtc->base.primary));
14949         bool modeset = needs_modeset(new_crtc_state);
14950
14951         if (new_crtc_state->update_pipe && !new_crtc_state->enable_fbc)
14952                 intel_fbc_disable(crtc);
14953         else if (new_plane_state)
14954                 intel_fbc_enable(crtc, new_crtc_state, new_plane_state);
14955
14956         /* Perform vblank evasion around commit operation */
14957         intel_pipe_update_start(new_crtc_state);
14958         commit_pipe_config(state, old_crtc_state, new_crtc_state);
14959         skl_update_planes_on_crtc(state, crtc);
14960         intel_pipe_update_end(new_crtc_state);
14961
14962         /*
14963          * We usually enable FIFO underrun interrupts as part of the
14964          * CRTC enable sequence during modesets.  But when we inherit a
14965          * valid pipe configuration from the BIOS we need to take care
14966          * of enabling them on the CRTC's first fastset.
14967          */
14968         if (new_crtc_state->update_pipe && !modeset &&
14969             old_crtc_state->hw.mode.private_flags & I915_MODE_FLAG_INHERITED)
14970                 intel_crtc_arm_fifo_underrun(crtc, new_crtc_state);
14971 }
14972
14973 static void intel_update_trans_port_sync_crtcs(struct intel_crtc *crtc,
14974                                                struct intel_atomic_state *state,
14975                                                struct intel_crtc_state *old_crtc_state,
14976                                                struct intel_crtc_state *new_crtc_state)
14977 {
14978         struct intel_crtc *slave_crtc = intel_get_slave_crtc(new_crtc_state);
14979         struct intel_crtc_state *new_slave_crtc_state =
14980                 intel_atomic_get_new_crtc_state(state, slave_crtc);
14981         struct intel_crtc_state *old_slave_crtc_state =
14982                 intel_atomic_get_old_crtc_state(state, slave_crtc);
14983
14984         WARN_ON(!slave_crtc || !new_slave_crtc_state ||
14985                 !old_slave_crtc_state);
14986
14987         DRM_DEBUG_KMS("Updating Transcoder Port Sync Master CRTC = %d %s and Slave CRTC %d %s\n",
14988                       crtc->base.base.id, crtc->base.name, slave_crtc->base.base.id,
14989                       slave_crtc->base.name);
14990
14991         /* Enable seq for slave with with DP_TP_CTL left Idle until the
14992          * master is ready
14993          */
14994         intel_crtc_enable_trans_port_sync(slave_crtc,
14995                                           state,
14996                                           new_slave_crtc_state);
14997
14998         /* Enable seq for master with with DP_TP_CTL left Idle */
14999         intel_crtc_enable_trans_port_sync(crtc,
15000                                           state,
15001                                           new_crtc_state);
15002
15003         /* Set Slave's DP_TP_CTL to Normal */
15004         intel_set_dp_tp_ctl_normal(slave_crtc,
15005                                    state);
15006
15007         /* Set Master's DP_TP_CTL To Normal */
15008         usleep_range(200, 400);
15009         intel_set_dp_tp_ctl_normal(crtc,
15010                                    state);
15011
15012         /* Now do the post crtc enable for all master and slaves */
15013         intel_post_crtc_enable_updates(slave_crtc,
15014                                        state);
15015         intel_post_crtc_enable_updates(crtc,
15016                                        state);
15017 }
15018
15019 static void skl_commit_modeset_enables(struct intel_atomic_state *state)
15020 {
15021         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
15022         struct intel_crtc *crtc;
15023         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
15024         u8 hw_enabled_slices = dev_priv->wm.skl_hw.ddb.enabled_slices;
15025         u8 required_slices = state->wm_results.ddb.enabled_slices;
15026         struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
15027         const u8 num_pipes = INTEL_NUM_PIPES(dev_priv);
15028         u8 update_pipes = 0, modeset_pipes = 0;
15029         int i;
15030
15031         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
15032                 if (!new_crtc_state->hw.active)
15033                         continue;
15034
15035                 /* ignore allocations for crtc's that have been turned off. */
15036                 if (!needs_modeset(new_crtc_state)) {
15037                         entries[i] = old_crtc_state->wm.skl.ddb;
15038                         update_pipes |= BIT(crtc->pipe);
15039                 } else {
15040                         modeset_pipes |= BIT(crtc->pipe);
15041                 }
15042         }
15043
15044         /* If 2nd DBuf slice required, enable it here */
15045         if (INTEL_GEN(dev_priv) >= 11 && required_slices > hw_enabled_slices)
15046                 icl_dbuf_slices_update(dev_priv, required_slices);
15047
15048         /*
15049          * Whenever the number of active pipes changes, we need to make sure we
15050          * update the pipes in the right order so that their ddb allocations
15051          * never overlap with each other between CRTC updates. Otherwise we'll
15052          * cause pipe underruns and other bad stuff.
15053          *
15054          * So first lets enable all pipes that do not need a fullmodeset as
15055          * those don't have any external dependency.
15056          */
15057         while (update_pipes) {
15058                 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
15059                                                     new_crtc_state, i) {
15060                         enum pipe pipe = crtc->pipe;
15061
15062                         if ((update_pipes & BIT(pipe)) == 0)
15063                                 continue;
15064
15065                         if (skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
15066                                                         entries, num_pipes, i))
15067                                 continue;
15068
15069                         entries[i] = new_crtc_state->wm.skl.ddb;
15070                         update_pipes &= ~BIT(pipe);
15071
15072                         intel_update_crtc(crtc, state, old_crtc_state,
15073                                           new_crtc_state);
15074
15075                         /*
15076                          * If this is an already active pipe, it's DDB changed,
15077                          * and this isn't the last pipe that needs updating
15078                          * then we need to wait for a vblank to pass for the
15079                          * new ddb allocation to take effect.
15080                          */
15081                         if (!skl_ddb_entry_equal(&new_crtc_state->wm.skl.ddb,
15082                                                  &old_crtc_state->wm.skl.ddb) &&
15083                             (update_pipes | modeset_pipes))
15084                                 intel_wait_for_vblank(dev_priv, pipe);
15085                 }
15086         }
15087
15088         /*
15089          * Enable all pipes that needs a modeset and do not depends on other
15090          * pipes
15091          */
15092         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
15093                                             new_crtc_state, i) {
15094                 enum pipe pipe = crtc->pipe;
15095
15096                 if ((modeset_pipes & BIT(pipe)) == 0)
15097                         continue;
15098
15099                 if (intel_dp_mst_is_slave_trans(new_crtc_state) ||
15100                     is_trans_port_sync_slave(new_crtc_state))
15101                         continue;
15102
15103                 WARN_ON(skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
15104                                                     entries, num_pipes, i));
15105
15106                 entries[i] = new_crtc_state->wm.skl.ddb;
15107                 modeset_pipes &= ~BIT(pipe);
15108
15109                 if (is_trans_port_sync_mode(new_crtc_state)) {
15110                         struct intel_crtc *slave_crtc;
15111
15112                         intel_update_trans_port_sync_crtcs(crtc, state,
15113                                                            old_crtc_state,
15114                                                            new_crtc_state);
15115
15116                         slave_crtc = intel_get_slave_crtc(new_crtc_state);
15117                         /* TODO: update entries[] of slave */
15118                         modeset_pipes &= ~BIT(slave_crtc->pipe);
15119
15120                 } else {
15121                         intel_update_crtc(crtc, state, old_crtc_state,
15122                                           new_crtc_state);
15123                 }
15124         }
15125
15126         /*
15127          * Finally enable all pipes that needs a modeset and depends on
15128          * other pipes, right now it is only MST slaves as both port sync slave
15129          * and master are enabled together
15130          */
15131         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
15132                                             new_crtc_state, i) {
15133                 enum pipe pipe = crtc->pipe;
15134
15135                 if ((modeset_pipes & BIT(pipe)) == 0)
15136                         continue;
15137
15138                 WARN_ON(skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
15139                                                     entries, num_pipes, i));
15140
15141                 entries[i] = new_crtc_state->wm.skl.ddb;
15142                 modeset_pipes &= ~BIT(pipe);
15143
15144                 intel_update_crtc(crtc, state, old_crtc_state, new_crtc_state);
15145         }
15146
15147         WARN_ON(modeset_pipes);
15148
15149         /* If 2nd DBuf slice is no more required disable it */
15150         if (INTEL_GEN(dev_priv) >= 11 && required_slices < hw_enabled_slices)
15151                 icl_dbuf_slices_update(dev_priv, required_slices);
15152 }
15153
15154 static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv)
15155 {
15156         struct intel_atomic_state *state, *next;
15157         struct llist_node *freed;
15158
15159         freed = llist_del_all(&dev_priv->atomic_helper.free_list);
15160         llist_for_each_entry_safe(state, next, freed, freed)
15161                 drm_atomic_state_put(&state->base);
15162 }
15163
15164 static void intel_atomic_helper_free_state_worker(struct work_struct *work)
15165 {
15166         struct drm_i915_private *dev_priv =
15167                 container_of(work, typeof(*dev_priv), atomic_helper.free_work);
15168
15169         intel_atomic_helper_free_state(dev_priv);
15170 }
15171
15172 static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state)
15173 {
15174         struct wait_queue_entry wait_fence, wait_reset;
15175         struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev);
15176
15177         init_wait_entry(&wait_fence, 0);
15178         init_wait_entry(&wait_reset, 0);
15179         for (;;) {
15180                 prepare_to_wait(&intel_state->commit_ready.wait,
15181                                 &wait_fence, TASK_UNINTERRUPTIBLE);
15182                 prepare_to_wait(bit_waitqueue(&dev_priv->gt.reset.flags,
15183                                               I915_RESET_MODESET),
15184                                 &wait_reset, TASK_UNINTERRUPTIBLE);
15185
15186
15187                 if (i915_sw_fence_done(&intel_state->commit_ready) ||
15188                     test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags))
15189                         break;
15190
15191                 schedule();
15192         }
15193         finish_wait(&intel_state->commit_ready.wait, &wait_fence);
15194         finish_wait(bit_waitqueue(&dev_priv->gt.reset.flags,
15195                                   I915_RESET_MODESET),
15196                     &wait_reset);
15197 }
15198
15199 static void intel_atomic_cleanup_work(struct work_struct *work)
15200 {
15201         struct drm_atomic_state *state =
15202                 container_of(work, struct drm_atomic_state, commit_work);
15203         struct drm_i915_private *i915 = to_i915(state->dev);
15204
15205         drm_atomic_helper_cleanup_planes(&i915->drm, state);
15206         drm_atomic_helper_commit_cleanup_done(state);
15207         drm_atomic_state_put(state);
15208
15209         intel_atomic_helper_free_state(i915);
15210 }
15211
15212 static void intel_atomic_commit_tail(struct intel_atomic_state *state)
15213 {
15214         struct drm_device *dev = state->base.dev;
15215         struct drm_i915_private *dev_priv = to_i915(dev);
15216         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
15217         struct intel_crtc *crtc;
15218         u64 put_domains[I915_MAX_PIPES] = {};
15219         intel_wakeref_t wakeref = 0;
15220         int i;
15221
15222         intel_atomic_commit_fence_wait(state);
15223
15224         drm_atomic_helper_wait_for_dependencies(&state->base);
15225
15226         if (state->modeset)
15227                 wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
15228
15229         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
15230                                             new_crtc_state, i) {
15231                 if (needs_modeset(new_crtc_state) ||
15232                     new_crtc_state->update_pipe) {
15233
15234                         put_domains[crtc->pipe] =
15235                                 modeset_get_crtc_power_domains(new_crtc_state);
15236                 }
15237         }
15238
15239         intel_commit_modeset_disables(state);
15240
15241         /* FIXME: Eventually get rid of our crtc->config pointer */
15242         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
15243                 crtc->config = new_crtc_state;
15244
15245         if (state->modeset) {
15246                 drm_atomic_helper_update_legacy_modeset_state(dev, &state->base);
15247
15248                 intel_set_cdclk_pre_plane_update(dev_priv,
15249                                                  &state->cdclk.actual,
15250                                                  &dev_priv->cdclk.actual,
15251                                                  state->cdclk.pipe);
15252
15253                 /*
15254                  * SKL workaround: bspec recommends we disable the SAGV when we
15255                  * have more then one pipe enabled
15256                  */
15257                 if (!intel_can_enable_sagv(state))
15258                         intel_disable_sagv(dev_priv);
15259
15260                 intel_modeset_verify_disabled(dev_priv, state);
15261         }
15262
15263         /* Complete the events for pipes that have now been disabled */
15264         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
15265                 bool modeset = needs_modeset(new_crtc_state);
15266
15267                 /* Complete events for now disable pipes here. */
15268                 if (modeset && !new_crtc_state->hw.active && new_crtc_state->uapi.event) {
15269                         spin_lock_irq(&dev->event_lock);
15270                         drm_crtc_send_vblank_event(&crtc->base,
15271                                                    new_crtc_state->uapi.event);
15272                         spin_unlock_irq(&dev->event_lock);
15273
15274                         new_crtc_state->uapi.event = NULL;
15275                 }
15276         }
15277
15278         if (state->modeset)
15279                 intel_encoders_update_prepare(state);
15280
15281         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
15282         dev_priv->display.commit_modeset_enables(state);
15283
15284         if (state->modeset) {
15285                 intel_encoders_update_complete(state);
15286
15287                 intel_set_cdclk_post_plane_update(dev_priv,
15288                                                   &state->cdclk.actual,
15289                                                   &dev_priv->cdclk.actual,
15290                                                   state->cdclk.pipe);
15291         }
15292
15293         /* FIXME: We should call drm_atomic_helper_commit_hw_done() here
15294          * already, but still need the state for the delayed optimization. To
15295          * fix this:
15296          * - wrap the optimization/post_plane_update stuff into a per-crtc work.
15297          * - schedule that vblank worker _before_ calling hw_done
15298          * - at the start of commit_tail, cancel it _synchrously
15299          * - switch over to the vblank wait helper in the core after that since
15300          *   we don't need out special handling any more.
15301          */
15302         drm_atomic_helper_wait_for_flip_done(dev, &state->base);
15303
15304         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
15305                 if (new_crtc_state->hw.active &&
15306                     !needs_modeset(new_crtc_state) &&
15307                     !new_crtc_state->preload_luts &&
15308                     (new_crtc_state->uapi.color_mgmt_changed ||
15309                      new_crtc_state->update_pipe))
15310                         intel_color_load_luts(new_crtc_state);
15311         }
15312
15313         /*
15314          * Now that the vblank has passed, we can go ahead and program the
15315          * optimal watermarks on platforms that need two-step watermark
15316          * programming.
15317          *
15318          * TODO: Move this (and other cleanup) to an async worker eventually.
15319          */
15320         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
15321                                             new_crtc_state, i) {
15322                 /*
15323                  * Gen2 reports pipe underruns whenever all planes are disabled.
15324                  * So re-enable underrun reporting after some planes get enabled.
15325                  *
15326                  * We do this before .optimize_watermarks() so that we have a
15327                  * chance of catching underruns with the intermediate watermarks
15328                  * vs. the new plane configuration.
15329                  */
15330                 if (IS_GEN(dev_priv, 2) && planes_enabling(old_crtc_state, new_crtc_state))
15331                         intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
15332
15333                 if (dev_priv->display.optimize_watermarks)
15334                         dev_priv->display.optimize_watermarks(state, crtc);
15335         }
15336
15337         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
15338                 intel_post_plane_update(state, crtc);
15339
15340                 if (put_domains[i])
15341                         modeset_put_power_domains(dev_priv, put_domains[i]);
15342
15343                 intel_modeset_verify_crtc(crtc, state, old_crtc_state, new_crtc_state);
15344         }
15345
15346         /* Underruns don't always raise interrupts, so check manually */
15347         intel_check_cpu_fifo_underruns(dev_priv);
15348         intel_check_pch_fifo_underruns(dev_priv);
15349
15350         if (state->modeset)
15351                 intel_verify_planes(state);
15352
15353         if (state->modeset && intel_can_enable_sagv(state))
15354                 intel_enable_sagv(dev_priv);
15355
15356         drm_atomic_helper_commit_hw_done(&state->base);
15357
15358         if (state->modeset) {
15359                 /* As one of the primary mmio accessors, KMS has a high
15360                  * likelihood of triggering bugs in unclaimed access. After we
15361                  * finish modesetting, see if an error has been flagged, and if
15362                  * so enable debugging for the next modeset - and hope we catch
15363                  * the culprit.
15364                  */
15365                 intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore);
15366                 intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET, wakeref);
15367         }
15368         intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
15369
15370         /*
15371          * Defer the cleanup of the old state to a separate worker to not
15372          * impede the current task (userspace for blocking modesets) that
15373          * are executed inline. For out-of-line asynchronous modesets/flips,
15374          * deferring to a new worker seems overkill, but we would place a
15375          * schedule point (cond_resched()) here anyway to keep latencies
15376          * down.
15377          */
15378         INIT_WORK(&state->base.commit_work, intel_atomic_cleanup_work);
15379         queue_work(system_highpri_wq, &state->base.commit_work);
15380 }
15381
15382 static void intel_atomic_commit_work(struct work_struct *work)
15383 {
15384         struct intel_atomic_state *state =
15385                 container_of(work, struct intel_atomic_state, base.commit_work);
15386
15387         intel_atomic_commit_tail(state);
15388 }
15389
15390 static int __i915_sw_fence_call
15391 intel_atomic_commit_ready(struct i915_sw_fence *fence,
15392                           enum i915_sw_fence_notify notify)
15393 {
15394         struct intel_atomic_state *state =
15395                 container_of(fence, struct intel_atomic_state, commit_ready);
15396
15397         switch (notify) {
15398         case FENCE_COMPLETE:
15399                 /* we do blocking waits in the worker, nothing to do here */
15400                 break;
15401         case FENCE_FREE:
15402                 {
15403                         struct intel_atomic_helper *helper =
15404                                 &to_i915(state->base.dev)->atomic_helper;
15405
15406                         if (llist_add(&state->freed, &helper->free_list))
15407                                 schedule_work(&helper->free_work);
15408                         break;
15409                 }
15410         }
15411
15412         return NOTIFY_DONE;
15413 }
15414
15415 static void intel_atomic_track_fbs(struct intel_atomic_state *state)
15416 {
15417         struct intel_plane_state *old_plane_state, *new_plane_state;
15418         struct intel_plane *plane;
15419         int i;
15420
15421         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
15422                                              new_plane_state, i)
15423                 intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->hw.fb),
15424                                         to_intel_frontbuffer(new_plane_state->hw.fb),
15425                                         plane->frontbuffer_bit);
15426 }
15427
15428 static void assert_global_state_locked(struct drm_i915_private *dev_priv)
15429 {
15430         struct intel_crtc *crtc;
15431
15432         for_each_intel_crtc(&dev_priv->drm, crtc)
15433                 drm_modeset_lock_assert_held(&crtc->base.mutex);
15434 }
15435
15436 static int intel_atomic_commit(struct drm_device *dev,
15437                                struct drm_atomic_state *_state,
15438                                bool nonblock)
15439 {
15440         struct intel_atomic_state *state = to_intel_atomic_state(_state);
15441         struct drm_i915_private *dev_priv = to_i915(dev);
15442         int ret = 0;
15443
15444         state->wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
15445
15446         drm_atomic_state_get(&state->base);
15447         i915_sw_fence_init(&state->commit_ready,
15448                            intel_atomic_commit_ready);
15449
15450         /*
15451          * The intel_legacy_cursor_update() fast path takes care
15452          * of avoiding the vblank waits for simple cursor
15453          * movement and flips. For cursor on/off and size changes,
15454          * we want to perform the vblank waits so that watermark
15455          * updates happen during the correct frames. Gen9+ have
15456          * double buffered watermarks and so shouldn't need this.
15457          *
15458          * Unset state->legacy_cursor_update before the call to
15459          * drm_atomic_helper_setup_commit() because otherwise
15460          * drm_atomic_helper_wait_for_flip_done() is a noop and
15461          * we get FIFO underruns because we didn't wait
15462          * for vblank.
15463          *
15464          * FIXME doing watermarks and fb cleanup from a vblank worker
15465          * (assuming we had any) would solve these problems.
15466          */
15467         if (INTEL_GEN(dev_priv) < 9 && state->base.legacy_cursor_update) {
15468                 struct intel_crtc_state *new_crtc_state;
15469                 struct intel_crtc *crtc;
15470                 int i;
15471
15472                 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
15473                         if (new_crtc_state->wm.need_postvbl_update ||
15474                             new_crtc_state->update_wm_post)
15475                                 state->base.legacy_cursor_update = false;
15476         }
15477
15478         ret = intel_atomic_prepare_commit(state);
15479         if (ret) {
15480                 DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret);
15481                 i915_sw_fence_commit(&state->commit_ready);
15482                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
15483                 return ret;
15484         }
15485
15486         ret = drm_atomic_helper_setup_commit(&state->base, nonblock);
15487         if (!ret)
15488                 ret = drm_atomic_helper_swap_state(&state->base, true);
15489
15490         if (ret) {
15491                 i915_sw_fence_commit(&state->commit_ready);
15492
15493                 drm_atomic_helper_cleanup_planes(dev, &state->base);
15494                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
15495                 return ret;
15496         }
15497         dev_priv->wm.distrust_bios_wm = false;
15498         intel_shared_dpll_swap_state(state);
15499         intel_atomic_track_fbs(state);
15500
15501         if (state->global_state_changed) {
15502                 assert_global_state_locked(dev_priv);
15503
15504                 memcpy(dev_priv->min_cdclk, state->min_cdclk,
15505                        sizeof(state->min_cdclk));
15506                 memcpy(dev_priv->min_voltage_level, state->min_voltage_level,
15507                        sizeof(state->min_voltage_level));
15508                 dev_priv->active_pipes = state->active_pipes;
15509                 dev_priv->cdclk.force_min_cdclk = state->cdclk.force_min_cdclk;
15510
15511                 intel_cdclk_swap_state(state);
15512         }
15513
15514         drm_atomic_state_get(&state->base);
15515         INIT_WORK(&state->base.commit_work, intel_atomic_commit_work);
15516
15517         i915_sw_fence_commit(&state->commit_ready);
15518         if (nonblock && state->modeset) {
15519                 queue_work(dev_priv->modeset_wq, &state->base.commit_work);
15520         } else if (nonblock) {
15521                 queue_work(dev_priv->flip_wq, &state->base.commit_work);
15522         } else {
15523                 if (state->modeset)
15524                         flush_workqueue(dev_priv->modeset_wq);
15525                 intel_atomic_commit_tail(state);
15526         }
15527
15528         return 0;
15529 }
15530
15531 struct wait_rps_boost {
15532         struct wait_queue_entry wait;
15533
15534         struct drm_crtc *crtc;
15535         struct i915_request *request;
15536 };
15537
15538 static int do_rps_boost(struct wait_queue_entry *_wait,
15539                         unsigned mode, int sync, void *key)
15540 {
15541         struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait);
15542         struct i915_request *rq = wait->request;
15543
15544         /*
15545          * If we missed the vblank, but the request is already running it
15546          * is reasonable to assume that it will complete before the next
15547          * vblank without our intervention, so leave RPS alone.
15548          */
15549         if (!i915_request_started(rq))
15550                 intel_rps_boost(rq);
15551         i915_request_put(rq);
15552
15553         drm_crtc_vblank_put(wait->crtc);
15554
15555         list_del(&wait->wait.entry);
15556         kfree(wait);
15557         return 1;
15558 }
15559
15560 static void add_rps_boost_after_vblank(struct drm_crtc *crtc,
15561                                        struct dma_fence *fence)
15562 {
15563         struct wait_rps_boost *wait;
15564
15565         if (!dma_fence_is_i915(fence))
15566                 return;
15567
15568         if (INTEL_GEN(to_i915(crtc->dev)) < 6)
15569                 return;
15570
15571         if (drm_crtc_vblank_get(crtc))
15572                 return;
15573
15574         wait = kmalloc(sizeof(*wait), GFP_KERNEL);
15575         if (!wait) {
15576                 drm_crtc_vblank_put(crtc);
15577                 return;
15578         }
15579
15580         wait->request = to_request(dma_fence_get(fence));
15581         wait->crtc = crtc;
15582
15583         wait->wait.func = do_rps_boost;
15584         wait->wait.flags = 0;
15585
15586         add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait);
15587 }
15588
15589 static int intel_plane_pin_fb(struct intel_plane_state *plane_state)
15590 {
15591         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
15592         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
15593         struct drm_framebuffer *fb = plane_state->hw.fb;
15594         struct i915_vma *vma;
15595
15596         if (plane->id == PLANE_CURSOR &&
15597             INTEL_INFO(dev_priv)->display.cursor_needs_physical) {
15598                 struct drm_i915_gem_object *obj = intel_fb_obj(fb);
15599                 const int align = intel_cursor_alignment(dev_priv);
15600                 int err;
15601
15602                 err = i915_gem_object_attach_phys(obj, align);
15603                 if (err)
15604                         return err;
15605         }
15606
15607         vma = intel_pin_and_fence_fb_obj(fb,
15608                                          &plane_state->view,
15609                                          intel_plane_uses_fence(plane_state),
15610                                          &plane_state->flags);
15611         if (IS_ERR(vma))
15612                 return PTR_ERR(vma);
15613
15614         plane_state->vma = vma;
15615
15616         return 0;
15617 }
15618
15619 static void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
15620 {
15621         struct i915_vma *vma;
15622
15623         vma = fetch_and_zero(&old_plane_state->vma);
15624         if (vma)
15625                 intel_unpin_fb_vma(vma, old_plane_state->flags);
15626 }
15627
15628 static void fb_obj_bump_render_priority(struct drm_i915_gem_object *obj)
15629 {
15630         struct i915_sched_attr attr = {
15631                 .priority = I915_USER_PRIORITY(I915_PRIORITY_DISPLAY),
15632         };
15633
15634         i915_gem_object_wait_priority(obj, 0, &attr);
15635 }
15636
15637 /**
15638  * intel_prepare_plane_fb - Prepare fb for usage on plane
15639  * @plane: drm plane to prepare for
15640  * @_new_plane_state: the plane state being prepared
15641  *
15642  * Prepares a framebuffer for usage on a display plane.  Generally this
15643  * involves pinning the underlying object and updating the frontbuffer tracking
15644  * bits.  Some older platforms need special physical address handling for
15645  * cursor planes.
15646  *
15647  * Returns 0 on success, negative error code on failure.
15648  */
15649 int
15650 intel_prepare_plane_fb(struct drm_plane *plane,
15651                        struct drm_plane_state *_new_plane_state)
15652 {
15653         struct intel_plane_state *new_plane_state =
15654                 to_intel_plane_state(_new_plane_state);
15655         struct intel_atomic_state *intel_state =
15656                 to_intel_atomic_state(new_plane_state->uapi.state);
15657         struct drm_i915_private *dev_priv = to_i915(plane->dev);
15658         struct drm_framebuffer *fb = new_plane_state->hw.fb;
15659         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
15660         struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb);
15661         int ret;
15662
15663         if (old_obj) {
15664                 struct intel_crtc_state *crtc_state =
15665                         intel_atomic_get_new_crtc_state(intel_state,
15666                                                         to_intel_crtc(plane->state->crtc));
15667
15668                 /* Big Hammer, we also need to ensure that any pending
15669                  * MI_WAIT_FOR_EVENT inside a user batch buffer on the
15670                  * current scanout is retired before unpinning the old
15671                  * framebuffer. Note that we rely on userspace rendering
15672                  * into the buffer attached to the pipe they are waiting
15673                  * on. If not, userspace generates a GPU hang with IPEHR
15674                  * point to the MI_WAIT_FOR_EVENT.
15675                  *
15676                  * This should only fail upon a hung GPU, in which case we
15677                  * can safely continue.
15678                  */
15679                 if (needs_modeset(crtc_state)) {
15680                         ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
15681                                                               old_obj->base.resv, NULL,
15682                                                               false, 0,
15683                                                               GFP_KERNEL);
15684                         if (ret < 0)
15685                                 return ret;
15686                 }
15687         }
15688
15689         if (new_plane_state->uapi.fence) { /* explicit fencing */
15690                 ret = i915_sw_fence_await_dma_fence(&intel_state->commit_ready,
15691                                                     new_plane_state->uapi.fence,
15692                                                     I915_FENCE_TIMEOUT,
15693                                                     GFP_KERNEL);
15694                 if (ret < 0)
15695                         return ret;
15696         }
15697
15698         if (!obj)
15699                 return 0;
15700
15701         ret = i915_gem_object_pin_pages(obj);
15702         if (ret)
15703                 return ret;
15704
15705         ret = intel_plane_pin_fb(new_plane_state);
15706
15707         i915_gem_object_unpin_pages(obj);
15708         if (ret)
15709                 return ret;
15710
15711         fb_obj_bump_render_priority(obj);
15712         i915_gem_object_flush_frontbuffer(obj, ORIGIN_DIRTYFB);
15713
15714         if (!new_plane_state->uapi.fence) { /* implicit fencing */
15715                 struct dma_fence *fence;
15716
15717                 ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
15718                                                       obj->base.resv, NULL,
15719                                                       false, I915_FENCE_TIMEOUT,
15720                                                       GFP_KERNEL);
15721                 if (ret < 0)
15722                         return ret;
15723
15724                 fence = dma_resv_get_excl_rcu(obj->base.resv);
15725                 if (fence) {
15726                         add_rps_boost_after_vblank(new_plane_state->hw.crtc,
15727                                                    fence);
15728                         dma_fence_put(fence);
15729                 }
15730         } else {
15731                 add_rps_boost_after_vblank(new_plane_state->hw.crtc,
15732                                            new_plane_state->uapi.fence);
15733         }
15734
15735         /*
15736          * We declare pageflips to be interactive and so merit a small bias
15737          * towards upclocking to deliver the frame on time. By only changing
15738          * the RPS thresholds to sample more regularly and aim for higher
15739          * clocks we can hopefully deliver low power workloads (like kodi)
15740          * that are not quite steady state without resorting to forcing
15741          * maximum clocks following a vblank miss (see do_rps_boost()).
15742          */
15743         if (!intel_state->rps_interactive) {
15744                 intel_rps_mark_interactive(&dev_priv->gt.rps, true);
15745                 intel_state->rps_interactive = true;
15746         }
15747
15748         return 0;
15749 }
15750
15751 /**
15752  * intel_cleanup_plane_fb - Cleans up an fb after plane use
15753  * @plane: drm plane to clean up for
15754  * @_old_plane_state: the state from the previous modeset
15755  *
15756  * Cleans up a framebuffer that has just been removed from a plane.
15757  */
15758 void
15759 intel_cleanup_plane_fb(struct drm_plane *plane,
15760                        struct drm_plane_state *_old_plane_state)
15761 {
15762         struct intel_plane_state *old_plane_state =
15763                 to_intel_plane_state(_old_plane_state);
15764         struct intel_atomic_state *intel_state =
15765                 to_intel_atomic_state(old_plane_state->uapi.state);
15766         struct drm_i915_private *dev_priv = to_i915(plane->dev);
15767
15768         if (intel_state->rps_interactive) {
15769                 intel_rps_mark_interactive(&dev_priv->gt.rps, false);
15770                 intel_state->rps_interactive = false;
15771         }
15772
15773         /* Should only be called after a successful intel_prepare_plane_fb()! */
15774         intel_plane_unpin_fb(old_plane_state);
15775 }
15776
15777 /**
15778  * intel_plane_destroy - destroy a plane
15779  * @plane: plane to destroy
15780  *
15781  * Common destruction function for all types of planes (primary, cursor,
15782  * sprite).
15783  */
15784 void intel_plane_destroy(struct drm_plane *plane)
15785 {
15786         drm_plane_cleanup(plane);
15787         kfree(to_intel_plane(plane));
15788 }
15789
15790 static bool i8xx_plane_format_mod_supported(struct drm_plane *_plane,
15791                                             u32 format, u64 modifier)
15792 {
15793         switch (modifier) {
15794         case DRM_FORMAT_MOD_LINEAR:
15795         case I915_FORMAT_MOD_X_TILED:
15796                 break;
15797         default:
15798                 return false;
15799         }
15800
15801         switch (format) {
15802         case DRM_FORMAT_C8:
15803         case DRM_FORMAT_RGB565:
15804         case DRM_FORMAT_XRGB1555:
15805         case DRM_FORMAT_XRGB8888:
15806                 return modifier == DRM_FORMAT_MOD_LINEAR ||
15807                         modifier == I915_FORMAT_MOD_X_TILED;
15808         default:
15809                 return false;
15810         }
15811 }
15812
15813 static bool i965_plane_format_mod_supported(struct drm_plane *_plane,
15814                                             u32 format, u64 modifier)
15815 {
15816         switch (modifier) {
15817         case DRM_FORMAT_MOD_LINEAR:
15818         case I915_FORMAT_MOD_X_TILED:
15819                 break;
15820         default:
15821                 return false;
15822         }
15823
15824         switch (format) {
15825         case DRM_FORMAT_C8:
15826         case DRM_FORMAT_RGB565:
15827         case DRM_FORMAT_XRGB8888:
15828         case DRM_FORMAT_XBGR8888:
15829         case DRM_FORMAT_ARGB8888:
15830         case DRM_FORMAT_ABGR8888:
15831         case DRM_FORMAT_XRGB2101010:
15832         case DRM_FORMAT_XBGR2101010:
15833         case DRM_FORMAT_ARGB2101010:
15834         case DRM_FORMAT_ABGR2101010:
15835         case DRM_FORMAT_XBGR16161616F:
15836                 return modifier == DRM_FORMAT_MOD_LINEAR ||
15837                         modifier == I915_FORMAT_MOD_X_TILED;
15838         default:
15839                 return false;
15840         }
15841 }
15842
15843 static bool intel_cursor_format_mod_supported(struct drm_plane *_plane,
15844                                               u32 format, u64 modifier)
15845 {
15846         return modifier == DRM_FORMAT_MOD_LINEAR &&
15847                 format == DRM_FORMAT_ARGB8888;
15848 }
15849
15850 static const struct drm_plane_funcs i965_plane_funcs = {
15851         .update_plane = drm_atomic_helper_update_plane,
15852         .disable_plane = drm_atomic_helper_disable_plane,
15853         .destroy = intel_plane_destroy,
15854         .atomic_duplicate_state = intel_plane_duplicate_state,
15855         .atomic_destroy_state = intel_plane_destroy_state,
15856         .format_mod_supported = i965_plane_format_mod_supported,
15857 };
15858
15859 static const struct drm_plane_funcs i8xx_plane_funcs = {
15860         .update_plane = drm_atomic_helper_update_plane,
15861         .disable_plane = drm_atomic_helper_disable_plane,
15862         .destroy = intel_plane_destroy,
15863         .atomic_duplicate_state = intel_plane_duplicate_state,
15864         .atomic_destroy_state = intel_plane_destroy_state,
15865         .format_mod_supported = i8xx_plane_format_mod_supported,
15866 };
15867
15868 static int
15869 intel_legacy_cursor_update(struct drm_plane *_plane,
15870                            struct drm_crtc *_crtc,
15871                            struct drm_framebuffer *fb,
15872                            int crtc_x, int crtc_y,
15873                            unsigned int crtc_w, unsigned int crtc_h,
15874                            u32 src_x, u32 src_y,
15875                            u32 src_w, u32 src_h,
15876                            struct drm_modeset_acquire_ctx *ctx)
15877 {
15878         struct intel_plane *plane = to_intel_plane(_plane);
15879         struct intel_crtc *crtc = to_intel_crtc(_crtc);
15880         struct intel_plane_state *old_plane_state =
15881                 to_intel_plane_state(plane->base.state);
15882         struct intel_plane_state *new_plane_state;
15883         struct intel_crtc_state *crtc_state =
15884                 to_intel_crtc_state(crtc->base.state);
15885         struct intel_crtc_state *new_crtc_state;
15886         int ret;
15887
15888         /*
15889          * When crtc is inactive or there is a modeset pending,
15890          * wait for it to complete in the slowpath
15891          */
15892         if (!crtc_state->hw.active || needs_modeset(crtc_state) ||
15893             crtc_state->update_pipe)
15894                 goto slow;
15895
15896         /*
15897          * Don't do an async update if there is an outstanding commit modifying
15898          * the plane.  This prevents our async update's changes from getting
15899          * overridden by a previous synchronous update's state.
15900          */
15901         if (old_plane_state->uapi.commit &&
15902             !try_wait_for_completion(&old_plane_state->uapi.commit->hw_done))
15903                 goto slow;
15904
15905         /*
15906          * If any parameters change that may affect watermarks,
15907          * take the slowpath. Only changing fb or position should be
15908          * in the fastpath.
15909          */
15910         if (old_plane_state->uapi.crtc != &crtc->base ||
15911             old_plane_state->uapi.src_w != src_w ||
15912             old_plane_state->uapi.src_h != src_h ||
15913             old_plane_state->uapi.crtc_w != crtc_w ||
15914             old_plane_state->uapi.crtc_h != crtc_h ||
15915             !old_plane_state->uapi.fb != !fb)
15916                 goto slow;
15917
15918         new_plane_state = to_intel_plane_state(intel_plane_duplicate_state(&plane->base));
15919         if (!new_plane_state)
15920                 return -ENOMEM;
15921
15922         new_crtc_state = to_intel_crtc_state(intel_crtc_duplicate_state(&crtc->base));
15923         if (!new_crtc_state) {
15924                 ret = -ENOMEM;
15925                 goto out_free;
15926         }
15927
15928         drm_atomic_set_fb_for_plane(&new_plane_state->uapi, fb);
15929
15930         new_plane_state->uapi.src_x = src_x;
15931         new_plane_state->uapi.src_y = src_y;
15932         new_plane_state->uapi.src_w = src_w;
15933         new_plane_state->uapi.src_h = src_h;
15934         new_plane_state->uapi.crtc_x = crtc_x;
15935         new_plane_state->uapi.crtc_y = crtc_y;
15936         new_plane_state->uapi.crtc_w = crtc_w;
15937         new_plane_state->uapi.crtc_h = crtc_h;
15938
15939         ret = intel_plane_atomic_check_with_state(crtc_state, new_crtc_state,
15940                                                   old_plane_state, new_plane_state);
15941         if (ret)
15942                 goto out_free;
15943
15944         ret = intel_plane_pin_fb(new_plane_state);
15945         if (ret)
15946                 goto out_free;
15947
15948         intel_frontbuffer_flush(to_intel_frontbuffer(new_plane_state->hw.fb),
15949                                 ORIGIN_FLIP);
15950         intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->hw.fb),
15951                                 to_intel_frontbuffer(new_plane_state->hw.fb),
15952                                 plane->frontbuffer_bit);
15953
15954         /* Swap plane state */
15955         plane->base.state = &new_plane_state->uapi;
15956
15957         /*
15958          * We cannot swap crtc_state as it may be in use by an atomic commit or
15959          * page flip that's running simultaneously. If we swap crtc_state and
15960          * destroy the old state, we will cause a use-after-free there.
15961          *
15962          * Only update active_planes, which is needed for our internal
15963          * bookkeeping. Either value will do the right thing when updating
15964          * planes atomically. If the cursor was part of the atomic update then
15965          * we would have taken the slowpath.
15966          */
15967         crtc_state->active_planes = new_crtc_state->active_planes;
15968
15969         if (new_plane_state->uapi.visible)
15970                 intel_update_plane(plane, crtc_state, new_plane_state);
15971         else
15972                 intel_disable_plane(plane, crtc_state);
15973
15974         intel_plane_unpin_fb(old_plane_state);
15975
15976 out_free:
15977         if (new_crtc_state)
15978                 intel_crtc_destroy_state(&crtc->base, &new_crtc_state->uapi);
15979         if (ret)
15980                 intel_plane_destroy_state(&plane->base, &new_plane_state->uapi);
15981         else
15982                 intel_plane_destroy_state(&plane->base, &old_plane_state->uapi);
15983         return ret;
15984
15985 slow:
15986         return drm_atomic_helper_update_plane(&plane->base, &crtc->base, fb,
15987                                               crtc_x, crtc_y, crtc_w, crtc_h,
15988                                               src_x, src_y, src_w, src_h, ctx);
15989 }
15990
15991 static const struct drm_plane_funcs intel_cursor_plane_funcs = {
15992         .update_plane = intel_legacy_cursor_update,
15993         .disable_plane = drm_atomic_helper_disable_plane,
15994         .destroy = intel_plane_destroy,
15995         .atomic_duplicate_state = intel_plane_duplicate_state,
15996         .atomic_destroy_state = intel_plane_destroy_state,
15997         .format_mod_supported = intel_cursor_format_mod_supported,
15998 };
15999
16000 static bool i9xx_plane_has_fbc(struct drm_i915_private *dev_priv,
16001                                enum i9xx_plane_id i9xx_plane)
16002 {
16003         if (!HAS_FBC(dev_priv))
16004                 return false;
16005
16006         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
16007                 return i9xx_plane == PLANE_A; /* tied to pipe A */
16008         else if (IS_IVYBRIDGE(dev_priv))
16009                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B ||
16010                         i9xx_plane == PLANE_C;
16011         else if (INTEL_GEN(dev_priv) >= 4)
16012                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B;
16013         else
16014                 return i9xx_plane == PLANE_A;
16015 }
16016
16017 static struct intel_plane *
16018 intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
16019 {
16020         struct intel_plane *plane;
16021         const struct drm_plane_funcs *plane_funcs;
16022         unsigned int supported_rotations;
16023         unsigned int possible_crtcs;
16024         const u32 *formats;
16025         int num_formats;
16026         int ret, zpos;
16027
16028         if (INTEL_GEN(dev_priv) >= 9)
16029                 return skl_universal_plane_create(dev_priv, pipe,
16030                                                   PLANE_PRIMARY);
16031
16032         plane = intel_plane_alloc();
16033         if (IS_ERR(plane))
16034                 return plane;
16035
16036         plane->pipe = pipe;
16037         /*
16038          * On gen2/3 only plane A can do FBC, but the panel fitter and LVDS
16039          * port is hooked to pipe B. Hence we want plane A feeding pipe B.
16040          */
16041         if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4)
16042                 plane->i9xx_plane = (enum i9xx_plane_id) !pipe;
16043         else
16044                 plane->i9xx_plane = (enum i9xx_plane_id) pipe;
16045         plane->id = PLANE_PRIMARY;
16046         plane->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, plane->id);
16047
16048         plane->has_fbc = i9xx_plane_has_fbc(dev_priv, plane->i9xx_plane);
16049         if (plane->has_fbc) {
16050                 struct intel_fbc *fbc = &dev_priv->fbc;
16051
16052                 fbc->possible_framebuffer_bits |= plane->frontbuffer_bit;
16053         }
16054
16055         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
16056                 formats = vlv_primary_formats;
16057                 num_formats = ARRAY_SIZE(vlv_primary_formats);
16058         } else if (INTEL_GEN(dev_priv) >= 4) {
16059                 /*
16060                  * WaFP16GammaEnabling:ivb
16061                  * "Workaround : When using the 64-bit format, the plane
16062                  *  output on each color channel has one quarter amplitude.
16063                  *  It can be brought up to full amplitude by using pipe
16064                  *  gamma correction or pipe color space conversion to
16065                  *  multiply the plane output by four."
16066                  *
16067                  * There is no dedicated plane gamma for the primary plane,
16068                  * and using the pipe gamma/csc could conflict with other
16069                  * planes, so we choose not to expose fp16 on IVB primary
16070                  * planes. HSW primary planes no longer have this problem.
16071                  */
16072                 if (IS_IVYBRIDGE(dev_priv)) {
16073                         formats = ivb_primary_formats;
16074                         num_formats = ARRAY_SIZE(ivb_primary_formats);
16075                 } else {
16076                         formats = i965_primary_formats;
16077                         num_formats = ARRAY_SIZE(i965_primary_formats);
16078                 }
16079         } else {
16080                 formats = i8xx_primary_formats;
16081                 num_formats = ARRAY_SIZE(i8xx_primary_formats);
16082         }
16083
16084         if (INTEL_GEN(dev_priv) >= 4)
16085                 plane_funcs = &i965_plane_funcs;
16086         else
16087                 plane_funcs = &i8xx_plane_funcs;
16088
16089         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
16090                 plane->min_cdclk = vlv_plane_min_cdclk;
16091         else if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
16092                 plane->min_cdclk = hsw_plane_min_cdclk;
16093         else if (IS_IVYBRIDGE(dev_priv))
16094                 plane->min_cdclk = ivb_plane_min_cdclk;
16095         else
16096                 plane->min_cdclk = i9xx_plane_min_cdclk;
16097
16098         plane->max_stride = i9xx_plane_max_stride;
16099         plane->update_plane = i9xx_update_plane;
16100         plane->disable_plane = i9xx_disable_plane;
16101         plane->get_hw_state = i9xx_plane_get_hw_state;
16102         plane->check_plane = i9xx_plane_check;
16103
16104         possible_crtcs = BIT(pipe);
16105
16106         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
16107                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
16108                                                possible_crtcs, plane_funcs,
16109                                                formats, num_formats,
16110                                                i9xx_format_modifiers,
16111                                                DRM_PLANE_TYPE_PRIMARY,
16112                                                "primary %c", pipe_name(pipe));
16113         else
16114                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
16115                                                possible_crtcs, plane_funcs,
16116                                                formats, num_formats,
16117                                                i9xx_format_modifiers,
16118                                                DRM_PLANE_TYPE_PRIMARY,
16119                                                "plane %c",
16120                                                plane_name(plane->i9xx_plane));
16121         if (ret)
16122                 goto fail;
16123
16124         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
16125                 supported_rotations =
16126                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180 |
16127                         DRM_MODE_REFLECT_X;
16128         } else if (INTEL_GEN(dev_priv) >= 4) {
16129                 supported_rotations =
16130                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180;
16131         } else {
16132                 supported_rotations = DRM_MODE_ROTATE_0;
16133         }
16134
16135         if (INTEL_GEN(dev_priv) >= 4)
16136                 drm_plane_create_rotation_property(&plane->base,
16137                                                    DRM_MODE_ROTATE_0,
16138                                                    supported_rotations);
16139
16140         zpos = 0;
16141         drm_plane_create_zpos_immutable_property(&plane->base, zpos);
16142
16143         drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs);
16144
16145         return plane;
16146
16147 fail:
16148         intel_plane_free(plane);
16149
16150         return ERR_PTR(ret);
16151 }
16152
16153 static struct intel_plane *
16154 intel_cursor_plane_create(struct drm_i915_private *dev_priv,
16155                           enum pipe pipe)
16156 {
16157         unsigned int possible_crtcs;
16158         struct intel_plane *cursor;
16159         int ret, zpos;
16160
16161         cursor = intel_plane_alloc();
16162         if (IS_ERR(cursor))
16163                 return cursor;
16164
16165         cursor->pipe = pipe;
16166         cursor->i9xx_plane = (enum i9xx_plane_id) pipe;
16167         cursor->id = PLANE_CURSOR;
16168         cursor->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, cursor->id);
16169
16170         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
16171                 cursor->max_stride = i845_cursor_max_stride;
16172                 cursor->update_plane = i845_update_cursor;
16173                 cursor->disable_plane = i845_disable_cursor;
16174                 cursor->get_hw_state = i845_cursor_get_hw_state;
16175                 cursor->check_plane = i845_check_cursor;
16176         } else {
16177                 cursor->max_stride = i9xx_cursor_max_stride;
16178                 cursor->update_plane = i9xx_update_cursor;
16179                 cursor->disable_plane = i9xx_disable_cursor;
16180                 cursor->get_hw_state = i9xx_cursor_get_hw_state;
16181                 cursor->check_plane = i9xx_check_cursor;
16182         }
16183
16184         cursor->cursor.base = ~0;
16185         cursor->cursor.cntl = ~0;
16186
16187         if (IS_I845G(dev_priv) || IS_I865G(dev_priv) || HAS_CUR_FBC(dev_priv))
16188                 cursor->cursor.size = ~0;
16189
16190         possible_crtcs = BIT(pipe);
16191
16192         ret = drm_universal_plane_init(&dev_priv->drm, &cursor->base,
16193                                        possible_crtcs, &intel_cursor_plane_funcs,
16194                                        intel_cursor_formats,
16195                                        ARRAY_SIZE(intel_cursor_formats),
16196                                        cursor_format_modifiers,
16197                                        DRM_PLANE_TYPE_CURSOR,
16198                                        "cursor %c", pipe_name(pipe));
16199         if (ret)
16200                 goto fail;
16201
16202         if (INTEL_GEN(dev_priv) >= 4)
16203                 drm_plane_create_rotation_property(&cursor->base,
16204                                                    DRM_MODE_ROTATE_0,
16205                                                    DRM_MODE_ROTATE_0 |
16206                                                    DRM_MODE_ROTATE_180);
16207
16208         zpos = RUNTIME_INFO(dev_priv)->num_sprites[pipe] + 1;
16209         drm_plane_create_zpos_immutable_property(&cursor->base, zpos);
16210
16211         drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs);
16212
16213         return cursor;
16214
16215 fail:
16216         intel_plane_free(cursor);
16217
16218         return ERR_PTR(ret);
16219 }
16220
16221 #define INTEL_CRTC_FUNCS \
16222         .gamma_set = drm_atomic_helper_legacy_gamma_set, \
16223         .set_config = drm_atomic_helper_set_config, \
16224         .destroy = intel_crtc_destroy, \
16225         .page_flip = drm_atomic_helper_page_flip, \
16226         .atomic_duplicate_state = intel_crtc_duplicate_state, \
16227         .atomic_destroy_state = intel_crtc_destroy_state, \
16228         .set_crc_source = intel_crtc_set_crc_source, \
16229         .verify_crc_source = intel_crtc_verify_crc_source, \
16230         .get_crc_sources = intel_crtc_get_crc_sources
16231
16232 static const struct drm_crtc_funcs bdw_crtc_funcs = {
16233         INTEL_CRTC_FUNCS,
16234
16235         .get_vblank_counter = g4x_get_vblank_counter,
16236         .enable_vblank = bdw_enable_vblank,
16237         .disable_vblank = bdw_disable_vblank,
16238 };
16239
16240 static const struct drm_crtc_funcs ilk_crtc_funcs = {
16241         INTEL_CRTC_FUNCS,
16242
16243         .get_vblank_counter = g4x_get_vblank_counter,
16244         .enable_vblank = ilk_enable_vblank,
16245         .disable_vblank = ilk_disable_vblank,
16246 };
16247
16248 static const struct drm_crtc_funcs g4x_crtc_funcs = {
16249         INTEL_CRTC_FUNCS,
16250
16251         .get_vblank_counter = g4x_get_vblank_counter,
16252         .enable_vblank = i965_enable_vblank,
16253         .disable_vblank = i965_disable_vblank,
16254 };
16255
16256 static const struct drm_crtc_funcs i965_crtc_funcs = {
16257         INTEL_CRTC_FUNCS,
16258
16259         .get_vblank_counter = i915_get_vblank_counter,
16260         .enable_vblank = i965_enable_vblank,
16261         .disable_vblank = i965_disable_vblank,
16262 };
16263
16264 static const struct drm_crtc_funcs i915gm_crtc_funcs = {
16265         INTEL_CRTC_FUNCS,
16266
16267         .get_vblank_counter = i915_get_vblank_counter,
16268         .enable_vblank = i915gm_enable_vblank,
16269         .disable_vblank = i915gm_disable_vblank,
16270 };
16271
16272 static const struct drm_crtc_funcs i915_crtc_funcs = {
16273         INTEL_CRTC_FUNCS,
16274
16275         .get_vblank_counter = i915_get_vblank_counter,
16276         .enable_vblank = i8xx_enable_vblank,
16277         .disable_vblank = i8xx_disable_vblank,
16278 };
16279
16280 static const struct drm_crtc_funcs i8xx_crtc_funcs = {
16281         INTEL_CRTC_FUNCS,
16282
16283         /* no hw vblank counter */
16284         .enable_vblank = i8xx_enable_vblank,
16285         .disable_vblank = i8xx_disable_vblank,
16286 };
16287
16288 static struct intel_crtc *intel_crtc_alloc(void)
16289 {
16290         struct intel_crtc_state *crtc_state;
16291         struct intel_crtc *crtc;
16292
16293         crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
16294         if (!crtc)
16295                 return ERR_PTR(-ENOMEM);
16296
16297         crtc_state = intel_crtc_state_alloc(crtc);
16298         if (!crtc_state) {
16299                 kfree(crtc);
16300                 return ERR_PTR(-ENOMEM);
16301         }
16302
16303         crtc->base.state = &crtc_state->uapi;
16304         crtc->config = crtc_state;
16305
16306         return crtc;
16307 }
16308
16309 static void intel_crtc_free(struct intel_crtc *crtc)
16310 {
16311         intel_crtc_destroy_state(&crtc->base, crtc->base.state);
16312         kfree(crtc);
16313 }
16314
16315 static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
16316 {
16317         struct intel_plane *primary, *cursor;
16318         const struct drm_crtc_funcs *funcs;
16319         struct intel_crtc *crtc;
16320         int sprite, ret;
16321
16322         crtc = intel_crtc_alloc();
16323         if (IS_ERR(crtc))
16324                 return PTR_ERR(crtc);
16325
16326         crtc->pipe = pipe;
16327         crtc->num_scalers = RUNTIME_INFO(dev_priv)->num_scalers[pipe];
16328
16329         primary = intel_primary_plane_create(dev_priv, pipe);
16330         if (IS_ERR(primary)) {
16331                 ret = PTR_ERR(primary);
16332                 goto fail;
16333         }
16334         crtc->plane_ids_mask |= BIT(primary->id);
16335
16336         for_each_sprite(dev_priv, pipe, sprite) {
16337                 struct intel_plane *plane;
16338
16339                 plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
16340                 if (IS_ERR(plane)) {
16341                         ret = PTR_ERR(plane);
16342                         goto fail;
16343                 }
16344                 crtc->plane_ids_mask |= BIT(plane->id);
16345         }
16346
16347         cursor = intel_cursor_plane_create(dev_priv, pipe);
16348         if (IS_ERR(cursor)) {
16349                 ret = PTR_ERR(cursor);
16350                 goto fail;
16351         }
16352         crtc->plane_ids_mask |= BIT(cursor->id);
16353
16354         if (HAS_GMCH(dev_priv)) {
16355                 if (IS_CHERRYVIEW(dev_priv) ||
16356                     IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
16357                         funcs = &g4x_crtc_funcs;
16358                 else if (IS_GEN(dev_priv, 4))
16359                         funcs = &i965_crtc_funcs;
16360                 else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv))
16361                         funcs = &i915gm_crtc_funcs;
16362                 else if (IS_GEN(dev_priv, 3))
16363                         funcs = &i915_crtc_funcs;
16364                 else
16365                         funcs = &i8xx_crtc_funcs;
16366         } else {
16367                 if (INTEL_GEN(dev_priv) >= 8)
16368                         funcs = &bdw_crtc_funcs;
16369                 else
16370                         funcs = &ilk_crtc_funcs;
16371         }
16372
16373         ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base,
16374                                         &primary->base, &cursor->base,
16375                                         funcs, "pipe %c", pipe_name(pipe));
16376         if (ret)
16377                 goto fail;
16378
16379         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->pipe_to_crtc_mapping) ||
16380                dev_priv->pipe_to_crtc_mapping[pipe] != NULL);
16381         dev_priv->pipe_to_crtc_mapping[pipe] = crtc;
16382
16383         if (INTEL_GEN(dev_priv) < 9) {
16384                 enum i9xx_plane_id i9xx_plane = primary->i9xx_plane;
16385
16386                 BUG_ON(i9xx_plane >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
16387                        dev_priv->plane_to_crtc_mapping[i9xx_plane] != NULL);
16388                 dev_priv->plane_to_crtc_mapping[i9xx_plane] = crtc;
16389         }
16390
16391         intel_color_init(crtc);
16392
16393         WARN_ON(drm_crtc_index(&crtc->base) != crtc->pipe);
16394
16395         return 0;
16396
16397 fail:
16398         intel_crtc_free(crtc);
16399
16400         return ret;
16401 }
16402
16403 int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
16404                                       struct drm_file *file)
16405 {
16406         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
16407         struct drm_crtc *drmmode_crtc;
16408         struct intel_crtc *crtc;
16409
16410         drmmode_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
16411         if (!drmmode_crtc)
16412                 return -ENOENT;
16413
16414         crtc = to_intel_crtc(drmmode_crtc);
16415         pipe_from_crtc_id->pipe = crtc->pipe;
16416
16417         return 0;
16418 }
16419
16420 static u32 intel_encoder_possible_clones(struct intel_encoder *encoder)
16421 {
16422         struct drm_device *dev = encoder->base.dev;
16423         struct intel_encoder *source_encoder;
16424         u32 possible_clones = 0;
16425
16426         for_each_intel_encoder(dev, source_encoder) {
16427                 if (encoders_cloneable(encoder, source_encoder))
16428                         possible_clones |= drm_encoder_mask(&source_encoder->base);
16429         }
16430
16431         return possible_clones;
16432 }
16433
16434 static u32 intel_encoder_possible_crtcs(struct intel_encoder *encoder)
16435 {
16436         struct drm_device *dev = encoder->base.dev;
16437         struct intel_crtc *crtc;
16438         u32 possible_crtcs = 0;
16439
16440         for_each_intel_crtc(dev, crtc) {
16441                 if (encoder->pipe_mask & BIT(crtc->pipe))
16442                         possible_crtcs |= drm_crtc_mask(&crtc->base);
16443         }
16444
16445         return possible_crtcs;
16446 }
16447
16448 static bool ilk_has_edp_a(struct drm_i915_private *dev_priv)
16449 {
16450         if (!IS_MOBILE(dev_priv))
16451                 return false;
16452
16453         if ((I915_READ(DP_A) & DP_DETECTED) == 0)
16454                 return false;
16455
16456         if (IS_GEN(dev_priv, 5) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
16457                 return false;
16458
16459         return true;
16460 }
16461
16462 static bool intel_ddi_crt_present(struct drm_i915_private *dev_priv)
16463 {
16464         if (INTEL_GEN(dev_priv) >= 9)
16465                 return false;
16466
16467         if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
16468                 return false;
16469
16470         if (HAS_PCH_LPT_H(dev_priv) &&
16471             I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
16472                 return false;
16473
16474         /* DDI E can't be used if DDI A requires 4 lanes */
16475         if (I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
16476                 return false;
16477
16478         if (!dev_priv->vbt.int_crt_support)
16479                 return false;
16480
16481         return true;
16482 }
16483
16484 void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv)
16485 {
16486         int pps_num;
16487         int pps_idx;
16488
16489         if (HAS_DDI(dev_priv))
16490                 return;
16491         /*
16492          * This w/a is needed at least on CPT/PPT, but to be sure apply it
16493          * everywhere where registers can be write protected.
16494          */
16495         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
16496                 pps_num = 2;
16497         else
16498                 pps_num = 1;
16499
16500         for (pps_idx = 0; pps_idx < pps_num; pps_idx++) {
16501                 u32 val = I915_READ(PP_CONTROL(pps_idx));
16502
16503                 val = (val & ~PANEL_UNLOCK_MASK) | PANEL_UNLOCK_REGS;
16504                 I915_WRITE(PP_CONTROL(pps_idx), val);
16505         }
16506 }
16507
16508 static void intel_pps_init(struct drm_i915_private *dev_priv)
16509 {
16510         if (HAS_PCH_SPLIT(dev_priv) || IS_GEN9_LP(dev_priv))
16511                 dev_priv->pps_mmio_base = PCH_PPS_BASE;
16512         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
16513                 dev_priv->pps_mmio_base = VLV_PPS_BASE;
16514         else
16515                 dev_priv->pps_mmio_base = PPS_BASE;
16516
16517         intel_pps_unlock_regs_wa(dev_priv);
16518 }
16519
16520 static void intel_setup_outputs(struct drm_i915_private *dev_priv)
16521 {
16522         struct intel_encoder *encoder;
16523         bool dpd_is_edp = false;
16524
16525         intel_pps_init(dev_priv);
16526
16527         if (!HAS_DISPLAY(dev_priv) || !INTEL_DISPLAY_ENABLED(dev_priv))
16528                 return;
16529
16530         if (INTEL_GEN(dev_priv) >= 12) {
16531                 intel_ddi_init(dev_priv, PORT_A);
16532                 intel_ddi_init(dev_priv, PORT_B);
16533                 intel_ddi_init(dev_priv, PORT_D);
16534                 intel_ddi_init(dev_priv, PORT_E);
16535                 intel_ddi_init(dev_priv, PORT_F);
16536                 intel_ddi_init(dev_priv, PORT_G);
16537                 intel_ddi_init(dev_priv, PORT_H);
16538                 intel_ddi_init(dev_priv, PORT_I);
16539                 icl_dsi_init(dev_priv);
16540         } else if (IS_ELKHARTLAKE(dev_priv)) {
16541                 intel_ddi_init(dev_priv, PORT_A);
16542                 intel_ddi_init(dev_priv, PORT_B);
16543                 intel_ddi_init(dev_priv, PORT_C);
16544                 intel_ddi_init(dev_priv, PORT_D);
16545                 icl_dsi_init(dev_priv);
16546         } else if (IS_GEN(dev_priv, 11)) {
16547                 intel_ddi_init(dev_priv, PORT_A);
16548                 intel_ddi_init(dev_priv, PORT_B);
16549                 intel_ddi_init(dev_priv, PORT_C);
16550                 intel_ddi_init(dev_priv, PORT_D);
16551                 intel_ddi_init(dev_priv, PORT_E);
16552                 /*
16553                  * On some ICL SKUs port F is not present. No strap bits for
16554                  * this, so rely on VBT.
16555                  * Work around broken VBTs on SKUs known to have no port F.
16556                  */
16557                 if (IS_ICL_WITH_PORT_F(dev_priv) &&
16558                     intel_bios_is_port_present(dev_priv, PORT_F))
16559                         intel_ddi_init(dev_priv, PORT_F);
16560
16561                 icl_dsi_init(dev_priv);
16562         } else if (IS_GEN9_LP(dev_priv)) {
16563                 /*
16564                  * FIXME: Broxton doesn't support port detection via the
16565                  * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to
16566                  * detect the ports.
16567                  */
16568                 intel_ddi_init(dev_priv, PORT_A);
16569                 intel_ddi_init(dev_priv, PORT_B);
16570                 intel_ddi_init(dev_priv, PORT_C);
16571
16572                 vlv_dsi_init(dev_priv);
16573         } else if (HAS_DDI(dev_priv)) {
16574                 int found;
16575
16576                 if (intel_ddi_crt_present(dev_priv))
16577                         intel_crt_init(dev_priv);
16578
16579                 /*
16580                  * Haswell uses DDI functions to detect digital outputs.
16581                  * On SKL pre-D0 the strap isn't connected, so we assume
16582                  * it's there.
16583                  */
16584                 found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
16585                 /* WaIgnoreDDIAStrap: skl */
16586                 if (found || IS_GEN9_BC(dev_priv))
16587                         intel_ddi_init(dev_priv, PORT_A);
16588
16589                 /* DDI B, C, D, and F detection is indicated by the SFUSE_STRAP
16590                  * register */
16591                 found = I915_READ(SFUSE_STRAP);
16592
16593                 if (found & SFUSE_STRAP_DDIB_DETECTED)
16594                         intel_ddi_init(dev_priv, PORT_B);
16595                 if (found & SFUSE_STRAP_DDIC_DETECTED)
16596                         intel_ddi_init(dev_priv, PORT_C);
16597                 if (found & SFUSE_STRAP_DDID_DETECTED)
16598                         intel_ddi_init(dev_priv, PORT_D);
16599                 if (found & SFUSE_STRAP_DDIF_DETECTED)
16600                         intel_ddi_init(dev_priv, PORT_F);
16601                 /*
16602                  * On SKL we don't have a way to detect DDI-E so we rely on VBT.
16603                  */
16604                 if (IS_GEN9_BC(dev_priv) &&
16605                     intel_bios_is_port_present(dev_priv, PORT_E))
16606                         intel_ddi_init(dev_priv, PORT_E);
16607
16608         } else if (HAS_PCH_SPLIT(dev_priv)) {
16609                 int found;
16610
16611                 /*
16612                  * intel_edp_init_connector() depends on this completing first,
16613                  * to prevent the registration of both eDP and LVDS and the
16614                  * incorrect sharing of the PPS.
16615                  */
16616                 intel_lvds_init(dev_priv);
16617                 intel_crt_init(dev_priv);
16618
16619                 dpd_is_edp = intel_dp_is_port_edp(dev_priv, PORT_D);
16620
16621                 if (ilk_has_edp_a(dev_priv))
16622                         intel_dp_init(dev_priv, DP_A, PORT_A);
16623
16624                 if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) {
16625                         /* PCH SDVOB multiplex with HDMIB */
16626                         found = intel_sdvo_init(dev_priv, PCH_SDVOB, PORT_B);
16627                         if (!found)
16628                                 intel_hdmi_init(dev_priv, PCH_HDMIB, PORT_B);
16629                         if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
16630                                 intel_dp_init(dev_priv, PCH_DP_B, PORT_B);
16631                 }
16632
16633                 if (I915_READ(PCH_HDMIC) & SDVO_DETECTED)
16634                         intel_hdmi_init(dev_priv, PCH_HDMIC, PORT_C);
16635
16636                 if (!dpd_is_edp && I915_READ(PCH_HDMID) & SDVO_DETECTED)
16637                         intel_hdmi_init(dev_priv, PCH_HDMID, PORT_D);
16638
16639                 if (I915_READ(PCH_DP_C) & DP_DETECTED)
16640                         intel_dp_init(dev_priv, PCH_DP_C, PORT_C);
16641
16642                 if (I915_READ(PCH_DP_D) & DP_DETECTED)
16643                         intel_dp_init(dev_priv, PCH_DP_D, PORT_D);
16644         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
16645                 bool has_edp, has_port;
16646
16647                 if (IS_VALLEYVIEW(dev_priv) && dev_priv->vbt.int_crt_support)
16648                         intel_crt_init(dev_priv);
16649
16650                 /*
16651                  * The DP_DETECTED bit is the latched state of the DDC
16652                  * SDA pin at boot. However since eDP doesn't require DDC
16653                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
16654                  * eDP ports may have been muxed to an alternate function.
16655                  * Thus we can't rely on the DP_DETECTED bit alone to detect
16656                  * eDP ports. Consult the VBT as well as DP_DETECTED to
16657                  * detect eDP ports.
16658                  *
16659                  * Sadly the straps seem to be missing sometimes even for HDMI
16660                  * ports (eg. on Voyo V3 - CHT x7-Z8700), so check both strap
16661                  * and VBT for the presence of the port. Additionally we can't
16662                  * trust the port type the VBT declares as we've seen at least
16663                  * HDMI ports that the VBT claim are DP or eDP.
16664                  */
16665                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_B);
16666                 has_port = intel_bios_is_port_present(dev_priv, PORT_B);
16667                 if (I915_READ(VLV_DP_B) & DP_DETECTED || has_port)
16668                         has_edp &= intel_dp_init(dev_priv, VLV_DP_B, PORT_B);
16669                 if ((I915_READ(VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp)
16670                         intel_hdmi_init(dev_priv, VLV_HDMIB, PORT_B);
16671
16672                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_C);
16673                 has_port = intel_bios_is_port_present(dev_priv, PORT_C);
16674                 if (I915_READ(VLV_DP_C) & DP_DETECTED || has_port)
16675                         has_edp &= intel_dp_init(dev_priv, VLV_DP_C, PORT_C);
16676                 if ((I915_READ(VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp)
16677                         intel_hdmi_init(dev_priv, VLV_HDMIC, PORT_C);
16678
16679                 if (IS_CHERRYVIEW(dev_priv)) {
16680                         /*
16681                          * eDP not supported on port D,
16682                          * so no need to worry about it
16683                          */
16684                         has_port = intel_bios_is_port_present(dev_priv, PORT_D);
16685                         if (I915_READ(CHV_DP_D) & DP_DETECTED || has_port)
16686                                 intel_dp_init(dev_priv, CHV_DP_D, PORT_D);
16687                         if (I915_READ(CHV_HDMID) & SDVO_DETECTED || has_port)
16688                                 intel_hdmi_init(dev_priv, CHV_HDMID, PORT_D);
16689                 }
16690
16691                 vlv_dsi_init(dev_priv);
16692         } else if (IS_PINEVIEW(dev_priv)) {
16693                 intel_lvds_init(dev_priv);
16694                 intel_crt_init(dev_priv);
16695         } else if (IS_GEN_RANGE(dev_priv, 3, 4)) {
16696                 bool found = false;
16697
16698                 if (IS_MOBILE(dev_priv))
16699                         intel_lvds_init(dev_priv);
16700
16701                 intel_crt_init(dev_priv);
16702
16703                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
16704                         DRM_DEBUG_KMS("probing SDVOB\n");
16705                         found = intel_sdvo_init(dev_priv, GEN3_SDVOB, PORT_B);
16706                         if (!found && IS_G4X(dev_priv)) {
16707                                 DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
16708                                 intel_hdmi_init(dev_priv, GEN4_HDMIB, PORT_B);
16709                         }
16710
16711                         if (!found && IS_G4X(dev_priv))
16712                                 intel_dp_init(dev_priv, DP_B, PORT_B);
16713                 }
16714
16715                 /* Before G4X SDVOC doesn't have its own detect register */
16716
16717                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
16718                         DRM_DEBUG_KMS("probing SDVOC\n");
16719                         found = intel_sdvo_init(dev_priv, GEN3_SDVOC, PORT_C);
16720                 }
16721
16722                 if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) {
16723
16724                         if (IS_G4X(dev_priv)) {
16725                                 DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
16726                                 intel_hdmi_init(dev_priv, GEN4_HDMIC, PORT_C);
16727                         }
16728                         if (IS_G4X(dev_priv))
16729                                 intel_dp_init(dev_priv, DP_C, PORT_C);
16730                 }
16731
16732                 if (IS_G4X(dev_priv) && (I915_READ(DP_D) & DP_DETECTED))
16733                         intel_dp_init(dev_priv, DP_D, PORT_D);
16734
16735                 if (SUPPORTS_TV(dev_priv))
16736                         intel_tv_init(dev_priv);
16737         } else if (IS_GEN(dev_priv, 2)) {
16738                 if (IS_I85X(dev_priv))
16739                         intel_lvds_init(dev_priv);
16740
16741                 intel_crt_init(dev_priv);
16742                 intel_dvo_init(dev_priv);
16743         }
16744
16745         intel_psr_init(dev_priv);
16746
16747         for_each_intel_encoder(&dev_priv->drm, encoder) {
16748                 encoder->base.possible_crtcs =
16749                         intel_encoder_possible_crtcs(encoder);
16750                 encoder->base.possible_clones =
16751                         intel_encoder_possible_clones(encoder);
16752         }
16753
16754         intel_init_pch_refclk(dev_priv);
16755
16756         drm_helper_move_panel_connectors_to_head(&dev_priv->drm);
16757 }
16758
16759 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
16760 {
16761         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
16762
16763         drm_framebuffer_cleanup(fb);
16764         intel_frontbuffer_put(intel_fb->frontbuffer);
16765
16766         kfree(intel_fb);
16767 }
16768
16769 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
16770                                                 struct drm_file *file,
16771                                                 unsigned int *handle)
16772 {
16773         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
16774
16775         if (obj->userptr.mm) {
16776                 DRM_DEBUG("attempting to use a userptr for a framebuffer, denied\n");
16777                 return -EINVAL;
16778         }
16779
16780         return drm_gem_handle_create(file, &obj->base, handle);
16781 }
16782
16783 static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
16784                                         struct drm_file *file,
16785                                         unsigned flags, unsigned color,
16786                                         struct drm_clip_rect *clips,
16787                                         unsigned num_clips)
16788 {
16789         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
16790
16791         i915_gem_object_flush_if_display(obj);
16792         intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB);
16793
16794         return 0;
16795 }
16796
16797 static const struct drm_framebuffer_funcs intel_fb_funcs = {
16798         .destroy = intel_user_framebuffer_destroy,
16799         .create_handle = intel_user_framebuffer_create_handle,
16800         .dirty = intel_user_framebuffer_dirty,
16801 };
16802
16803 static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
16804                                   struct drm_i915_gem_object *obj,
16805                                   struct drm_mode_fb_cmd2 *mode_cmd)
16806 {
16807         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
16808         struct drm_framebuffer *fb = &intel_fb->base;
16809         u32 max_stride;
16810         unsigned int tiling, stride;
16811         int ret = -EINVAL;
16812         int i;
16813
16814         intel_fb->frontbuffer = intel_frontbuffer_get(obj);
16815         if (!intel_fb->frontbuffer)
16816                 return -ENOMEM;
16817
16818         i915_gem_object_lock(obj);
16819         tiling = i915_gem_object_get_tiling(obj);
16820         stride = i915_gem_object_get_stride(obj);
16821         i915_gem_object_unlock(obj);
16822
16823         if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
16824                 /*
16825                  * If there's a fence, enforce that
16826                  * the fb modifier and tiling mode match.
16827                  */
16828                 if (tiling != I915_TILING_NONE &&
16829                     tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
16830                         DRM_DEBUG_KMS("tiling_mode doesn't match fb modifier\n");
16831                         goto err;
16832                 }
16833         } else {
16834                 if (tiling == I915_TILING_X) {
16835                         mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
16836                 } else if (tiling == I915_TILING_Y) {
16837                         DRM_DEBUG_KMS("No Y tiling for legacy addfb\n");
16838                         goto err;
16839                 }
16840         }
16841
16842         if (!drm_any_plane_has_format(&dev_priv->drm,
16843                                       mode_cmd->pixel_format,
16844                                       mode_cmd->modifier[0])) {
16845                 struct drm_format_name_buf format_name;
16846
16847                 DRM_DEBUG_KMS("unsupported pixel format %s / modifier 0x%llx\n",
16848                               drm_get_format_name(mode_cmd->pixel_format,
16849                                                   &format_name),
16850                               mode_cmd->modifier[0]);
16851                 goto err;
16852         }
16853
16854         /*
16855          * gen2/3 display engine uses the fence if present,
16856          * so the tiling mode must match the fb modifier exactly.
16857          */
16858         if (INTEL_GEN(dev_priv) < 4 &&
16859             tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
16860                 DRM_DEBUG_KMS("tiling_mode must match fb modifier exactly on gen2/3\n");
16861                 goto err;
16862         }
16863
16864         max_stride = intel_fb_max_stride(dev_priv, mode_cmd->pixel_format,
16865                                          mode_cmd->modifier[0]);
16866         if (mode_cmd->pitches[0] > max_stride) {
16867                 DRM_DEBUG_KMS("%s pitch (%u) must be at most %d\n",
16868                               mode_cmd->modifier[0] != DRM_FORMAT_MOD_LINEAR ?
16869                               "tiled" : "linear",
16870                               mode_cmd->pitches[0], max_stride);
16871                 goto err;
16872         }
16873
16874         /*
16875          * If there's a fence, enforce that
16876          * the fb pitch and fence stride match.
16877          */
16878         if (tiling != I915_TILING_NONE && mode_cmd->pitches[0] != stride) {
16879                 DRM_DEBUG_KMS("pitch (%d) must match tiling stride (%d)\n",
16880                               mode_cmd->pitches[0], stride);
16881                 goto err;
16882         }
16883
16884         /* FIXME need to adjust LINOFF/TILEOFF accordingly. */
16885         if (mode_cmd->offsets[0] != 0)
16886                 goto err;
16887
16888         drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd);
16889
16890         for (i = 0; i < fb->format->num_planes; i++) {
16891                 u32 stride_alignment;
16892
16893                 if (mode_cmd->handles[i] != mode_cmd->handles[0]) {
16894                         DRM_DEBUG_KMS("bad plane %d handle\n", i);
16895                         goto err;
16896                 }
16897
16898                 stride_alignment = intel_fb_stride_alignment(fb, i);
16899                 if (fb->pitches[i] & (stride_alignment - 1)) {
16900                         DRM_DEBUG_KMS("plane %d pitch (%d) must be at least %u byte aligned\n",
16901                                       i, fb->pitches[i], stride_alignment);
16902                         goto err;
16903                 }
16904
16905                 if (is_gen12_ccs_plane(fb, i)) {
16906                         int ccs_aux_stride = gen12_ccs_aux_stride(fb, i);
16907
16908                         if (fb->pitches[i] != ccs_aux_stride) {
16909                                 DRM_DEBUG_KMS("ccs aux plane %d pitch (%d) must be %d\n",
16910                                               i,
16911                                               fb->pitches[i], ccs_aux_stride);
16912                                 goto err;
16913                         }
16914                 }
16915
16916                 fb->obj[i] = &obj->base;
16917         }
16918
16919         ret = intel_fill_fb_info(dev_priv, fb);
16920         if (ret)
16921                 goto err;
16922
16923         ret = drm_framebuffer_init(&dev_priv->drm, fb, &intel_fb_funcs);
16924         if (ret) {
16925                 DRM_ERROR("framebuffer init failed %d\n", ret);
16926                 goto err;
16927         }
16928
16929         return 0;
16930
16931 err:
16932         intel_frontbuffer_put(intel_fb->frontbuffer);
16933         return ret;
16934 }
16935
16936 static struct drm_framebuffer *
16937 intel_user_framebuffer_create(struct drm_device *dev,
16938                               struct drm_file *filp,
16939                               const struct drm_mode_fb_cmd2 *user_mode_cmd)
16940 {
16941         struct drm_framebuffer *fb;
16942         struct drm_i915_gem_object *obj;
16943         struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
16944
16945         obj = i915_gem_object_lookup(filp, mode_cmd.handles[0]);
16946         if (!obj)
16947                 return ERR_PTR(-ENOENT);
16948
16949         fb = intel_framebuffer_create(obj, &mode_cmd);
16950         i915_gem_object_put(obj);
16951
16952         return fb;
16953 }
16954
16955 static void intel_atomic_state_free(struct drm_atomic_state *state)
16956 {
16957         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
16958
16959         drm_atomic_state_default_release(state);
16960
16961         i915_sw_fence_fini(&intel_state->commit_ready);
16962
16963         kfree(state);
16964 }
16965
16966 static enum drm_mode_status
16967 intel_mode_valid(struct drm_device *dev,
16968                  const struct drm_display_mode *mode)
16969 {
16970         struct drm_i915_private *dev_priv = to_i915(dev);
16971         int hdisplay_max, htotal_max;
16972         int vdisplay_max, vtotal_max;
16973
16974         /*
16975          * Can't reject DBLSCAN here because Xorg ddxen can add piles
16976          * of DBLSCAN modes to the output's mode list when they detect
16977          * the scaling mode property on the connector. And they don't
16978          * ask the kernel to validate those modes in any way until
16979          * modeset time at which point the client gets a protocol error.
16980          * So in order to not upset those clients we silently ignore the
16981          * DBLSCAN flag on such connectors. For other connectors we will
16982          * reject modes with the DBLSCAN flag in encoder->compute_config().
16983          * And we always reject DBLSCAN modes in connector->mode_valid()
16984          * as we never want such modes on the connector's mode list.
16985          */
16986
16987         if (mode->vscan > 1)
16988                 return MODE_NO_VSCAN;
16989
16990         if (mode->flags & DRM_MODE_FLAG_HSKEW)
16991                 return MODE_H_ILLEGAL;
16992
16993         if (mode->flags & (DRM_MODE_FLAG_CSYNC |
16994                            DRM_MODE_FLAG_NCSYNC |
16995                            DRM_MODE_FLAG_PCSYNC))
16996                 return MODE_HSYNC;
16997
16998         if (mode->flags & (DRM_MODE_FLAG_BCAST |
16999                            DRM_MODE_FLAG_PIXMUX |
17000                            DRM_MODE_FLAG_CLKDIV2))
17001                 return MODE_BAD;
17002
17003         /* Transcoder timing limits */
17004         if (INTEL_GEN(dev_priv) >= 11) {
17005                 hdisplay_max = 16384;
17006                 vdisplay_max = 8192;
17007                 htotal_max = 16384;
17008                 vtotal_max = 8192;
17009         } else if (INTEL_GEN(dev_priv) >= 9 ||
17010                    IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
17011                 hdisplay_max = 8192; /* FDI max 4096 handled elsewhere */
17012                 vdisplay_max = 4096;
17013                 htotal_max = 8192;
17014                 vtotal_max = 8192;
17015         } else if (INTEL_GEN(dev_priv) >= 3) {
17016                 hdisplay_max = 4096;
17017                 vdisplay_max = 4096;
17018                 htotal_max = 8192;
17019                 vtotal_max = 8192;
17020         } else {
17021                 hdisplay_max = 2048;
17022                 vdisplay_max = 2048;
17023                 htotal_max = 4096;
17024                 vtotal_max = 4096;
17025         }
17026
17027         if (mode->hdisplay > hdisplay_max ||
17028             mode->hsync_start > htotal_max ||
17029             mode->hsync_end > htotal_max ||
17030             mode->htotal > htotal_max)
17031                 return MODE_H_ILLEGAL;
17032
17033         if (mode->vdisplay > vdisplay_max ||
17034             mode->vsync_start > vtotal_max ||
17035             mode->vsync_end > vtotal_max ||
17036             mode->vtotal > vtotal_max)
17037                 return MODE_V_ILLEGAL;
17038
17039         if (INTEL_GEN(dev_priv) >= 5) {
17040                 if (mode->hdisplay < 64 ||
17041                     mode->htotal - mode->hdisplay < 32)
17042                         return MODE_H_ILLEGAL;
17043
17044                 if (mode->vtotal - mode->vdisplay < 5)
17045                         return MODE_V_ILLEGAL;
17046         } else {
17047                 if (mode->htotal - mode->hdisplay < 32)
17048                         return MODE_H_ILLEGAL;
17049
17050                 if (mode->vtotal - mode->vdisplay < 3)
17051                         return MODE_V_ILLEGAL;
17052         }
17053
17054         return MODE_OK;
17055 }
17056
17057 enum drm_mode_status
17058 intel_mode_valid_max_plane_size(struct drm_i915_private *dev_priv,
17059                                 const struct drm_display_mode *mode)
17060 {
17061         int plane_width_max, plane_height_max;
17062
17063         /*
17064          * intel_mode_valid() should be
17065          * sufficient on older platforms.
17066          */
17067         if (INTEL_GEN(dev_priv) < 9)
17068                 return MODE_OK;
17069
17070         /*
17071          * Most people will probably want a fullscreen
17072          * plane so let's not advertize modes that are
17073          * too big for that.
17074          */
17075         if (INTEL_GEN(dev_priv) >= 11) {
17076                 plane_width_max = 5120;
17077                 plane_height_max = 4320;
17078         } else {
17079                 plane_width_max = 5120;
17080                 plane_height_max = 4096;
17081         }
17082
17083         if (mode->hdisplay > plane_width_max)
17084                 return MODE_H_ILLEGAL;
17085
17086         if (mode->vdisplay > plane_height_max)
17087                 return MODE_V_ILLEGAL;
17088
17089         return MODE_OK;
17090 }
17091
17092 static const struct drm_mode_config_funcs intel_mode_funcs = {
17093         .fb_create = intel_user_framebuffer_create,
17094         .get_format_info = intel_get_format_info,
17095         .output_poll_changed = intel_fbdev_output_poll_changed,
17096         .mode_valid = intel_mode_valid,
17097         .atomic_check = intel_atomic_check,
17098         .atomic_commit = intel_atomic_commit,
17099         .atomic_state_alloc = intel_atomic_state_alloc,
17100         .atomic_state_clear = intel_atomic_state_clear,
17101         .atomic_state_free = intel_atomic_state_free,
17102 };
17103
17104 /**
17105  * intel_init_display_hooks - initialize the display modesetting hooks
17106  * @dev_priv: device private
17107  */
17108 void intel_init_display_hooks(struct drm_i915_private *dev_priv)
17109 {
17110         intel_init_cdclk_hooks(dev_priv);
17111
17112         if (INTEL_GEN(dev_priv) >= 9) {
17113                 dev_priv->display.get_pipe_config = hsw_get_pipe_config;
17114                 dev_priv->display.get_initial_plane_config =
17115                         skl_get_initial_plane_config;
17116                 dev_priv->display.crtc_compute_clock = hsw_crtc_compute_clock;
17117                 dev_priv->display.crtc_enable = hsw_crtc_enable;
17118                 dev_priv->display.crtc_disable = hsw_crtc_disable;
17119         } else if (HAS_DDI(dev_priv)) {
17120                 dev_priv->display.get_pipe_config = hsw_get_pipe_config;
17121                 dev_priv->display.get_initial_plane_config =
17122                         i9xx_get_initial_plane_config;
17123                 dev_priv->display.crtc_compute_clock =
17124                         hsw_crtc_compute_clock;
17125                 dev_priv->display.crtc_enable = hsw_crtc_enable;
17126                 dev_priv->display.crtc_disable = hsw_crtc_disable;
17127         } else if (HAS_PCH_SPLIT(dev_priv)) {
17128                 dev_priv->display.get_pipe_config = ilk_get_pipe_config;
17129                 dev_priv->display.get_initial_plane_config =
17130                         i9xx_get_initial_plane_config;
17131                 dev_priv->display.crtc_compute_clock =
17132                         ilk_crtc_compute_clock;
17133                 dev_priv->display.crtc_enable = ilk_crtc_enable;
17134                 dev_priv->display.crtc_disable = ilk_crtc_disable;
17135         } else if (IS_CHERRYVIEW(dev_priv)) {
17136                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
17137                 dev_priv->display.get_initial_plane_config =
17138                         i9xx_get_initial_plane_config;
17139                 dev_priv->display.crtc_compute_clock = chv_crtc_compute_clock;
17140                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
17141                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
17142         } else if (IS_VALLEYVIEW(dev_priv)) {
17143                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
17144                 dev_priv->display.get_initial_plane_config =
17145                         i9xx_get_initial_plane_config;
17146                 dev_priv->display.crtc_compute_clock = vlv_crtc_compute_clock;
17147                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
17148                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
17149         } else if (IS_G4X(dev_priv)) {
17150                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
17151                 dev_priv->display.get_initial_plane_config =
17152                         i9xx_get_initial_plane_config;
17153                 dev_priv->display.crtc_compute_clock = g4x_crtc_compute_clock;
17154                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
17155                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
17156         } else if (IS_PINEVIEW(dev_priv)) {
17157                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
17158                 dev_priv->display.get_initial_plane_config =
17159                         i9xx_get_initial_plane_config;
17160                 dev_priv->display.crtc_compute_clock = pnv_crtc_compute_clock;
17161                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
17162                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
17163         } else if (!IS_GEN(dev_priv, 2)) {
17164                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
17165                 dev_priv->display.get_initial_plane_config =
17166                         i9xx_get_initial_plane_config;
17167                 dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
17168                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
17169                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
17170         } else {
17171                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
17172                 dev_priv->display.get_initial_plane_config =
17173                         i9xx_get_initial_plane_config;
17174                 dev_priv->display.crtc_compute_clock = i8xx_crtc_compute_clock;
17175                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
17176                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
17177         }
17178
17179         if (IS_GEN(dev_priv, 5)) {
17180                 dev_priv->display.fdi_link_train = ilk_fdi_link_train;
17181         } else if (IS_GEN(dev_priv, 6)) {
17182                 dev_priv->display.fdi_link_train = gen6_fdi_link_train;
17183         } else if (IS_IVYBRIDGE(dev_priv)) {
17184                 /* FIXME: detect B0+ stepping and use auto training */
17185                 dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
17186         }
17187
17188         if (INTEL_GEN(dev_priv) >= 9)
17189                 dev_priv->display.commit_modeset_enables = skl_commit_modeset_enables;
17190         else
17191                 dev_priv->display.commit_modeset_enables = intel_commit_modeset_enables;
17192
17193 }
17194
17195 void intel_modeset_init_hw(struct drm_i915_private *i915)
17196 {
17197         intel_update_cdclk(i915);
17198         intel_dump_cdclk_state(&i915->cdclk.hw, "Current CDCLK");
17199         i915->cdclk.logical = i915->cdclk.actual = i915->cdclk.hw;
17200 }
17201
17202 /*
17203  * Calculate what we think the watermarks should be for the state we've read
17204  * out of the hardware and then immediately program those watermarks so that
17205  * we ensure the hardware settings match our internal state.
17206  *
17207  * We can calculate what we think WM's should be by creating a duplicate of the
17208  * current state (which was constructed during hardware readout) and running it
17209  * through the atomic check code to calculate new watermark values in the
17210  * state object.
17211  */
17212 static void sanitize_watermarks(struct drm_device *dev)
17213 {
17214         struct drm_i915_private *dev_priv = to_i915(dev);
17215         struct drm_atomic_state *state;
17216         struct intel_atomic_state *intel_state;
17217         struct intel_crtc *crtc;
17218         struct intel_crtc_state *crtc_state;
17219         struct drm_modeset_acquire_ctx ctx;
17220         int ret;
17221         int i;
17222
17223         /* Only supported on platforms that use atomic watermark design */
17224         if (!dev_priv->display.optimize_watermarks)
17225                 return;
17226
17227         /*
17228          * We need to hold connection_mutex before calling duplicate_state so
17229          * that the connector loop is protected.
17230          */
17231         drm_modeset_acquire_init(&ctx, 0);
17232 retry:
17233         ret = drm_modeset_lock_all_ctx(dev, &ctx);
17234         if (ret == -EDEADLK) {
17235                 drm_modeset_backoff(&ctx);
17236                 goto retry;
17237         } else if (WARN_ON(ret)) {
17238                 goto fail;
17239         }
17240
17241         state = drm_atomic_helper_duplicate_state(dev, &ctx);
17242         if (WARN_ON(IS_ERR(state)))
17243                 goto fail;
17244
17245         intel_state = to_intel_atomic_state(state);
17246
17247         /*
17248          * Hardware readout is the only time we don't want to calculate
17249          * intermediate watermarks (since we don't trust the current
17250          * watermarks).
17251          */
17252         if (!HAS_GMCH(dev_priv))
17253                 intel_state->skip_intermediate_wm = true;
17254
17255         ret = intel_atomic_check(dev, state);
17256         if (ret) {
17257                 /*
17258                  * If we fail here, it means that the hardware appears to be
17259                  * programmed in a way that shouldn't be possible, given our
17260                  * understanding of watermark requirements.  This might mean a
17261                  * mistake in the hardware readout code or a mistake in the
17262                  * watermark calculations for a given platform.  Raise a WARN
17263                  * so that this is noticeable.
17264                  *
17265                  * If this actually happens, we'll have to just leave the
17266                  * BIOS-programmed watermarks untouched and hope for the best.
17267                  */
17268                 WARN(true, "Could not determine valid watermarks for inherited state\n");
17269                 goto put_state;
17270         }
17271
17272         /* Write calculated watermark values back */
17273         for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
17274                 crtc_state->wm.need_postvbl_update = true;
17275                 dev_priv->display.optimize_watermarks(intel_state, crtc);
17276
17277                 to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
17278         }
17279
17280 put_state:
17281         drm_atomic_state_put(state);
17282 fail:
17283         drm_modeset_drop_locks(&ctx);
17284         drm_modeset_acquire_fini(&ctx);
17285 }
17286
17287 static void intel_update_fdi_pll_freq(struct drm_i915_private *dev_priv)
17288 {
17289         if (IS_GEN(dev_priv, 5)) {
17290                 u32 fdi_pll_clk =
17291                         I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK;
17292
17293                 dev_priv->fdi_pll_freq = (fdi_pll_clk + 2) * 10000;
17294         } else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) {
17295                 dev_priv->fdi_pll_freq = 270000;
17296         } else {
17297                 return;
17298         }
17299
17300         DRM_DEBUG_DRIVER("FDI PLL freq=%d\n", dev_priv->fdi_pll_freq);
17301 }
17302
17303 static int intel_initial_commit(struct drm_device *dev)
17304 {
17305         struct drm_atomic_state *state = NULL;
17306         struct drm_modeset_acquire_ctx ctx;
17307         struct intel_crtc *crtc;
17308         int ret = 0;
17309
17310         state = drm_atomic_state_alloc(dev);
17311         if (!state)
17312                 return -ENOMEM;
17313
17314         drm_modeset_acquire_init(&ctx, 0);
17315
17316 retry:
17317         state->acquire_ctx = &ctx;
17318
17319         for_each_intel_crtc(dev, crtc) {
17320                 struct intel_crtc_state *crtc_state =
17321                         intel_atomic_get_crtc_state(state, crtc);
17322
17323                 if (IS_ERR(crtc_state)) {
17324                         ret = PTR_ERR(crtc_state);
17325                         goto out;
17326                 }
17327
17328                 if (crtc_state->hw.active) {
17329                         ret = drm_atomic_add_affected_planes(state, &crtc->base);
17330                         if (ret)
17331                                 goto out;
17332
17333                         /*
17334                          * FIXME hack to force a LUT update to avoid the
17335                          * plane update forcing the pipe gamma on without
17336                          * having a proper LUT loaded. Remove once we
17337                          * have readout for pipe gamma enable.
17338                          */
17339                         crtc_state->uapi.color_mgmt_changed = true;
17340                 }
17341         }
17342
17343         ret = drm_atomic_commit(state);
17344
17345 out:
17346         if (ret == -EDEADLK) {
17347                 drm_atomic_state_clear(state);
17348                 drm_modeset_backoff(&ctx);
17349                 goto retry;
17350         }
17351
17352         drm_atomic_state_put(state);
17353
17354         drm_modeset_drop_locks(&ctx);
17355         drm_modeset_acquire_fini(&ctx);
17356
17357         return ret;
17358 }
17359
17360 static void intel_mode_config_init(struct drm_i915_private *i915)
17361 {
17362         struct drm_mode_config *mode_config = &i915->drm.mode_config;
17363
17364         drm_mode_config_init(&i915->drm);
17365
17366         mode_config->min_width = 0;
17367         mode_config->min_height = 0;
17368
17369         mode_config->preferred_depth = 24;
17370         mode_config->prefer_shadow = 1;
17371
17372         mode_config->allow_fb_modifiers = true;
17373
17374         mode_config->funcs = &intel_mode_funcs;
17375
17376         /*
17377          * Maximum framebuffer dimensions, chosen to match
17378          * the maximum render engine surface size on gen4+.
17379          */
17380         if (INTEL_GEN(i915) >= 7) {
17381                 mode_config->max_width = 16384;
17382                 mode_config->max_height = 16384;
17383         } else if (INTEL_GEN(i915) >= 4) {
17384                 mode_config->max_width = 8192;
17385                 mode_config->max_height = 8192;
17386         } else if (IS_GEN(i915, 3)) {
17387                 mode_config->max_width = 4096;
17388                 mode_config->max_height = 4096;
17389         } else {
17390                 mode_config->max_width = 2048;
17391                 mode_config->max_height = 2048;
17392         }
17393
17394         if (IS_I845G(i915) || IS_I865G(i915)) {
17395                 mode_config->cursor_width = IS_I845G(i915) ? 64 : 512;
17396                 mode_config->cursor_height = 1023;
17397         } else if (IS_GEN(i915, 2)) {
17398                 mode_config->cursor_width = 64;
17399                 mode_config->cursor_height = 64;
17400         } else {
17401                 mode_config->cursor_width = 256;
17402                 mode_config->cursor_height = 256;
17403         }
17404 }
17405
17406 int intel_modeset_init(struct drm_i915_private *i915)
17407 {
17408         struct drm_device *dev = &i915->drm;
17409         enum pipe pipe;
17410         struct intel_crtc *crtc;
17411         int ret;
17412
17413         i915->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0);
17414         i915->flip_wq = alloc_workqueue("i915_flip", WQ_HIGHPRI |
17415                                         WQ_UNBOUND, WQ_UNBOUND_MAX_ACTIVE);
17416
17417         intel_mode_config_init(i915);
17418
17419         ret = intel_bw_init(i915);
17420         if (ret)
17421                 return ret;
17422
17423         init_llist_head(&i915->atomic_helper.free_list);
17424         INIT_WORK(&i915->atomic_helper.free_work,
17425                   intel_atomic_helper_free_state_worker);
17426
17427         intel_init_quirks(i915);
17428
17429         intel_fbc_init(i915);
17430
17431         intel_init_pm(i915);
17432
17433         intel_panel_sanitize_ssc(i915);
17434
17435         intel_gmbus_setup(i915);
17436
17437         DRM_DEBUG_KMS("%d display pipe%s available.\n",
17438                       INTEL_NUM_PIPES(i915),
17439                       INTEL_NUM_PIPES(i915) > 1 ? "s" : "");
17440
17441         if (HAS_DISPLAY(i915) && INTEL_DISPLAY_ENABLED(i915)) {
17442                 for_each_pipe(i915, pipe) {
17443                         ret = intel_crtc_init(i915, pipe);
17444                         if (ret) {
17445                                 drm_mode_config_cleanup(dev);
17446                                 return ret;
17447                         }
17448                 }
17449         }
17450
17451         intel_shared_dpll_init(dev);
17452         intel_update_fdi_pll_freq(i915);
17453
17454         intel_update_czclk(i915);
17455         intel_modeset_init_hw(i915);
17456
17457         intel_hdcp_component_init(i915);
17458
17459         if (i915->max_cdclk_freq == 0)
17460                 intel_update_max_cdclk(i915);
17461
17462         /* Just disable it once at startup */
17463         intel_vga_disable(i915);
17464         intel_setup_outputs(i915);
17465
17466         drm_modeset_lock_all(dev);
17467         intel_modeset_setup_hw_state(dev, dev->mode_config.acquire_ctx);
17468         drm_modeset_unlock_all(dev);
17469
17470         for_each_intel_crtc(dev, crtc) {
17471                 struct intel_initial_plane_config plane_config = {};
17472
17473                 if (!crtc->active)
17474                         continue;
17475
17476                 /*
17477                  * Note that reserving the BIOS fb up front prevents us
17478                  * from stuffing other stolen allocations like the ring
17479                  * on top.  This prevents some ugliness at boot time, and
17480                  * can even allow for smooth boot transitions if the BIOS
17481                  * fb is large enough for the active pipe configuration.
17482                  */
17483                 i915->display.get_initial_plane_config(crtc, &plane_config);
17484
17485                 /*
17486                  * If the fb is shared between multiple heads, we'll
17487                  * just get the first one.
17488                  */
17489                 intel_find_initial_plane_obj(crtc, &plane_config);
17490         }
17491
17492         /*
17493          * Make sure hardware watermarks really match the state we read out.
17494          * Note that we need to do this after reconstructing the BIOS fb's
17495          * since the watermark calculation done here will use pstate->fb.
17496          */
17497         if (!HAS_GMCH(i915))
17498                 sanitize_watermarks(dev);
17499
17500         /*
17501          * Force all active planes to recompute their states. So that on
17502          * mode_setcrtc after probe, all the intel_plane_state variables
17503          * are already calculated and there is no assert_plane warnings
17504          * during bootup.
17505          */
17506         ret = intel_initial_commit(dev);
17507         if (ret)
17508                 DRM_DEBUG_KMS("Initial commit in probe failed.\n");
17509
17510         return 0;
17511 }
17512
17513 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
17514 {
17515         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
17516         /* 640x480@60Hz, ~25175 kHz */
17517         struct dpll clock = {
17518                 .m1 = 18,
17519                 .m2 = 7,
17520                 .p1 = 13,
17521                 .p2 = 4,
17522                 .n = 2,
17523         };
17524         u32 dpll, fp;
17525         int i;
17526
17527         WARN_ON(i9xx_calc_dpll_params(48000, &clock) != 25154);
17528
17529         DRM_DEBUG_KMS("enabling pipe %c due to force quirk (vco=%d dot=%d)\n",
17530                       pipe_name(pipe), clock.vco, clock.dot);
17531
17532         fp = i9xx_dpll_compute_fp(&clock);
17533         dpll = DPLL_DVO_2X_MODE |
17534                 DPLL_VGA_MODE_DIS |
17535                 ((clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT) |
17536                 PLL_P2_DIVIDE_BY_4 |
17537                 PLL_REF_INPUT_DREFCLK |
17538                 DPLL_VCO_ENABLE;
17539
17540         I915_WRITE(FP0(pipe), fp);
17541         I915_WRITE(FP1(pipe), fp);
17542
17543         I915_WRITE(HTOTAL(pipe), (640 - 1) | ((800 - 1) << 16));
17544         I915_WRITE(HBLANK(pipe), (640 - 1) | ((800 - 1) << 16));
17545         I915_WRITE(HSYNC(pipe), (656 - 1) | ((752 - 1) << 16));
17546         I915_WRITE(VTOTAL(pipe), (480 - 1) | ((525 - 1) << 16));
17547         I915_WRITE(VBLANK(pipe), (480 - 1) | ((525 - 1) << 16));
17548         I915_WRITE(VSYNC(pipe), (490 - 1) | ((492 - 1) << 16));
17549         I915_WRITE(PIPESRC(pipe), ((640 - 1) << 16) | (480 - 1));
17550
17551         /*
17552          * Apparently we need to have VGA mode enabled prior to changing
17553          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
17554          * dividers, even though the register value does change.
17555          */
17556         I915_WRITE(DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS);
17557         I915_WRITE(DPLL(pipe), dpll);
17558
17559         /* Wait for the clocks to stabilize. */
17560         POSTING_READ(DPLL(pipe));
17561         udelay(150);
17562
17563         /* The pixel multiplier can only be updated once the
17564          * DPLL is enabled and the clocks are stable.
17565          *
17566          * So write it again.
17567          */
17568         I915_WRITE(DPLL(pipe), dpll);
17569
17570         /* We do this three times for luck */
17571         for (i = 0; i < 3 ; i++) {
17572                 I915_WRITE(DPLL(pipe), dpll);
17573                 POSTING_READ(DPLL(pipe));
17574                 udelay(150); /* wait for warmup */
17575         }
17576
17577         I915_WRITE(PIPECONF(pipe), PIPECONF_ENABLE | PIPECONF_PROGRESSIVE);
17578         POSTING_READ(PIPECONF(pipe));
17579
17580         intel_wait_for_pipe_scanline_moving(crtc);
17581 }
17582
17583 void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
17584 {
17585         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
17586
17587         DRM_DEBUG_KMS("disabling pipe %c due to force quirk\n",
17588                       pipe_name(pipe));
17589
17590         WARN_ON(I915_READ(DSPCNTR(PLANE_A)) & DISPLAY_PLANE_ENABLE);
17591         WARN_ON(I915_READ(DSPCNTR(PLANE_B)) & DISPLAY_PLANE_ENABLE);
17592         WARN_ON(I915_READ(DSPCNTR(PLANE_C)) & DISPLAY_PLANE_ENABLE);
17593         WARN_ON(I915_READ(CURCNTR(PIPE_A)) & MCURSOR_MODE);
17594         WARN_ON(I915_READ(CURCNTR(PIPE_B)) & MCURSOR_MODE);
17595
17596         I915_WRITE(PIPECONF(pipe), 0);
17597         POSTING_READ(PIPECONF(pipe));
17598
17599         intel_wait_for_pipe_scanline_stopped(crtc);
17600
17601         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
17602         POSTING_READ(DPLL(pipe));
17603 }
17604
17605 static void
17606 intel_sanitize_plane_mapping(struct drm_i915_private *dev_priv)
17607 {
17608         struct intel_crtc *crtc;
17609
17610         if (INTEL_GEN(dev_priv) >= 4)
17611                 return;
17612
17613         for_each_intel_crtc(&dev_priv->drm, crtc) {
17614                 struct intel_plane *plane =
17615                         to_intel_plane(crtc->base.primary);
17616                 struct intel_crtc *plane_crtc;
17617                 enum pipe pipe;
17618
17619                 if (!plane->get_hw_state(plane, &pipe))
17620                         continue;
17621
17622                 if (pipe == crtc->pipe)
17623                         continue;
17624
17625                 DRM_DEBUG_KMS("[PLANE:%d:%s] attached to the wrong pipe, disabling plane\n",
17626                               plane->base.base.id, plane->base.name);
17627
17628                 plane_crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
17629                 intel_plane_disable_noatomic(plane_crtc, plane);
17630         }
17631 }
17632
17633 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
17634 {
17635         struct drm_device *dev = crtc->base.dev;
17636         struct intel_encoder *encoder;
17637
17638         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
17639                 return true;
17640
17641         return false;
17642 }
17643
17644 static struct intel_connector *intel_encoder_find_connector(struct intel_encoder *encoder)
17645 {
17646         struct drm_device *dev = encoder->base.dev;
17647         struct intel_connector *connector;
17648
17649         for_each_connector_on_encoder(dev, &encoder->base, connector)
17650                 return connector;
17651
17652         return NULL;
17653 }
17654
17655 static bool has_pch_trancoder(struct drm_i915_private *dev_priv,
17656                               enum pipe pch_transcoder)
17657 {
17658         return HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
17659                 (HAS_PCH_LPT_H(dev_priv) && pch_transcoder == PIPE_A);
17660 }
17661
17662 static void intel_sanitize_frame_start_delay(const struct intel_crtc_state *crtc_state)
17663 {
17664         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
17665         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
17666         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
17667
17668         if (INTEL_GEN(dev_priv) >= 9 ||
17669             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
17670                 i915_reg_t reg = CHICKEN_TRANS(cpu_transcoder);
17671                 u32 val;
17672
17673                 if (transcoder_is_dsi(cpu_transcoder))
17674                         return;
17675
17676                 val = I915_READ(reg);
17677                 val &= ~HSW_FRAME_START_DELAY_MASK;
17678                 val |= HSW_FRAME_START_DELAY(0);
17679                 I915_WRITE(reg, val);
17680         } else {
17681                 i915_reg_t reg = PIPECONF(cpu_transcoder);
17682                 u32 val;
17683
17684                 val = I915_READ(reg);
17685                 val &= ~PIPECONF_FRAME_START_DELAY_MASK;
17686                 val |= PIPECONF_FRAME_START_DELAY(0);
17687                 I915_WRITE(reg, val);
17688         }
17689
17690         if (!crtc_state->has_pch_encoder)
17691                 return;
17692
17693         if (HAS_PCH_IBX(dev_priv)) {
17694                 i915_reg_t reg = PCH_TRANSCONF(crtc->pipe);
17695                 u32 val;
17696
17697                 val = I915_READ(reg);
17698                 val &= ~TRANS_FRAME_START_DELAY_MASK;
17699                 val |= TRANS_FRAME_START_DELAY(0);
17700                 I915_WRITE(reg, val);
17701         } else {
17702                 enum pipe pch_transcoder = intel_crtc_pch_transcoder(crtc);
17703                 i915_reg_t reg = TRANS_CHICKEN2(pch_transcoder);
17704                 u32 val;
17705
17706                 val = I915_READ(reg);
17707                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
17708                 val |= TRANS_CHICKEN2_FRAME_START_DELAY(0);
17709                 I915_WRITE(reg, val);
17710         }
17711 }
17712
17713 static void intel_sanitize_crtc(struct intel_crtc *crtc,
17714                                 struct drm_modeset_acquire_ctx *ctx)
17715 {
17716         struct drm_device *dev = crtc->base.dev;
17717         struct drm_i915_private *dev_priv = to_i915(dev);
17718         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
17719
17720         if (crtc_state->hw.active) {
17721                 struct intel_plane *plane;
17722
17723                 /* Clear any frame start delays used for debugging left by the BIOS */
17724                 intel_sanitize_frame_start_delay(crtc_state);
17725
17726                 /* Disable everything but the primary plane */
17727                 for_each_intel_plane_on_crtc(dev, crtc, plane) {
17728                         const struct intel_plane_state *plane_state =
17729                                 to_intel_plane_state(plane->base.state);
17730
17731                         if (plane_state->uapi.visible &&
17732                             plane->base.type != DRM_PLANE_TYPE_PRIMARY)
17733                                 intel_plane_disable_noatomic(crtc, plane);
17734                 }
17735
17736                 /*
17737                  * Disable any background color set by the BIOS, but enable the
17738                  * gamma and CSC to match how we program our planes.
17739                  */
17740                 if (INTEL_GEN(dev_priv) >= 9)
17741                         I915_WRITE(SKL_BOTTOM_COLOR(crtc->pipe),
17742                                    SKL_BOTTOM_COLOR_GAMMA_ENABLE |
17743                                    SKL_BOTTOM_COLOR_CSC_ENABLE);
17744         }
17745
17746         /* Adjust the state of the output pipe according to whether we
17747          * have active connectors/encoders. */
17748         if (crtc_state->hw.active && !intel_crtc_has_encoders(crtc))
17749                 intel_crtc_disable_noatomic(crtc, ctx);
17750
17751         if (crtc_state->hw.active || HAS_GMCH(dev_priv)) {
17752                 /*
17753                  * We start out with underrun reporting disabled to avoid races.
17754                  * For correct bookkeeping mark this on active crtcs.
17755                  *
17756                  * Also on gmch platforms we dont have any hardware bits to
17757                  * disable the underrun reporting. Which means we need to start
17758                  * out with underrun reporting disabled also on inactive pipes,
17759                  * since otherwise we'll complain about the garbage we read when
17760                  * e.g. coming up after runtime pm.
17761                  *
17762                  * No protection against concurrent access is required - at
17763                  * worst a fifo underrun happens which also sets this to false.
17764                  */
17765                 crtc->cpu_fifo_underrun_disabled = true;
17766                 /*
17767                  * We track the PCH trancoder underrun reporting state
17768                  * within the crtc. With crtc for pipe A housing the underrun
17769                  * reporting state for PCH transcoder A, crtc for pipe B housing
17770                  * it for PCH transcoder B, etc. LPT-H has only PCH transcoder A,
17771                  * and marking underrun reporting as disabled for the non-existing
17772                  * PCH transcoders B and C would prevent enabling the south
17773                  * error interrupt (see cpt_can_enable_serr_int()).
17774                  */
17775                 if (has_pch_trancoder(dev_priv, crtc->pipe))
17776                         crtc->pch_fifo_underrun_disabled = true;
17777         }
17778 }
17779
17780 static bool has_bogus_dpll_config(const struct intel_crtc_state *crtc_state)
17781 {
17782         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
17783
17784         /*
17785          * Some SNB BIOSen (eg. ASUS K53SV) are known to misprogram
17786          * the hardware when a high res displays plugged in. DPLL P
17787          * divider is zero, and the pipe timings are bonkers. We'll
17788          * try to disable everything in that case.
17789          *
17790          * FIXME would be nice to be able to sanitize this state
17791          * without several WARNs, but for now let's take the easy
17792          * road.
17793          */
17794         return IS_GEN(dev_priv, 6) &&
17795                 crtc_state->hw.active &&
17796                 crtc_state->shared_dpll &&
17797                 crtc_state->port_clock == 0;
17798 }
17799
17800 static void intel_sanitize_encoder(struct intel_encoder *encoder)
17801 {
17802         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
17803         struct intel_connector *connector;
17804         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
17805         struct intel_crtc_state *crtc_state = crtc ?
17806                 to_intel_crtc_state(crtc->base.state) : NULL;
17807
17808         /* We need to check both for a crtc link (meaning that the
17809          * encoder is active and trying to read from a pipe) and the
17810          * pipe itself being active. */
17811         bool has_active_crtc = crtc_state &&
17812                 crtc_state->hw.active;
17813
17814         if (crtc_state && has_bogus_dpll_config(crtc_state)) {
17815                 DRM_DEBUG_KMS("BIOS has misprogrammed the hardware. Disabling pipe %c\n",
17816                               pipe_name(crtc->pipe));
17817                 has_active_crtc = false;
17818         }
17819
17820         connector = intel_encoder_find_connector(encoder);
17821         if (connector && !has_active_crtc) {
17822                 DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n",
17823                               encoder->base.base.id,
17824                               encoder->base.name);
17825
17826                 /* Connector is active, but has no active pipe. This is
17827                  * fallout from our resume register restoring. Disable
17828                  * the encoder manually again. */
17829                 if (crtc_state) {
17830                         struct drm_encoder *best_encoder;
17831
17832                         DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n",
17833                                       encoder->base.base.id,
17834                                       encoder->base.name);
17835
17836                         /* avoid oopsing in case the hooks consult best_encoder */
17837                         best_encoder = connector->base.state->best_encoder;
17838                         connector->base.state->best_encoder = &encoder->base;
17839
17840                         if (encoder->disable)
17841                                 encoder->disable(encoder, crtc_state,
17842                                                  connector->base.state);
17843                         if (encoder->post_disable)
17844                                 encoder->post_disable(encoder, crtc_state,
17845                                                       connector->base.state);
17846
17847                         connector->base.state->best_encoder = best_encoder;
17848                 }
17849                 encoder->base.crtc = NULL;
17850
17851                 /* Inconsistent output/port/pipe state happens presumably due to
17852                  * a bug in one of the get_hw_state functions. Or someplace else
17853                  * in our code, like the register restore mess on resume. Clamp
17854                  * things to off as a safer default. */
17855
17856                 connector->base.dpms = DRM_MODE_DPMS_OFF;
17857                 connector->base.encoder = NULL;
17858         }
17859
17860         /* notify opregion of the sanitized encoder state */
17861         intel_opregion_notify_encoder(encoder, connector && has_active_crtc);
17862
17863         if (INTEL_GEN(dev_priv) >= 11)
17864                 icl_sanitize_encoder_pll_mapping(encoder);
17865 }
17866
17867 /* FIXME read out full plane state for all planes */
17868 static void readout_plane_state(struct drm_i915_private *dev_priv)
17869 {
17870         struct intel_plane *plane;
17871         struct intel_crtc *crtc;
17872
17873         for_each_intel_plane(&dev_priv->drm, plane) {
17874                 struct intel_plane_state *plane_state =
17875                         to_intel_plane_state(plane->base.state);
17876                 struct intel_crtc_state *crtc_state;
17877                 enum pipe pipe = PIPE_A;
17878                 bool visible;
17879
17880                 visible = plane->get_hw_state(plane, &pipe);
17881
17882                 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
17883                 crtc_state = to_intel_crtc_state(crtc->base.state);
17884
17885                 intel_set_plane_visible(crtc_state, plane_state, visible);
17886
17887                 DRM_DEBUG_KMS("[PLANE:%d:%s] hw state readout: %s, pipe %c\n",
17888                               plane->base.base.id, plane->base.name,
17889                               enableddisabled(visible), pipe_name(pipe));
17890         }
17891
17892         for_each_intel_crtc(&dev_priv->drm, crtc) {
17893                 struct intel_crtc_state *crtc_state =
17894                         to_intel_crtc_state(crtc->base.state);
17895
17896                 fixup_active_planes(crtc_state);
17897         }
17898 }
17899
17900 static void intel_modeset_readout_hw_state(struct drm_device *dev)
17901 {
17902         struct drm_i915_private *dev_priv = to_i915(dev);
17903         enum pipe pipe;
17904         struct intel_crtc *crtc;
17905         struct intel_encoder *encoder;
17906         struct intel_connector *connector;
17907         struct drm_connector_list_iter conn_iter;
17908         int i;
17909
17910         dev_priv->active_pipes = 0;
17911
17912         for_each_intel_crtc(dev, crtc) {
17913                 struct intel_crtc_state *crtc_state =
17914                         to_intel_crtc_state(crtc->base.state);
17915
17916                 __drm_atomic_helper_crtc_destroy_state(&crtc_state->uapi);
17917                 intel_crtc_free_hw_state(crtc_state);
17918                 intel_crtc_state_reset(crtc_state, crtc);
17919
17920                 crtc_state->hw.active = crtc_state->hw.enable =
17921                         dev_priv->display.get_pipe_config(crtc, crtc_state);
17922
17923                 crtc->base.enabled = crtc_state->hw.enable;
17924                 crtc->active = crtc_state->hw.active;
17925
17926                 if (crtc_state->hw.active)
17927                         dev_priv->active_pipes |= BIT(crtc->pipe);
17928
17929                 DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n",
17930                               crtc->base.base.id, crtc->base.name,
17931                               enableddisabled(crtc_state->hw.active));
17932         }
17933
17934         readout_plane_state(dev_priv);
17935
17936         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
17937                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
17938
17939                 pll->on = pll->info->funcs->get_hw_state(dev_priv, pll,
17940                                                         &pll->state.hw_state);
17941
17942                 if (IS_ELKHARTLAKE(dev_priv) && pll->on &&
17943                     pll->info->id == DPLL_ID_EHL_DPLL4) {
17944                         pll->wakeref = intel_display_power_get(dev_priv,
17945                                                                POWER_DOMAIN_DPLL_DC_OFF);
17946                 }
17947
17948                 pll->state.crtc_mask = 0;
17949                 for_each_intel_crtc(dev, crtc) {
17950                         struct intel_crtc_state *crtc_state =
17951                                 to_intel_crtc_state(crtc->base.state);
17952
17953                         if (crtc_state->hw.active &&
17954                             crtc_state->shared_dpll == pll)
17955                                 pll->state.crtc_mask |= 1 << crtc->pipe;
17956                 }
17957                 pll->active_mask = pll->state.crtc_mask;
17958
17959                 DRM_DEBUG_KMS("%s hw state readout: crtc_mask 0x%08x, on %i\n",
17960                               pll->info->name, pll->state.crtc_mask, pll->on);
17961         }
17962
17963         for_each_intel_encoder(dev, encoder) {
17964                 pipe = 0;
17965
17966                 if (encoder->get_hw_state(encoder, &pipe)) {
17967                         struct intel_crtc_state *crtc_state;
17968
17969                         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
17970                         crtc_state = to_intel_crtc_state(crtc->base.state);
17971
17972                         encoder->base.crtc = &crtc->base;
17973                         encoder->get_config(encoder, crtc_state);
17974                 } else {
17975                         encoder->base.crtc = NULL;
17976                 }
17977
17978                 DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
17979                               encoder->base.base.id, encoder->base.name,
17980                               enableddisabled(encoder->base.crtc),
17981                               pipe_name(pipe));
17982         }
17983
17984         drm_connector_list_iter_begin(dev, &conn_iter);
17985         for_each_intel_connector_iter(connector, &conn_iter) {
17986                 if (connector->get_hw_state(connector)) {
17987                         struct intel_crtc_state *crtc_state;
17988                         struct intel_crtc *crtc;
17989
17990                         connector->base.dpms = DRM_MODE_DPMS_ON;
17991
17992                         encoder = connector->encoder;
17993                         connector->base.encoder = &encoder->base;
17994
17995                         crtc = to_intel_crtc(encoder->base.crtc);
17996                         crtc_state = crtc ? to_intel_crtc_state(crtc->base.state) : NULL;
17997
17998                         if (crtc_state && crtc_state->hw.active) {
17999                                 /*
18000                                  * This has to be done during hardware readout
18001                                  * because anything calling .crtc_disable may
18002                                  * rely on the connector_mask being accurate.
18003                                  */
18004                                 crtc_state->uapi.connector_mask |=
18005                                         drm_connector_mask(&connector->base);
18006                                 crtc_state->uapi.encoder_mask |=
18007                                         drm_encoder_mask(&encoder->base);
18008                         }
18009                 } else {
18010                         connector->base.dpms = DRM_MODE_DPMS_OFF;
18011                         connector->base.encoder = NULL;
18012                 }
18013                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n",
18014                               connector->base.base.id, connector->base.name,
18015                               enableddisabled(connector->base.encoder));
18016         }
18017         drm_connector_list_iter_end(&conn_iter);
18018
18019         for_each_intel_crtc(dev, crtc) {
18020                 struct intel_bw_state *bw_state =
18021                         to_intel_bw_state(dev_priv->bw_obj.state);
18022                 struct intel_crtc_state *crtc_state =
18023                         to_intel_crtc_state(crtc->base.state);
18024                 struct intel_plane *plane;
18025                 int min_cdclk = 0;
18026
18027                 if (crtc_state->hw.active) {
18028                         struct drm_display_mode *mode = &crtc_state->hw.mode;
18029
18030                         intel_mode_from_pipe_config(&crtc_state->hw.adjusted_mode,
18031                                                     crtc_state);
18032
18033                         *mode = crtc_state->hw.adjusted_mode;
18034                         mode->hdisplay = crtc_state->pipe_src_w;
18035                         mode->vdisplay = crtc_state->pipe_src_h;
18036
18037                         /*
18038                          * The initial mode needs to be set in order to keep
18039                          * the atomic core happy. It wants a valid mode if the
18040                          * crtc's enabled, so we do the above call.
18041                          *
18042                          * But we don't set all the derived state fully, hence
18043                          * set a flag to indicate that a full recalculation is
18044                          * needed on the next commit.
18045                          */
18046                         mode->private_flags = I915_MODE_FLAG_INHERITED;
18047
18048                         intel_crtc_compute_pixel_rate(crtc_state);
18049
18050                         intel_crtc_update_active_timings(crtc_state);
18051
18052                         intel_crtc_copy_hw_to_uapi_state(crtc_state);
18053                 }
18054
18055                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
18056                         const struct intel_plane_state *plane_state =
18057                                 to_intel_plane_state(plane->base.state);
18058
18059                         /*
18060                          * FIXME don't have the fb yet, so can't
18061                          * use intel_plane_data_rate() :(
18062                          */
18063                         if (plane_state->uapi.visible)
18064                                 crtc_state->data_rate[plane->id] =
18065                                         4 * crtc_state->pixel_rate;
18066                         /*
18067                          * FIXME don't have the fb yet, so can't
18068                          * use plane->min_cdclk() :(
18069                          */
18070                         if (plane_state->uapi.visible && plane->min_cdclk) {
18071                                 if (crtc_state->double_wide ||
18072                                     INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
18073                                         crtc_state->min_cdclk[plane->id] =
18074                                                 DIV_ROUND_UP(crtc_state->pixel_rate, 2);
18075                                 else
18076                                         crtc_state->min_cdclk[plane->id] =
18077                                                 crtc_state->pixel_rate;
18078                         }
18079                         DRM_DEBUG_KMS("[PLANE:%d:%s] min_cdclk %d kHz\n",
18080                                       plane->base.base.id, plane->base.name,
18081                                       crtc_state->min_cdclk[plane->id]);
18082                 }
18083
18084                 if (crtc_state->hw.active) {
18085                         min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
18086                         if (WARN_ON(min_cdclk < 0))
18087                                 min_cdclk = 0;
18088                 }
18089
18090                 dev_priv->min_cdclk[crtc->pipe] = min_cdclk;
18091                 dev_priv->min_voltage_level[crtc->pipe] =
18092                         crtc_state->min_voltage_level;
18093
18094                 intel_bw_crtc_update(bw_state, crtc_state);
18095
18096                 intel_pipe_config_sanity_check(dev_priv, crtc_state);
18097         }
18098 }
18099
18100 static void
18101 get_encoder_power_domains(struct drm_i915_private *dev_priv)
18102 {
18103         struct intel_encoder *encoder;
18104
18105         for_each_intel_encoder(&dev_priv->drm, encoder) {
18106                 struct intel_crtc_state *crtc_state;
18107
18108                 if (!encoder->get_power_domains)
18109                         continue;
18110
18111                 /*
18112                  * MST-primary and inactive encoders don't have a crtc state
18113                  * and neither of these require any power domain references.
18114                  */
18115                 if (!encoder->base.crtc)
18116                         continue;
18117
18118                 crtc_state = to_intel_crtc_state(encoder->base.crtc->state);
18119                 encoder->get_power_domains(encoder, crtc_state);
18120         }
18121 }
18122
18123 static void intel_early_display_was(struct drm_i915_private *dev_priv)
18124 {
18125         /*
18126          * Display WA #1185 WaDisableDARBFClkGating:cnl,glk,icl,ehl,tgl
18127          * Also known as Wa_14010480278.
18128          */
18129         if (IS_GEN_RANGE(dev_priv, 10, 12) || IS_GEMINILAKE(dev_priv))
18130                 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
18131                            DARBF_GATING_DIS);
18132
18133         if (IS_HASWELL(dev_priv)) {
18134                 /*
18135                  * WaRsPkgCStateDisplayPMReq:hsw
18136                  * System hang if this isn't done before disabling all planes!
18137                  */
18138                 I915_WRITE(CHICKEN_PAR1_1,
18139                            I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
18140         }
18141 }
18142
18143 static void ibx_sanitize_pch_hdmi_port(struct drm_i915_private *dev_priv,
18144                                        enum port port, i915_reg_t hdmi_reg)
18145 {
18146         u32 val = I915_READ(hdmi_reg);
18147
18148         if (val & SDVO_ENABLE ||
18149             (val & SDVO_PIPE_SEL_MASK) == SDVO_PIPE_SEL(PIPE_A))
18150                 return;
18151
18152         DRM_DEBUG_KMS("Sanitizing transcoder select for HDMI %c\n",
18153                       port_name(port));
18154
18155         val &= ~SDVO_PIPE_SEL_MASK;
18156         val |= SDVO_PIPE_SEL(PIPE_A);
18157
18158         I915_WRITE(hdmi_reg, val);
18159 }
18160
18161 static void ibx_sanitize_pch_dp_port(struct drm_i915_private *dev_priv,
18162                                      enum port port, i915_reg_t dp_reg)
18163 {
18164         u32 val = I915_READ(dp_reg);
18165
18166         if (val & DP_PORT_EN ||
18167             (val & DP_PIPE_SEL_MASK) == DP_PIPE_SEL(PIPE_A))
18168                 return;
18169
18170         DRM_DEBUG_KMS("Sanitizing transcoder select for DP %c\n",
18171                       port_name(port));
18172
18173         val &= ~DP_PIPE_SEL_MASK;
18174         val |= DP_PIPE_SEL(PIPE_A);
18175
18176         I915_WRITE(dp_reg, val);
18177 }
18178
18179 static void ibx_sanitize_pch_ports(struct drm_i915_private *dev_priv)
18180 {
18181         /*
18182          * The BIOS may select transcoder B on some of the PCH
18183          * ports even it doesn't enable the port. This would trip
18184          * assert_pch_dp_disabled() and assert_pch_hdmi_disabled().
18185          * Sanitize the transcoder select bits to prevent that. We
18186          * assume that the BIOS never actually enabled the port,
18187          * because if it did we'd actually have to toggle the port
18188          * on and back off to make the transcoder A select stick
18189          * (see. intel_dp_link_down(), intel_disable_hdmi(),
18190          * intel_disable_sdvo()).
18191          */
18192         ibx_sanitize_pch_dp_port(dev_priv, PORT_B, PCH_DP_B);
18193         ibx_sanitize_pch_dp_port(dev_priv, PORT_C, PCH_DP_C);
18194         ibx_sanitize_pch_dp_port(dev_priv, PORT_D, PCH_DP_D);
18195
18196         /* PCH SDVOB multiplex with HDMIB */
18197         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_B, PCH_HDMIB);
18198         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_C, PCH_HDMIC);
18199         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_D, PCH_HDMID);
18200 }
18201
18202 /* Scan out the current hw modeset state,
18203  * and sanitizes it to the current state
18204  */
18205 static void
18206 intel_modeset_setup_hw_state(struct drm_device *dev,
18207                              struct drm_modeset_acquire_ctx *ctx)
18208 {
18209         struct drm_i915_private *dev_priv = to_i915(dev);
18210         struct intel_encoder *encoder;
18211         struct intel_crtc *crtc;
18212         intel_wakeref_t wakeref;
18213         int i;
18214
18215         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
18216
18217         intel_early_display_was(dev_priv);
18218         intel_modeset_readout_hw_state(dev);
18219
18220         /* HW state is read out, now we need to sanitize this mess. */
18221
18222         /* Sanitize the TypeC port mode upfront, encoders depend on this */
18223         for_each_intel_encoder(dev, encoder) {
18224                 enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
18225
18226                 /* We need to sanitize only the MST primary port. */
18227                 if (encoder->type != INTEL_OUTPUT_DP_MST &&
18228                     intel_phy_is_tc(dev_priv, phy))
18229                         intel_tc_port_sanitize(enc_to_dig_port(&encoder->base));
18230         }
18231
18232         get_encoder_power_domains(dev_priv);
18233
18234         if (HAS_PCH_IBX(dev_priv))
18235                 ibx_sanitize_pch_ports(dev_priv);
18236
18237         /*
18238          * intel_sanitize_plane_mapping() may need to do vblank
18239          * waits, so we need vblank interrupts restored beforehand.
18240          */
18241         for_each_intel_crtc(&dev_priv->drm, crtc) {
18242                 struct intel_crtc_state *crtc_state =
18243                         to_intel_crtc_state(crtc->base.state);
18244
18245                 drm_crtc_vblank_reset(&crtc->base);
18246
18247                 if (crtc_state->hw.active)
18248                         intel_crtc_vblank_on(crtc_state);
18249         }
18250
18251         intel_sanitize_plane_mapping(dev_priv);
18252
18253         for_each_intel_encoder(dev, encoder)
18254                 intel_sanitize_encoder(encoder);
18255
18256         for_each_intel_crtc(&dev_priv->drm, crtc) {
18257                 struct intel_crtc_state *crtc_state =
18258                         to_intel_crtc_state(crtc->base.state);
18259
18260                 intel_sanitize_crtc(crtc, ctx);
18261                 intel_dump_pipe_config(crtc_state, NULL, "[setup_hw_state]");
18262         }
18263
18264         intel_modeset_update_connector_atomic_state(dev);
18265
18266         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
18267                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
18268
18269                 if (!pll->on || pll->active_mask)
18270                         continue;
18271
18272                 DRM_DEBUG_KMS("%s enabled but not in use, disabling\n",
18273                               pll->info->name);
18274
18275                 pll->info->funcs->disable(dev_priv, pll);
18276                 pll->on = false;
18277         }
18278
18279         if (IS_G4X(dev_priv)) {
18280                 g4x_wm_get_hw_state(dev_priv);
18281                 g4x_wm_sanitize(dev_priv);
18282         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
18283                 vlv_wm_get_hw_state(dev_priv);
18284                 vlv_wm_sanitize(dev_priv);
18285         } else if (INTEL_GEN(dev_priv) >= 9) {
18286                 skl_wm_get_hw_state(dev_priv);
18287         } else if (HAS_PCH_SPLIT(dev_priv)) {
18288                 ilk_wm_get_hw_state(dev_priv);
18289         }
18290
18291         for_each_intel_crtc(dev, crtc) {
18292                 struct intel_crtc_state *crtc_state =
18293                         to_intel_crtc_state(crtc->base.state);
18294                 u64 put_domains;
18295
18296                 put_domains = modeset_get_crtc_power_domains(crtc_state);
18297                 if (WARN_ON(put_domains))
18298                         modeset_put_power_domains(dev_priv, put_domains);
18299         }
18300
18301         intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref);
18302 }
18303
18304 void intel_display_resume(struct drm_device *dev)
18305 {
18306         struct drm_i915_private *dev_priv = to_i915(dev);
18307         struct drm_atomic_state *state = dev_priv->modeset_restore_state;
18308         struct drm_modeset_acquire_ctx ctx;
18309         int ret;
18310
18311         dev_priv->modeset_restore_state = NULL;
18312         if (state)
18313                 state->acquire_ctx = &ctx;
18314
18315         drm_modeset_acquire_init(&ctx, 0);
18316
18317         while (1) {
18318                 ret = drm_modeset_lock_all_ctx(dev, &ctx);
18319                 if (ret != -EDEADLK)
18320                         break;
18321
18322                 drm_modeset_backoff(&ctx);
18323         }
18324
18325         if (!ret)
18326                 ret = __intel_display_resume(dev, state, &ctx);
18327
18328         intel_enable_ipc(dev_priv);
18329         drm_modeset_drop_locks(&ctx);
18330         drm_modeset_acquire_fini(&ctx);
18331
18332         if (ret)
18333                 DRM_ERROR("Restoring old state failed with %i\n", ret);
18334         if (state)
18335                 drm_atomic_state_put(state);
18336 }
18337
18338 static void intel_hpd_poll_fini(struct drm_i915_private *i915)
18339 {
18340         struct intel_connector *connector;
18341         struct drm_connector_list_iter conn_iter;
18342
18343         /* Kill all the work that may have been queued by hpd. */
18344         drm_connector_list_iter_begin(&i915->drm, &conn_iter);
18345         for_each_intel_connector_iter(connector, &conn_iter) {
18346                 if (connector->modeset_retry_work.func)
18347                         cancel_work_sync(&connector->modeset_retry_work);
18348                 if (connector->hdcp.shim) {
18349                         cancel_delayed_work_sync(&connector->hdcp.check_work);
18350                         cancel_work_sync(&connector->hdcp.prop_work);
18351                 }
18352         }
18353         drm_connector_list_iter_end(&conn_iter);
18354 }
18355
18356 void intel_modeset_driver_remove(struct drm_i915_private *i915)
18357 {
18358         flush_workqueue(i915->flip_wq);
18359         flush_workqueue(i915->modeset_wq);
18360
18361         flush_work(&i915->atomic_helper.free_work);
18362         WARN_ON(!llist_empty(&i915->atomic_helper.free_list));
18363
18364         /*
18365          * Interrupts and polling as the first thing to avoid creating havoc.
18366          * Too much stuff here (turning of connectors, ...) would
18367          * experience fancy races otherwise.
18368          */
18369         intel_irq_uninstall(i915);
18370
18371         /*
18372          * Due to the hpd irq storm handling the hotplug work can re-arm the
18373          * poll handlers. Hence disable polling after hpd handling is shut down.
18374          */
18375         intel_hpd_poll_fini(i915);
18376
18377         /*
18378          * MST topology needs to be suspended so we don't have any calls to
18379          * fbdev after it's finalized. MST will be destroyed later as part of
18380          * drm_mode_config_cleanup()
18381          */
18382         intel_dp_mst_suspend(i915);
18383
18384         /* poll work can call into fbdev, hence clean that up afterwards */
18385         intel_fbdev_fini(i915);
18386
18387         intel_unregister_dsm_handler();
18388
18389         intel_fbc_global_disable(i915);
18390
18391         /* flush any delayed tasks or pending work */
18392         flush_scheduled_work();
18393
18394         intel_hdcp_component_fini(i915);
18395
18396         drm_mode_config_cleanup(&i915->drm);
18397
18398         intel_overlay_cleanup(i915);
18399
18400         intel_gmbus_teardown(i915);
18401
18402         intel_bw_cleanup(i915);
18403
18404         destroy_workqueue(i915->flip_wq);
18405         destroy_workqueue(i915->modeset_wq);
18406
18407         intel_fbc_cleanup_cfb(i915);
18408 }
18409
18410 #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
18411
18412 struct intel_display_error_state {
18413
18414         u32 power_well_driver;
18415
18416         struct intel_cursor_error_state {
18417                 u32 control;
18418                 u32 position;
18419                 u32 base;
18420                 u32 size;
18421         } cursor[I915_MAX_PIPES];
18422
18423         struct intel_pipe_error_state {
18424                 bool power_domain_on;
18425                 u32 source;
18426                 u32 stat;
18427         } pipe[I915_MAX_PIPES];
18428
18429         struct intel_plane_error_state {
18430                 u32 control;
18431                 u32 stride;
18432                 u32 size;
18433                 u32 pos;
18434                 u32 addr;
18435                 u32 surface;
18436                 u32 tile_offset;
18437         } plane[I915_MAX_PIPES];
18438
18439         struct intel_transcoder_error_state {
18440                 bool available;
18441                 bool power_domain_on;
18442                 enum transcoder cpu_transcoder;
18443
18444                 u32 conf;
18445
18446                 u32 htotal;
18447                 u32 hblank;
18448                 u32 hsync;
18449                 u32 vtotal;
18450                 u32 vblank;
18451                 u32 vsync;
18452         } transcoder[5];
18453 };
18454
18455 struct intel_display_error_state *
18456 intel_display_capture_error_state(struct drm_i915_private *dev_priv)
18457 {
18458         struct intel_display_error_state *error;
18459         int transcoders[] = {
18460                 TRANSCODER_A,
18461                 TRANSCODER_B,
18462                 TRANSCODER_C,
18463                 TRANSCODER_D,
18464                 TRANSCODER_EDP,
18465         };
18466         int i;
18467
18468         BUILD_BUG_ON(ARRAY_SIZE(transcoders) != ARRAY_SIZE(error->transcoder));
18469
18470         if (!HAS_DISPLAY(dev_priv) || !INTEL_DISPLAY_ENABLED(dev_priv))
18471                 return NULL;
18472
18473         error = kzalloc(sizeof(*error), GFP_ATOMIC);
18474         if (error == NULL)
18475                 return NULL;
18476
18477         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
18478                 error->power_well_driver = I915_READ(HSW_PWR_WELL_CTL2);
18479
18480         for_each_pipe(dev_priv, i) {
18481                 error->pipe[i].power_domain_on =
18482                         __intel_display_power_is_enabled(dev_priv,
18483                                                          POWER_DOMAIN_PIPE(i));
18484                 if (!error->pipe[i].power_domain_on)
18485                         continue;
18486
18487                 error->cursor[i].control = I915_READ(CURCNTR(i));
18488                 error->cursor[i].position = I915_READ(CURPOS(i));
18489                 error->cursor[i].base = I915_READ(CURBASE(i));
18490
18491                 error->plane[i].control = I915_READ(DSPCNTR(i));
18492                 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
18493                 if (INTEL_GEN(dev_priv) <= 3) {
18494                         error->plane[i].size = I915_READ(DSPSIZE(i));
18495                         error->plane[i].pos = I915_READ(DSPPOS(i));
18496                 }
18497                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
18498                         error->plane[i].addr = I915_READ(DSPADDR(i));
18499                 if (INTEL_GEN(dev_priv) >= 4) {
18500                         error->plane[i].surface = I915_READ(DSPSURF(i));
18501                         error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
18502                 }
18503
18504                 error->pipe[i].source = I915_READ(PIPESRC(i));
18505
18506                 if (HAS_GMCH(dev_priv))
18507                         error->pipe[i].stat = I915_READ(PIPESTAT(i));
18508         }
18509
18510         for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
18511                 enum transcoder cpu_transcoder = transcoders[i];
18512
18513                 if (!INTEL_INFO(dev_priv)->trans_offsets[cpu_transcoder])
18514                         continue;
18515
18516                 error->transcoder[i].available = true;
18517                 error->transcoder[i].power_domain_on =
18518                         __intel_display_power_is_enabled(dev_priv,
18519                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder));
18520                 if (!error->transcoder[i].power_domain_on)
18521                         continue;
18522
18523                 error->transcoder[i].cpu_transcoder = cpu_transcoder;
18524
18525                 error->transcoder[i].conf = I915_READ(PIPECONF(cpu_transcoder));
18526                 error->transcoder[i].htotal = I915_READ(HTOTAL(cpu_transcoder));
18527                 error->transcoder[i].hblank = I915_READ(HBLANK(cpu_transcoder));
18528                 error->transcoder[i].hsync = I915_READ(HSYNC(cpu_transcoder));
18529                 error->transcoder[i].vtotal = I915_READ(VTOTAL(cpu_transcoder));
18530                 error->transcoder[i].vblank = I915_READ(VBLANK(cpu_transcoder));
18531                 error->transcoder[i].vsync = I915_READ(VSYNC(cpu_transcoder));
18532         }
18533
18534         return error;
18535 }
18536
18537 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
18538
18539 void
18540 intel_display_print_error_state(struct drm_i915_error_state_buf *m,
18541                                 struct intel_display_error_state *error)
18542 {
18543         struct drm_i915_private *dev_priv = m->i915;
18544         int i;
18545
18546         if (!error)
18547                 return;
18548
18549         err_printf(m, "Num Pipes: %d\n", INTEL_NUM_PIPES(dev_priv));
18550         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
18551                 err_printf(m, "PWR_WELL_CTL2: %08x\n",
18552                            error->power_well_driver);
18553         for_each_pipe(dev_priv, i) {
18554                 err_printf(m, "Pipe [%d]:\n", i);
18555                 err_printf(m, "  Power: %s\n",
18556                            onoff(error->pipe[i].power_domain_on));
18557                 err_printf(m, "  SRC: %08x\n", error->pipe[i].source);
18558                 err_printf(m, "  STAT: %08x\n", error->pipe[i].stat);
18559
18560                 err_printf(m, "Plane [%d]:\n", i);
18561                 err_printf(m, "  CNTR: %08x\n", error->plane[i].control);
18562                 err_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
18563                 if (INTEL_GEN(dev_priv) <= 3) {
18564                         err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
18565                         err_printf(m, "  POS: %08x\n", error->plane[i].pos);
18566                 }
18567                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
18568                         err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
18569                 if (INTEL_GEN(dev_priv) >= 4) {
18570                         err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
18571                         err_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
18572                 }
18573
18574                 err_printf(m, "Cursor [%d]:\n", i);
18575                 err_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
18576                 err_printf(m, "  POS: %08x\n", error->cursor[i].position);
18577                 err_printf(m, "  BASE: %08x\n", error->cursor[i].base);
18578         }
18579
18580         for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
18581                 if (!error->transcoder[i].available)
18582                         continue;
18583
18584                 err_printf(m, "CPU transcoder: %s\n",
18585                            transcoder_name(error->transcoder[i].cpu_transcoder));
18586                 err_printf(m, "  Power: %s\n",
18587                            onoff(error->transcoder[i].power_domain_on));
18588                 err_printf(m, "  CONF: %08x\n", error->transcoder[i].conf);
18589                 err_printf(m, "  HTOTAL: %08x\n", error->transcoder[i].htotal);
18590                 err_printf(m, "  HBLANK: %08x\n", error->transcoder[i].hblank);
18591                 err_printf(m, "  HSYNC: %08x\n", error->transcoder[i].hsync);
18592                 err_printf(m, "  VTOTAL: %08x\n", error->transcoder[i].vtotal);
18593                 err_printf(m, "  VBLANK: %08x\n", error->transcoder[i].vblank);
18594                 err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
18595         }
18596 }
18597
18598 #endif