drivers/gpu/drm/i915/display/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/i2c.h>
  28 #include <linux/input.h>
  29 #include <linux/intel-iommu.h>
  30 #include <linux/kernel.h>
  31 #include <linux/module.h>
  32 #include <linux/dma-resv.h>
  33 #include <linux/slab.h>
  34
  35 #include <drm/drm_atomic.h>
  36 #include <drm/drm_atomic_helper.h>
  37 #include <drm/drm_atomic_uapi.h>
  38 #include <drm/drm_dp_helper.h>
  39 #include <drm/drm_edid.h>
  40 #include <drm/drm_fourcc.h>
  41 #include <drm/drm_plane_helper.h>
  42 #include <drm/drm_probe_helper.h>
  43 #include <drm/drm_rect.h>
  44 #include <drm/i915_drm.h>
  45
  46 #include "display/intel_crt.h"
  47 #include "display/intel_ddi.h"
  48 #include "display/intel_dp.h"
  49 #include "display/intel_dp_mst.h"
  50 #include "display/intel_dsi.h"
  51 #include "display/intel_dvo.h"
  52 #include "display/intel_gmbus.h"
  53 #include "display/intel_hdmi.h"
  54 #include "display/intel_lvds.h"
  55 #include "display/intel_sdvo.h"
  56 #include "display/intel_tv.h"
  57 #include "display/intel_vdsc.h"
  58
  59 #include "gt/intel_rps.h"
  60
  61 #include "i915_drv.h"
  62 #include "i915_trace.h"
  63 #include "intel_acpi.h"
  64 #include "intel_atomic.h"
  65 #include "intel_atomic_plane.h"
  66 #include "intel_bw.h"
  67 #include "intel_cdclk.h"
  68 #include "intel_color.h"
  69 #include "intel_display_types.h"
  70 #include "intel_dp_link_training.h"
  71 #include "intel_fbc.h"
  72 #include "intel_fbdev.h"
  73 #include "intel_fifo_underrun.h"
  74 #include "intel_frontbuffer.h"
  75 #include "intel_hdcp.h"
  76 #include "intel_hotplug.h"
  77 #include "intel_overlay.h"
  78 #include "intel_pipe_crc.h"
  79 #include "intel_pm.h"
  80 #include "intel_psr.h"
  81 #include "intel_quirks.h"
  82 #include "intel_sideband.h"
  83 #include "intel_sprite.h"
  84 #include "intel_tc.h"
  85 #include "intel_vga.h"
  86
  87 /* Primary plane formats for gen <= 3 */
  88 static const u32 i8xx_primary_formats[] = {
  89         DRM_FORMAT_C8,
  90         DRM_FORMAT_XRGB1555,
  91         DRM_FORMAT_RGB565,
  92         DRM_FORMAT_XRGB8888,
  93 };
  94
  95 /* Primary plane formats for ivb (no fp16 due to hw issue) */
  96 static const u32 ivb_primary_formats[] = {
  97         DRM_FORMAT_C8,
  98         DRM_FORMAT_RGB565,
  99         DRM_FORMAT_XRGB8888,
 100         DRM_FORMAT_XBGR8888,
 101         DRM_FORMAT_XRGB2101010,
 102         DRM_FORMAT_XBGR2101010,
 103 };
 104
 105 /* Primary plane formats for gen >= 4, except ivb */
 106 static const u32 i965_primary_formats[] = {
 107         DRM_FORMAT_C8,
 108         DRM_FORMAT_RGB565,
 109         DRM_FORMAT_XRGB8888,
 110         DRM_FORMAT_XBGR8888,
 111         DRM_FORMAT_XRGB2101010,
 112         DRM_FORMAT_XBGR2101010,
 113         DRM_FORMAT_XBGR16161616F,
 114 };
 115
 116 /* Primary plane formats for vlv/chv */
 117 static const u32 vlv_primary_formats[] = {
 118         DRM_FORMAT_C8,
 119         DRM_FORMAT_RGB565,
 120         DRM_FORMAT_XRGB8888,
 121         DRM_FORMAT_XBGR8888,
 122         DRM_FORMAT_ARGB8888,
 123         DRM_FORMAT_ABGR8888,
 124         DRM_FORMAT_XRGB2101010,
 125         DRM_FORMAT_XBGR2101010,
 126         DRM_FORMAT_ARGB2101010,
 127         DRM_FORMAT_ABGR2101010,
 128         DRM_FORMAT_XBGR16161616F,
 129 };
 130
 131 static const u64 i9xx_format_modifiers[] = {
 132         I915_FORMAT_MOD_X_TILED,
 133         DRM_FORMAT_MOD_LINEAR,
 134         DRM_FORMAT_MOD_INVALID
 135 };
 136
 137 /* Cursor formats */
 138 static const u32 intel_cursor_formats[] = {
 139         DRM_FORMAT_ARGB8888,
 140 };
 141
 142 static const u64 cursor_format_modifiers[] = {
 143         DRM_FORMAT_MOD_LINEAR,
 144         DRM_FORMAT_MOD_INVALID
 145 };
 146
 147 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
 148                                 struct intel_crtc_state *pipe_config);
 149 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
 150                                    struct intel_crtc_state *pipe_config);
 151
 152 static int intel_framebuffer_init(struct intel_framebuffer *ifb,
 153                                   struct drm_i915_gem_object *obj,
 154                                   struct drm_mode_fb_cmd2 *mode_cmd);
 155 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state);
 156 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state);
 157 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
 158                                          const struct intel_link_m_n *m_n,
 159                                          const struct intel_link_m_n *m2_n2);
 160 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state);
 161 static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state);
 162 static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state);
 163 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state);
 164 static void vlv_prepare_pll(struct intel_crtc *crtc,
 165                             const struct intel_crtc_state *pipe_config);
 166 static void chv_prepare_pll(struct intel_crtc *crtc,
 167                             const struct intel_crtc_state *pipe_config);
 168 static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state);
 169 static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state);
 170 static void intel_modeset_setup_hw_state(struct drm_device *dev,
 171                                          struct drm_modeset_acquire_ctx *ctx);
 172 static struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc);
 173
 174 struct intel_limit {
 175         struct {
 176                 int min, max;
 177         } dot, vco, n, m, m1, m2, p, p1;
 178
 179         struct {
 180                 int dot_limit;
 181                 int p2_slow, p2_fast;
 182         } p2;
 183 };
 184
 185 /* returns HPLL frequency in kHz */
 186 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv)
 187 {
 188         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 189
 190         /* Obtain SKU information */
 191         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
 192                 CCK_FUSE_HPLL_FREQ_MASK;
 193
 194         return vco_freq[hpll_freq] * 1000;
 195 }
 196
 197 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
 198                       const char *name, u32 reg, int ref_freq)
 199 {
 200         u32 val;
 201         int divider;
 202
 203         val = vlv_cck_read(dev_priv, reg);
 204         divider = val & CCK_FREQUENCY_VALUES;
 205
 206         WARN((val & CCK_FREQUENCY_STATUS) !=
 207              (divider << CCK_FREQUENCY_STATUS_SHIFT),
 208              "%s change in progress\n", name);
 209
 210         return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1);
 211 }
 212
 213 int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 214                            const char *name, u32 reg)
 215 {
 216         int hpll;
 217
 218         vlv_cck_get(dev_priv);
 219
 220         if (dev_priv->hpll_freq == 0)
 221                 dev_priv->hpll_freq = vlv_get_hpll_vco(dev_priv);
 222
 223         hpll = vlv_get_cck_clock(dev_priv, name, reg, dev_priv->hpll_freq);
 224
 225         vlv_cck_put(dev_priv);
 226
 227         return hpll;
 228 }
 229
 230 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 231 {
 232         if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
 233                 return;
 234
 235         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
 236                                                       CCK_CZ_CLOCK_CONTROL);
 237
 238         DRM_DEBUG_DRIVER("CZ clock rate: %d kHz\n", dev_priv->czclk_freq);
 239 }
 240
 241 static inline u32 /* units of 100MHz */
 242 intel_fdi_link_freq(struct drm_i915_private *dev_priv,
 243                     const struct intel_crtc_state *pipe_config)
 244 {
 245         if (HAS_DDI(dev_priv))
 246                 return pipe_config->port_clock; /* SPLL */
 247         else
 248                 return dev_priv->fdi_pll_freq;
 249 }
 250
 251 static const struct intel_limit intel_limits_i8xx_dac = {
 252         .dot = { .min = 25000, .max = 350000 },
 253         .vco = { .min = 908000, .max = 1512000 },
 254         .n = { .min = 2, .max = 16 },
 255         .m = { .min = 96, .max = 140 },
 256         .m1 = { .min = 18, .max = 26 },
 257         .m2 = { .min = 6, .max = 16 },
 258         .p = { .min = 4, .max = 128 },
 259         .p1 = { .min = 2, .max = 33 },
 260         .p2 = { .dot_limit = 165000,
 261                 .p2_slow = 4, .p2_fast = 2 },
 262 };
 263
 264 static const struct intel_limit intel_limits_i8xx_dvo = {
 265         .dot = { .min = 25000, .max = 350000 },
 266         .vco = { .min = 908000, .max = 1512000 },
 267         .n = { .min = 2, .max = 16 },
 268         .m = { .min = 96, .max = 140 },
 269         .m1 = { .min = 18, .max = 26 },
 270         .m2 = { .min = 6, .max = 16 },
 271         .p = { .min = 4, .max = 128 },
 272         .p1 = { .min = 2, .max = 33 },
 273         .p2 = { .dot_limit = 165000,
 274                 .p2_slow = 4, .p2_fast = 4 },
 275 };
 276
 277 static const struct intel_limit intel_limits_i8xx_lvds = {
 278         .dot = { .min = 25000, .max = 350000 },
 279         .vco = { .min = 908000, .max = 1512000 },
 280         .n = { .min = 2, .max = 16 },
 281         .m = { .min = 96, .max = 140 },
 282         .m1 = { .min = 18, .max = 26 },
 283         .m2 = { .min = 6, .max = 16 },
 284         .p = { .min = 4, .max = 128 },
 285         .p1 = { .min = 1, .max = 6 },
 286         .p2 = { .dot_limit = 165000,
 287                 .p2_slow = 14, .p2_fast = 7 },
 288 };
 289
 290 static const struct intel_limit intel_limits_i9xx_sdvo = {
 291         .dot = { .min = 20000, .max = 400000 },
 292         .vco = { .min = 1400000, .max = 2800000 },
 293         .n = { .min = 1, .max = 6 },
 294         .m = { .min = 70, .max = 120 },
 295         .m1 = { .min = 8, .max = 18 },
 296         .m2 = { .min = 3, .max = 7 },
 297         .p = { .min = 5, .max = 80 },
 298         .p1 = { .min = 1, .max = 8 },
 299         .p2 = { .dot_limit = 200000,
 300                 .p2_slow = 10, .p2_fast = 5 },
 301 };
 302
 303 static const struct intel_limit intel_limits_i9xx_lvds = {
 304         .dot = { .min = 20000, .max = 400000 },
 305         .vco = { .min = 1400000, .max = 2800000 },
 306         .n = { .min = 1, .max = 6 },
 307         .m = { .min = 70, .max = 120 },
 308         .m1 = { .min = 8, .max = 18 },
 309         .m2 = { .min = 3, .max = 7 },
 310         .p = { .min = 7, .max = 98 },
 311         .p1 = { .min = 1, .max = 8 },
 312         .p2 = { .dot_limit = 112000,
 313                 .p2_slow = 14, .p2_fast = 7 },
 314 };
 315
 316
 317 static const struct intel_limit intel_limits_g4x_sdvo = {
 318         .dot = { .min = 25000, .max = 270000 },
 319         .vco = { .min = 1750000, .max = 3500000},
 320         .n = { .min = 1, .max = 4 },
 321         .m = { .min = 104, .max = 138 },
 322         .m1 = { .min = 17, .max = 23 },
 323         .m2 = { .min = 5, .max = 11 },
 324         .p = { .min = 10, .max = 30 },
 325         .p1 = { .min = 1, .max = 3},
 326         .p2 = { .dot_limit = 270000,
 327                 .p2_slow = 10,
 328                 .p2_fast = 10
 329         },
 330 };
 331
 332 static const struct intel_limit intel_limits_g4x_hdmi = {
 333         .dot = { .min = 22000, .max = 400000 },
 334         .vco = { .min = 1750000, .max = 3500000},
 335         .n = { .min = 1, .max = 4 },
 336         .m = { .min = 104, .max = 138 },
 337         .m1 = { .min = 16, .max = 23 },
 338         .m2 = { .min = 5, .max = 11 },
 339         .p = { .min = 5, .max = 80 },
 340         .p1 = { .min = 1, .max = 8},
 341         .p2 = { .dot_limit = 165000,
 342                 .p2_slow = 10, .p2_fast = 5 },
 343 };
 344
 345 static const struct intel_limit intel_limits_g4x_single_channel_lvds = {
 346         .dot = { .min = 20000, .max = 115000 },
 347         .vco = { .min = 1750000, .max = 3500000 },
 348         .n = { .min = 1, .max = 3 },
 349         .m = { .min = 104, .max = 138 },
 350         .m1 = { .min = 17, .max = 23 },
 351         .m2 = { .min = 5, .max = 11 },
 352         .p = { .min = 28, .max = 112 },
 353         .p1 = { .min = 2, .max = 8 },
 354         .p2 = { .dot_limit = 0,
 355                 .p2_slow = 14, .p2_fast = 14
 356         },
 357 };
 358
 359 static const struct intel_limit intel_limits_g4x_dual_channel_lvds = {
 360         .dot = { .min = 80000, .max = 224000 },
 361         .vco = { .min = 1750000, .max = 3500000 },
 362         .n = { .min = 1, .max = 3 },
 363         .m = { .min = 104, .max = 138 },
 364         .m1 = { .min = 17, .max = 23 },
 365         .m2 = { .min = 5, .max = 11 },
 366         .p = { .min = 14, .max = 42 },
 367         .p1 = { .min = 2, .max = 6 },
 368         .p2 = { .dot_limit = 0,
 369                 .p2_slow = 7, .p2_fast = 7
 370         },
 371 };
 372
 373 static const struct intel_limit intel_limits_pineview_sdvo = {
 374         .dot = { .min = 20000, .max = 400000},
 375         .vco = { .min = 1700000, .max = 3500000 },
 376         /* Pineview's Ncounter is a ring counter */
 377         .n = { .min = 3, .max = 6 },
 378         .m = { .min = 2, .max = 256 },
 379         /* Pineview only has one combined m divider, which we treat as m2. */
 380         .m1 = { .min = 0, .max = 0 },
 381         .m2 = { .min = 0, .max = 254 },
 382         .p = { .min = 5, .max = 80 },
 383         .p1 = { .min = 1, .max = 8 },
 384         .p2 = { .dot_limit = 200000,
 385                 .p2_slow = 10, .p2_fast = 5 },
 386 };
 387
 388 static const struct intel_limit intel_limits_pineview_lvds = {
 389         .dot = { .min = 20000, .max = 400000 },
 390         .vco = { .min = 1700000, .max = 3500000 },
 391         .n = { .min = 3, .max = 6 },
 392         .m = { .min = 2, .max = 256 },
 393         .m1 = { .min = 0, .max = 0 },
 394         .m2 = { .min = 0, .max = 254 },
 395         .p = { .min = 7, .max = 112 },
 396         .p1 = { .min = 1, .max = 8 },
 397         .p2 = { .dot_limit = 112000,
 398                 .p2_slow = 14, .p2_fast = 14 },
 399 };
 400
 401 /* Ironlake / Sandybridge
 402  *
 403  * We calculate clock using (register_value + 2) for N/M1/M2, so here
 404  * the range value for them is (actual_value - 2).
 405  */
 406 static const struct intel_limit intel_limits_ironlake_dac = {
 407         .dot = { .min = 25000, .max = 350000 },
 408         .vco = { .min = 1760000, .max = 3510000 },
 409         .n = { .min = 1, .max = 5 },
 410         .m = { .min = 79, .max = 127 },
 411         .m1 = { .min = 12, .max = 22 },
 412         .m2 = { .min = 5, .max = 9 },
 413         .p = { .min = 5, .max = 80 },
 414         .p1 = { .min = 1, .max = 8 },
 415         .p2 = { .dot_limit = 225000,
 416                 .p2_slow = 10, .p2_fast = 5 },
 417 };
 418
 419 static const struct intel_limit intel_limits_ironlake_single_lvds = {
 420         .dot = { .min = 25000, .max = 350000 },
 421         .vco = { .min = 1760000, .max = 3510000 },
 422         .n = { .min = 1, .max = 3 },
 423         .m = { .min = 79, .max = 118 },
 424         .m1 = { .min = 12, .max = 22 },
 425         .m2 = { .min = 5, .max = 9 },
 426         .p = { .min = 28, .max = 112 },
 427         .p1 = { .min = 2, .max = 8 },
 428         .p2 = { .dot_limit = 225000,
 429                 .p2_slow = 14, .p2_fast = 14 },
 430 };
 431
 432 static const struct intel_limit intel_limits_ironlake_dual_lvds = {
 433         .dot = { .min = 25000, .max = 350000 },
 434         .vco = { .min = 1760000, .max = 3510000 },
 435         .n = { .min = 1, .max = 3 },
 436         .m = { .min = 79, .max = 127 },
 437         .m1 = { .min = 12, .max = 22 },
 438         .m2 = { .min = 5, .max = 9 },
 439         .p = { .min = 14, .max = 56 },
 440         .p1 = { .min = 2, .max = 8 },
 441         .p2 = { .dot_limit = 225000,
 442                 .p2_slow = 7, .p2_fast = 7 },
 443 };
 444
 445 /* LVDS 100mhz refclk limits. */
 446 static const struct intel_limit intel_limits_ironlake_single_lvds_100m = {
 447         .dot = { .min = 25000, .max = 350000 },
 448         .vco = { .min = 1760000, .max = 3510000 },
 449         .n = { .min = 1, .max = 2 },
 450         .m = { .min = 79, .max = 126 },
 451         .m1 = { .min = 12, .max = 22 },
 452         .m2 = { .min = 5, .max = 9 },
 453         .p = { .min = 28, .max = 112 },
 454         .p1 = { .min = 2, .max = 8 },
 455         .p2 = { .dot_limit = 225000,
 456                 .p2_slow = 14, .p2_fast = 14 },
 457 };
 458
 459 static const struct intel_limit intel_limits_ironlake_dual_lvds_100m = {
 460         .dot = { .min = 25000, .max = 350000 },
 461         .vco = { .min = 1760000, .max = 3510000 },
 462         .n = { .min = 1, .max = 3 },
 463         .m = { .min = 79, .max = 126 },
 464         .m1 = { .min = 12, .max = 22 },
 465         .m2 = { .min = 5, .max = 9 },
 466         .p = { .min = 14, .max = 42 },
 467         .p1 = { .min = 2, .max = 6 },
 468         .p2 = { .dot_limit = 225000,
 469                 .p2_slow = 7, .p2_fast = 7 },
 470 };
 471
 472 static const struct intel_limit intel_limits_vlv = {
 473          /*
 474           * These are the data rate limits (measured in fast clocks)
 475           * since those are the strictest limits we have. The fast
 476           * clock and actual rate limits are more relaxed, so checking
 477           * them would make no difference.
 478           */
 479         .dot = { .min = 25000 * 5, .max = 270000 * 5 },
 480         .vco = { .min = 4000000, .max = 6000000 },
 481         .n = { .min = 1, .max = 7 },
 482         .m1 = { .min = 2, .max = 3 },
 483         .m2 = { .min = 11, .max = 156 },
 484         .p1 = { .min = 2, .max = 3 },
 485         .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */
 486 };
 487
 488 static const struct intel_limit intel_limits_chv = {
 489         /*
 490          * These are the data rate limits (measured in fast clocks)
 491          * since those are the strictest limits we have.  The fast
 492          * clock and actual rate limits are more relaxed, so checking
 493          * them would make no difference.
 494          */
 495         .dot = { .min = 25000 * 5, .max = 540000 * 5},
 496         .vco = { .min = 4800000, .max = 6480000 },
 497         .n = { .min = 1, .max = 1 },
 498         .m1 = { .min = 2, .max = 2 },
 499         .m2 = { .min = 24 << 22, .max = 175 << 22 },
 500         .p1 = { .min = 2, .max = 4 },
 501         .p2 = { .p2_slow = 1, .p2_fast = 14 },
 502 };
 503
 504 static const struct intel_limit intel_limits_bxt = {
 505         /* FIXME: find real dot limits */
 506         .dot = { .min = 0, .max = INT_MAX },
 507         .vco = { .min = 4800000, .max = 6700000 },
 508         .n = { .min = 1, .max = 1 },
 509         .m1 = { .min = 2, .max = 2 },
 510         /* FIXME: find real m2 limits */
 511         .m2 = { .min = 2 << 22, .max = 255 << 22 },
 512         .p1 = { .min = 2, .max = 4 },
 513         .p2 = { .p2_slow = 1, .p2_fast = 20 },
 514 };
 515
 516 /* WA Display #0827: Gen9:all */
 517 static void
 518 skl_wa_827(struct drm_i915_private *dev_priv, enum pipe pipe, bool enable)
 519 {
 520         if (enable)
 521                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 522                            I915_READ(CLKGATE_DIS_PSL(pipe)) |
 523                            DUPS1_GATING_DIS | DUPS2_GATING_DIS);
 524         else
 525                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 526                            I915_READ(CLKGATE_DIS_PSL(pipe)) &
 527                            ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS));
 528 }
 529
 530 /* Wa_2006604312:icl */
 531 static void
 532 icl_wa_scalerclkgating(struct drm_i915_private *dev_priv, enum pipe pipe,
 533                        bool enable)
 534 {
 535         if (enable)
 536                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 537                            I915_READ(CLKGATE_DIS_PSL(pipe)) | DPFR_GATING_DIS);
 538         else
 539                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 540                            I915_READ(CLKGATE_DIS_PSL(pipe)) & ~DPFR_GATING_DIS);
 541 }
 542
 543 static bool
 544 needs_modeset(const struct intel_crtc_state *state)
 545 {
 546         return drm_atomic_crtc_needs_modeset(&state->uapi);
 547 }
 548
 549 bool
 550 is_trans_port_sync_mode(const struct intel_crtc_state *crtc_state)
 551 {
 552         return (crtc_state->master_transcoder != INVALID_TRANSCODER ||
 553                 crtc_state->sync_mode_slaves_mask);
 554 }
 555
 556 static bool
 557 is_trans_port_sync_master(const struct intel_crtc_state *crtc_state)
 558 {
 559         return (crtc_state->master_transcoder == INVALID_TRANSCODER &&
 560                 crtc_state->sync_mode_slaves_mask);
 561 }
 562
 563 static bool
 564 is_trans_port_sync_slave(const struct intel_crtc_state *crtc_state)
 565 {
 566         return crtc_state->master_transcoder != INVALID_TRANSCODER;
 567 }
 568
 569 /*
 570  * Platform specific helpers to calculate the port PLL loopback- (clock.m),
 571  * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast
 572  * (clock.dot) clock rates. This fast dot clock is fed to the port's IO logic.
 573  * The helpers' return value is the rate of the clock that is fed to the
 574  * display engine's pipe which can be the above fast dot clock rate or a
 575  * divided-down version of it.
 576  */
 577 /* m1 is reserved as 0 in Pineview, n is a ring counter */
 578 static int pnv_calc_dpll_params(int refclk, struct dpll *clock)
 579 {
 580         clock->m = clock->m2 + 2;
 581         clock->p = clock->p1 * clock->p2;
 582         if (WARN_ON(clock->n == 0 || clock->p == 0))
 583                 return 0;
 584         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 585         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 586
 587         return clock->dot;
 588 }
 589
 590 static u32 i9xx_dpll_compute_m(struct dpll *dpll)
 591 {
 592         return 5 * (dpll->m1 + 2) + (dpll->m2 + 2);
 593 }
 594
 595 static int i9xx_calc_dpll_params(int refclk, struct dpll *clock)
 596 {
 597         clock->m = i9xx_dpll_compute_m(clock);
 598         clock->p = clock->p1 * clock->p2;
 599         if (WARN_ON(clock->n + 2 == 0 || clock->p == 0))
 600                 return 0;
 601         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n + 2);
 602         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 603
 604         return clock->dot;
 605 }
 606
 607 static int vlv_calc_dpll_params(int refclk, struct dpll *clock)
 608 {
 609         clock->m = clock->m1 * clock->m2;
 610         clock->p = clock->p1 * clock->p2;
 611         if (WARN_ON(clock->n == 0 || clock->p == 0))
 612                 return 0;
 613         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 614         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 615
 616         return clock->dot / 5;
 617 }
 618
 619 int chv_calc_dpll_params(int refclk, struct dpll *clock)
 620 {
 621         clock->m = clock->m1 * clock->m2;
 622         clock->p = clock->p1 * clock->p2;
 623         if (WARN_ON(clock->n == 0 || clock->p == 0))
 624                 return 0;
 625         clock->vco = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(refclk, clock->m),
 626                                            clock->n << 22);
 627         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 628
 629         return clock->dot / 5;
 630 }
 631
 632 #define INTELPllInvalid(s)   do { /* DRM_DEBUG(s); */ return false; } while (0)
 633
 634 /*
 635  * Returns whether the given set of divisors are valid for a given refclk with
 636  * the given connectors.
 637  */
 638 static bool intel_PLL_is_valid(struct drm_i915_private *dev_priv,
 639                                const struct intel_limit *limit,
 640                                const struct dpll *clock)
 641 {
 642         if (clock->n   < limit->n.min   || limit->n.max   < clock->n)
 643                 INTELPllInvalid("n out of range\n");
 644         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
 645                 INTELPllInvalid("p1 out of range\n");
 646         if (clock->m2  < limit->m2.min  || limit->m2.max  < clock->m2)
 647                 INTELPllInvalid("m2 out of range\n");
 648         if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
 649                 INTELPllInvalid("m1 out of range\n");
 650
 651         if (!IS_PINEVIEW(dev_priv) && !IS_VALLEYVIEW(dev_priv) &&
 652             !IS_CHERRYVIEW(dev_priv) && !IS_GEN9_LP(dev_priv))
 653                 if (clock->m1 <= clock->m2)
 654                         INTELPllInvalid("m1 <= m2\n");
 655
 656         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
 657             !IS_GEN9_LP(dev_priv)) {
 658                 if (clock->p < limit->p.min || limit->p.max < clock->p)
 659                         INTELPllInvalid("p out of range\n");
 660                 if (clock->m < limit->m.min || limit->m.max < clock->m)
 661                         INTELPllInvalid("m out of range\n");
 662         }
 663
 664         if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
 665                 INTELPllInvalid("vco out of range\n");
 666         /* XXX: We may need to be checking "Dot clock" depending on the multiplier,
 667          * connector, etc., rather than just a single range.
 668          */
 669         if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
 670                 INTELPllInvalid("dot out of range\n");
 671
 672         return true;
 673 }
 674
 675 static int
 676 i9xx_select_p2_div(const struct intel_limit *limit,
 677                    const struct intel_crtc_state *crtc_state,
 678                    int target)
 679 {
 680         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 681
 682         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 683                 /*
 684                  * For LVDS just rely on its current settings for dual-channel.
 685                  * We haven't figured out how to reliably set up different
 686                  * single/dual channel state, if we even can.
 687                  */
 688                 if (intel_is_dual_link_lvds(dev_priv))
 689                         return limit->p2.p2_fast;
 690                 else
 691                         return limit->p2.p2_slow;
 692         } else {
 693                 if (target < limit->p2.dot_limit)
 694                         return limit->p2.p2_slow;
 695                 else
 696                         return limit->p2.p2_fast;
 697         }
 698 }
 699
 700 /*
 701  * Returns a set of divisors for the desired target clock with the given
 702  * refclk, or FALSE.  The returned values represent the clock equation:
 703  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 704  *
 705  * Target and reference clocks are specified in kHz.
 706  *
 707  * If match_clock is provided, then best_clock P divider must match the P
 708  * divider from @match_clock used for LVDS downclocking.
 709  */
 710 static bool
 711 i9xx_find_best_dpll(const struct intel_limit *limit,
 712                     struct intel_crtc_state *crtc_state,
 713                     int target, int refclk, struct dpll *match_clock,
 714                     struct dpll *best_clock)
 715 {
 716         struct drm_device *dev = crtc_state->uapi.crtc->dev;
 717         struct dpll clock;
 718         int err = target;
 719
 720         memset(best_clock, 0, sizeof(*best_clock));
 721
 722         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 723
 724         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 725              clock.m1++) {
 726                 for (clock.m2 = limit->m2.min;
 727                      clock.m2 <= limit->m2.max; clock.m2++) {
 728                         if (clock.m2 >= clock.m1)
 729                                 break;
 730                         for (clock.n = limit->n.min;
 731                              clock.n <= limit->n.max; clock.n++) {
 732                                 for (clock.p1 = limit->p1.min;
 733                                         clock.p1 <= limit->p1.max; clock.p1++) {
 734                                         int this_err;
 735
 736                                         i9xx_calc_dpll_params(refclk, &clock);
 737                                         if (!intel_PLL_is_valid(to_i915(dev),
 738                                                                 limit,
 739                                                                 &clock))
 740                                                 continue;
 741                                         if (match_clock &&
 742                                             clock.p != match_clock->p)
 743                                                 continue;
 744
 745                                         this_err = abs(clock.dot - target);
 746                                         if (this_err < err) {
 747                                                 *best_clock = clock;
 748                                                 err = this_err;
 749                                         }
 750                                 }
 751                         }
 752                 }
 753         }
 754
 755         return (err != target);
 756 }
 757
 758 /*
 759  * Returns a set of divisors for the desired target clock with the given
 760  * refclk, or FALSE.  The returned values represent the clock equation:
 761  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 762  *
 763  * Target and reference clocks are specified in kHz.
 764  *
 765  * If match_clock is provided, then best_clock P divider must match the P
 766  * divider from @match_clock used for LVDS downclocking.
 767  */
 768 static bool
 769 pnv_find_best_dpll(const struct intel_limit *limit,
 770                    struct intel_crtc_state *crtc_state,
 771                    int target, int refclk, struct dpll *match_clock,
 772                    struct dpll *best_clock)
 773 {
 774         struct drm_device *dev = crtc_state->uapi.crtc->dev;
 775         struct dpll clock;
 776         int err = target;
 777
 778         memset(best_clock, 0, sizeof(*best_clock));
 779
 780         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 781
 782         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 783              clock.m1++) {
 784                 for (clock.m2 = limit->m2.min;
 785                      clock.m2 <= limit->m2.max; clock.m2++) {
 786                         for (clock.n = limit->n.min;
 787                              clock.n <= limit->n.max; clock.n++) {
 788                                 for (clock.p1 = limit->p1.min;
 789                                         clock.p1 <= limit->p1.max; clock.p1++) {
 790                                         int this_err;
 791
 792                                         pnv_calc_dpll_params(refclk, &clock);
 793                                         if (!intel_PLL_is_valid(to_i915(dev),
 794                                                                 limit,
 795                                                                 &clock))
 796                                                 continue;
 797                                         if (match_clock &&
 798                                             clock.p != match_clock->p)
 799                                                 continue;
 800
 801                                         this_err = abs(clock.dot - target);
 802                                         if (this_err < err) {
 803                                                 *best_clock = clock;
 804                                                 err = this_err;
 805                                         }
 806                                 }
 807                         }
 808                 }
 809         }
 810
 811         return (err != target);
 812 }
 813
 814 /*
 815  * Returns a set of divisors for the desired target clock with the given
 816  * refclk, or FALSE.  The returned values represent the clock equation:
 817  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 818  *
 819  * Target and reference clocks are specified in kHz.
 820  *
 821  * If match_clock is provided, then best_clock P divider must match the P
 822  * divider from @match_clock used for LVDS downclocking.
 823  */
 824 static bool
 825 g4x_find_best_dpll(const struct intel_limit *limit,
 826                    struct intel_crtc_state *crtc_state,
 827                    int target, int refclk, struct dpll *match_clock,
 828                    struct dpll *best_clock)
 829 {
 830         struct drm_device *dev = crtc_state->uapi.crtc->dev;
 831         struct dpll clock;
 832         int max_n;
 833         bool found = false;
 834         /* approximately equals target * 0.00585 */
 835         int err_most = (target >> 8) + (target >> 9);
 836
 837         memset(best_clock, 0, sizeof(*best_clock));
 838
 839         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 840
 841         max_n = limit->n.max;
 842         /* based on hardware requirement, prefer smaller n to precision */
 843         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 844                 /* based on hardware requirement, prefere larger m1,m2 */
 845                 for (clock.m1 = limit->m1.max;
 846                      clock.m1 >= limit->m1.min; clock.m1--) {
 847                         for (clock.m2 = limit->m2.max;
 848                              clock.m2 >= limit->m2.min; clock.m2--) {
 849                                 for (clock.p1 = limit->p1.max;
 850                                      clock.p1 >= limit->p1.min; clock.p1--) {
 851                                         int this_err;
 852
 853                                         i9xx_calc_dpll_params(refclk, &clock);
 854                                         if (!intel_PLL_is_valid(to_i915(dev),
 855                                                                 limit,
 856                                                                 &clock))
 857                                                 continue;
 858
 859                                         this_err = abs(clock.dot - target);
 860                                         if (this_err < err_most) {
 861                                                 *best_clock = clock;
 862                                                 err_most = this_err;
 863                                                 max_n = clock.n;
 864                                                 found = true;
 865                                         }
 866                                 }
 867                         }
 868                 }
 869         }
 870         return found;
 871 }
 872
 873 /*
 874  * Check if the calculated PLL configuration is more optimal compared to the
 875  * best configuration and error found so far. Return the calculated error.
 876  */
 877 static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
 878                                const struct dpll *calculated_clock,
 879                                const struct dpll *best_clock,
 880                                unsigned int best_error_ppm,
 881                                unsigned int *error_ppm)
 882 {
 883         /*
 884          * For CHV ignore the error and consider only the P value.
 885          * Prefer a bigger P value based on HW requirements.
 886          */
 887         if (IS_CHERRYVIEW(to_i915(dev))) {
 888                 *error_ppm = 0;
 889
 890                 return calculated_clock->p > best_clock->p;
 891         }
 892
 893         if (WARN_ON_ONCE(!target_freq))
 894                 return false;
 895
 896         *error_ppm = div_u64(1000000ULL *
 897                                 abs(target_freq - calculated_clock->dot),
 898                              target_freq);
 899         /*
 900          * Prefer a better P value over a better (smaller) error if the error
 901          * is small. Ensure this preference for future configurations too by
 902          * setting the error to 0.
 903          */
 904         if (*error_ppm < 100 && calculated_clock->p > best_clock->p) {
 905                 *error_ppm = 0;
 906
 907                 return true;
 908         }
 909
 910         return *error_ppm + 10 < best_error_ppm;
 911 }
 912
 913 /*
 914  * Returns a set of divisors for the desired target clock with the given
 915  * refclk, or FALSE.  The returned values represent the clock equation:
 916  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 917  */
 918 static bool
 919 vlv_find_best_dpll(const struct intel_limit *limit,
 920                    struct intel_crtc_state *crtc_state,
 921                    int target, int refclk, struct dpll *match_clock,
 922                    struct dpll *best_clock)
 923 {
 924         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 925         struct drm_device *dev = crtc->base.dev;
 926         struct dpll clock;
 927         unsigned int bestppm = 1000000;
 928         /* min update 19.2 MHz */
 929         int max_n = min(limit->n.max, refclk / 19200);
 930         bool found = false;
 931
 932         target *= 5; /* fast clock */
 933
 934         memset(best_clock, 0, sizeof(*best_clock));
 935
 936         /* based on hardware requirement, prefer smaller n to precision */
 937         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 938                 for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 939                         for (clock.p2 = limit->p2.p2_fast; clock.p2 >= limit->p2.p2_slow;
 940                              clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 941                                 clock.p = clock.p1 * clock.p2;
 942                                 /* based on hardware requirement, prefer bigger m1,m2 values */
 943                                 for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) {
 944                                         unsigned int ppm;
 945
 946                                         clock.m2 = DIV_ROUND_CLOSEST(target * clock.p * clock.n,
 947                                                                      refclk * clock.m1);
 948
 949                                         vlv_calc_dpll_params(refclk, &clock);
 950
 951                                         if (!intel_PLL_is_valid(to_i915(dev),
 952                                                                 limit,
 953                                                                 &clock))
 954                                                 continue;
 955
 956                                         if (!vlv_PLL_is_optimal(dev, target,
 957                                                                 &clock,
 958                                                                 best_clock,
 959                                                                 bestppm, &ppm))
 960                                                 continue;
 961
 962                                         *best_clock = clock;
 963                                         bestppm = ppm;
 964                                         found = true;
 965                                 }
 966                         }
 967                 }
 968         }
 969
 970         return found;
 971 }
 972
 973 /*
 974  * Returns a set of divisors for the desired target clock with the given
 975  * refclk, or FALSE.  The returned values represent the clock equation:
 976  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 977  */
 978 static bool
 979 chv_find_best_dpll(const struct intel_limit *limit,
 980                    struct intel_crtc_state *crtc_state,
 981                    int target, int refclk, struct dpll *match_clock,
 982                    struct dpll *best_clock)
 983 {
 984         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 985         struct drm_device *dev = crtc->base.dev;
 986         unsigned int best_error_ppm;
 987         struct dpll clock;
 988         u64 m2;
 989         int found = false;
 990
 991         memset(best_clock, 0, sizeof(*best_clock));
 992         best_error_ppm = 1000000;
 993
 994         /*
 995          * Based on hardware doc, the n always set to 1, and m1 always
 996          * set to 2.  If requires to support 200Mhz refclk, we need to
 997          * revisit this because n may not 1 anymore.
 998          */
 999         clock.n = 1, clock.m1 = 2;
1000         target *= 5;    /* fast clock */
1001
1002         for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
1003                 for (clock.p2 = limit->p2.p2_fast;
1004                                 clock.p2 >= limit->p2.p2_slow;
1005                                 clock.p2 -= clock.p2 > 10 ? 2 : 1) {
1006                         unsigned int error_ppm;
1007
1008                         clock.p = clock.p1 * clock.p2;
1009
1010                         m2 = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(target, clock.p * clock.n) << 22,
1011                                                    refclk * clock.m1);
1012
1013                         if (m2 > INT_MAX/clock.m1)
1014                                 continue;
1015
1016                         clock.m2 = m2;
1017
1018                         chv_calc_dpll_params(refclk, &clock);
1019
1020                         if (!intel_PLL_is_valid(to_i915(dev), limit, &clock))
1021                                 continue;
1022
1023                         if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock,
1024                                                 best_error_ppm, &error_ppm))
1025                                 continue;
1026
1027                         *best_clock = clock;
1028                         best_error_ppm = error_ppm;
1029                         found = true;
1030                 }
1031         }
1032
1033         return found;
1034 }
1035
1036 bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state,
1037                         struct dpll *best_clock)
1038 {
1039         int refclk = 100000;
1040         const struct intel_limit *limit = &intel_limits_bxt;
1041
1042         return chv_find_best_dpll(limit, crtc_state,
1043                                   crtc_state->port_clock, refclk,
1044                                   NULL, best_clock);
1045 }
1046
1047 static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv,
1048                                     enum pipe pipe)
1049 {
1050         i915_reg_t reg = PIPEDSL(pipe);
1051         u32 line1, line2;
1052         u32 line_mask;
1053
1054         if (IS_GEN(dev_priv, 2))
1055                 line_mask = DSL_LINEMASK_GEN2;
1056         else
1057                 line_mask = DSL_LINEMASK_GEN3;
1058
1059         line1 = I915_READ(reg) & line_mask;
1060         msleep(5);
1061         line2 = I915_READ(reg) & line_mask;
1062
1063         return line1 != line2;
1064 }
1065
1066 static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state)
1067 {
1068         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1069         enum pipe pipe = crtc->pipe;
1070
1071         /* Wait for the display line to settle/start moving */
1072         if (wait_for(pipe_scanline_is_moving(dev_priv, pipe) == state, 100))
1073                 DRM_ERROR("pipe %c scanline %s wait timed out\n",
1074                           pipe_name(pipe), onoff(state));
1075 }
1076
1077 static void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc)
1078 {
1079         wait_for_pipe_scanline_moving(crtc, false);
1080 }
1081
1082 static void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
1083 {
1084         wait_for_pipe_scanline_moving(crtc, true);
1085 }
1086
1087 static void
1088 intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
1089 {
1090         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
1091         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1092
1093         if (INTEL_GEN(dev_priv) >= 4) {
1094                 enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1095                 i915_reg_t reg = PIPECONF(cpu_transcoder);
1096
1097                 /* Wait for the Pipe State to go off */
1098                 if (intel_de_wait_for_clear(dev_priv, reg,
1099                                             I965_PIPECONF_ACTIVE, 100))
1100                         WARN(1, "pipe_off wait timed out\n");
1101         } else {
1102                 intel_wait_for_pipe_scanline_stopped(crtc);
1103         }
1104 }
1105
1106 /* Only for pre-ILK configs */
1107 void assert_pll(struct drm_i915_private *dev_priv,
1108                 enum pipe pipe, bool state)
1109 {
1110         u32 val;
1111         bool cur_state;
1112
1113         val = I915_READ(DPLL(pipe));
1114         cur_state = !!(val & DPLL_VCO_ENABLE);
1115         I915_STATE_WARN(cur_state != state,
1116              "PLL state assertion failure (expected %s, current %s)\n",
1117                         onoff(state), onoff(cur_state));
1118 }
1119
1120 /* XXX: the dsi pll is shared between MIPI DSI ports */
1121 void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state)
1122 {
1123         u32 val;
1124         bool cur_state;
1125
1126         vlv_cck_get(dev_priv);
1127         val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL);
1128         vlv_cck_put(dev_priv);
1129
1130         cur_state = val & DSI_PLL_VCO_EN;
1131         I915_STATE_WARN(cur_state != state,
1132              "DSI PLL state assertion failure (expected %s, current %s)\n",
1133                         onoff(state), onoff(cur_state));
1134 }
1135
1136 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
1137                           enum pipe pipe, bool state)
1138 {
1139         bool cur_state;
1140
1141         if (HAS_DDI(dev_priv)) {
1142                 /*
1143                  * DDI does not have a specific FDI_TX register.
1144                  *
1145                  * FDI is never fed from EDP transcoder
1146                  * so pipe->transcoder cast is fine here.
1147                  */
1148                 enum transcoder cpu_transcoder = (enum transcoder)pipe;
1149                 u32 val = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
1150                 cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
1151         } else {
1152                 u32 val = I915_READ(FDI_TX_CTL(pipe));
1153                 cur_state = !!(val & FDI_TX_ENABLE);
1154         }
1155         I915_STATE_WARN(cur_state != state,
1156              "FDI TX state assertion failure (expected %s, current %s)\n",
1157                         onoff(state), onoff(cur_state));
1158 }
1159 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
1160 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
1161
1162 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
1163                           enum pipe pipe, bool state)
1164 {
1165         u32 val;
1166         bool cur_state;
1167
1168         val = I915_READ(FDI_RX_CTL(pipe));
1169         cur_state = !!(val & FDI_RX_ENABLE);
1170         I915_STATE_WARN(cur_state != state,
1171              "FDI RX state assertion failure (expected %s, current %s)\n",
1172                         onoff(state), onoff(cur_state));
1173 }
1174 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
1175 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
1176
1177 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
1178                                       enum pipe pipe)
1179 {
1180         u32 val;
1181
1182         /* ILK FDI PLL is always enabled */
1183         if (IS_GEN(dev_priv, 5))
1184                 return;
1185
1186         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
1187         if (HAS_DDI(dev_priv))
1188                 return;
1189
1190         val = I915_READ(FDI_TX_CTL(pipe));
1191         I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
1192 }
1193
1194 void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
1195                        enum pipe pipe, bool state)
1196 {
1197         u32 val;
1198         bool cur_state;
1199
1200         val = I915_READ(FDI_RX_CTL(pipe));
1201         cur_state = !!(val & FDI_RX_PLL_ENABLE);
1202         I915_STATE_WARN(cur_state != state,
1203              "FDI RX PLL assertion failure (expected %s, current %s)\n",
1204                         onoff(state), onoff(cur_state));
1205 }
1206
1207 void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe)
1208 {
1209         i915_reg_t pp_reg;
1210         u32 val;
1211         enum pipe panel_pipe = INVALID_PIPE;
1212         bool locked = true;
1213
1214         if (WARN_ON(HAS_DDI(dev_priv)))
1215                 return;
1216
1217         if (HAS_PCH_SPLIT(dev_priv)) {
1218                 u32 port_sel;
1219
1220                 pp_reg = PP_CONTROL(0);
1221                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1222
1223                 switch (port_sel) {
1224                 case PANEL_PORT_SELECT_LVDS:
1225                         intel_lvds_port_enabled(dev_priv, PCH_LVDS, &panel_pipe);
1226                         break;
1227                 case PANEL_PORT_SELECT_DPA:
1228                         intel_dp_port_enabled(dev_priv, DP_A, PORT_A, &panel_pipe);
1229                         break;
1230                 case PANEL_PORT_SELECT_DPC:
1231                         intel_dp_port_enabled(dev_priv, PCH_DP_C, PORT_C, &panel_pipe);
1232                         break;
1233                 case PANEL_PORT_SELECT_DPD:
1234                         intel_dp_port_enabled(dev_priv, PCH_DP_D, PORT_D, &panel_pipe);
1235                         break;
1236                 default:
1237                         MISSING_CASE(port_sel);
1238                         break;
1239                 }
1240         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1241                 /* presumably write lock depends on pipe, not port select */
1242                 pp_reg = PP_CONTROL(pipe);
1243                 panel_pipe = pipe;
1244         } else {
1245                 u32 port_sel;
1246
1247                 pp_reg = PP_CONTROL(0);
1248                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1249
1250                 WARN_ON(port_sel != PANEL_PORT_SELECT_LVDS);
1251                 intel_lvds_port_enabled(dev_priv, LVDS, &panel_pipe);
1252         }
1253
1254         val = I915_READ(pp_reg);
1255         if (!(val & PANEL_POWER_ON) ||
1256             ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
1257                 locked = false;
1258
1259         I915_STATE_WARN(panel_pipe == pipe && locked,
1260              "panel assertion failure, pipe %c regs locked\n",
1261              pipe_name(pipe));
1262 }
1263
1264 void assert_pipe(struct drm_i915_private *dev_priv,
1265                  enum transcoder cpu_transcoder, bool state)
1266 {
1267         bool cur_state;
1268         enum intel_display_power_domain power_domain;
1269         intel_wakeref_t wakeref;
1270
1271         /* we keep both pipes enabled on 830 */
1272         if (IS_I830(dev_priv))
1273                 state = true;
1274
1275         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
1276         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
1277         if (wakeref) {
1278                 u32 val = I915_READ(PIPECONF(cpu_transcoder));
1279                 cur_state = !!(val & PIPECONF_ENABLE);
1280
1281                 intel_display_power_put(dev_priv, power_domain, wakeref);
1282         } else {
1283                 cur_state = false;
1284         }
1285
1286         I915_STATE_WARN(cur_state != state,
1287                         "transcoder %s assertion failure (expected %s, current %s)\n",
1288                         transcoder_name(cpu_transcoder),
1289                         onoff(state), onoff(cur_state));
1290 }
1291
1292 static void assert_plane(struct intel_plane *plane, bool state)
1293 {
1294         enum pipe pipe;
1295         bool cur_state;
1296
1297         cur_state = plane->get_hw_state(plane, &pipe);
1298
1299         I915_STATE_WARN(cur_state != state,
1300                         "%s assertion failure (expected %s, current %s)\n",
1301                         plane->base.name, onoff(state), onoff(cur_state));
1302 }
1303
1304 #define assert_plane_enabled(p) assert_plane(p, true)
1305 #define assert_plane_disabled(p) assert_plane(p, false)
1306
1307 static void assert_planes_disabled(struct intel_crtc *crtc)
1308 {
1309         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1310         struct intel_plane *plane;
1311
1312         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
1313                 assert_plane_disabled(plane);
1314 }
1315
1316 static void assert_vblank_disabled(struct drm_crtc *crtc)
1317 {
1318         if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
1319                 drm_crtc_vblank_put(crtc);
1320 }
1321
1322 void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
1323                                     enum pipe pipe)
1324 {
1325         u32 val;
1326         bool enabled;
1327
1328         val = I915_READ(PCH_TRANSCONF(pipe));
1329         enabled = !!(val & TRANS_ENABLE);
1330         I915_STATE_WARN(enabled,
1331              "transcoder assertion failed, should be off on pipe %c but is still active\n",
1332              pipe_name(pipe));
1333 }
1334
1335 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
1336                                    enum pipe pipe, enum port port,
1337                                    i915_reg_t dp_reg)
1338 {
1339         enum pipe port_pipe;
1340         bool state;
1341
1342         state = intel_dp_port_enabled(dev_priv, dp_reg, port, &port_pipe);
1343
1344         I915_STATE_WARN(state && port_pipe == pipe,
1345                         "PCH DP %c enabled on transcoder %c, should be disabled\n",
1346                         port_name(port), pipe_name(pipe));
1347
1348         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1349                         "IBX PCH DP %c still using transcoder B\n",
1350                         port_name(port));
1351 }
1352
1353 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
1354                                      enum pipe pipe, enum port port,
1355                                      i915_reg_t hdmi_reg)
1356 {
1357         enum pipe port_pipe;
1358         bool state;
1359
1360         state = intel_sdvo_port_enabled(dev_priv, hdmi_reg, &port_pipe);
1361
1362         I915_STATE_WARN(state && port_pipe == pipe,
1363                         "PCH HDMI %c enabled on transcoder %c, should be disabled\n",
1364                         port_name(port), pipe_name(pipe));
1365
1366         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1367                         "IBX PCH HDMI %c still using transcoder B\n",
1368                         port_name(port));
1369 }
1370
1371 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
1372                                       enum pipe pipe)
1373 {
1374         enum pipe port_pipe;
1375
1376         assert_pch_dp_disabled(dev_priv, pipe, PORT_B, PCH_DP_B);
1377         assert_pch_dp_disabled(dev_priv, pipe, PORT_C, PCH_DP_C);
1378         assert_pch_dp_disabled(dev_priv, pipe, PORT_D, PCH_DP_D);
1379
1380         I915_STATE_WARN(intel_crt_port_enabled(dev_priv, PCH_ADPA, &port_pipe) &&
1381                         port_pipe == pipe,
1382                         "PCH VGA enabled on transcoder %c, should be disabled\n",
1383                         pipe_name(pipe));
1384
1385         I915_STATE_WARN(intel_lvds_port_enabled(dev_priv, PCH_LVDS, &port_pipe) &&
1386                         port_pipe == pipe,
1387                         "PCH LVDS enabled on transcoder %c, should be disabled\n",
1388                         pipe_name(pipe));
1389
1390         /* PCH SDVOB multiplex with HDMIB */
1391         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_B, PCH_HDMIB);
1392         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_C, PCH_HDMIC);
1393         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_D, PCH_HDMID);
1394 }
1395
1396 static void _vlv_enable_pll(struct intel_crtc *crtc,
1397                             const struct intel_crtc_state *pipe_config)
1398 {
1399         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1400         enum pipe pipe = crtc->pipe;
1401
1402         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1403         POSTING_READ(DPLL(pipe));
1404         udelay(150);
1405
1406         if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1))
1407                 DRM_ERROR("DPLL %d failed to lock\n", pipe);
1408 }
1409
1410 static void vlv_enable_pll(struct intel_crtc *crtc,
1411                            const struct intel_crtc_state *pipe_config)
1412 {
1413         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1414         enum pipe pipe = crtc->pipe;
1415
1416         assert_pipe_disabled(dev_priv, pipe_config->cpu_transcoder);
1417
1418         /* PLL is protected by panel, make sure we can write it */
1419         assert_panel_unlocked(dev_priv, pipe);
1420
1421         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1422                 _vlv_enable_pll(crtc, pipe_config);
1423
1424         I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1425         POSTING_READ(DPLL_MD(pipe));
1426 }
1427
1428
1429 static void _chv_enable_pll(struct intel_crtc *crtc,
1430                             const struct intel_crtc_state *pipe_config)
1431 {
1432         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1433         enum pipe pipe = crtc->pipe;
1434         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1435         u32 tmp;
1436
1437         vlv_dpio_get(dev_priv);
1438
1439         /* Enable back the 10bit clock to display controller */
1440         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1441         tmp |= DPIO_DCLKP_EN;
1442         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp);
1443
1444         vlv_dpio_put(dev_priv);
1445
1446         /*
1447          * Need to wait > 100ns between dclkp clock enable bit and PLL enable.
1448          */
1449         udelay(1);
1450
1451         /* Enable PLL */
1452         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1453
1454         /* Check PLL is locked */
1455         if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1))
1456                 DRM_ERROR("PLL %d failed to lock\n", pipe);
1457 }
1458
1459 static void chv_enable_pll(struct intel_crtc *crtc,
1460                            const struct intel_crtc_state *pipe_config)
1461 {
1462         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1463         enum pipe pipe = crtc->pipe;
1464
1465         assert_pipe_disabled(dev_priv, pipe_config->cpu_transcoder);
1466
1467         /* PLL is protected by panel, make sure we can write it */
1468         assert_panel_unlocked(dev_priv, pipe);
1469
1470         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1471                 _chv_enable_pll(crtc, pipe_config);
1472
1473         if (pipe != PIPE_A) {
1474                 /*
1475                  * WaPixelRepeatModeFixForC0:chv
1476                  *
1477                  * DPLLCMD is AWOL. Use chicken bits to propagate
1478                  * the value from DPLLBMD to either pipe B or C.
1479                  */
1480                 I915_WRITE(CBR4_VLV, CBR_DPLLBMD_PIPE(pipe));
1481                 I915_WRITE(DPLL_MD(PIPE_B), pipe_config->dpll_hw_state.dpll_md);
1482                 I915_WRITE(CBR4_VLV, 0);
1483                 dev_priv->chv_dpll_md[pipe] = pipe_config->dpll_hw_state.dpll_md;
1484
1485                 /*
1486                  * DPLLB VGA mode also seems to cause problems.
1487                  * We should always have it disabled.
1488                  */
1489                 WARN_ON((I915_READ(DPLL(PIPE_B)) & DPLL_VGA_MODE_DIS) == 0);
1490         } else {
1491                 I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1492                 POSTING_READ(DPLL_MD(pipe));
1493         }
1494 }
1495
1496 static bool i9xx_has_pps(struct drm_i915_private *dev_priv)
1497 {
1498         if (IS_I830(dev_priv))
1499                 return false;
1500
1501         return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
1502 }
1503
1504 static void i9xx_enable_pll(struct intel_crtc *crtc,
1505                             const struct intel_crtc_state *crtc_state)
1506 {
1507         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1508         i915_reg_t reg = DPLL(crtc->pipe);
1509         u32 dpll = crtc_state->dpll_hw_state.dpll;
1510         int i;
1511
1512         assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder);
1513
1514         /* PLL is protected by panel, make sure we can write it */
1515         if (i9xx_has_pps(dev_priv))
1516                 assert_panel_unlocked(dev_priv, crtc->pipe);
1517
1518         /*
1519          * Apparently we need to have VGA mode enabled prior to changing
1520          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
1521          * dividers, even though the register value does change.
1522          */
1523         I915_WRITE(reg, dpll & ~DPLL_VGA_MODE_DIS);
1524         I915_WRITE(reg, dpll);
1525
1526         /* Wait for the clocks to stabilize. */
1527         POSTING_READ(reg);
1528         udelay(150);
1529
1530         if (INTEL_GEN(dev_priv) >= 4) {
1531                 I915_WRITE(DPLL_MD(crtc->pipe),
1532                            crtc_state->dpll_hw_state.dpll_md);
1533         } else {
1534                 /* The pixel multiplier can only be updated once the
1535                  * DPLL is enabled and the clocks are stable.
1536                  *
1537                  * So write it again.
1538                  */
1539                 I915_WRITE(reg, dpll);
1540         }
1541
1542         /* We do this three times for luck */
1543         for (i = 0; i < 3; i++) {
1544                 I915_WRITE(reg, dpll);
1545                 POSTING_READ(reg);
1546                 udelay(150); /* wait for warmup */
1547         }
1548 }
1549
1550 static void i9xx_disable_pll(const struct intel_crtc_state *crtc_state)
1551 {
1552         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1553         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1554         enum pipe pipe = crtc->pipe;
1555
1556         /* Don't disable pipe or pipe PLLs if needed */
1557         if (IS_I830(dev_priv))
1558                 return;
1559
1560         /* Make sure the pipe isn't still relying on us */
1561         assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder);
1562
1563         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
1564         POSTING_READ(DPLL(pipe));
1565 }
1566
1567 static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1568 {
1569         u32 val;
1570
1571         /* Make sure the pipe isn't still relying on us */
1572         assert_pipe_disabled(dev_priv, (enum transcoder)pipe);
1573
1574         val = DPLL_INTEGRATED_REF_CLK_VLV |
1575                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1576         if (pipe != PIPE_A)
1577                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1578
1579         I915_WRITE(DPLL(pipe), val);
1580         POSTING_READ(DPLL(pipe));
1581 }
1582
1583 static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1584 {
1585         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1586         u32 val;
1587
1588         /* Make sure the pipe isn't still relying on us */
1589         assert_pipe_disabled(dev_priv, (enum transcoder)pipe);
1590
1591         val = DPLL_SSC_REF_CLK_CHV |
1592                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1593         if (pipe != PIPE_A)
1594                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1595
1596         I915_WRITE(DPLL(pipe), val);
1597         POSTING_READ(DPLL(pipe));
1598
1599         vlv_dpio_get(dev_priv);
1600
1601         /* Disable 10bit clock to display controller */
1602         val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1603         val &= ~DPIO_DCLKP_EN;
1604         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
1605
1606         vlv_dpio_put(dev_priv);
1607 }
1608
1609 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
1610                          struct intel_digital_port *dport,
1611                          unsigned int expected_mask)
1612 {
1613         u32 port_mask;
1614         i915_reg_t dpll_reg;
1615
1616         switch (dport->base.port) {
1617         case PORT_B:
1618                 port_mask = DPLL_PORTB_READY_MASK;
1619                 dpll_reg = DPLL(0);
1620                 break;
1621         case PORT_C:
1622                 port_mask = DPLL_PORTC_READY_MASK;
1623                 dpll_reg = DPLL(0);
1624                 expected_mask <<= 4;
1625                 break;
1626         case PORT_D:
1627                 port_mask = DPLL_PORTD_READY_MASK;
1628                 dpll_reg = DPIO_PHY_STATUS;
1629                 break;
1630         default:
1631                 BUG();
1632         }
1633
1634         if (intel_de_wait_for_register(dev_priv, dpll_reg,
1635                                        port_mask, expected_mask, 1000))
1636                 WARN(1, "timed out waiting for [ENCODER:%d:%s] port ready: got 0x%x, expected 0x%x\n",
1637                      dport->base.base.base.id, dport->base.base.name,
1638                      I915_READ(dpll_reg) & port_mask, expected_mask);
1639 }
1640
1641 static void ironlake_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
1642 {
1643         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1644         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1645         enum pipe pipe = crtc->pipe;
1646         i915_reg_t reg;
1647         u32 val, pipeconf_val;
1648
1649         /* Make sure PCH DPLL is enabled */
1650         assert_shared_dpll_enabled(dev_priv, crtc_state->shared_dpll);
1651
1652         /* FDI must be feeding us bits for PCH ports */
1653         assert_fdi_tx_enabled(dev_priv, pipe);
1654         assert_fdi_rx_enabled(dev_priv, pipe);
1655
1656         if (HAS_PCH_CPT(dev_priv)) {
1657                 reg = TRANS_CHICKEN2(pipe);
1658                 val = I915_READ(reg);
1659                 /*
1660                  * Workaround: Set the timing override bit
1661                  * before enabling the pch transcoder.
1662                  */
1663                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1664                 /* Configure frame start delay to match the CPU */
1665                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
1666                 val |= TRANS_CHICKEN2_FRAME_START_DELAY(0);
1667                 I915_WRITE(reg, val);
1668         }
1669
1670         reg = PCH_TRANSCONF(pipe);
1671         val = I915_READ(reg);
1672         pipeconf_val = I915_READ(PIPECONF(pipe));
1673
1674         if (HAS_PCH_IBX(dev_priv)) {
1675                 /* Configure frame start delay to match the CPU */
1676                 val &= ~TRANS_FRAME_START_DELAY_MASK;
1677                 val |= TRANS_FRAME_START_DELAY(0);
1678
1679                 /*
1680                  * Make the BPC in transcoder be consistent with
1681                  * that in pipeconf reg. For HDMI we must use 8bpc
1682                  * here for both 8bpc and 12bpc.
1683                  */
1684                 val &= ~PIPECONF_BPC_MASK;
1685                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
1686                         val |= PIPECONF_8BPC;
1687                 else
1688                         val |= pipeconf_val & PIPECONF_BPC_MASK;
1689         }
1690
1691         val &= ~TRANS_INTERLACE_MASK;
1692         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK) {
1693                 if (HAS_PCH_IBX(dev_priv) &&
1694                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
1695                         val |= TRANS_LEGACY_INTERLACED_ILK;
1696                 else
1697                         val |= TRANS_INTERLACED;
1698         } else {
1699                 val |= TRANS_PROGRESSIVE;
1700         }
1701
1702         I915_WRITE(reg, val | TRANS_ENABLE);
1703         if (intel_de_wait_for_set(dev_priv, reg, TRANS_STATE_ENABLE, 100))
1704                 DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe));
1705 }
1706
1707 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1708                                       enum transcoder cpu_transcoder)
1709 {
1710         u32 val, pipeconf_val;
1711
1712         /* FDI must be feeding us bits for PCH ports */
1713         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
1714         assert_fdi_rx_enabled(dev_priv, PIPE_A);
1715
1716         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1717         /* Workaround: set timing override bit. */
1718         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1719         /* Configure frame start delay to match the CPU */
1720         val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
1721         val |= TRANS_CHICKEN2_FRAME_START_DELAY(0);
1722         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1723
1724         val = TRANS_ENABLE;
1725         pipeconf_val = I915_READ(PIPECONF(cpu_transcoder));
1726
1727         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
1728             PIPECONF_INTERLACED_ILK)
1729                 val |= TRANS_INTERLACED;
1730         else
1731                 val |= TRANS_PROGRESSIVE;
1732
1733         I915_WRITE(LPT_TRANSCONF, val);
1734         if (intel_de_wait_for_set(dev_priv, LPT_TRANSCONF,
1735                                   TRANS_STATE_ENABLE, 100))
1736                 DRM_ERROR("Failed to enable PCH transcoder\n");
1737 }
1738
1739 static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
1740                                             enum pipe pipe)
1741 {
1742         i915_reg_t reg;
1743         u32 val;
1744
1745         /* FDI relies on the transcoder */
1746         assert_fdi_tx_disabled(dev_priv, pipe);
1747         assert_fdi_rx_disabled(dev_priv, pipe);
1748
1749         /* Ports must be off as well */
1750         assert_pch_ports_disabled(dev_priv, pipe);
1751
1752         reg = PCH_TRANSCONF(pipe);
1753         val = I915_READ(reg);
1754         val &= ~TRANS_ENABLE;
1755         I915_WRITE(reg, val);
1756         /* wait for PCH transcoder off, transcoder state */
1757         if (intel_de_wait_for_clear(dev_priv, reg, TRANS_STATE_ENABLE, 50))
1758                 DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
1759
1760         if (HAS_PCH_CPT(dev_priv)) {
1761                 /* Workaround: Clear the timing override chicken bit again. */
1762                 reg = TRANS_CHICKEN2(pipe);
1763                 val = I915_READ(reg);
1764                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1765                 I915_WRITE(reg, val);
1766         }
1767 }
1768
1769 void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
1770 {
1771         u32 val;
1772
1773         val = I915_READ(LPT_TRANSCONF);
1774         val &= ~TRANS_ENABLE;
1775         I915_WRITE(LPT_TRANSCONF, val);
1776         /* wait for PCH transcoder off, transcoder state */
1777         if (intel_de_wait_for_clear(dev_priv, LPT_TRANSCONF,
1778                                     TRANS_STATE_ENABLE, 50))
1779                 DRM_ERROR("Failed to disable PCH transcoder\n");
1780
1781         /* Workaround: clear timing override bit. */
1782         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1783         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1784         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1785 }
1786
1787 enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
1788 {
1789         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1790
1791         if (HAS_PCH_LPT(dev_priv))
1792                 return PIPE_A;
1793         else
1794                 return crtc->pipe;
1795 }
1796
1797 static u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
1798 {
1799         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1800
1801         /*
1802          * On i965gm the hardware frame counter reads
1803          * zero when the TV encoder is enabled :(
1804          */
1805         if (IS_I965GM(dev_priv) &&
1806             (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
1807                 return 0;
1808
1809         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1810                 return 0xffffffff; /* full 32 bit counter */
1811         else if (INTEL_GEN(dev_priv) >= 3)
1812                 return 0xffffff; /* only 24 bits of frame count */
1813         else
1814                 return 0; /* Gen2 doesn't have a hardware frame counter */
1815 }
1816
1817 static void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
1818 {
1819         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1820
1821         assert_vblank_disabled(&crtc->base);
1822         drm_crtc_set_max_vblank_count(&crtc->base,
1823                                       intel_crtc_max_vblank_count(crtc_state));
1824         drm_crtc_vblank_on(&crtc->base);
1825 }
1826
1827 void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
1828 {
1829         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1830
1831         drm_crtc_vblank_off(&crtc->base);
1832         assert_vblank_disabled(&crtc->base);
1833 }
1834
1835 static void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state)
1836 {
1837         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
1838         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1839         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
1840         enum pipe pipe = crtc->pipe;
1841         i915_reg_t reg;
1842         u32 val;
1843
1844         DRM_DEBUG_KMS("enabling pipe %c\n", pipe_name(pipe));
1845
1846         assert_planes_disabled(crtc);
1847
1848         /*
1849          * A pipe without a PLL won't actually be able to drive bits from
1850          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
1851          * need the check.
1852          */
1853         if (HAS_GMCH(dev_priv)) {
1854                 if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
1855                         assert_dsi_pll_enabled(dev_priv);
1856                 else
1857                         assert_pll_enabled(dev_priv, pipe);
1858         } else {
1859                 if (new_crtc_state->has_pch_encoder) {
1860                         /* if driving the PCH, we need FDI enabled */
1861                         assert_fdi_rx_pll_enabled(dev_priv,
1862                                                   intel_crtc_pch_transcoder(crtc));
1863                         assert_fdi_tx_pll_enabled(dev_priv,
1864                                                   (enum pipe) cpu_transcoder);
1865                 }
1866                 /* FIXME: assert CPU port conditions for SNB+ */
1867         }
1868
1869         trace_intel_pipe_enable(crtc);
1870
1871         reg = PIPECONF(cpu_transcoder);
1872         val = I915_READ(reg);
1873         if (val & PIPECONF_ENABLE) {
1874                 /* we keep both pipes enabled on 830 */
1875                 WARN_ON(!IS_I830(dev_priv));
1876                 return;
1877         }
1878
1879         I915_WRITE(reg, val | PIPECONF_ENABLE);
1880         POSTING_READ(reg);
1881
1882         /*
1883          * Until the pipe starts PIPEDSL reads will return a stale value,
1884          * which causes an apparent vblank timestamp jump when PIPEDSL
1885          * resets to its proper value. That also messes up the frame count
1886          * when it's derived from the timestamps. So let's wait for the
1887          * pipe to start properly before we call drm_crtc_vblank_on()
1888          */
1889         if (intel_crtc_max_vblank_count(new_crtc_state) == 0)
1890                 intel_wait_for_pipe_scanline_moving(crtc);
1891 }
1892
1893 void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state)
1894 {
1895         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
1896         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1897         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1898         enum pipe pipe = crtc->pipe;
1899         i915_reg_t reg;
1900         u32 val;
1901
1902         DRM_DEBUG_KMS("disabling pipe %c\n", pipe_name(pipe));
1903
1904         /*
1905          * Make sure planes won't keep trying to pump pixels to us,
1906          * or we might hang the display.
1907          */
1908         assert_planes_disabled(crtc);
1909
1910         trace_intel_pipe_disable(crtc);
1911
1912         reg = PIPECONF(cpu_transcoder);
1913         val = I915_READ(reg);
1914         if ((val & PIPECONF_ENABLE) == 0)
1915                 return;
1916
1917         /*
1918          * Double wide has implications for planes
1919          * so best keep it disabled when not needed.
1920          */
1921         if (old_crtc_state->double_wide)
1922                 val &= ~PIPECONF_DOUBLE_WIDE;
1923
1924         /* Don't disable pipe or pipe PLLs if needed */
1925         if (!IS_I830(dev_priv))
1926                 val &= ~PIPECONF_ENABLE;
1927
1928         I915_WRITE(reg, val);
1929         if ((val & PIPECONF_ENABLE) == 0)
1930                 intel_wait_for_pipe_off(old_crtc_state);
1931 }
1932
1933 static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv)
1934 {
1935         return IS_GEN(dev_priv, 2) ? 2048 : 4096;
1936 }
1937
1938 static bool is_ccs_plane(const struct drm_framebuffer *fb, int plane)
1939 {
1940         if (!is_ccs_modifier(fb->modifier))
1941                 return false;
1942
1943         return plane >= fb->format->num_planes / 2;
1944 }
1945
1946 static bool is_gen12_ccs_modifier(u64 modifier)
1947 {
1948         return modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS;
1949 }
1950
1951 static bool is_gen12_ccs_plane(const struct drm_framebuffer *fb, int plane)
1952 {
1953         return is_gen12_ccs_modifier(fb->modifier) && is_ccs_plane(fb, plane);
1954 }
1955
1956 static bool is_aux_plane(const struct drm_framebuffer *fb, int plane)
1957 {
1958         if (is_ccs_modifier(fb->modifier))
1959                 return is_ccs_plane(fb, plane);
1960
1961         return plane == 1;
1962 }
1963
1964 static int main_to_ccs_plane(const struct drm_framebuffer *fb, int main_plane)
1965 {
1966         WARN_ON(!is_ccs_modifier(fb->modifier) ||
1967                 (main_plane && main_plane >= fb->format->num_planes / 2));
1968
1969         return fb->format->num_planes / 2 + main_plane;
1970 }
1971
1972 static int ccs_to_main_plane(const struct drm_framebuffer *fb, int ccs_plane)
1973 {
1974         WARN_ON(!is_ccs_modifier(fb->modifier) ||
1975                 ccs_plane < fb->format->num_planes / 2);
1976
1977         return ccs_plane - fb->format->num_planes / 2;
1978 }
1979
1980 /* Return either the main plane's CCS or - if not a CCS FB - UV plane */
1981 static int
1982 intel_main_to_aux_plane(const struct drm_framebuffer *fb, int main_plane)
1983 {
1984         if (is_ccs_modifier(fb->modifier))
1985                 return main_to_ccs_plane(fb, main_plane);
1986
1987         return 1;
1988 }
1989
1990 bool
1991 intel_format_info_is_yuv_semiplanar(const struct drm_format_info *info,
1992                                     uint64_t modifier)
1993 {
1994         return info->is_yuv &&
1995                info->num_planes == (is_ccs_modifier(modifier) ? 4 : 2);
1996 }
1997
1998 static unsigned int
1999 intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane)
2000 {
2001         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2002         unsigned int cpp = fb->format->cpp[color_plane];
2003
2004         switch (fb->modifier) {
2005         case DRM_FORMAT_MOD_LINEAR:
2006                 return intel_tile_size(dev_priv);
2007         case I915_FORMAT_MOD_X_TILED:
2008                 if (IS_GEN(dev_priv, 2))
2009                         return 128;
2010                 else
2011                         return 512;
2012         case I915_FORMAT_MOD_Y_TILED_CCS:
2013                 if (is_ccs_plane(fb, color_plane))
2014                         return 128;
2015                 /* fall through */
2016         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
2017                 if (is_ccs_plane(fb, color_plane))
2018                         return 64;
2019                 /* fall through */
2020         case I915_FORMAT_MOD_Y_TILED:
2021                 if (IS_GEN(dev_priv, 2) || HAS_128_BYTE_Y_TILING(dev_priv))
2022                         return 128;
2023                 else
2024                         return 512;
2025         case I915_FORMAT_MOD_Yf_TILED_CCS:
2026                 if (is_ccs_plane(fb, color_plane))
2027                         return 128;
2028                 /* fall through */
2029         case I915_FORMAT_MOD_Yf_TILED:
2030                 switch (cpp) {
2031                 case 1:
2032                         return 64;
2033                 case 2:
2034                 case 4:
2035                         return 128;
2036                 case 8:
2037                 case 16:
2038                         return 256;
2039                 default:
2040                         MISSING_CASE(cpp);
2041                         return cpp;
2042                 }
2043                 break;
2044         default:
2045                 MISSING_CASE(fb->modifier);
2046                 return cpp;
2047         }
2048 }
2049
2050 static unsigned int
2051 intel_tile_height(const struct drm_framebuffer *fb, int color_plane)
2052 {
2053         if (is_gen12_ccs_plane(fb, color_plane))
2054                 return 1;
2055
2056         return intel_tile_size(to_i915(fb->dev)) /
2057                 intel_tile_width_bytes(fb, color_plane);
2058 }
2059
2060 /* Return the tile dimensions in pixel units */
2061 static void intel_tile_dims(const struct drm_framebuffer *fb, int color_plane,
2062                             unsigned int *tile_width,
2063                             unsigned int *tile_height)
2064 {
2065         unsigned int tile_width_bytes = intel_tile_width_bytes(fb, color_plane);
2066         unsigned int cpp = fb->format->cpp[color_plane];
2067
2068         *tile_width = tile_width_bytes / cpp;
2069         *tile_height = intel_tile_height(fb, color_plane);
2070 }
2071
2072 unsigned int
2073 intel_fb_align_height(const struct drm_framebuffer *fb,
2074                       int color_plane, unsigned int height)
2075 {
2076         unsigned int tile_height = intel_tile_height(fb, color_plane);
2077
2078         return ALIGN(height, tile_height);
2079 }
2080
2081 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info)
2082 {
2083         unsigned int size = 0;
2084         int i;
2085
2086         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
2087                 size += rot_info->plane[i].width * rot_info->plane[i].height;
2088
2089         return size;
2090 }
2091
2092 unsigned int intel_remapped_info_size(const struct intel_remapped_info *rem_info)
2093 {
2094         unsigned int size = 0;
2095         int i;
2096
2097         for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++)
2098                 size += rem_info->plane[i].width * rem_info->plane[i].height;
2099
2100         return size;
2101 }
2102
2103 static void
2104 intel_fill_fb_ggtt_view(struct i915_ggtt_view *view,
2105                         const struct drm_framebuffer *fb,
2106                         unsigned int rotation)
2107 {
2108         view->type = I915_GGTT_VIEW_NORMAL;
2109         if (drm_rotation_90_or_270(rotation)) {
2110                 view->type = I915_GGTT_VIEW_ROTATED;
2111                 view->rotated = to_intel_framebuffer(fb)->rot_info;
2112         }
2113 }
2114
2115 static unsigned int intel_cursor_alignment(const struct drm_i915_private *dev_priv)
2116 {
2117         if (IS_I830(dev_priv))
2118                 return 16 * 1024;
2119         else if (IS_I85X(dev_priv))
2120                 return 256;
2121         else if (IS_I845G(dev_priv) || IS_I865G(dev_priv))
2122                 return 32;
2123         else
2124                 return 4 * 1024;
2125 }
2126
2127 static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv)
2128 {
2129         if (INTEL_GEN(dev_priv) >= 9)
2130                 return 256 * 1024;
2131         else if (IS_I965G(dev_priv) || IS_I965GM(dev_priv) ||
2132                  IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2133                 return 128 * 1024;
2134         else if (INTEL_GEN(dev_priv) >= 4)
2135                 return 4 * 1024;
2136         else
2137                 return 0;
2138 }
2139
2140 static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb,
2141                                          int color_plane)
2142 {
2143         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2144
2145         /* AUX_DIST needs only 4K alignment */
2146         if (is_aux_plane(fb, color_plane))
2147                 return 4096;
2148
2149         switch (fb->modifier) {
2150         case DRM_FORMAT_MOD_LINEAR:
2151                 return intel_linear_alignment(dev_priv);
2152         case I915_FORMAT_MOD_X_TILED:
2153                 if (INTEL_GEN(dev_priv) >= 9)
2154                         return 256 * 1024;
2155                 return 0;
2156         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
2157                 return 16 * 1024;
2158         case I915_FORMAT_MOD_Y_TILED_CCS:
2159         case I915_FORMAT_MOD_Yf_TILED_CCS:
2160         case I915_FORMAT_MOD_Y_TILED:
2161         case I915_FORMAT_MOD_Yf_TILED:
2162                 return 1 * 1024 * 1024;
2163         default:
2164                 MISSING_CASE(fb->modifier);
2165                 return 0;
2166         }
2167 }
2168
2169 static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
2170 {
2171         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2172         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2173
2174         return INTEL_GEN(dev_priv) < 4 ||
2175                 (plane->has_fbc &&
2176                  plane_state->view.type == I915_GGTT_VIEW_NORMAL);
2177 }
2178
2179 struct i915_vma *
2180 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
2181                            const struct i915_ggtt_view *view,
2182                            bool uses_fence,
2183                            unsigned long *out_flags)
2184 {
2185         struct drm_device *dev = fb->dev;
2186         struct drm_i915_private *dev_priv = to_i915(dev);
2187         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2188         intel_wakeref_t wakeref;
2189         struct i915_vma *vma;
2190         unsigned int pinctl;
2191         u32 alignment;
2192
2193         if (WARN_ON(!i915_gem_object_is_framebuffer(obj)))
2194                 return ERR_PTR(-EINVAL);
2195
2196         alignment = intel_surf_alignment(fb, 0);
2197
2198         /* Note that the w/a also requires 64 PTE of padding following the
2199          * bo. We currently fill all unused PTE with the shadow page and so
2200          * we should always have valid PTE following the scanout preventing
2201          * the VT-d warning.
2202          */
2203         if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024)
2204                 alignment = 256 * 1024;
2205
2206         /*
2207          * Global gtt pte registers are special registers which actually forward
2208          * writes to a chunk of system memory. Which means that there is no risk
2209          * that the register values disappear as soon as we call
2210          * intel_runtime_pm_put(), so it is correct to wrap only the
2211          * pin/unpin/fence and not more.
2212          */
2213         wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
2214
2215         atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
2216
2217         /*
2218          * Valleyview is definitely limited to scanning out the first
2219          * 512MiB. Lets presume this behaviour was inherited from the
2220          * g4x display engine and that all earlier gen are similarly
2221          * limited. Testing suggests that it is a little more
2222          * complicated than this. For example, Cherryview appears quite
2223          * happy to scanout from anywhere within its global aperture.
2224          */
2225         pinctl = 0;
2226         if (HAS_GMCH(dev_priv))
2227                 pinctl |= PIN_MAPPABLE;
2228
2229         vma = i915_gem_object_pin_to_display_plane(obj,
2230                                                    alignment, view, pinctl);
2231         if (IS_ERR(vma))
2232                 goto err;
2233
2234         if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
2235                 int ret;
2236
2237                 /*
2238                  * Install a fence for tiled scan-out. Pre-i965 always needs a
2239                  * fence, whereas 965+ only requires a fence if using
2240                  * framebuffer compression.  For simplicity, we always, when
2241                  * possible, install a fence as the cost is not that onerous.
2242                  *
2243                  * If we fail to fence the tiled scanout, then either the
2244                  * modeset will reject the change (which is highly unlikely as
2245                  * the affected systems, all but one, do not have unmappable
2246                  * space) or we will not be able to enable full powersaving
2247                  * techniques (also likely not to apply due to various limits
2248                  * FBC and the like impose on the size of the buffer, which
2249                  * presumably we violated anyway with this unmappable buffer).
2250                  * Anyway, it is presumably better to stumble onwards with
2251                  * something and try to run the system in a "less than optimal"
2252                  * mode that matches the user configuration.
2253                  */
2254                 ret = i915_vma_pin_fence(vma);
2255                 if (ret != 0 && INTEL_GEN(dev_priv) < 4) {
2256                         i915_gem_object_unpin_from_display_plane(vma);
2257                         vma = ERR_PTR(ret);
2258                         goto err;
2259                 }
2260
2261                 if (ret == 0 && vma->fence)
2262                         *out_flags |= PLANE_HAS_FENCE;
2263         }
2264
2265         i915_vma_get(vma);
2266 err:
2267         atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
2268         intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
2269         return vma;
2270 }
2271
2272 void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
2273 {
2274         i915_gem_object_lock(vma->obj);
2275         if (flags & PLANE_HAS_FENCE)
2276                 i915_vma_unpin_fence(vma);
2277         i915_gem_object_unpin_from_display_plane(vma);
2278         i915_gem_object_unlock(vma->obj);
2279
2280         i915_vma_put(vma);
2281 }
2282
2283 static int intel_fb_pitch(const struct drm_framebuffer *fb, int color_plane,
2284                           unsigned int rotation)
2285 {
2286         if (drm_rotation_90_or_270(rotation))
2287                 return to_intel_framebuffer(fb)->rotated[color_plane].pitch;
2288         else
2289                 return fb->pitches[color_plane];
2290 }
2291
2292 /*
2293  * Convert the x/y offsets into a linear offset.
2294  * Only valid with 0/180 degree rotation, which is fine since linear
2295  * offset is only used with linear buffers on pre-hsw and tiled buffers
2296  * with gen2/3, and 90/270 degree rotations isn't supported on any of them.
2297  */
2298 u32 intel_fb_xy_to_linear(int x, int y,
2299                           const struct intel_plane_state *state,
2300                           int color_plane)
2301 {
2302         const struct drm_framebuffer *fb = state->hw.fb;
2303         unsigned int cpp = fb->format->cpp[color_plane];
2304         unsigned int pitch = state->color_plane[color_plane].stride;
2305
2306         return y * pitch + x * cpp;
2307 }
2308
2309 /*
2310  * Add the x/y offsets derived from fb->offsets[] to the user
2311  * specified plane src x/y offsets. The resulting x/y offsets
2312  * specify the start of scanout from the beginning of the gtt mapping.
2313  */
2314 void intel_add_fb_offsets(int *x, int *y,
2315                           const struct intel_plane_state *state,
2316                           int color_plane)
2317
2318 {
2319         *x += state->color_plane[color_plane].x;
2320         *y += state->color_plane[color_plane].y;
2321 }
2322
2323 static u32 intel_adjust_tile_offset(int *x, int *y,
2324                                     unsigned int tile_width,
2325                                     unsigned int tile_height,
2326                                     unsigned int tile_size,
2327                                     unsigned int pitch_tiles,
2328                                     u32 old_offset,
2329                                     u32 new_offset)
2330 {
2331         unsigned int pitch_pixels = pitch_tiles * tile_width;
2332         unsigned int tiles;
2333
2334         WARN_ON(old_offset & (tile_size - 1));
2335         WARN_ON(new_offset & (tile_size - 1));
2336         WARN_ON(new_offset > old_offset);
2337
2338         tiles = (old_offset - new_offset) / tile_size;
2339
2340         *y += tiles / pitch_tiles * tile_height;
2341         *x += tiles % pitch_tiles * tile_width;
2342
2343         /* minimize x in case it got needlessly big */
2344         *y += *x / pitch_pixels * tile_height;
2345         *x %= pitch_pixels;
2346
2347         return new_offset;
2348 }
2349
2350 static bool is_surface_linear(const struct drm_framebuffer *fb, int color_plane)
2351 {
2352         return fb->modifier == DRM_FORMAT_MOD_LINEAR ||
2353                is_gen12_ccs_plane(fb, color_plane);
2354 }
2355
2356 static u32 intel_adjust_aligned_offset(int *x, int *y,
2357                                        const struct drm_framebuffer *fb,
2358                                        int color_plane,
2359                                        unsigned int rotation,
2360                                        unsigned int pitch,
2361                                        u32 old_offset, u32 new_offset)
2362 {
2363         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2364         unsigned int cpp = fb->format->cpp[color_plane];
2365
2366         WARN_ON(new_offset > old_offset);
2367
2368         if (!is_surface_linear(fb, color_plane)) {
2369                 unsigned int tile_size, tile_width, tile_height;
2370                 unsigned int pitch_tiles;
2371
2372                 tile_size = intel_tile_size(dev_priv);
2373                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2374
2375                 if (drm_rotation_90_or_270(rotation)) {
2376                         pitch_tiles = pitch / tile_height;
2377                         swap(tile_width, tile_height);
2378                 } else {
2379                         pitch_tiles = pitch / (tile_width * cpp);
2380                 }
2381
2382                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2383                                          tile_size, pitch_tiles,
2384                                          old_offset, new_offset);
2385         } else {
2386                 old_offset += *y * pitch + *x * cpp;
2387
2388                 *y = (old_offset - new_offset) / pitch;
2389                 *x = ((old_offset - new_offset) - *y * pitch) / cpp;
2390         }
2391
2392         return new_offset;
2393 }
2394
2395 /*
2396  * Adjust the tile offset by moving the difference into
2397  * the x/y offsets.
2398  */
2399 static u32 intel_plane_adjust_aligned_offset(int *x, int *y,
2400                                              const struct intel_plane_state *state,
2401                                              int color_plane,
2402                                              u32 old_offset, u32 new_offset)
2403 {
2404         return intel_adjust_aligned_offset(x, y, state->hw.fb, color_plane,
2405                                            state->hw.rotation,
2406                                            state->color_plane[color_plane].stride,
2407                                            old_offset, new_offset);
2408 }
2409
2410 /*
2411  * Computes the aligned offset to the base tile and adjusts
2412  * x, y. bytes per pixel is assumed to be a power-of-two.
2413  *
2414  * In the 90/270 rotated case, x and y are assumed
2415  * to be already rotated to match the rotated GTT view, and
2416  * pitch is the tile_height aligned framebuffer height.
2417  *
2418  * This function is used when computing the derived information
2419  * under intel_framebuffer, so using any of that information
2420  * here is not allowed. Anything under drm_framebuffer can be
2421  * used. This is why the user has to pass in the pitch since it
2422  * is specified in the rotated orientation.
2423  */
2424 static u32 intel_compute_aligned_offset(struct drm_i915_private *dev_priv,
2425                                         int *x, int *y,
2426                                         const struct drm_framebuffer *fb,
2427                                         int color_plane,
2428                                         unsigned int pitch,
2429                                         unsigned int rotation,
2430                                         u32 alignment)
2431 {
2432         unsigned int cpp = fb->format->cpp[color_plane];
2433         u32 offset, offset_aligned;
2434
2435         if (alignment)
2436                 alignment--;
2437
2438         if (!is_surface_linear(fb, color_plane)) {
2439                 unsigned int tile_size, tile_width, tile_height;
2440                 unsigned int tile_rows, tiles, pitch_tiles;
2441
2442                 tile_size = intel_tile_size(dev_priv);
2443                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2444
2445                 if (drm_rotation_90_or_270(rotation)) {
2446                         pitch_tiles = pitch / tile_height;
2447                         swap(tile_width, tile_height);
2448                 } else {
2449                         pitch_tiles = pitch / (tile_width * cpp);
2450                 }
2451
2452                 tile_rows = *y / tile_height;
2453                 *y %= tile_height;
2454
2455                 tiles = *x / tile_width;
2456                 *x %= tile_width;
2457
2458                 offset = (tile_rows * pitch_tiles + tiles) * tile_size;
2459                 offset_aligned = offset & ~alignment;
2460
2461                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2462                                          tile_size, pitch_tiles,
2463                                          offset, offset_aligned);
2464         } else {
2465                 offset = *y * pitch + *x * cpp;
2466                 offset_aligned = offset & ~alignment;
2467
2468                 *y = (offset & alignment) / pitch;
2469                 *x = ((offset & alignment) - *y * pitch) / cpp;
2470         }
2471
2472         return offset_aligned;
2473 }
2474
2475 static u32 intel_plane_compute_aligned_offset(int *x, int *y,
2476                                               const struct intel_plane_state *state,
2477                                               int color_plane)
2478 {
2479         struct intel_plane *intel_plane = to_intel_plane(state->uapi.plane);
2480         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
2481         const struct drm_framebuffer *fb = state->hw.fb;
2482         unsigned int rotation = state->hw.rotation;
2483         int pitch = state->color_plane[color_plane].stride;
2484         u32 alignment;
2485
2486         if (intel_plane->id == PLANE_CURSOR)
2487                 alignment = intel_cursor_alignment(dev_priv);
2488         else
2489                 alignment = intel_surf_alignment(fb, color_plane);
2490
2491         return intel_compute_aligned_offset(dev_priv, x, y, fb, color_plane,
2492                                             pitch, rotation, alignment);
2493 }
2494
2495 /* Convert the fb->offset[] into x/y offsets */
2496 static int intel_fb_offset_to_xy(int *x, int *y,
2497                                  const struct drm_framebuffer *fb,
2498                                  int color_plane)
2499 {
2500         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2501         unsigned int height;
2502
2503         if (fb->modifier != DRM_FORMAT_MOD_LINEAR &&
2504             fb->offsets[color_plane] % intel_tile_size(dev_priv)) {
2505                 DRM_DEBUG_KMS("Misaligned offset 0x%08x for color plane %d\n",
2506                               fb->offsets[color_plane], color_plane);
2507                 return -EINVAL;
2508         }
2509
2510         height = drm_framebuffer_plane_height(fb->height, fb, color_plane);
2511         height = ALIGN(height, intel_tile_height(fb, color_plane));
2512
2513         /* Catch potential overflows early */
2514         if (add_overflows_t(u32, mul_u32_u32(height, fb->pitches[color_plane]),
2515                             fb->offsets[color_plane])) {
2516                 DRM_DEBUG_KMS("Bad offset 0x%08x or pitch %d for color plane %d\n",
2517                               fb->offsets[color_plane], fb->pitches[color_plane],
2518                               color_plane);
2519                 return -ERANGE;
2520         }
2521
2522         *x = 0;
2523         *y = 0;
2524
2525         intel_adjust_aligned_offset(x, y,
2526                                     fb, color_plane, DRM_MODE_ROTATE_0,
2527                                     fb->pitches[color_plane],
2528                                     fb->offsets[color_plane], 0);
2529
2530         return 0;
2531 }
2532
2533 static unsigned int intel_fb_modifier_to_tiling(u64 fb_modifier)
2534 {
2535         switch (fb_modifier) {
2536         case I915_FORMAT_MOD_X_TILED:
2537                 return I915_TILING_X;
2538         case I915_FORMAT_MOD_Y_TILED:
2539         case I915_FORMAT_MOD_Y_TILED_CCS:
2540         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
2541                 return I915_TILING_Y;
2542         default:
2543                 return I915_TILING_NONE;
2544         }
2545 }
2546
2547 /*
2548  * From the Sky Lake PRM:
2549  * "The Color Control Surface (CCS) contains the compression status of
2550  *  the cache-line pairs. The compression state of the cache-line pair
2551  *  is specified by 2 bits in the CCS. Each CCS cache-line represents
2552  *  an area on the main surface of 16 x16 sets of 128 byte Y-tiled
2553  *  cache-line-pairs. CCS is always Y tiled."
2554  *
2555  * Since cache line pairs refers to horizontally adjacent cache lines,
2556  * each cache line in the CCS corresponds to an area of 32x16 cache
2557  * lines on the main surface. Since each pixel is 4 bytes, this gives
2558  * us a ratio of one byte in the CCS for each 8x16 pixels in the
2559  * main surface.
2560  */
2561 static const struct drm_format_info skl_ccs_formats[] = {
2562         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
2563           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2564         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
2565           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2566         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
2567           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
2568         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
2569           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
2570 };
2571
2572 /*
2573  * Gen-12 compression uses 4 bits of CCS data for each cache line pair in the
2574  * main surface. And each 64B CCS cache line represents an area of 4x1 Y-tiles
2575  * in the main surface. With 4 byte pixels and each Y-tile having dimensions of
2576  * 32x32 pixels, the ratio turns out to 1B in the CCS for every 2x32 pixels in
2577  * the main surface.
2578  */
2579 static const struct drm_format_info gen12_ccs_formats[] = {
2580         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
2581           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2582           .hsub = 1, .vsub = 1, },
2583         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
2584           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2585           .hsub = 1, .vsub = 1, },
2586         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
2587           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2588           .hsub = 1, .vsub = 1, .has_alpha = true },
2589         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
2590           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2591           .hsub = 1, .vsub = 1, .has_alpha = true },
2592 };
2593
2594 static const struct drm_format_info *
2595 lookup_format_info(const struct drm_format_info formats[],
2596                    int num_formats, u32 format)
2597 {
2598         int i;
2599
2600         for (i = 0; i < num_formats; i++) {
2601                 if (formats[i].format == format)
2602                         return &formats[i];
2603         }
2604
2605         return NULL;
2606 }
2607
2608 static const struct drm_format_info *
2609 intel_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
2610 {
2611         switch (cmd->modifier[0]) {
2612         case I915_FORMAT_MOD_Y_TILED_CCS:
2613         case I915_FORMAT_MOD_Yf_TILED_CCS:
2614                 return lookup_format_info(skl_ccs_formats,
2615                                           ARRAY_SIZE(skl_ccs_formats),
2616                                           cmd->pixel_format);
2617         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
2618                 return lookup_format_info(gen12_ccs_formats,
2619                                           ARRAY_SIZE(gen12_ccs_formats),
2620                                           cmd->pixel_format);
2621         default:
2622                 return NULL;
2623         }
2624 }
2625
2626 bool is_ccs_modifier(u64 modifier)
2627 {
2628         return modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS ||
2629                modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
2630                modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
2631 }
2632
2633 static int gen12_ccs_aux_stride(struct drm_framebuffer *fb, int ccs_plane)
2634 {
2635         return DIV_ROUND_UP(fb->pitches[ccs_to_main_plane(fb, ccs_plane)],
2636                             512) * 64;
2637 }
2638
2639 u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv,
2640                               u32 pixel_format, u64 modifier)
2641 {
2642         struct intel_crtc *crtc;
2643         struct intel_plane *plane;
2644
2645         /*
2646          * We assume the primary plane for pipe A has
2647          * the highest stride limits of them all.
2648          */
2649         crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A);
2650         if (!crtc)
2651                 return 0;
2652
2653         plane = to_intel_plane(crtc->base.primary);
2654
2655         return plane->max_stride(plane, pixel_format, modifier,
2656                                  DRM_MODE_ROTATE_0);
2657 }
2658
2659 static
2660 u32 intel_fb_max_stride(struct drm_i915_private *dev_priv,
2661                         u32 pixel_format, u64 modifier)
2662 {
2663         /*
2664          * Arbitrary limit for gen4+ chosen to match the
2665          * render engine max stride.
2666          *
2667          * The new CCS hash mode makes remapping impossible
2668          */
2669         if (!is_ccs_modifier(modifier)) {
2670                 if (INTEL_GEN(dev_priv) >= 7)
2671                         return 256*1024;
2672                 else if (INTEL_GEN(dev_priv) >= 4)
2673                         return 128*1024;
2674         }
2675
2676         return intel_plane_fb_max_stride(dev_priv, pixel_format, modifier);
2677 }
2678
2679 static u32
2680 intel_fb_stride_alignment(const struct drm_framebuffer *fb, int color_plane)
2681 {
2682         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2683         u32 tile_width;
2684
2685         if (is_surface_linear(fb, color_plane)) {
2686                 u32 max_stride = intel_plane_fb_max_stride(dev_priv,
2687                                                            fb->format->format,
2688                                                            fb->modifier);
2689
2690                 /*
2691                  * To make remapping with linear generally feasible
2692                  * we need the stride to be page aligned.
2693                  */
2694                 if (fb->pitches[color_plane] > max_stride &&
2695                     !is_ccs_modifier(fb->modifier))
2696                         return intel_tile_size(dev_priv);
2697                 else
2698                         return 64;
2699         }
2700
2701         tile_width = intel_tile_width_bytes(fb, color_plane);
2702         if (is_ccs_modifier(fb->modifier) && color_plane == 0) {
2703                 /*
2704                  * Display WA #0531: skl,bxt,kbl,glk
2705                  *
2706                  * Render decompression and plane width > 3840
2707                  * combined with horizontal panning requires the
2708                  * plane stride to be a multiple of 4. We'll just
2709                  * require the entire fb to accommodate that to avoid
2710                  * potential runtime errors at plane configuration time.
2711                  */
2712                 if (IS_GEN(dev_priv, 9) && fb->width > 3840)
2713                         tile_width *= 4;
2714                 /*
2715                  * The main surface pitch must be padded to a multiple of four
2716                  * tile widths.
2717                  */
2718                 else if (INTEL_GEN(dev_priv) >= 12)
2719                         tile_width *= 4;
2720         }
2721         return tile_width;
2722 }
2723
2724 bool intel_plane_can_remap(const struct intel_plane_state *plane_state)
2725 {
2726         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2727         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2728         const struct drm_framebuffer *fb = plane_state->hw.fb;
2729         int i;
2730
2731         /* We don't want to deal with remapping with cursors */
2732         if (plane->id == PLANE_CURSOR)
2733                 return false;
2734
2735         /*
2736          * The display engine limits already match/exceed the
2737          * render engine limits, so not much point in remapping.
2738          * Would also need to deal with the fence POT alignment
2739          * and gen2 2KiB GTT tile size.
2740          */
2741         if (INTEL_GEN(dev_priv) < 4)
2742                 return false;
2743
2744         /*
2745          * The new CCS hash mode isn't compatible with remapping as
2746          * the virtual address of the pages affects the compressed data.
2747          */
2748         if (is_ccs_modifier(fb->modifier))
2749                 return false;
2750
2751         /* Linear needs a page aligned stride for remapping */
2752         if (fb->modifier == DRM_FORMAT_MOD_LINEAR) {
2753                 unsigned int alignment = intel_tile_size(dev_priv) - 1;
2754
2755                 for (i = 0; i < fb->format->num_planes; i++) {
2756                         if (fb->pitches[i] & alignment)
2757                                 return false;
2758                 }
2759         }
2760
2761         return true;
2762 }
2763
2764 static bool intel_plane_needs_remap(const struct intel_plane_state *plane_state)
2765 {
2766         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2767         const struct drm_framebuffer *fb = plane_state->hw.fb;
2768         unsigned int rotation = plane_state->hw.rotation;
2769         u32 stride, max_stride;
2770
2771         /*
2772          * No remapping for invisible planes since we don't have
2773          * an actual source viewport to remap.
2774          */
2775         if (!plane_state->uapi.visible)
2776                 return false;
2777
2778         if (!intel_plane_can_remap(plane_state))
2779                 return false;
2780
2781         /*
2782          * FIXME: aux plane limits on gen9+ are
2783          * unclear in Bspec, for now no checking.
2784          */
2785         stride = intel_fb_pitch(fb, 0, rotation);
2786         max_stride = plane->max_stride(plane, fb->format->format,
2787                                        fb->modifier, rotation);
2788
2789         return stride > max_stride;
2790 }
2791
2792 static void
2793 intel_fb_plane_get_subsampling(int *hsub, int *vsub,
2794                                const struct drm_framebuffer *fb,
2795                                int color_plane)
2796 {
2797         int main_plane;
2798
2799         if (color_plane == 0) {
2800                 *hsub = 1;
2801                 *vsub = 1;
2802
2803                 return;
2804         }
2805
2806         /*
2807          * TODO: Deduct the subsampling from the char block for all CCS
2808          * formats and planes.
2809          */
2810         if (!is_gen12_ccs_plane(fb, color_plane)) {
2811                 *hsub = fb->format->hsub;
2812                 *vsub = fb->format->vsub;
2813
2814                 return;
2815         }
2816
2817         main_plane = ccs_to_main_plane(fb, color_plane);
2818         *hsub = drm_format_info_block_width(fb->format, color_plane) /
2819                 drm_format_info_block_width(fb->format, main_plane);
2820
2821         /*
2822          * The min stride check in the core framebuffer_check() function
2823          * assumes that format->hsub applies to every plane except for the
2824          * first plane. That's incorrect for the CCS AUX plane of the first
2825          * plane, but for the above check to pass we must define the block
2826          * width with that subsampling applied to it. Adjust the width here
2827          * accordingly, so we can calculate the actual subsampling factor.
2828          */
2829         if (main_plane == 0)
2830                 *hsub *= fb->format->hsub;
2831
2832         *vsub = 32;
2833 }
2834 static int
2835 intel_fb_check_ccs_xy(struct drm_framebuffer *fb, int ccs_plane, int x, int y)
2836 {
2837         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2838         int main_plane;
2839         int hsub, vsub;
2840         int tile_width, tile_height;
2841         int ccs_x, ccs_y;
2842         int main_x, main_y;
2843
2844         if (!is_ccs_plane(fb, ccs_plane))
2845                 return 0;
2846
2847         intel_tile_dims(fb, ccs_plane, &tile_width, &tile_height);
2848         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, ccs_plane);
2849
2850         tile_width *= hsub;
2851         tile_height *= vsub;
2852
2853         ccs_x = (x * hsub) % tile_width;
2854         ccs_y = (y * vsub) % tile_height;
2855
2856         main_plane = ccs_to_main_plane(fb, ccs_plane);
2857         main_x = intel_fb->normal[main_plane].x % tile_width;
2858         main_y = intel_fb->normal[main_plane].y % tile_height;
2859
2860         /*
2861          * CCS doesn't have its own x/y offset register, so the intra CCS tile
2862          * x/y offsets must match between CCS and the main surface.
2863          */
2864         if (main_x != ccs_x || main_y != ccs_y) {
2865                 DRM_DEBUG_KMS("Bad CCS x/y (main %d,%d ccs %d,%d) full (main %d,%d ccs %d,%d)\n",
2866                               main_x, main_y,
2867                               ccs_x, ccs_y,
2868                               intel_fb->normal[main_plane].x,
2869                               intel_fb->normal[main_plane].y,
2870                               x, y);
2871                 return -EINVAL;
2872         }
2873
2874         return 0;
2875 }
2876
2877 static void
2878 intel_fb_plane_dims(int *w, int *h, struct drm_framebuffer *fb, int color_plane)
2879 {
2880         int hsub, vsub;
2881
2882         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, color_plane);
2883         *w = fb->width / hsub;
2884         *h = fb->height / vsub;
2885 }
2886
2887 /*
2888  * Setup the rotated view for an FB plane and return the size the GTT mapping
2889  * requires for this view.
2890  */
2891 static u32
2892 setup_fb_rotation(int plane, const struct intel_remapped_plane_info *plane_info,
2893                   u32 gtt_offset_rotated, int x, int y,
2894                   unsigned int width, unsigned int height,
2895                   unsigned int tile_size,
2896                   unsigned int tile_width, unsigned int tile_height,
2897                   struct drm_framebuffer *fb)
2898 {
2899         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2900         struct intel_rotation_info *rot_info = &intel_fb->rot_info;
2901         unsigned int pitch_tiles;
2902         struct drm_rect r;
2903
2904         /* Y or Yf modifiers required for 90/270 rotation */
2905         if (fb->modifier != I915_FORMAT_MOD_Y_TILED &&
2906             fb->modifier != I915_FORMAT_MOD_Yf_TILED)
2907                 return 0;
2908
2909         if (WARN_ON(plane >= ARRAY_SIZE(rot_info->plane)))
2910                 return 0;
2911
2912         rot_info->plane[plane] = *plane_info;
2913
2914         intel_fb->rotated[plane].pitch = plane_info->height * tile_height;
2915
2916         /* rotate the x/y offsets to match the GTT view */
2917         drm_rect_init(&r, x, y, width, height);
2918         drm_rect_rotate(&r,
2919                         plane_info->width * tile_width,
2920                         plane_info->height * tile_height,
2921                         DRM_MODE_ROTATE_270);
2922         x = r.x1;
2923         y = r.y1;
2924
2925         /* rotate the tile dimensions to match the GTT view */
2926         pitch_tiles = intel_fb->rotated[plane].pitch / tile_height;
2927         swap(tile_width, tile_height);
2928
2929         /*
2930          * We only keep the x/y offsets, so push all of the
2931          * gtt offset into the x/y offsets.
2932          */
2933         intel_adjust_tile_offset(&x, &y,
2934                                  tile_width, tile_height,
2935                                  tile_size, pitch_tiles,
2936                                  gtt_offset_rotated * tile_size, 0);
2937
2938         /*
2939          * First pixel of the framebuffer from
2940          * the start of the rotated gtt mapping.
2941          */
2942         intel_fb->rotated[plane].x = x;
2943         intel_fb->rotated[plane].y = y;
2944
2945         return plane_info->width * plane_info->height;
2946 }
2947
2948 static int
2949 intel_fill_fb_info(struct drm_i915_private *dev_priv,
2950                    struct drm_framebuffer *fb)
2951 {
2952         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2953         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2954         u32 gtt_offset_rotated = 0;
2955         unsigned int max_size = 0;
2956         int i, num_planes = fb->format->num_planes;
2957         unsigned int tile_size = intel_tile_size(dev_priv);
2958
2959         for (i = 0; i < num_planes; i++) {
2960                 unsigned int width, height;
2961                 unsigned int cpp, size;
2962                 u32 offset;
2963                 int x, y;
2964                 int ret;
2965
2966                 cpp = fb->format->cpp[i];
2967                 intel_fb_plane_dims(&width, &height, fb, i);
2968
2969                 ret = intel_fb_offset_to_xy(&x, &y, fb, i);
2970                 if (ret) {
2971                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2972                                       i, fb->offsets[i]);
2973                         return ret;
2974                 }
2975
2976                 ret = intel_fb_check_ccs_xy(fb, i, x, y);
2977                 if (ret)
2978                         return ret;
2979
2980                 /*
2981                  * The fence (if used) is aligned to the start of the object
2982                  * so having the framebuffer wrap around across the edge of the
2983                  * fenced region doesn't really work. We have no API to configure
2984                  * the fence start offset within the object (nor could we probably
2985                  * on gen2/3). So it's just easier if we just require that the
2986                  * fb layout agrees with the fence layout. We already check that the
2987                  * fb stride matches the fence stride elsewhere.
2988                  */
2989                 if (i == 0 && i915_gem_object_is_tiled(obj) &&
2990                     (x + width) * cpp > fb->pitches[i]) {
2991                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2992                                       i, fb->offsets[i]);
2993                         return -EINVAL;
2994                 }
2995
2996                 /*
2997                  * First pixel of the framebuffer from
2998                  * the start of the normal gtt mapping.
2999                  */
3000                 intel_fb->normal[i].x = x;
3001                 intel_fb->normal[i].y = y;
3002
3003                 offset = intel_compute_aligned_offset(dev_priv, &x, &y, fb, i,
3004                                                       fb->pitches[i],
3005                                                       DRM_MODE_ROTATE_0,
3006                                                       tile_size);
3007                 offset /= tile_size;
3008
3009                 if (!is_surface_linear(fb, i)) {
3010                         struct intel_remapped_plane_info plane_info;
3011                         unsigned int tile_width, tile_height;
3012
3013                         intel_tile_dims(fb, i, &tile_width, &tile_height);
3014
3015                         plane_info.offset = offset;
3016                         plane_info.stride = DIV_ROUND_UP(fb->pitches[i],
3017                                                          tile_width * cpp);
3018                         plane_info.width = DIV_ROUND_UP(x + width, tile_width);
3019                         plane_info.height = DIV_ROUND_UP(y + height,
3020                                                          tile_height);
3021
3022                         /* how many tiles does this plane need */
3023                         size = plane_info.stride * plane_info.height;
3024                         /*
3025                          * If the plane isn't horizontally tile aligned,
3026                          * we need one more tile.
3027                          */
3028                         if (x != 0)
3029                                 size++;
3030
3031                         gtt_offset_rotated +=
3032                                 setup_fb_rotation(i, &plane_info,
3033                                                   gtt_offset_rotated,
3034                                                   x, y, width, height,
3035                                                   tile_size,
3036                                                   tile_width, tile_height,
3037                                                   fb);
3038                 } else {
3039                         size = DIV_ROUND_UP((y + height) * fb->pitches[i] +
3040                                             x * cpp, tile_size);
3041                 }
3042
3043                 /* how many tiles in total needed in the bo */
3044                 max_size = max(max_size, offset + size);
3045         }
3046
3047         if (mul_u32_u32(max_size, tile_size) > obj->base.size) {
3048                 DRM_DEBUG_KMS("fb too big for bo (need %llu bytes, have %zu bytes)\n",
3049                               mul_u32_u32(max_size, tile_size), obj->base.size);
3050                 return -EINVAL;
3051         }
3052
3053         return 0;
3054 }
3055
3056 static void
3057 intel_plane_remap_gtt(struct intel_plane_state *plane_state)
3058 {
3059         struct drm_i915_private *dev_priv =
3060                 to_i915(plane_state->uapi.plane->dev);
3061         struct drm_framebuffer *fb = plane_state->hw.fb;
3062         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
3063         struct intel_rotation_info *info = &plane_state->view.rotated;
3064         unsigned int rotation = plane_state->hw.rotation;
3065         int i, num_planes = fb->format->num_planes;
3066         unsigned int tile_size = intel_tile_size(dev_priv);
3067         unsigned int src_x, src_y;
3068         unsigned int src_w, src_h;
3069         u32 gtt_offset = 0;
3070
3071         memset(&plane_state->view, 0, sizeof(plane_state->view));
3072         plane_state->view.type = drm_rotation_90_or_270(rotation) ?
3073                 I915_GGTT_VIEW_ROTATED : I915_GGTT_VIEW_REMAPPED;
3074
3075         src_x = plane_state->uapi.src.x1 >> 16;
3076         src_y = plane_state->uapi.src.y1 >> 16;
3077         src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
3078         src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
3079
3080         WARN_ON(is_ccs_modifier(fb->modifier));
3081
3082         /* Make src coordinates relative to the viewport */
3083         drm_rect_translate(&plane_state->uapi.src,
3084                            -(src_x << 16), -(src_y << 16));
3085
3086         /* Rotate src coordinates to match rotated GTT view */
3087         if (drm_rotation_90_or_270(rotation))
3088                 drm_rect_rotate(&plane_state->uapi.src,
3089                                 src_w << 16, src_h << 16,
3090                                 DRM_MODE_ROTATE_270);
3091
3092         for (i = 0; i < num_planes; i++) {
3093                 unsigned int hsub = i ? fb->format->hsub : 1;
3094                 unsigned int vsub = i ? fb->format->vsub : 1;
3095                 unsigned int cpp = fb->format->cpp[i];
3096                 unsigned int tile_width, tile_height;
3097                 unsigned int width, height;
3098                 unsigned int pitch_tiles;
3099                 unsigned int x, y;
3100                 u32 offset;
3101
3102                 intel_tile_dims(fb, i, &tile_width, &tile_height);
3103
3104                 x = src_x / hsub;
3105                 y = src_y / vsub;
3106                 width = src_w / hsub;
3107                 height = src_h / vsub;
3108
3109                 /*
3110                  * First pixel of the src viewport from the
3111                  * start of the normal gtt mapping.
3112                  */
3113                 x += intel_fb->normal[i].x;
3114                 y += intel_fb->normal[i].y;
3115
3116                 offset = intel_compute_aligned_offset(dev_priv, &x, &y,
3117                                                       fb, i, fb->pitches[i],
3118                                                       DRM_MODE_ROTATE_0, tile_size);
3119                 offset /= tile_size;
3120
3121                 WARN_ON(i >= ARRAY_SIZE(info->plane));
3122                 info->plane[i].offset = offset;
3123                 info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i],
3124                                                      tile_width * cpp);
3125                 info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
3126                 info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
3127
3128                 if (drm_rotation_90_or_270(rotation)) {
3129                         struct drm_rect r;
3130
3131                         /* rotate the x/y offsets to match the GTT view */
3132                         drm_rect_init(&r, x, y, width, height);
3133                         drm_rect_rotate(&r,
3134                                         info->plane[i].width * tile_width,
3135                                         info->plane[i].height * tile_height,
3136                                         DRM_MODE_ROTATE_270);
3137                         x = r.x1;
3138                         y = r.y1;
3139
3140                         pitch_tiles = info->plane[i].height;
3141                         plane_state->color_plane[i].stride = pitch_tiles * tile_height;
3142
3143                         /* rotate the tile dimensions to match the GTT view */
3144                         swap(tile_width, tile_height);
3145                 } else {
3146                         pitch_tiles = info->plane[i].width;
3147                         plane_state->color_plane[i].stride = pitch_tiles * tile_width * cpp;
3148                 }
3149
3150                 /*
3151                  * We only keep the x/y offsets, so push all of the
3152                  * gtt offset into the x/y offsets.
3153                  */
3154                 intel_adjust_tile_offset(&x, &y,
3155                                          tile_width, tile_height,
3156                                          tile_size, pitch_tiles,
3157                                          gtt_offset * tile_size, 0);
3158
3159                 gtt_offset += info->plane[i].width * info->plane[i].height;
3160
3161                 plane_state->color_plane[i].offset = 0;
3162                 plane_state->color_plane[i].x = x;
3163                 plane_state->color_plane[i].y = y;
3164         }
3165 }
3166
3167 static int
3168 intel_plane_compute_gtt(struct intel_plane_state *plane_state)
3169 {
3170         const struct intel_framebuffer *fb =
3171                 to_intel_framebuffer(plane_state->hw.fb);
3172         unsigned int rotation = plane_state->hw.rotation;
3173         int i, num_planes;
3174
3175         if (!fb)
3176                 return 0;
3177
3178         num_planes = fb->base.format->num_planes;
3179
3180         if (intel_plane_needs_remap(plane_state)) {
3181                 intel_plane_remap_gtt(plane_state);
3182
3183                 /*
3184                  * Sometimes even remapping can't overcome
3185                  * the stride limitations :( Can happen with
3186                  * big plane sizes and suitably misaligned
3187                  * offsets.
3188                  */
3189                 return intel_plane_check_stride(plane_state);
3190         }
3191
3192         intel_fill_fb_ggtt_view(&plane_state->view, &fb->base, rotation);
3193
3194         for (i = 0; i < num_planes; i++) {
3195                 plane_state->color_plane[i].stride = intel_fb_pitch(&fb->base, i, rotation);
3196                 plane_state->color_plane[i].offset = 0;
3197
3198                 if (drm_rotation_90_or_270(rotation)) {
3199                         plane_state->color_plane[i].x = fb->rotated[i].x;
3200                         plane_state->color_plane[i].y = fb->rotated[i].y;
3201                 } else {
3202                         plane_state->color_plane[i].x = fb->normal[i].x;
3203                         plane_state->color_plane[i].y = fb->normal[i].y;
3204                 }
3205         }
3206
3207         /* Rotate src coordinates to match rotated GTT view */
3208         if (drm_rotation_90_or_270(rotation))
3209                 drm_rect_rotate(&plane_state->uapi.src,
3210                                 fb->base.width << 16, fb->base.height << 16,
3211                                 DRM_MODE_ROTATE_270);
3212
3213         return intel_plane_check_stride(plane_state);
3214 }
3215
3216 static int i9xx_format_to_fourcc(int format)
3217 {
3218         switch (format) {
3219         case DISPPLANE_8BPP:
3220                 return DRM_FORMAT_C8;
3221         case DISPPLANE_BGRA555:
3222                 return DRM_FORMAT_ARGB1555;
3223         case DISPPLANE_BGRX555:
3224                 return DRM_FORMAT_XRGB1555;
3225         case DISPPLANE_BGRX565:
3226                 return DRM_FORMAT_RGB565;
3227         default:
3228         case DISPPLANE_BGRX888:
3229                 return DRM_FORMAT_XRGB8888;
3230         case DISPPLANE_RGBX888:
3231                 return DRM_FORMAT_XBGR8888;
3232         case DISPPLANE_BGRA888:
3233                 return DRM_FORMAT_ARGB8888;
3234         case DISPPLANE_RGBA888:
3235                 return DRM_FORMAT_ABGR8888;
3236         case DISPPLANE_BGRX101010:
3237                 return DRM_FORMAT_XRGB2101010;
3238         case DISPPLANE_RGBX101010:
3239                 return DRM_FORMAT_XBGR2101010;
3240         case DISPPLANE_BGRA101010:
3241                 return DRM_FORMAT_ARGB2101010;
3242         case DISPPLANE_RGBA101010:
3243                 return DRM_FORMAT_ABGR2101010;
3244         case DISPPLANE_RGBX161616:
3245                 return DRM_FORMAT_XBGR16161616F;
3246         }
3247 }
3248
3249 int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
3250 {
3251         switch (format) {
3252         case PLANE_CTL_FORMAT_RGB_565:
3253                 return DRM_FORMAT_RGB565;
3254         case PLANE_CTL_FORMAT_NV12:
3255                 return DRM_FORMAT_NV12;
3256         case PLANE_CTL_FORMAT_P010:
3257                 return DRM_FORMAT_P010;
3258         case PLANE_CTL_FORMAT_P012:
3259                 return DRM_FORMAT_P012;
3260         case PLANE_CTL_FORMAT_P016:
3261                 return DRM_FORMAT_P016;
3262         case PLANE_CTL_FORMAT_Y210:
3263                 return DRM_FORMAT_Y210;
3264         case PLANE_CTL_FORMAT_Y212:
3265                 return DRM_FORMAT_Y212;
3266         case PLANE_CTL_FORMAT_Y216:
3267                 return DRM_FORMAT_Y216;
3268         case PLANE_CTL_FORMAT_Y410:
3269                 return DRM_FORMAT_XVYU2101010;
3270         case PLANE_CTL_FORMAT_Y412:
3271                 return DRM_FORMAT_XVYU12_16161616;
3272         case PLANE_CTL_FORMAT_Y416:
3273                 return DRM_FORMAT_XVYU16161616;
3274         default:
3275         case PLANE_CTL_FORMAT_XRGB_8888:
3276                 if (rgb_order) {
3277                         if (alpha)
3278                                 return DRM_FORMAT_ABGR8888;
3279                         else
3280                                 return DRM_FORMAT_XBGR8888;
3281                 } else {
3282                         if (alpha)
3283                                 return DRM_FORMAT_ARGB8888;
3284                         else
3285                                 return DRM_FORMAT_XRGB8888;
3286                 }
3287         case PLANE_CTL_FORMAT_XRGB_2101010:
3288                 if (rgb_order) {
3289                         if (alpha)
3290                                 return DRM_FORMAT_ABGR2101010;
3291                         else
3292                                 return DRM_FORMAT_XBGR2101010;
3293                 } else {
3294                         if (alpha)
3295                                 return DRM_FORMAT_ARGB2101010;
3296                         else
3297                                 return DRM_FORMAT_XRGB2101010;
3298                 }
3299         case PLANE_CTL_FORMAT_XRGB_16161616F:
3300                 if (rgb_order) {
3301                         if (alpha)
3302                                 return DRM_FORMAT_ABGR16161616F;
3303                         else
3304                                 return DRM_FORMAT_XBGR16161616F;
3305                 } else {
3306                         if (alpha)
3307                                 return DRM_FORMAT_ARGB16161616F;
3308                         else
3309                                 return DRM_FORMAT_XRGB16161616F;
3310                 }
3311         }
3312 }
3313
3314 static bool
3315 intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
3316                               struct intel_initial_plane_config *plane_config)
3317 {
3318         struct drm_device *dev = crtc->base.dev;
3319         struct drm_i915_private *dev_priv = to_i915(dev);
3320         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
3321         struct drm_framebuffer *fb = &plane_config->fb->base;
3322         u32 base_aligned = round_down(plane_config->base, PAGE_SIZE);
3323         u32 size_aligned = round_up(plane_config->base + plane_config->size,
3324                                     PAGE_SIZE);
3325         struct drm_i915_gem_object *obj;
3326         bool ret = false;
3327
3328         size_aligned -= base_aligned;
3329
3330         if (plane_config->size == 0)
3331                 return false;
3332
3333         /* If the FB is too big, just don't use it since fbdev is not very
3334          * important and we should probably use that space with FBC or other
3335          * features. */
3336         if (size_aligned * 2 > dev_priv->stolen_usable_size)
3337                 return false;
3338
3339         switch (fb->modifier) {
3340         case DRM_FORMAT_MOD_LINEAR:
3341         case I915_FORMAT_MOD_X_TILED:
3342         case I915_FORMAT_MOD_Y_TILED:
3343                 break;
3344         default:
3345                 DRM_DEBUG_DRIVER("Unsupported modifier for initial FB: 0x%llx\n",
3346                                  fb->modifier);
3347                 return false;
3348         }
3349
3350         obj = i915_gem_object_create_stolen_for_preallocated(dev_priv,
3351                                                              base_aligned,
3352                                                              base_aligned,
3353                                                              size_aligned);
3354         if (IS_ERR(obj))
3355                 return false;
3356
3357         switch (plane_config->tiling) {
3358         case I915_TILING_NONE:
3359                 break;
3360         case I915_TILING_X:
3361         case I915_TILING_Y:
3362                 obj->tiling_and_stride = fb->pitches[0] | plane_config->tiling;
3363                 break;
3364         default:
3365                 MISSING_CASE(plane_config->tiling);
3366                 goto out;
3367         }
3368
3369         mode_cmd.pixel_format = fb->format->format;
3370         mode_cmd.width = fb->width;
3371         mode_cmd.height = fb->height;
3372         mode_cmd.pitches[0] = fb->pitches[0];
3373         mode_cmd.modifier[0] = fb->modifier;
3374         mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
3375
3376         if (intel_framebuffer_init(to_intel_framebuffer(fb), obj, &mode_cmd)) {
3377                 DRM_DEBUG_KMS("intel fb init failed\n");
3378                 goto out;
3379         }
3380
3381
3382         DRM_DEBUG_KMS("initial plane fb obj %p\n", obj);
3383         ret = true;
3384 out:
3385         i915_gem_object_put(obj);
3386         return ret;
3387 }
3388
3389 static void
3390 intel_set_plane_visible(struct intel_crtc_state *crtc_state,
3391                         struct intel_plane_state *plane_state,
3392                         bool visible)
3393 {
3394         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
3395
3396         plane_state->uapi.visible = visible;
3397
3398         if (visible)
3399                 crtc_state->uapi.plane_mask |= drm_plane_mask(&plane->base);
3400         else
3401                 crtc_state->uapi.plane_mask &= ~drm_plane_mask(&plane->base);
3402 }
3403
3404 static void fixup_active_planes(struct intel_crtc_state *crtc_state)
3405 {
3406         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3407         struct drm_plane *plane;
3408
3409         /*
3410          * Active_planes aliases if multiple "primary" or cursor planes
3411          * have been used on the same (or wrong) pipe. plane_mask uses
3412          * unique ids, hence we can use that to reconstruct active_planes.
3413          */
3414         crtc_state->active_planes = 0;
3415
3416         drm_for_each_plane_mask(plane, &dev_priv->drm,
3417                                 crtc_state->uapi.plane_mask)
3418                 crtc_state->active_planes |= BIT(to_intel_plane(plane)->id);
3419 }
3420
3421 static void intel_plane_disable_noatomic(struct intel_crtc *crtc,
3422                                          struct intel_plane *plane)
3423 {
3424         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3425         struct intel_crtc_state *crtc_state =
3426                 to_intel_crtc_state(crtc->base.state);
3427         struct intel_plane_state *plane_state =
3428                 to_intel_plane_state(plane->base.state);
3429
3430         DRM_DEBUG_KMS("Disabling [PLANE:%d:%s] on [CRTC:%d:%s]\n",
3431                       plane->base.base.id, plane->base.name,
3432                       crtc->base.base.id, crtc->base.name);
3433
3434         intel_set_plane_visible(crtc_state, plane_state, false);
3435         fixup_active_planes(crtc_state);
3436         crtc_state->data_rate[plane->id] = 0;
3437         crtc_state->min_cdclk[plane->id] = 0;
3438
3439         if (plane->id == PLANE_PRIMARY)
3440                 hsw_disable_ips(crtc_state);
3441
3442         /*
3443          * Vblank time updates from the shadow to live plane control register
3444          * are blocked if the memory self-refresh mode is active at that
3445          * moment. So to make sure the plane gets truly disabled, disable
3446          * first the self-refresh mode. The self-refresh enable bit in turn
3447          * will be checked/applied by the HW only at the next frame start
3448          * event which is after the vblank start event, so we need to have a
3449          * wait-for-vblank between disabling the plane and the pipe.
3450          */
3451         if (HAS_GMCH(dev_priv) &&
3452             intel_set_memory_cxsr(dev_priv, false))
3453                 intel_wait_for_vblank(dev_priv, crtc->pipe);
3454
3455         /*
3456          * Gen2 reports pipe underruns whenever all planes are disabled.
3457          * So disable underrun reporting before all the planes get disabled.
3458          */
3459         if (IS_GEN(dev_priv, 2) && !crtc_state->active_planes)
3460                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
3461
3462         intel_disable_plane(plane, crtc_state);
3463 }
3464
3465 static struct intel_frontbuffer *
3466 to_intel_frontbuffer(struct drm_framebuffer *fb)
3467 {
3468         return fb ? to_intel_framebuffer(fb)->frontbuffer : NULL;
3469 }
3470
3471 static void
3472 intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
3473                              struct intel_initial_plane_config *plane_config)
3474 {
3475         struct drm_device *dev = intel_crtc->base.dev;
3476         struct drm_i915_private *dev_priv = to_i915(dev);
3477         struct drm_crtc *c;
3478         struct drm_plane *primary = intel_crtc->base.primary;
3479         struct drm_plane_state *plane_state = primary->state;
3480         struct intel_plane *intel_plane = to_intel_plane(primary);
3481         struct intel_plane_state *intel_state =
3482                 to_intel_plane_state(plane_state);
3483         struct drm_framebuffer *fb;
3484
3485         if (!plane_config->fb)
3486                 return;
3487
3488         if (intel_alloc_initial_plane_obj(intel_crtc, plane_config)) {
3489                 fb = &plane_config->fb->base;
3490                 goto valid_fb;
3491         }
3492
3493         kfree(plane_config->fb);
3494
3495         /*
3496          * Failed to alloc the obj, check to see if we should share
3497          * an fb with another CRTC instead
3498          */
3499         for_each_crtc(dev, c) {
3500                 struct intel_plane_state *state;
3501
3502                 if (c == &intel_crtc->base)
3503                         continue;
3504
3505                 if (!to_intel_crtc(c)->active)
3506                         continue;
3507
3508                 state = to_intel_plane_state(c->primary->state);
3509                 if (!state->vma)
3510                         continue;
3511
3512                 if (intel_plane_ggtt_offset(state) == plane_config->base) {
3513                         fb = state->hw.fb;
3514                         drm_framebuffer_get(fb);
3515                         goto valid_fb;
3516                 }
3517         }
3518
3519         /*
3520          * We've failed to reconstruct the BIOS FB.  Current display state
3521          * indicates that the primary plane is visible, but has a NULL FB,
3522          * which will lead to problems later if we don't fix it up.  The
3523          * simplest solution is to just disable the primary plane now and
3524          * pretend the BIOS never had it enabled.
3525          */
3526         intel_plane_disable_noatomic(intel_crtc, intel_plane);
3527
3528         return;
3529
3530 valid_fb:
3531         intel_state->hw.rotation = plane_config->rotation;
3532         intel_fill_fb_ggtt_view(&intel_state->view, fb,
3533                                 intel_state->hw.rotation);
3534         intel_state->color_plane[0].stride =
3535                 intel_fb_pitch(fb, 0, intel_state->hw.rotation);
3536
3537         intel_state->vma =
3538                 intel_pin_and_fence_fb_obj(fb,
3539                                            &intel_state->view,
3540                                            intel_plane_uses_fence(intel_state),
3541                                            &intel_state->flags);
3542         if (IS_ERR(intel_state->vma)) {
3543                 DRM_ERROR("failed to pin boot fb on pipe %d: %li\n",
3544                           intel_crtc->pipe, PTR_ERR(intel_state->vma));
3545
3546                 intel_state->vma = NULL;
3547                 drm_framebuffer_put(fb);
3548                 return;
3549         }
3550
3551         intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB);
3552
3553         plane_state->src_x = 0;
3554         plane_state->src_y = 0;
3555         plane_state->src_w = fb->width << 16;
3556         plane_state->src_h = fb->height << 16;
3557
3558         plane_state->crtc_x = 0;
3559         plane_state->crtc_y = 0;
3560         plane_state->crtc_w = fb->width;
3561         plane_state->crtc_h = fb->height;
3562
3563         intel_state->uapi.src = drm_plane_state_src(plane_state);
3564         intel_state->uapi.dst = drm_plane_state_dest(plane_state);
3565
3566         if (plane_config->tiling)
3567                 dev_priv->preserve_bios_swizzle = true;
3568
3569         plane_state->fb = fb;
3570         plane_state->crtc = &intel_crtc->base;
3571         intel_plane_copy_uapi_to_hw_state(intel_state, intel_state);
3572
3573         atomic_or(to_intel_plane(primary)->frontbuffer_bit,
3574                   &to_intel_frontbuffer(fb)->bits);
3575 }
3576
3577 static int skl_max_plane_width(const struct drm_framebuffer *fb,
3578                                int color_plane,
3579                                unsigned int rotation)
3580 {
3581         int cpp = fb->format->cpp[color_plane];
3582
3583         switch (fb->modifier) {
3584         case DRM_FORMAT_MOD_LINEAR:
3585         case I915_FORMAT_MOD_X_TILED:
3586                 /*
3587                  * Validated limit is 4k, but has 5k should
3588                  * work apart from the following features:
3589                  * - Ytile (already limited to 4k)
3590                  * - FP16 (already limited to 4k)
3591                  * - render compression (already limited to 4k)
3592                  * - KVMR sprite and cursor (don't care)
3593                  * - horizontal panning (TODO verify this)
3594                  * - pipe and plane scaling (TODO verify this)
3595                  */
3596                 if (cpp == 8)
3597                         return 4096;
3598                 else
3599                         return 5120;
3600         case I915_FORMAT_MOD_Y_TILED_CCS:
3601         case I915_FORMAT_MOD_Yf_TILED_CCS:
3602                 /* FIXME AUX plane? */
3603         case I915_FORMAT_MOD_Y_TILED:
3604         case I915_FORMAT_MOD_Yf_TILED:
3605                 if (cpp == 8)
3606                         return 2048;
3607                 else
3608                         return 4096;
3609         default:
3610                 MISSING_CASE(fb->modifier);
3611                 return 2048;
3612         }
3613 }
3614
3615 static int glk_max_plane_width(const struct drm_framebuffer *fb,
3616                                int color_plane,
3617                                unsigned int rotation)
3618 {
3619         int cpp = fb->format->cpp[color_plane];
3620
3621         switch (fb->modifier) {
3622         case DRM_FORMAT_MOD_LINEAR:
3623         case I915_FORMAT_MOD_X_TILED:
3624                 if (cpp == 8)
3625                         return 4096;
3626                 else
3627                         return 5120;
3628         case I915_FORMAT_MOD_Y_TILED_CCS:
3629         case I915_FORMAT_MOD_Yf_TILED_CCS:
3630                 /* FIXME AUX plane? */
3631         case I915_FORMAT_MOD_Y_TILED:
3632         case I915_FORMAT_MOD_Yf_TILED:
3633                 if (cpp == 8)
3634                         return 2048;
3635                 else
3636                         return 5120;
3637         default:
3638                 MISSING_CASE(fb->modifier);
3639                 return 2048;
3640         }
3641 }
3642
3643 static int icl_max_plane_width(const struct drm_framebuffer *fb,
3644                                int color_plane,
3645                                unsigned int rotation)
3646 {
3647         return 5120;
3648 }
3649
3650 static int skl_max_plane_height(void)
3651 {
3652         return 4096;
3653 }
3654
3655 static int icl_max_plane_height(void)
3656 {
3657         return 4320;
3658 }
3659
3660 static bool skl_check_main_ccs_coordinates(struct intel_plane_state *plane_state,
3661                                            int main_x, int main_y, u32 main_offset)
3662 {
3663         const struct drm_framebuffer *fb = plane_state->hw.fb;
3664         int ccs_plane = main_to_ccs_plane(fb, 0);
3665         int aux_x = plane_state->color_plane[ccs_plane].x;
3666         int aux_y = plane_state->color_plane[ccs_plane].y;
3667         u32 aux_offset = plane_state->color_plane[ccs_plane].offset;
3668         u32 alignment = intel_surf_alignment(fb, ccs_plane);
3669         int hsub;
3670         int vsub;
3671
3672         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, ccs_plane);
3673         while (aux_offset >= main_offset && aux_y <= main_y) {
3674                 int x, y;
3675
3676                 if (aux_x == main_x && aux_y == main_y)
3677                         break;
3678
3679                 if (aux_offset == 0)
3680                         break;
3681
3682                 x = aux_x / hsub;
3683                 y = aux_y / vsub;
3684                 aux_offset = intel_plane_adjust_aligned_offset(&x, &y,
3685                                                                plane_state,
3686                                                                ccs_plane,
3687                                                                aux_offset,
3688                                                                aux_offset -
3689                                                                 alignment);
3690                 aux_x = x * hsub + aux_x % hsub;
3691                 aux_y = y * vsub + aux_y % vsub;
3692         }
3693
3694         if (aux_x != main_x || aux_y != main_y)
3695                 return false;
3696
3697         plane_state->color_plane[ccs_plane].offset = aux_offset;
3698         plane_state->color_plane[ccs_plane].x = aux_x;
3699         plane_state->color_plane[ccs_plane].y = aux_y;
3700
3701         return true;
3702 }
3703
3704 static int skl_check_main_surface(struct intel_plane_state *plane_state)
3705 {
3706         struct drm_i915_private *dev_priv = to_i915(plane_state->uapi.plane->dev);
3707         const struct drm_framebuffer *fb = plane_state->hw.fb;
3708         unsigned int rotation = plane_state->hw.rotation;
3709         int x = plane_state->uapi.src.x1 >> 16;
3710         int y = plane_state->uapi.src.y1 >> 16;
3711         int w = drm_rect_width(&plane_state->uapi.src) >> 16;
3712         int h = drm_rect_height(&plane_state->uapi.src) >> 16;
3713         int max_width;
3714         int max_height;
3715         u32 alignment;
3716         u32 offset;
3717         int aux_plane = intel_main_to_aux_plane(fb, 0);
3718         u32 aux_offset = plane_state->color_plane[aux_plane].offset;
3719
3720         if (INTEL_GEN(dev_priv) >= 11)
3721                 max_width = icl_max_plane_width(fb, 0, rotation);
3722         else if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
3723                 max_width = glk_max_plane_width(fb, 0, rotation);
3724         else
3725                 max_width = skl_max_plane_width(fb, 0, rotation);
3726
3727         if (INTEL_GEN(dev_priv) >= 11)
3728                 max_height = icl_max_plane_height();
3729         else
3730                 max_height = skl_max_plane_height();
3731
3732         if (w > max_width || h > max_height) {
3733                 DRM_DEBUG_KMS("requested Y/RGB source size %dx%d too big (limit %dx%d)\n",
3734                               w, h, max_width, max_height);
3735                 return -EINVAL;
3736         }
3737
3738         intel_add_fb_offsets(&x, &y, plane_state, 0);
3739         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 0);
3740         alignment = intel_surf_alignment(fb, 0);
3741
3742         /*
3743          * AUX surface offset is specified as the distance from the
3744          * main surface offset, and it must be non-negative. Make
3745          * sure that is what we will get.
3746          */
3747         if (offset > aux_offset)
3748                 offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3749                                                            offset, aux_offset & ~(alignment - 1));
3750
3751         /*
3752          * When using an X-tiled surface, the plane blows up
3753          * if the x offset + width exceed the stride.
3754          *
3755          * TODO: linear and Y-tiled seem fine, Yf untested,
3756          */
3757         if (fb->modifier == I915_FORMAT_MOD_X_TILED) {
3758                 int cpp = fb->format->cpp[0];
3759
3760                 while ((x + w) * cpp > plane_state->color_plane[0].stride) {
3761                         if (offset == 0) {
3762                                 DRM_DEBUG_KMS("Unable to find suitable display surface offset due to X-tiling\n");
3763                                 return -EINVAL;
3764                         }
3765
3766                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3767                                                                    offset, offset - alignment);
3768                 }
3769         }
3770
3771         /*
3772          * CCS AUX surface doesn't have its own x/y offsets, we must make sure
3773          * they match with the main surface x/y offsets.
3774          */
3775         if (is_ccs_modifier(fb->modifier)) {
3776                 while (!skl_check_main_ccs_coordinates(plane_state, x, y, offset)) {
3777                         if (offset == 0)
3778                                 break;
3779
3780                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3781                                                                    offset, offset - alignment);
3782                 }
3783
3784                 if (x != plane_state->color_plane[aux_plane].x ||
3785                     y != plane_state->color_plane[aux_plane].y) {
3786                         DRM_DEBUG_KMS("Unable to find suitable display surface offset due to CCS\n");
3787                         return -EINVAL;
3788                 }
3789         }
3790
3791         plane_state->color_plane[0].offset = offset;
3792         plane_state->color_plane[0].x = x;
3793         plane_state->color_plane[0].y = y;
3794
3795         /*
3796          * Put the final coordinates back so that the src
3797          * coordinate checks will see the right values.
3798          */
3799         drm_rect_translate_to(&plane_state->uapi.src,
3800                               x << 16, y << 16);
3801
3802         return 0;
3803 }
3804
3805 static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
3806 {
3807         const struct drm_framebuffer *fb = plane_state->hw.fb;
3808         unsigned int rotation = plane_state->hw.rotation;
3809         int max_width = skl_max_plane_width(fb, 1, rotation);
3810         int max_height = 4096;
3811         int x = plane_state->uapi.src.x1 >> 17;
3812         int y = plane_state->uapi.src.y1 >> 17;
3813         int w = drm_rect_width(&plane_state->uapi.src) >> 17;
3814         int h = drm_rect_height(&plane_state->uapi.src) >> 17;
3815         u32 offset;
3816
3817         intel_add_fb_offsets(&x, &y, plane_state, 1);
3818         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3819
3820         /* FIXME not quite sure how/if these apply to the chroma plane */
3821         if (w > max_width || h > max_height) {
3822                 DRM_DEBUG_KMS("CbCr source size %dx%d too big (limit %dx%d)\n",
3823                               w, h, max_width, max_height);
3824                 return -EINVAL;
3825         }
3826
3827         plane_state->color_plane[1].offset = offset;
3828         plane_state->color_plane[1].x = x;
3829         plane_state->color_plane[1].y = y;
3830
3831         return 0;
3832 }
3833
3834 static int skl_check_ccs_aux_surface(struct intel_plane_state *plane_state)
3835 {
3836         const struct drm_framebuffer *fb = plane_state->hw.fb;
3837         int src_x = plane_state->uapi.src.x1 >> 16;
3838         int src_y = plane_state->uapi.src.y1 >> 16;
3839         int hsub;
3840         int vsub;
3841         int x;
3842         int y;
3843         u32 offset;
3844
3845         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, 1);
3846         x = src_x / hsub;
3847         y = src_y / vsub;
3848         intel_add_fb_offsets(&x, &y, plane_state, 1);
3849         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3850
3851         plane_state->color_plane[1].offset = offset;
3852         plane_state->color_plane[1].x = x * hsub + src_x % hsub;
3853         plane_state->color_plane[1].y = y * vsub + src_y % vsub;
3854
3855         return 0;
3856 }
3857
3858 int skl_check_plane_surface(struct intel_plane_state *plane_state)
3859 {
3860         const struct drm_framebuffer *fb = plane_state->hw.fb;
3861         int ret;
3862
3863         ret = intel_plane_compute_gtt(plane_state);
3864         if (ret)
3865                 return ret;
3866
3867         if (!plane_state->uapi.visible)
3868                 return 0;
3869
3870         /*
3871          * Handle the AUX surface first since
3872          * the main surface setup depends on it.
3873          */
3874         if (intel_format_info_is_yuv_semiplanar(fb->format,
3875                                                 fb->modifier)) {
3876                 ret = skl_check_nv12_aux_surface(plane_state);
3877                 if (ret)
3878                         return ret;
3879         } else if (is_ccs_modifier(fb->modifier)) {
3880                 ret = skl_check_ccs_aux_surface(plane_state);
3881                 if (ret)
3882                         return ret;
3883         } else {
3884                 plane_state->color_plane[1].offset = ~0xfff;
3885                 plane_state->color_plane[1].x = 0;
3886                 plane_state->color_plane[1].y = 0;
3887         }
3888
3889         ret = skl_check_main_surface(plane_state);
3890         if (ret)
3891                 return ret;
3892
3893         return 0;
3894 }
3895
3896 static void i9xx_plane_ratio(const struct intel_crtc_state *crtc_state,
3897                              const struct intel_plane_state *plane_state,
3898                              unsigned int *num, unsigned int *den)
3899 {
3900         const struct drm_framebuffer *fb = plane_state->hw.fb;
3901         unsigned int cpp = fb->format->cpp[0];
3902
3903         /*
3904          * g4x bspec says 64bpp pixel rate can't exceed 80%
3905          * of cdclk when the sprite plane is enabled on the
3906          * same pipe. ilk/snb bspec says 64bpp pixel rate is
3907          * never allowed to exceed 80% of cdclk. Let's just go
3908          * with the ilk/snb limit always.
3909          */
3910         if (cpp == 8) {
3911                 *num = 10;
3912                 *den = 8;
3913         } else {
3914                 *num = 1;
3915                 *den = 1;
3916         }
3917 }
3918
3919 static int i9xx_plane_min_cdclk(const struct intel_crtc_state *crtc_state,
3920                                 const struct intel_plane_state *plane_state)
3921 {
3922         unsigned int pixel_rate;
3923         unsigned int num, den;
3924
3925         /*
3926          * Note that crtc_state->pixel_rate accounts for both
3927          * horizontal and vertical panel fitter downscaling factors.
3928          * Pre-HSW bspec tells us to only consider the horizontal
3929          * downscaling factor here. We ignore that and just consider
3930          * both for simplicity.
3931          */
3932         pixel_rate = crtc_state->pixel_rate;
3933
3934         i9xx_plane_ratio(crtc_state, plane_state, &num, &den);
3935
3936         /* two pixels per clock with double wide pipe */
3937         if (crtc_state->double_wide)
3938                 den *= 2;
3939
3940         return DIV_ROUND_UP(pixel_rate * num, den);
3941 }
3942
3943 unsigned int
3944 i9xx_plane_max_stride(struct intel_plane *plane,
3945                       u32 pixel_format, u64 modifier,
3946                       unsigned int rotation)
3947 {
3948         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3949
3950         if (!HAS_GMCH(dev_priv)) {
3951                 return 32*1024;
3952         } else if (INTEL_GEN(dev_priv) >= 4) {
3953                 if (modifier == I915_FORMAT_MOD_X_TILED)
3954                         return 16*1024;
3955                 else
3956                         return 32*1024;
3957         } else if (INTEL_GEN(dev_priv) >= 3) {
3958                 if (modifier == I915_FORMAT_MOD_X_TILED)
3959                         return 8*1024;
3960                 else
3961                         return 16*1024;
3962         } else {
3963                 if (plane->i9xx_plane == PLANE_C)
3964                         return 4*1024;
3965                 else
3966                         return 8*1024;
3967         }
3968 }
3969
3970 static u32 i9xx_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
3971 {
3972         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3973         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3974         u32 dspcntr = 0;
3975
3976         if (crtc_state->gamma_enable)
3977                 dspcntr |= DISPPLANE_GAMMA_ENABLE;
3978
3979         if (crtc_state->csc_enable)
3980                 dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
3981
3982         if (INTEL_GEN(dev_priv) < 5)
3983                 dspcntr |= DISPPLANE_SEL_PIPE(crtc->pipe);
3984
3985         return dspcntr;
3986 }
3987
3988 static u32 i9xx_plane_ctl(const struct intel_crtc_state *crtc_state,
3989                           const struct intel_plane_state *plane_state)
3990 {
3991         struct drm_i915_private *dev_priv =
3992                 to_i915(plane_state->uapi.plane->dev);
3993         const struct drm_framebuffer *fb = plane_state->hw.fb;
3994         unsigned int rotation = plane_state->hw.rotation;
3995         u32 dspcntr;
3996
3997         dspcntr = DISPLAY_PLANE_ENABLE;
3998
3999         if (IS_G4X(dev_priv) || IS_GEN(dev_priv, 5) ||
4000             IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
4001                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
4002
4003         switch (fb->format->format) {
4004         case DRM_FORMAT_C8:
4005                 dspcntr |= DISPPLANE_8BPP;
4006                 break;
4007         case DRM_FORMAT_XRGB1555:
4008                 dspcntr |= DISPPLANE_BGRX555;
4009                 break;
4010         case DRM_FORMAT_ARGB1555:
4011                 dspcntr |= DISPPLANE_BGRA555;
4012                 break;
4013         case DRM_FORMAT_RGB565:
4014                 dspcntr |= DISPPLANE_BGRX565;
4015                 break;
4016         case DRM_FORMAT_XRGB8888:
4017                 dspcntr |= DISPPLANE_BGRX888;
4018                 break;
4019         case DRM_FORMAT_XBGR8888:
4020                 dspcntr |= DISPPLANE_RGBX888;
4021                 break;
4022         case DRM_FORMAT_ARGB8888:
4023                 dspcntr |= DISPPLANE_BGRA888;
4024                 break;
4025         case DRM_FORMAT_ABGR8888:
4026                 dspcntr |= DISPPLANE_RGBA888;
4027                 break;
4028         case DRM_FORMAT_XRGB2101010:
4029                 dspcntr |= DISPPLANE_BGRX101010;
4030                 break;
4031         case DRM_FORMAT_XBGR2101010:
4032                 dspcntr |= DISPPLANE_RGBX101010;
4033                 break;
4034         case DRM_FORMAT_ARGB2101010:
4035                 dspcntr |= DISPPLANE_BGRA101010;
4036                 break;
4037         case DRM_FORMAT_ABGR2101010:
4038                 dspcntr |= DISPPLANE_RGBA101010;
4039                 break;
4040         case DRM_FORMAT_XBGR16161616F:
4041                 dspcntr |= DISPPLANE_RGBX161616;
4042                 break;
4043         default:
4044                 MISSING_CASE(fb->format->format);
4045                 return 0;
4046         }
4047
4048         if (INTEL_GEN(dev_priv) >= 4 &&
4049             fb->modifier == I915_FORMAT_MOD_X_TILED)
4050                 dspcntr |= DISPPLANE_TILED;
4051
4052         if (rotation & DRM_MODE_ROTATE_180)
4053                 dspcntr |= DISPPLANE_ROTATE_180;
4054
4055         if (rotation & DRM_MODE_REFLECT_X)
4056                 dspcntr |= DISPPLANE_MIRROR;
4057
4058         return dspcntr;
4059 }
4060
4061 int i9xx_check_plane_surface(struct intel_plane_state *plane_state)
4062 {
4063         struct drm_i915_private *dev_priv =
4064                 to_i915(plane_state->uapi.plane->dev);
4065         const struct drm_framebuffer *fb = plane_state->hw.fb;
4066         int src_x, src_y, src_w;
4067         u32 offset;
4068         int ret;
4069
4070         ret = intel_plane_compute_gtt(plane_state);
4071         if (ret)
4072                 return ret;
4073
4074         if (!plane_state->uapi.visible)
4075                 return 0;
4076
4077         src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4078         src_x = plane_state->uapi.src.x1 >> 16;
4079         src_y = plane_state->uapi.src.y1 >> 16;
4080
4081         /* Undocumented hardware limit on i965/g4x/vlv/chv */
4082         if (HAS_GMCH(dev_priv) && fb->format->cpp[0] == 8 && src_w > 2048)
4083                 return -EINVAL;
4084
4085         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
4086
4087         if (INTEL_GEN(dev_priv) >= 4)
4088                 offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
4089                                                             plane_state, 0);
4090         else
4091                 offset = 0;
4092
4093         /*
4094          * Put the final coordinates back so that the src
4095          * coordinate checks will see the right values.
4096          */
4097         drm_rect_translate_to(&plane_state->uapi.src,
4098                               src_x << 16, src_y << 16);
4099
4100         /* HSW/BDW do this automagically in hardware */
4101         if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) {
4102                 unsigned int rotation = plane_state->hw.rotation;
4103                 int src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4104                 int src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
4105
4106                 if (rotation & DRM_MODE_ROTATE_180) {
4107                         src_x += src_w - 1;
4108                         src_y += src_h - 1;
4109                 } else if (rotation & DRM_MODE_REFLECT_X) {
4110                         src_x += src_w - 1;
4111                 }
4112         }
4113
4114         plane_state->color_plane[0].offset = offset;
4115         plane_state->color_plane[0].x = src_x;
4116         plane_state->color_plane[0].y = src_y;
4117
4118         return 0;
4119 }
4120
4121 static bool i9xx_plane_has_windowing(struct intel_plane *plane)
4122 {
4123         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4124         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4125
4126         if (IS_CHERRYVIEW(dev_priv))
4127                 return i9xx_plane == PLANE_B;
4128         else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
4129                 return false;
4130         else if (IS_GEN(dev_priv, 4))
4131                 return i9xx_plane == PLANE_C;
4132         else
4133                 return i9xx_plane == PLANE_B ||
4134                         i9xx_plane == PLANE_C;
4135 }
4136
4137 static int
4138 i9xx_plane_check(struct intel_crtc_state *crtc_state,
4139                  struct intel_plane_state *plane_state)
4140 {
4141         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4142         int ret;
4143
4144         ret = chv_plane_check_rotation(plane_state);
4145         if (ret)
4146                 return ret;
4147
4148         ret = drm_atomic_helper_check_plane_state(&plane_state->uapi,
4149                                                   &crtc_state->uapi,
4150                                                   DRM_PLANE_HELPER_NO_SCALING,
4151                                                   DRM_PLANE_HELPER_NO_SCALING,
4152                                                   i9xx_plane_has_windowing(plane),
4153                                                   true);
4154         if (ret)
4155                 return ret;
4156
4157         ret = i9xx_check_plane_surface(plane_state);
4158         if (ret)
4159                 return ret;
4160
4161         if (!plane_state->uapi.visible)
4162                 return 0;
4163
4164         ret = intel_plane_check_src_coordinates(plane_state);
4165         if (ret)
4166                 return ret;
4167
4168         plane_state->ctl = i9xx_plane_ctl(crtc_state, plane_state);
4169
4170         return 0;
4171 }
4172
4173 static void i9xx_update_plane(struct intel_plane *plane,
4174                               const struct intel_crtc_state *crtc_state,
4175                               const struct intel_plane_state *plane_state)
4176 {
4177         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4178         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4179         u32 linear_offset;
4180         int x = plane_state->color_plane[0].x;
4181         int y = plane_state->color_plane[0].y;
4182         int crtc_x = plane_state->uapi.dst.x1;
4183         int crtc_y = plane_state->uapi.dst.y1;
4184         int crtc_w = drm_rect_width(&plane_state->uapi.dst);
4185         int crtc_h = drm_rect_height(&plane_state->uapi.dst);
4186         unsigned long irqflags;
4187         u32 dspaddr_offset;
4188         u32 dspcntr;
4189
4190         dspcntr = plane_state->ctl | i9xx_plane_ctl_crtc(crtc_state);
4191
4192         linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
4193
4194         if (INTEL_GEN(dev_priv) >= 4)
4195                 dspaddr_offset = plane_state->color_plane[0].offset;
4196         else
4197                 dspaddr_offset = linear_offset;
4198
4199         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
4200
4201         I915_WRITE_FW(DSPSTRIDE(i9xx_plane), plane_state->color_plane[0].stride);
4202
4203         if (INTEL_GEN(dev_priv) < 4) {
4204                 /*
4205                  * PLANE_A doesn't actually have a full window
4206                  * generator but let's assume we still need to
4207                  * program whatever is there.
4208                  */
4209                 I915_WRITE_FW(DSPPOS(i9xx_plane), (crtc_y << 16) | crtc_x);
4210                 I915_WRITE_FW(DSPSIZE(i9xx_plane),
4211                               ((crtc_h - 1) << 16) | (crtc_w - 1));
4212         } else if (IS_CHERRYVIEW(dev_priv) && i9xx_plane == PLANE_B) {
4213                 I915_WRITE_FW(PRIMPOS(i9xx_plane), (crtc_y << 16) | crtc_x);
4214                 I915_WRITE_FW(PRIMSIZE(i9xx_plane),
4215                               ((crtc_h - 1) << 16) | (crtc_w - 1));
4216                 I915_WRITE_FW(PRIMCNSTALPHA(i9xx_plane), 0);
4217         }
4218
4219         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
4220                 I915_WRITE_FW(DSPOFFSET(i9xx_plane), (y << 16) | x);
4221         } else if (INTEL_GEN(dev_priv) >= 4) {
4222                 I915_WRITE_FW(DSPLINOFF(i9xx_plane), linear_offset);
4223                 I915_WRITE_FW(DSPTILEOFF(i9xx_plane), (y << 16) | x);
4224         }
4225
4226         /*
4227          * The control register self-arms if the plane was previously
4228          * disabled. Try to make the plane enable atomic by writing
4229          * the control register just before the surface register.
4230          */
4231         I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr);
4232         if (INTEL_GEN(dev_priv) >= 4)
4233                 I915_WRITE_FW(DSPSURF(i9xx_plane),
4234                               intel_plane_ggtt_offset(plane_state) +
4235                               dspaddr_offset);
4236         else
4237                 I915_WRITE_FW(DSPADDR(i9xx_plane),
4238                               intel_plane_ggtt_offset(plane_state) +
4239                               dspaddr_offset);
4240
4241         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
4242 }
4243
4244 static void i9xx_disable_plane(struct intel_plane *plane,
4245                                const struct intel_crtc_state *crtc_state)
4246 {
4247         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4248         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4249         unsigned long irqflags;
4250         u32 dspcntr;
4251
4252         /*
4253          * DSPCNTR pipe gamma enable on g4x+ and pipe csc
4254          * enable on ilk+ affect the pipe bottom color as
4255          * well, so we must configure them even if the plane
4256          * is disabled.
4257          *
4258          * On pre-g4x there is no way to gamma correct the
4259          * pipe bottom color but we'll keep on doing this
4260          * anyway so that the crtc state readout works correctly.
4261          */
4262         dspcntr = i9xx_plane_ctl_crtc(crtc_state);
4263
4264         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
4265
4266         I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr);
4267         if (INTEL_GEN(dev_priv) >= 4)
4268                 I915_WRITE_FW(DSPSURF(i9xx_plane), 0);
4269         else
4270                 I915_WRITE_FW(DSPADDR(i9xx_plane), 0);
4271
4272         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
4273 }
4274
4275 static bool i9xx_plane_get_hw_state(struct intel_plane *plane,
4276                                     enum pipe *pipe)
4277 {
4278         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4279         enum intel_display_power_domain power_domain;
4280         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4281         intel_wakeref_t wakeref;
4282         bool ret;
4283         u32 val;
4284
4285         /*
4286          * Not 100% correct for planes that can move between pipes,
4287          * but that's only the case for gen2-4 which don't have any
4288          * display power wells.
4289          */
4290         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
4291         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4292         if (!wakeref)
4293                 return false;
4294
4295         val = I915_READ(DSPCNTR(i9xx_plane));
4296
4297         ret = val & DISPLAY_PLANE_ENABLE;
4298
4299         if (INTEL_GEN(dev_priv) >= 5)
4300                 *pipe = plane->pipe;
4301         else
4302                 *pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
4303                         DISPPLANE_SEL_PIPE_SHIFT;
4304
4305         intel_display_power_put(dev_priv, power_domain, wakeref);
4306
4307         return ret;
4308 }
4309
4310 static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
4311 {
4312         struct drm_device *dev = intel_crtc->base.dev;
4313         struct drm_i915_private *dev_priv = to_i915(dev);
4314
4315         I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, id), 0);
4316         I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, id), 0);
4317         I915_WRITE(SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0);
4318 }
4319
4320 /*
4321  * This function detaches (aka. unbinds) unused scalers in hardware
4322  */
4323 static void skl_detach_scalers(const struct intel_crtc_state *crtc_state)
4324 {
4325         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->uapi.crtc);
4326         const struct intel_crtc_scaler_state *scaler_state =
4327                 &crtc_state->scaler_state;
4328         int i;
4329
4330         /* loop through and disable scalers that aren't in use */
4331         for (i = 0; i < intel_crtc->num_scalers; i++) {
4332                 if (!scaler_state->scalers[i].in_use)
4333                         skl_detach_scaler(intel_crtc, i);
4334         }
4335 }
4336
4337 static unsigned int skl_plane_stride_mult(const struct drm_framebuffer *fb,
4338                                           int color_plane, unsigned int rotation)
4339 {
4340         /*
4341          * The stride is either expressed as a multiple of 64 bytes chunks for
4342          * linear buffers or in number of tiles for tiled buffers.
4343          */
4344         if (is_surface_linear(fb, color_plane))
4345                 return 64;
4346         else if (drm_rotation_90_or_270(rotation))
4347                 return intel_tile_height(fb, color_plane);
4348         else
4349                 return intel_tile_width_bytes(fb, color_plane);
4350 }
4351
4352 u32 skl_plane_stride(const struct intel_plane_state *plane_state,
4353                      int color_plane)
4354 {
4355         const struct drm_framebuffer *fb = plane_state->hw.fb;
4356         unsigned int rotation = plane_state->hw.rotation;
4357         u32 stride = plane_state->color_plane[color_plane].stride;
4358
4359         if (color_plane >= fb->format->num_planes)
4360                 return 0;
4361
4362         return stride / skl_plane_stride_mult(fb, color_plane, rotation);
4363 }
4364
4365 static u32 skl_plane_ctl_format(u32 pixel_format)
4366 {
4367         switch (pixel_format) {
4368         case DRM_FORMAT_C8:
4369                 return PLANE_CTL_FORMAT_INDEXED;
4370         case DRM_FORMAT_RGB565:
4371                 return PLANE_CTL_FORMAT_RGB_565;
4372         case DRM_FORMAT_XBGR8888:
4373         case DRM_FORMAT_ABGR8888:
4374                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX;
4375         case DRM_FORMAT_XRGB8888:
4376         case DRM_FORMAT_ARGB8888:
4377                 return PLANE_CTL_FORMAT_XRGB_8888;
4378         case DRM_FORMAT_XBGR2101010:
4379         case DRM_FORMAT_ABGR2101010:
4380                 return PLANE_CTL_FORMAT_XRGB_2101010 | PLANE_CTL_ORDER_RGBX;
4381         case DRM_FORMAT_XRGB2101010:
4382         case DRM_FORMAT_ARGB2101010:
4383                 return PLANE_CTL_FORMAT_XRGB_2101010;
4384         case DRM_FORMAT_XBGR16161616F:
4385         case DRM_FORMAT_ABGR16161616F:
4386                 return PLANE_CTL_FORMAT_XRGB_16161616F | PLANE_CTL_ORDER_RGBX;
4387         case DRM_FORMAT_XRGB16161616F:
4388         case DRM_FORMAT_ARGB16161616F:
4389                 return PLANE_CTL_FORMAT_XRGB_16161616F;
4390         case DRM_FORMAT_YUYV:
4391                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV;
4392         case DRM_FORMAT_YVYU:
4393                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YVYU;
4394         case DRM_FORMAT_UYVY:
4395                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_UYVY;
4396         case DRM_FORMAT_VYUY:
4397                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY;
4398         case DRM_FORMAT_NV12:
4399                 return PLANE_CTL_FORMAT_NV12;
4400         case DRM_FORMAT_P010:
4401                 return PLANE_CTL_FORMAT_P010;
4402         case DRM_FORMAT_P012:
4403                 return PLANE_CTL_FORMAT_P012;
4404         case DRM_FORMAT_P016:
4405                 return PLANE_CTL_FORMAT_P016;
4406         case DRM_FORMAT_Y210:
4407                 return PLANE_CTL_FORMAT_Y210;
4408         case DRM_FORMAT_Y212:
4409                 return PLANE_CTL_FORMAT_Y212;
4410         case DRM_FORMAT_Y216:
4411                 return PLANE_CTL_FORMAT_Y216;
4412         case DRM_FORMAT_XVYU2101010:
4413                 return PLANE_CTL_FORMAT_Y410;
4414         case DRM_FORMAT_XVYU12_16161616:
4415                 return PLANE_CTL_FORMAT_Y412;
4416         case DRM_FORMAT_XVYU16161616:
4417                 return PLANE_CTL_FORMAT_Y416;
4418         default:
4419                 MISSING_CASE(pixel_format);
4420         }
4421
4422         return 0;
4423 }
4424
4425 static u32 skl_plane_ctl_alpha(const struct intel_plane_state *plane_state)
4426 {
4427         if (!plane_state->hw.fb->format->has_alpha)
4428                 return PLANE_CTL_ALPHA_DISABLE;
4429
4430         switch (plane_state->hw.pixel_blend_mode) {
4431         case DRM_MODE_BLEND_PIXEL_NONE:
4432                 return PLANE_CTL_ALPHA_DISABLE;
4433         case DRM_MODE_BLEND_PREMULTI:
4434                 return PLANE_CTL_ALPHA_SW_PREMULTIPLY;
4435         case DRM_MODE_BLEND_COVERAGE:
4436                 return PLANE_CTL_ALPHA_HW_PREMULTIPLY;
4437         default:
4438                 MISSING_CASE(plane_state->hw.pixel_blend_mode);
4439                 return PLANE_CTL_ALPHA_DISABLE;
4440         }
4441 }
4442
4443 static u32 glk_plane_color_ctl_alpha(const struct intel_plane_state *plane_state)
4444 {
4445         if (!plane_state->hw.fb->format->has_alpha)
4446                 return PLANE_COLOR_ALPHA_DISABLE;
4447
4448         switch (plane_state->hw.pixel_blend_mode) {
4449         case DRM_MODE_BLEND_PIXEL_NONE:
4450                 return PLANE_COLOR_ALPHA_DISABLE;
4451         case DRM_MODE_BLEND_PREMULTI:
4452                 return PLANE_COLOR_ALPHA_SW_PREMULTIPLY;
4453         case DRM_MODE_BLEND_COVERAGE:
4454                 return PLANE_COLOR_ALPHA_HW_PREMULTIPLY;
4455         default:
4456                 MISSING_CASE(plane_state->hw.pixel_blend_mode);
4457                 return PLANE_COLOR_ALPHA_DISABLE;
4458         }
4459 }
4460
4461 static u32 skl_plane_ctl_tiling(u64 fb_modifier)
4462 {
4463         switch (fb_modifier) {
4464         case DRM_FORMAT_MOD_LINEAR:
4465                 break;
4466         case I915_FORMAT_MOD_X_TILED:
4467                 return PLANE_CTL_TILED_X;
4468         case I915_FORMAT_MOD_Y_TILED:
4469                 return PLANE_CTL_TILED_Y;
4470         case I915_FORMAT_MOD_Y_TILED_CCS:
4471                 return PLANE_CTL_TILED_Y | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
4472         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
4473                 return PLANE_CTL_TILED_Y |
4474                        PLANE_CTL_RENDER_DECOMPRESSION_ENABLE |
4475                        PLANE_CTL_CLEAR_COLOR_DISABLE;
4476         case I915_FORMAT_MOD_Yf_TILED:
4477                 return PLANE_CTL_TILED_YF;
4478         case I915_FORMAT_MOD_Yf_TILED_CCS:
4479                 return PLANE_CTL_TILED_YF | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
4480         default:
4481                 MISSING_CASE(fb_modifier);
4482         }
4483
4484         return 0;
4485 }
4486
4487 static u32 skl_plane_ctl_rotate(unsigned int rotate)
4488 {
4489         switch (rotate) {
4490         case DRM_MODE_ROTATE_0:
4491                 break;
4492         /*
4493          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
4494          * while i915 HW rotation is clockwise, thats why this swapping.
4495          */
4496         case DRM_MODE_ROTATE_90:
4497                 return PLANE_CTL_ROTATE_270;
4498         case DRM_MODE_ROTATE_180:
4499                 return PLANE_CTL_ROTATE_180;
4500         case DRM_MODE_ROTATE_270:
4501                 return PLANE_CTL_ROTATE_90;
4502         default:
4503                 MISSING_CASE(rotate);
4504         }
4505
4506         return 0;
4507 }
4508
4509 static u32 cnl_plane_ctl_flip(unsigned int reflect)
4510 {
4511         switch (reflect) {
4512         case 0:
4513                 break;
4514         case DRM_MODE_REFLECT_X:
4515                 return PLANE_CTL_FLIP_HORIZONTAL;
4516         case DRM_MODE_REFLECT_Y:
4517         default:
4518                 MISSING_CASE(reflect);
4519         }
4520
4521         return 0;
4522 }
4523
4524 u32 skl_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
4525 {
4526         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4527         u32 plane_ctl = 0;
4528
4529         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
4530                 return plane_ctl;
4531
4532         if (crtc_state->gamma_enable)
4533                 plane_ctl |= PLANE_CTL_PIPE_GAMMA_ENABLE;
4534
4535         if (crtc_state->csc_enable)
4536                 plane_ctl |= PLANE_CTL_PIPE_CSC_ENABLE;
4537
4538         return plane_ctl;
4539 }
4540
4541 u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
4542                   const struct intel_plane_state *plane_state)
4543 {
4544         struct drm_i915_private *dev_priv =
4545                 to_i915(plane_state->uapi.plane->dev);
4546         const struct drm_framebuffer *fb = plane_state->hw.fb;
4547         unsigned int rotation = plane_state->hw.rotation;
4548         const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
4549         u32 plane_ctl;
4550
4551         plane_ctl = PLANE_CTL_ENABLE;
4552
4553         if (INTEL_GEN(dev_priv) < 10 && !IS_GEMINILAKE(dev_priv)) {
4554                 plane_ctl |= skl_plane_ctl_alpha(plane_state);
4555                 plane_ctl |= PLANE_CTL_PLANE_GAMMA_DISABLE;
4556
4557                 if (plane_state->hw.color_encoding == DRM_COLOR_YCBCR_BT709)
4558                         plane_ctl |= PLANE_CTL_YUV_TO_RGB_CSC_FORMAT_BT709;
4559
4560                 if (plane_state->hw.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
4561                         plane_ctl |= PLANE_CTL_YUV_RANGE_CORRECTION_DISABLE;
4562         }
4563
4564         plane_ctl |= skl_plane_ctl_format(fb->format->format);
4565         plane_ctl |= skl_plane_ctl_tiling(fb->modifier);
4566         plane_ctl |= skl_plane_ctl_rotate(rotation & DRM_MODE_ROTATE_MASK);
4567
4568         if (INTEL_GEN(dev_priv) >= 10)
4569                 plane_ctl |= cnl_plane_ctl_flip(rotation &
4570                                                 DRM_MODE_REFLECT_MASK);
4571
4572         if (key->flags & I915_SET_COLORKEY_DESTINATION)
4573                 plane_ctl |= PLANE_CTL_KEY_ENABLE_DESTINATION;
4574         else if (key->flags & I915_SET_COLORKEY_SOURCE)
4575                 plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE;
4576
4577         return plane_ctl;
4578 }
4579
4580 u32 glk_plane_color_ctl_crtc(const struct intel_crtc_state *crtc_state)
4581 {
4582         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4583         u32 plane_color_ctl = 0;
4584
4585         if (INTEL_GEN(dev_priv) >= 11)
4586                 return plane_color_ctl;
4587
4588         if (crtc_state->gamma_enable)
4589                 plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE;
4590
4591         if (crtc_state->csc_enable)
4592                 plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE;
4593
4594         return plane_color_ctl;
4595 }
4596
4597 u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
4598                         const struct intel_plane_state *plane_state)
4599 {
4600         struct drm_i915_private *dev_priv =
4601                 to_i915(plane_state->uapi.plane->dev);
4602         const struct drm_framebuffer *fb = plane_state->hw.fb;
4603         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4604         u32 plane_color_ctl = 0;
4605
4606         plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE;
4607         plane_color_ctl |= glk_plane_color_ctl_alpha(plane_state);
4608
4609         if (fb->format->is_yuv && !icl_is_hdr_plane(dev_priv, plane->id)) {
4610                 if (plane_state->hw.color_encoding == DRM_COLOR_YCBCR_BT709)
4611                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709;
4612                 else
4613                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV601_TO_RGB709;
4614
4615                 if (plane_state->hw.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
4616                         plane_color_ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE;
4617         } else if (fb->format->is_yuv) {
4618                 plane_color_ctl |= PLANE_COLOR_INPUT_CSC_ENABLE;
4619         }
4620
4621         return plane_color_ctl;
4622 }
4623
4624 static int
4625 __intel_display_resume(struct drm_device *dev,
4626                        struct drm_atomic_state *state,
4627                        struct drm_modeset_acquire_ctx *ctx)
4628 {
4629         struct drm_crtc_state *crtc_state;
4630         struct drm_crtc *crtc;
4631         int i, ret;
4632
4633         intel_modeset_setup_hw_state(dev, ctx);
4634         intel_vga_redisable(to_i915(dev));
4635
4636         if (!state)
4637                 return 0;
4638
4639         /*
4640          * We've duplicated the state, pointers to the old state are invalid.
4641          *
4642          * Don't attempt to use the old state until we commit the duplicated state.
4643          */
4644         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
4645                 /*
4646                  * Force recalculation even if we restore
4647                  * current state. With fast modeset this may not result
4648                  * in a modeset when the state is compatible.
4649                  */
4650                 crtc_state->mode_changed = true;
4651         }
4652
4653         /* ignore any reset values/BIOS leftovers in the WM registers */
4654         if (!HAS_GMCH(to_i915(dev)))
4655                 to_intel_atomic_state(state)->skip_intermediate_wm = true;
4656
4657         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
4658
4659         WARN_ON(ret == -EDEADLK);
4660         return ret;
4661 }
4662
4663 static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv)
4664 {
4665         return (INTEL_INFO(dev_priv)->gpu_reset_clobbers_display &&
4666                 intel_has_gpu_reset(&dev_priv->gt));
4667 }
4668
4669 void intel_prepare_reset(struct drm_i915_private *dev_priv)
4670 {
4671         struct drm_device *dev = &dev_priv->drm;
4672         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
4673         struct drm_atomic_state *state;
4674         int ret;
4675
4676         /* reset doesn't touch the display */
4677         if (!i915_modparams.force_reset_modeset_test &&
4678             !gpu_reset_clobbers_display(dev_priv))
4679                 return;
4680
4681         /* We have a modeset vs reset deadlock, defensively unbreak it. */
4682         set_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags);
4683         smp_mb__after_atomic();
4684         wake_up_bit(&dev_priv->gt.reset.flags, I915_RESET_MODESET);
4685
4686         if (atomic_read(&dev_priv->gpu_error.pending_fb_pin)) {
4687                 DRM_DEBUG_KMS("Modeset potentially stuck, unbreaking through wedging\n");
4688                 intel_gt_set_wedged(&dev_priv->gt);
4689         }
4690
4691         /*
4692          * Need mode_config.mutex so that we don't
4693          * trample ongoing ->detect() and whatnot.
4694          */
4695         mutex_lock(&dev->mode_config.mutex);
4696         drm_modeset_acquire_init(ctx, 0);
4697         while (1) {
4698                 ret = drm_modeset_lock_all_ctx(dev, ctx);
4699                 if (ret != -EDEADLK)
4700                         break;
4701
4702                 drm_modeset_backoff(ctx);
4703         }
4704         /*
4705          * Disabling the crtcs gracefully seems nicer. Also the
4706          * g33 docs say we should at least disable all the planes.
4707          */
4708         state = drm_atomic_helper_duplicate_state(dev, ctx);
4709         if (IS_ERR(state)) {
4710                 ret = PTR_ERR(state);
4711                 DRM_ERROR("Duplicating state failed with %i\n", ret);
4712                 return;
4713         }
4714
4715         ret = drm_atomic_helper_disable_all(dev, ctx);
4716         if (ret) {
4717                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
4718                 drm_atomic_state_put(state);
4719                 return;
4720         }
4721
4722         dev_priv->modeset_restore_state = state;
4723         state->acquire_ctx = ctx;
4724 }
4725
4726 void intel_finish_reset(struct drm_i915_private *dev_priv)
4727 {
4728         struct drm_device *dev = &dev_priv->drm;
4729         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
4730         struct drm_atomic_state *state;
4731         int ret;
4732
4733         /* reset doesn't touch the display */
4734         if (!test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags))
4735                 return;
4736
4737         state = fetch_and_zero(&dev_priv->modeset_restore_state);
4738         if (!state)
4739                 goto unlock;
4740
4741         /* reset doesn't touch the display */
4742         if (!gpu_reset_clobbers_display(dev_priv)) {
4743                 /* for testing only restore the display */
4744                 ret = __intel_display_resume(dev, state, ctx);
4745                 if (ret)
4746                         DRM_ERROR("Restoring old state failed with %i\n", ret);
4747         } else {
4748                 /*
4749                  * The display has been reset as well,
4750                  * so need a full re-initialization.
4751                  */
4752                 intel_pps_unlock_regs_wa(dev_priv);
4753                 intel_modeset_init_hw(dev_priv);
4754                 intel_init_clock_gating(dev_priv);
4755
4756                 spin_lock_irq(&dev_priv->irq_lock);
4757                 if (dev_priv->display.hpd_irq_setup)
4758                         dev_priv->display.hpd_irq_setup(dev_priv);
4759                 spin_unlock_irq(&dev_priv->irq_lock);
4760
4761                 ret = __intel_display_resume(dev, state, ctx);
4762                 if (ret)
4763                         DRM_ERROR("Restoring old state failed with %i\n", ret);
4764
4765                 intel_hpd_init(dev_priv);
4766         }
4767
4768         drm_atomic_state_put(state);
4769 unlock:
4770         drm_modeset_drop_locks(ctx);
4771         drm_modeset_acquire_fini(ctx);
4772         mutex_unlock(&dev->mode_config.mutex);
4773
4774         clear_bit_unlock(I915_RESET_MODESET, &dev_priv->gt.reset.flags);
4775 }
4776
4777 static void icl_set_pipe_chicken(struct intel_crtc *crtc)
4778 {
4779         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4780         enum pipe pipe = crtc->pipe;
4781         u32 tmp;
4782
4783         tmp = I915_READ(PIPE_CHICKEN(pipe));
4784
4785         /*
4786          * Display WA #1153: icl
4787          * enable hardware to bypass the alpha math
4788          * and rounding for per-pixel values 00 and 0xff
4789          */
4790         tmp |= PER_PIXEL_ALPHA_BYPASS_EN;
4791         /*
4792          * Display WA # 1605353570: icl
4793          * Set the pixel rounding bit to 1 for allowing
4794          * passthrough of Frame buffer pixels unmodified
4795          * across pipe
4796          */
4797         tmp |= PIXEL_ROUNDING_TRUNC_FB_PASSTHRU;
4798         I915_WRITE(PIPE_CHICKEN(pipe), tmp);
4799 }
4800
4801 static void icl_enable_trans_port_sync(const struct intel_crtc_state *crtc_state)
4802 {
4803         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4804         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4805         u32 trans_ddi_func_ctl2_val;
4806         u8 master_select;
4807
4808         /*
4809          * Configure the master select and enable Transcoder Port Sync for
4810          * Slave CRTCs transcoder.
4811          */
4812         if (crtc_state->master_transcoder == INVALID_TRANSCODER)
4813                 return;
4814
4815         if (crtc_state->master_transcoder == TRANSCODER_EDP)
4816                 master_select = 0;
4817         else
4818                 master_select = crtc_state->master_transcoder + 1;
4819
4820         /* Set the master select bits for Tranascoder Port Sync */
4821         trans_ddi_func_ctl2_val = (PORT_SYNC_MODE_MASTER_SELECT(master_select) &
4822                                    PORT_SYNC_MODE_MASTER_SELECT_MASK) <<
4823                 PORT_SYNC_MODE_MASTER_SELECT_SHIFT;
4824         /* Enable Transcoder Port Sync */
4825         trans_ddi_func_ctl2_val |= PORT_SYNC_MODE_ENABLE;
4826
4827         I915_WRITE(TRANS_DDI_FUNC_CTL2(crtc_state->cpu_transcoder),
4828                    trans_ddi_func_ctl2_val);
4829 }
4830
4831 static void intel_fdi_normal_train(struct intel_crtc *crtc)
4832 {
4833         struct drm_device *dev = crtc->base.dev;
4834         struct drm_i915_private *dev_priv = to_i915(dev);
4835         enum pipe pipe = crtc->pipe;
4836         i915_reg_t reg;
4837         u32 temp;
4838
4839         /* enable normal train */
4840         reg = FDI_TX_CTL(pipe);
4841         temp = I915_READ(reg);
4842         if (IS_IVYBRIDGE(dev_priv)) {
4843                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
4844                 temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
4845         } else {
4846                 temp &= ~FDI_LINK_TRAIN_NONE;
4847                 temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
4848         }
4849         I915_WRITE(reg, temp);
4850
4851         reg = FDI_RX_CTL(pipe);
4852         temp = I915_READ(reg);
4853         if (HAS_PCH_CPT(dev_priv)) {
4854                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4855                 temp |= FDI_LINK_TRAIN_NORMAL_CPT;
4856         } else {
4857                 temp &= ~FDI_LINK_TRAIN_NONE;
4858                 temp |= FDI_LINK_TRAIN_NONE;
4859         }
4860         I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
4861
4862         /* wait one idle pattern time */
4863         POSTING_READ(reg);
4864         udelay(1000);
4865
4866         /* IVB wants error correction enabled */
4867         if (IS_IVYBRIDGE(dev_priv))
4868                 I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
4869                            FDI_FE_ERRC_ENABLE);
4870 }
4871
4872 /* The FDI link training functions for ILK/Ibexpeak. */
4873 static void ironlake_fdi_link_train(struct intel_crtc *crtc,
4874                                     const struct intel_crtc_state *crtc_state)
4875 {
4876         struct drm_device *dev = crtc->base.dev;
4877         struct drm_i915_private *dev_priv = to_i915(dev);
4878         enum pipe pipe = crtc->pipe;
4879         i915_reg_t reg;
4880         u32 temp, tries;
4881
4882         /* FDI needs bits from pipe first */
4883         assert_pipe_enabled(dev_priv, crtc_state->cpu_transcoder);
4884
4885         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4886            for train result */
4887         reg = FDI_RX_IMR(pipe);
4888         temp = I915_READ(reg);
4889         temp &= ~FDI_RX_SYMBOL_LOCK;
4890         temp &= ~FDI_RX_BIT_LOCK;
4891         I915_WRITE(reg, temp);
4892         I915_READ(reg);
4893         udelay(150);
4894
4895         /* enable CPU FDI TX and PCH FDI RX */
4896         reg = FDI_TX_CTL(pipe);
4897         temp = I915_READ(reg);
4898         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4899         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4900         temp &= ~FDI_LINK_TRAIN_NONE;
4901         temp |= FDI_LINK_TRAIN_PATTERN_1;
4902         I915_WRITE(reg, temp | FDI_TX_ENABLE);
4903
4904         reg = FDI_RX_CTL(pipe);
4905         temp = I915_READ(reg);
4906         temp &= ~FDI_LINK_TRAIN_NONE;
4907         temp |= FDI_LINK_TRAIN_PATTERN_1;
4908         I915_WRITE(reg, temp | FDI_RX_ENABLE);
4909
4910         POSTING_READ(reg);
4911         udelay(150);
4912
4913         /* Ironlake workaround, enable clock pointer after FDI enable*/
4914         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
4915         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR |
4916                    FDI_RX_PHASE_SYNC_POINTER_EN);
4917
4918         reg = FDI_RX_IIR(pipe);
4919         for (tries = 0; tries < 5; tries++) {
4920                 temp = I915_READ(reg);
4921                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4922
4923                 if ((temp & FDI_RX_BIT_LOCK)) {
4924                         DRM_DEBUG_KMS("FDI train 1 done.\n");
4925                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4926                         break;
4927                 }
4928         }
4929         if (tries == 5)
4930                 DRM_ERROR("FDI train 1 fail!\n");
4931
4932         /* Train 2 */
4933         reg = FDI_TX_CTL(pipe);
4934         temp = I915_READ(reg);
4935         temp &= ~FDI_LINK_TRAIN_NONE;
4936         temp |= FDI_LINK_TRAIN_PATTERN_2;
4937         I915_WRITE(reg, temp);
4938
4939         reg = FDI_RX_CTL(pipe);
4940         temp = I915_READ(reg);
4941         temp &= ~FDI_LINK_TRAIN_NONE;
4942         temp |= FDI_LINK_TRAIN_PATTERN_2;
4943         I915_WRITE(reg, temp);
4944
4945         POSTING_READ(reg);
4946         udelay(150);
4947
4948         reg = FDI_RX_IIR(pipe);
4949         for (tries = 0; tries < 5; tries++) {
4950                 temp = I915_READ(reg);
4951                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4952
4953                 if (temp & FDI_RX_SYMBOL_LOCK) {
4954                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4955                         DRM_DEBUG_KMS("FDI train 2 done.\n");
4956                         break;
4957                 }
4958         }
4959         if (tries == 5)
4960                 DRM_ERROR("FDI train 2 fail!\n");
4961
4962         DRM_DEBUG_KMS("FDI train done\n");
4963
4964 }
4965
4966 static const int snb_b_fdi_train_param[] = {
4967         FDI_LINK_TRAIN_400MV_0DB_SNB_B,
4968         FDI_LINK_TRAIN_400MV_6DB_SNB_B,
4969         FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
4970         FDI_LINK_TRAIN_800MV_0DB_SNB_B,
4971 };
4972
4973 /* The FDI link training functions for SNB/Cougarpoint. */
4974 static void gen6_fdi_link_train(struct intel_crtc *crtc,
4975                                 const struct intel_crtc_state *crtc_state)
4976 {
4977         struct drm_device *dev = crtc->base.dev;
4978         struct drm_i915_private *dev_priv = to_i915(dev);
4979         enum pipe pipe = crtc->pipe;
4980         i915_reg_t reg;
4981         u32 temp, i, retry;
4982
4983         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4984            for train result */
4985         reg = FDI_RX_IMR(pipe);
4986         temp = I915_READ(reg);
4987         temp &= ~FDI_RX_SYMBOL_LOCK;
4988         temp &= ~FDI_RX_BIT_LOCK;
4989         I915_WRITE(reg, temp);
4990
4991         POSTING_READ(reg);
4992         udelay(150);
4993
4994         /* enable CPU FDI TX and PCH FDI RX */
4995         reg = FDI_TX_CTL(pipe);
4996         temp = I915_READ(reg);
4997         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4998         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4999         temp &= ~FDI_LINK_TRAIN_NONE;
5000         temp |= FDI_LINK_TRAIN_PATTERN_1;
5001         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5002         /* SNB-B */
5003         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
5004         I915_WRITE(reg, temp | FDI_TX_ENABLE);
5005
5006         I915_WRITE(FDI_RX_MISC(pipe),
5007                    FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
5008
5009         reg = FDI_RX_CTL(pipe);
5010         temp = I915_READ(reg);
5011         if (HAS_PCH_CPT(dev_priv)) {
5012                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5013                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
5014         } else {
5015                 temp &= ~FDI_LINK_TRAIN_NONE;
5016                 temp |= FDI_LINK_TRAIN_PATTERN_1;
5017         }
5018         I915_WRITE(reg, temp | FDI_RX_ENABLE);
5019
5020         POSTING_READ(reg);
5021         udelay(150);
5022
5023         for (i = 0; i < 4; i++) {
5024                 reg = FDI_TX_CTL(pipe);
5025                 temp = I915_READ(reg);
5026                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5027                 temp |= snb_b_fdi_train_param[i];
5028                 I915_WRITE(reg, temp);
5029
5030                 POSTING_READ(reg);
5031                 udelay(500);
5032
5033                 for (retry = 0; retry < 5; retry++) {
5034                         reg = FDI_RX_IIR(pipe);
5035                         temp = I915_READ(reg);
5036                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
5037                         if (temp & FDI_RX_BIT_LOCK) {
5038                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
5039                                 DRM_DEBUG_KMS("FDI train 1 done.\n");
5040                                 break;
5041                         }
5042                         udelay(50);
5043                 }
5044                 if (retry < 5)
5045                         break;
5046         }
5047         if (i == 4)
5048                 DRM_ERROR("FDI train 1 fail!\n");
5049
5050         /* Train 2 */
5051         reg = FDI_TX_CTL(pipe);
5052         temp = I915_READ(reg);
5053         temp &= ~FDI_LINK_TRAIN_NONE;
5054         temp |= FDI_LINK_TRAIN_PATTERN_2;
5055         if (IS_GEN(dev_priv, 6)) {
5056                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5057                 /* SNB-B */
5058                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
5059         }
5060         I915_WRITE(reg, temp);
5061
5062         reg = FDI_RX_CTL(pipe);
5063         temp = I915_READ(reg);
5064         if (HAS_PCH_CPT(dev_priv)) {
5065                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5066                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
5067         } else {
5068                 temp &= ~FDI_LINK_TRAIN_NONE;
5069                 temp |= FDI_LINK_TRAIN_PATTERN_2;
5070         }
5071         I915_WRITE(reg, temp);
5072
5073         POSTING_READ(reg);
5074         udelay(150);
5075
5076         for (i = 0; i < 4; i++) {
5077                 reg = FDI_TX_CTL(pipe);
5078                 temp = I915_READ(reg);
5079                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5080                 temp |= snb_b_fdi_train_param[i];
5081                 I915_WRITE(reg, temp);
5082
5083                 POSTING_READ(reg);
5084                 udelay(500);
5085
5086                 for (retry = 0; retry < 5; retry++) {
5087                         reg = FDI_RX_IIR(pipe);
5088                         temp = I915_READ(reg);
5089                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
5090                         if (temp & FDI_RX_SYMBOL_LOCK) {
5091                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
5092                                 DRM_DEBUG_KMS("FDI train 2 done.\n");
5093                                 break;
5094                         }
5095                         udelay(50);
5096                 }
5097                 if (retry < 5)
5098                         break;
5099         }
5100         if (i == 4)
5101                 DRM_ERROR("FDI train 2 fail!\n");
5102
5103         DRM_DEBUG_KMS("FDI train done.\n");
5104 }
5105
5106 /* Manual link training for Ivy Bridge A0 parts */
5107 static void ivb_manual_fdi_link_train(struct intel_crtc *crtc,
5108                                       const struct intel_crtc_state *crtc_state)
5109 {
5110         struct drm_device *dev = crtc->base.dev;
5111         struct drm_i915_private *dev_priv = to_i915(dev);
5112         enum pipe pipe = crtc->pipe;
5113         i915_reg_t reg;
5114         u32 temp, i, j;
5115
5116         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
5117            for train result */
5118         reg = FDI_RX_IMR(pipe);
5119         temp = I915_READ(reg);
5120         temp &= ~FDI_RX_SYMBOL_LOCK;
5121         temp &= ~FDI_RX_BIT_LOCK;
5122         I915_WRITE(reg, temp);
5123
5124         POSTING_READ(reg);
5125         udelay(150);
5126
5127         DRM_DEBUG_KMS("FDI_RX_IIR before link train 0x%x\n",
5128                       I915_READ(FDI_RX_IIR(pipe)));
5129
5130         /* Try each vswing and preemphasis setting twice before moving on */
5131         for (j = 0; j < ARRAY_SIZE(snb_b_fdi_train_param) * 2; j++) {
5132                 /* disable first in case we need to retry */
5133                 reg = FDI_TX_CTL(pipe);
5134                 temp = I915_READ(reg);
5135                 temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
5136                 temp &= ~FDI_TX_ENABLE;
5137                 I915_WRITE(reg, temp);
5138
5139                 reg = FDI_RX_CTL(pipe);
5140                 temp = I915_READ(reg);
5141                 temp &= ~FDI_LINK_TRAIN_AUTO;
5142                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5143                 temp &= ~FDI_RX_ENABLE;
5144                 I915_WRITE(reg, temp);
5145
5146                 /* enable CPU FDI TX and PCH FDI RX */
5147                 reg = FDI_TX_CTL(pipe);
5148                 temp = I915_READ(reg);
5149                 temp &= ~FDI_DP_PORT_WIDTH_MASK;
5150                 temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
5151                 temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
5152                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5153                 temp |= snb_b_fdi_train_param[j/2];
5154                 temp |= FDI_COMPOSITE_SYNC;
5155                 I915_WRITE(reg, temp | FDI_TX_ENABLE);
5156
5157                 I915_WRITE(FDI_RX_MISC(pipe),
5158                            FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
5159
5160                 reg = FDI_RX_CTL(pipe);
5161                 temp = I915_READ(reg);
5162                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
5163                 temp |= FDI_COMPOSITE_SYNC;
5164                 I915_WRITE(reg, temp | FDI_RX_ENABLE);
5165
5166                 POSTING_READ(reg);
5167                 udelay(1); /* should be 0.5us */
5168
5169                 for (i = 0; i < 4; i++) {
5170                         reg = FDI_RX_IIR(pipe);
5171                         temp = I915_READ(reg);
5172                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
5173
5174                         if (temp & FDI_RX_BIT_LOCK ||
5175                             (I915_READ(reg) & FDI_RX_BIT_LOCK)) {
5176                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
5177                                 DRM_DEBUG_KMS("FDI train 1 done, level %i.\n",
5178                                               i);
5179                                 break;
5180                         }
5181                         udelay(1); /* should be 0.5us */
5182                 }
5183                 if (i == 4) {
5184                         DRM_DEBUG_KMS("FDI train 1 fail on vswing %d\n", j / 2);
5185                         continue;
5186                 }
5187
5188                 /* Train 2 */
5189                 reg = FDI_TX_CTL(pipe);
5190                 temp = I915_READ(reg);
5191                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
5192                 temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
5193                 I915_WRITE(reg, temp);
5194
5195                 reg = FDI_RX_CTL(pipe);
5196                 temp = I915_READ(reg);
5197                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5198                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
5199                 I915_WRITE(reg, temp);
5200
5201                 POSTING_READ(reg);
5202                 udelay(2); /* should be 1.5us */
5203
5204                 for (i = 0; i < 4; i++) {
5205                         reg = FDI_RX_IIR(pipe);
5206                         temp = I915_READ(reg);
5207                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
5208
5209                         if (temp & FDI_RX_SYMBOL_LOCK ||
5210                             (I915_READ(reg) & FDI_RX_SYMBOL_LOCK)) {
5211                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
5212                                 DRM_DEBUG_KMS("FDI train 2 done, level %i.\n",
5213                                               i);
5214                                 goto train_done;
5215                         }
5216                         udelay(2); /* should be 1.5us */
5217                 }
5218                 if (i == 4)
5219                         DRM_DEBUG_KMS("FDI train 2 fail on vswing %d\n", j / 2);
5220         }
5221
5222 train_done:
5223         DRM_DEBUG_KMS("FDI train done.\n");
5224 }
5225
5226 static void ironlake_fdi_pll_enable(const struct intel_crtc_state *crtc_state)
5227 {
5228         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->uapi.crtc);
5229         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
5230         enum pipe pipe = intel_crtc->pipe;
5231         i915_reg_t reg;
5232         u32 temp;
5233
5234         /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
5235         reg = FDI_RX_CTL(pipe);
5236         temp = I915_READ(reg);
5237         temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16));
5238         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
5239         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
5240         I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
5241
5242         POSTING_READ(reg);
5243         udelay(200);
5244
5245         /* Switch from Rawclk to PCDclk */
5246         temp = I915_READ(reg);
5247         I915_WRITE(reg, temp | FDI_PCDCLK);
5248
5249         POSTING_READ(reg);
5250         udelay(200);
5251
5252         /* Enable CPU FDI TX PLL, always on for Ironlake */
5253         reg = FDI_TX_CTL(pipe);
5254         temp = I915_READ(reg);
5255         if ((temp & FDI_TX_PLL_ENABLE) == 0) {
5256                 I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
5257
5258                 POSTING_READ(reg);
5259                 udelay(100);
5260         }
5261 }
5262
5263 static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc)
5264 {
5265         struct drm_device *dev = intel_crtc->base.dev;
5266         struct drm_i915_private *dev_priv = to_i915(dev);
5267         enum pipe pipe = intel_crtc->pipe;
5268         i915_reg_t reg;
5269         u32 temp;
5270
5271         /* Switch from PCDclk to Rawclk */
5272         reg = FDI_RX_CTL(pipe);
5273         temp = I915_READ(reg);
5274         I915_WRITE(reg, temp & ~FDI_PCDCLK);
5275
5276         /* Disable CPU FDI TX PLL */
5277         reg = FDI_TX_CTL(pipe);
5278         temp = I915_READ(reg);
5279         I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
5280
5281         POSTING_READ(reg);
5282         udelay(100);
5283
5284         reg = FDI_RX_CTL(pipe);
5285         temp = I915_READ(reg);
5286         I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
5287
5288         /* Wait for the clocks to turn off. */
5289         POSTING_READ(reg);
5290         udelay(100);
5291 }
5292
5293 static void ironlake_fdi_disable(struct intel_crtc *crtc)
5294 {
5295         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5296         enum pipe pipe = crtc->pipe;
5297         i915_reg_t reg;
5298         u32 temp;
5299
5300         /* disable CPU FDI tx and PCH FDI rx */
5301         reg = FDI_TX_CTL(pipe);
5302         temp = I915_READ(reg);
5303         I915_WRITE(reg, temp & ~FDI_TX_ENABLE);
5304         POSTING_READ(reg);
5305
5306         reg = FDI_RX_CTL(pipe);
5307         temp = I915_READ(reg);
5308         temp &= ~(0x7 << 16);
5309         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
5310         I915_WRITE(reg, temp & ~FDI_RX_ENABLE);
5311
5312         POSTING_READ(reg);
5313         udelay(100);
5314
5315         /* Ironlake workaround, disable clock pointer after downing FDI */
5316         if (HAS_PCH_IBX(dev_priv))
5317                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
5318
5319         /* still set train pattern 1 */
5320         reg = FDI_TX_CTL(pipe);
5321         temp = I915_READ(reg);
5322         temp &= ~FDI_LINK_TRAIN_NONE;
5323         temp |= FDI_LINK_TRAIN_PATTERN_1;
5324         I915_WRITE(reg, temp);
5325
5326         reg = FDI_RX_CTL(pipe);
5327         temp = I915_READ(reg);
5328         if (HAS_PCH_CPT(dev_priv)) {
5329                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5330                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
5331         } else {
5332                 temp &= ~FDI_LINK_TRAIN_NONE;
5333                 temp |= FDI_LINK_TRAIN_PATTERN_1;
5334         }
5335         /* BPC in FDI rx is consistent with that in PIPECONF */
5336         temp &= ~(0x07 << 16);
5337         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
5338         I915_WRITE(reg, temp);
5339
5340         POSTING_READ(reg);
5341         udelay(100);
5342 }
5343
5344 bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv)
5345 {
5346         struct drm_crtc *crtc;
5347         bool cleanup_done;
5348
5349         drm_for_each_crtc(crtc, &dev_priv->drm) {
5350                 struct drm_crtc_commit *commit;
5351                 spin_lock(&crtc->commit_lock);
5352                 commit = list_first_entry_or_null(&crtc->commit_list,
5353                                                   struct drm_crtc_commit, commit_entry);
5354                 cleanup_done = commit ?
5355                         try_wait_for_completion(&commit->cleanup_done) : true;
5356                 spin_unlock(&crtc->commit_lock);
5357
5358                 if (cleanup_done)
5359                         continue;
5360
5361                 drm_crtc_wait_one_vblank(crtc);
5362
5363                 return true;
5364         }
5365
5366         return false;
5367 }
5368
5369 void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
5370 {
5371         u32 temp;
5372
5373         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
5374
5375         mutex_lock(&dev_priv->sb_lock);
5376
5377         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5378         temp |= SBI_SSCCTL_DISABLE;
5379         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
5380
5381         mutex_unlock(&dev_priv->sb_lock);
5382 }
5383
5384 /* Program iCLKIP clock to the desired frequency */
5385 static void lpt_program_iclkip(const struct intel_crtc_state *crtc_state)
5386 {
5387         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5388         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5389         int clock = crtc_state->hw.adjusted_mode.crtc_clock;
5390         u32 divsel, phaseinc, auxdiv, phasedir = 0;
5391         u32 temp;
5392
5393         lpt_disable_iclkip(dev_priv);
5394
5395         /* The iCLK virtual clock root frequency is in MHz,
5396          * but the adjusted_mode->crtc_clock in in KHz. To get the
5397          * divisors, it is necessary to divide one by another, so we
5398          * convert the virtual clock precision to KHz here for higher
5399          * precision.
5400          */
5401         for (auxdiv = 0; auxdiv < 2; auxdiv++) {
5402                 u32 iclk_virtual_root_freq = 172800 * 1000;
5403                 u32 iclk_pi_range = 64;
5404                 u32 desired_divisor;
5405
5406                 desired_divisor = DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
5407                                                     clock << auxdiv);
5408                 divsel = (desired_divisor / iclk_pi_range) - 2;
5409                 phaseinc = desired_divisor % iclk_pi_range;
5410
5411                 /*
5412                  * Near 20MHz is a corner case which is
5413                  * out of range for the 7-bit divisor
5414                  */
5415                 if (divsel <= 0x7f)
5416                         break;
5417         }
5418
5419         /* This should not happen with any sane values */
5420         WARN_ON(SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
5421                 ~SBI_SSCDIVINTPHASE_DIVSEL_MASK);
5422         WARN_ON(SBI_SSCDIVINTPHASE_DIR(phasedir) &
5423                 ~SBI_SSCDIVINTPHASE_INCVAL_MASK);
5424
5425         DRM_DEBUG_KMS("iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
5426                         clock,
5427                         auxdiv,
5428                         divsel,
5429                         phasedir,
5430                         phaseinc);
5431
5432         mutex_lock(&dev_priv->sb_lock);
5433
5434         /* Program SSCDIVINTPHASE6 */
5435         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
5436         temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
5437         temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
5438         temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
5439         temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
5440         temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
5441         temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
5442         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK);
5443
5444         /* Program SSCAUXDIV */
5445         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
5446         temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
5447         temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
5448         intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK);
5449
5450         /* Enable modulator and associated divider */
5451         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5452         temp &= ~SBI_SSCCTL_DISABLE;
5453         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
5454
5455         mutex_unlock(&dev_priv->sb_lock);
5456
5457         /* Wait for initialization time */
5458         udelay(24);
5459
5460         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE);
5461 }
5462
5463 int lpt_get_iclkip(struct drm_i915_private *dev_priv)
5464 {
5465         u32 divsel, phaseinc, auxdiv;
5466         u32 iclk_virtual_root_freq = 172800 * 1000;
5467         u32 iclk_pi_range = 64;
5468         u32 desired_divisor;
5469         u32 temp;
5470
5471         if ((I915_READ(PIXCLK_GATE) & PIXCLK_GATE_UNGATE) == 0)
5472                 return 0;
5473
5474         mutex_lock(&dev_priv->sb_lock);
5475
5476         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5477         if (temp & SBI_SSCCTL_DISABLE) {
5478                 mutex_unlock(&dev_priv->sb_lock);
5479                 return 0;
5480         }
5481
5482         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
5483         divsel = (temp & SBI_SSCDIVINTPHASE_DIVSEL_MASK) >>
5484                 SBI_SSCDIVINTPHASE_DIVSEL_SHIFT;
5485         phaseinc = (temp & SBI_SSCDIVINTPHASE_INCVAL_MASK) >>
5486                 SBI_SSCDIVINTPHASE_INCVAL_SHIFT;
5487
5488         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
5489         auxdiv = (temp & SBI_SSCAUXDIV_FINALDIV2SEL_MASK) >>
5490                 SBI_SSCAUXDIV_FINALDIV2SEL_SHIFT;
5491
5492         mutex_unlock(&dev_priv->sb_lock);
5493
5494         desired_divisor = (divsel + 2) * iclk_pi_range + phaseinc;
5495
5496         return DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
5497                                  desired_divisor << auxdiv);
5498 }
5499
5500 static void ironlake_pch_transcoder_set_timings(const struct intel_crtc_state *crtc_state,
5501                                                 enum pipe pch_transcoder)
5502 {
5503         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5504         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5505         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
5506
5507         I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder),
5508                    I915_READ(HTOTAL(cpu_transcoder)));
5509         I915_WRITE(PCH_TRANS_HBLANK(pch_transcoder),
5510                    I915_READ(HBLANK(cpu_transcoder)));
5511         I915_WRITE(PCH_TRANS_HSYNC(pch_transcoder),
5512                    I915_READ(HSYNC(cpu_transcoder)));
5513
5514         I915_WRITE(PCH_TRANS_VTOTAL(pch_transcoder),
5515                    I915_READ(VTOTAL(cpu_transcoder)));
5516         I915_WRITE(PCH_TRANS_VBLANK(pch_transcoder),
5517                    I915_READ(VBLANK(cpu_transcoder)));
5518         I915_WRITE(PCH_TRANS_VSYNC(pch_transcoder),
5519                    I915_READ(VSYNC(cpu_transcoder)));
5520         I915_WRITE(PCH_TRANS_VSYNCSHIFT(pch_transcoder),
5521                    I915_READ(VSYNCSHIFT(cpu_transcoder)));
5522 }
5523
5524 static void cpt_set_fdi_bc_bifurcation(struct drm_i915_private *dev_priv, bool enable)
5525 {
5526         u32 temp;
5527
5528         temp = I915_READ(SOUTH_CHICKEN1);
5529         if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable)
5530                 return;
5531
5532         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_B)) & FDI_RX_ENABLE);
5533         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_C)) & FDI_RX_ENABLE);
5534
5535         temp &= ~FDI_BC_BIFURCATION_SELECT;
5536         if (enable)
5537                 temp |= FDI_BC_BIFURCATION_SELECT;
5538
5539         DRM_DEBUG_KMS("%sabling fdi C rx\n", enable ? "en" : "dis");
5540         I915_WRITE(SOUTH_CHICKEN1, temp);
5541         POSTING_READ(SOUTH_CHICKEN1);
5542 }
5543
5544 static void ivybridge_update_fdi_bc_bifurcation(const struct intel_crtc_state *crtc_state)
5545 {
5546         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5547         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5548
5549         switch (crtc->pipe) {
5550         case PIPE_A:
5551                 break;
5552         case PIPE_B:
5553                 if (crtc_state->fdi_lanes > 2)
5554                         cpt_set_fdi_bc_bifurcation(dev_priv, false);
5555                 else
5556                         cpt_set_fdi_bc_bifurcation(dev_priv, true);
5557
5558                 break;
5559         case PIPE_C:
5560                 cpt_set_fdi_bc_bifurcation(dev_priv, true);
5561
5562                 break;
5563         default:
5564                 BUG();
5565         }
5566 }
5567
5568 /*
5569  * Finds the encoder associated with the given CRTC. This can only be
5570  * used when we know that the CRTC isn't feeding multiple encoders!
5571  */
5572 static struct intel_encoder *
5573 intel_get_crtc_new_encoder(const struct intel_atomic_state *state,
5574                            const struct intel_crtc_state *crtc_state)
5575 {
5576         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5577         const struct drm_connector_state *connector_state;
5578         const struct drm_connector *connector;
5579         struct intel_encoder *encoder = NULL;
5580         int num_encoders = 0;
5581         int i;
5582
5583         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
5584                 if (connector_state->crtc != &crtc->base)
5585                         continue;
5586
5587                 encoder = to_intel_encoder(connector_state->best_encoder);
5588                 num_encoders++;
5589         }
5590
5591         WARN(num_encoders != 1, "%d encoders for pipe %c\n",
5592              num_encoders, pipe_name(crtc->pipe));
5593
5594         return encoder;
5595 }
5596
5597 /*
5598  * Enable PCH resources required for PCH ports:
5599  *   - PCH PLLs
5600  *   - FDI training & RX/TX
5601  *   - update transcoder timings
5602  *   - DP transcoding bits
5603  *   - transcoder
5604  */
5605 static void ironlake_pch_enable(const struct intel_atomic_state *state,
5606                                 const struct intel_crtc_state *crtc_state)
5607 {
5608         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5609         struct drm_device *dev = crtc->base.dev;
5610         struct drm_i915_private *dev_priv = to_i915(dev);
5611         enum pipe pipe = crtc->pipe;
5612         u32 temp;
5613
5614         assert_pch_transcoder_disabled(dev_priv, pipe);
5615
5616         if (IS_IVYBRIDGE(dev_priv))
5617                 ivybridge_update_fdi_bc_bifurcation(crtc_state);
5618
5619         /* Write the TU size bits before fdi link training, so that error
5620          * detection works. */
5621         I915_WRITE(FDI_RX_TUSIZE1(pipe),
5622                    I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
5623
5624         /* For PCH output, training FDI link */
5625         dev_priv->display.fdi_link_train(crtc, crtc_state);
5626
5627         /* We need to program the right clock selection before writing the pixel
5628          * mutliplier into the DPLL. */
5629         if (HAS_PCH_CPT(dev_priv)) {
5630                 u32 sel;
5631
5632                 temp = I915_READ(PCH_DPLL_SEL);
5633                 temp |= TRANS_DPLL_ENABLE(pipe);
5634                 sel = TRANS_DPLLB_SEL(pipe);
5635                 if (crtc_state->shared_dpll ==
5636                     intel_get_shared_dpll_by_id(dev_priv, DPLL_ID_PCH_PLL_B))
5637                         temp |= sel;
5638                 else
5639                         temp &= ~sel;
5640                 I915_WRITE(PCH_DPLL_SEL, temp);
5641         }
5642
5643         /* XXX: pch pll's can be enabled any time before we enable the PCH
5644          * transcoder, and we actually should do this to not upset any PCH
5645          * transcoder that already use the clock when we share it.
5646          *
5647          * Note that enable_shared_dpll tries to do the right thing, but
5648          * get_shared_dpll unconditionally resets the pll - we need that to have
5649          * the right LVDS enable sequence. */
5650         intel_enable_shared_dpll(crtc_state);
5651
5652         /* set transcoder timing, panel must allow it */
5653         assert_panel_unlocked(dev_priv, pipe);
5654         ironlake_pch_transcoder_set_timings(crtc_state, pipe);
5655
5656         intel_fdi_normal_train(crtc);
5657
5658         /* For PCH DP, enable TRANS_DP_CTL */
5659         if (HAS_PCH_CPT(dev_priv) &&
5660             intel_crtc_has_dp_encoder(crtc_state)) {
5661                 const struct drm_display_mode *adjusted_mode =
5662                         &crtc_state->hw.adjusted_mode;
5663                 u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
5664                 i915_reg_t reg = TRANS_DP_CTL(pipe);
5665                 enum port port;
5666
5667                 temp = I915_READ(reg);
5668                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
5669                           TRANS_DP_SYNC_MASK |
5670                           TRANS_DP_BPC_MASK);
5671                 temp |= TRANS_DP_OUTPUT_ENABLE;
5672                 temp |= bpc << 9; /* same format but at 11:9 */
5673
5674                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
5675                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
5676                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
5677                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
5678
5679                 port = intel_get_crtc_new_encoder(state, crtc_state)->port;
5680                 WARN_ON(port < PORT_B || port > PORT_D);
5681                 temp |= TRANS_DP_PORT_SEL(port);
5682
5683                 I915_WRITE(reg, temp);
5684         }
5685
5686         ironlake_enable_pch_transcoder(crtc_state);
5687 }
5688
5689 static void lpt_pch_enable(const struct intel_atomic_state *state,
5690                            const struct intel_crtc_state *crtc_state)
5691 {
5692         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5693         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5694         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
5695
5696         assert_pch_transcoder_disabled(dev_priv, PIPE_A);
5697
5698         lpt_program_iclkip(crtc_state);
5699
5700         /* Set transcoder timing. */
5701         ironlake_pch_transcoder_set_timings(crtc_state, PIPE_A);
5702
5703         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
5704 }
5705
5706 static void cpt_verify_modeset(struct drm_i915_private *dev_priv,
5707                                enum pipe pipe)
5708 {
5709         i915_reg_t dslreg = PIPEDSL(pipe);
5710         u32 temp;
5711
5712         temp = I915_READ(dslreg);
5713         udelay(500);
5714         if (wait_for(I915_READ(dslreg) != temp, 5)) {
5715                 if (wait_for(I915_READ(dslreg) != temp, 5))
5716                         DRM_ERROR("mode set failed: pipe %c stuck\n", pipe_name(pipe));
5717         }
5718 }
5719
5720 /*
5721  * The hardware phase 0.0 refers to the center of the pixel.
5722  * We want to start from the top/left edge which is phase
5723  * -0.5. That matches how the hardware calculates the scaling
5724  * factors (from top-left of the first pixel to bottom-right
5725  * of the last pixel, as opposed to the pixel centers).
5726  *
5727  * For 4:2:0 subsampled chroma planes we obviously have to
5728  * adjust that so that the chroma sample position lands in
5729  * the right spot.
5730  *
5731  * Note that for packed YCbCr 4:2:2 formats there is no way to
5732  * control chroma siting. The hardware simply replicates the
5733  * chroma samples for both of the luma samples, and thus we don't
5734  * actually get the expected MPEG2 chroma siting convention :(
5735  * The same behaviour is observed on pre-SKL platforms as well.
5736  *
5737  * Theory behind the formula (note that we ignore sub-pixel
5738  * source coordinates):
5739  * s = source sample position
5740  * d = destination sample position
5741  *
5742  * Downscaling 4:1:
5743  * -0.5
5744  * | 0.0
5745  * | |     1.5 (initial phase)
5746  * | |     |
5747  * v v     v
5748  * | s | s | s | s |
5749  * |       d       |
5750  *
5751  * Upscaling 1:4:
5752  * -0.5
5753  * | -0.375 (initial phase)
5754  * | |     0.0
5755  * | |     |
5756  * v v     v
5757  * |       s       |
5758  * | d | d | d | d |
5759  */
5760 u16 skl_scaler_calc_phase(int sub, int scale, bool chroma_cosited)
5761 {
5762         int phase = -0x8000;
5763         u16 trip = 0;
5764
5765         if (chroma_cosited)
5766                 phase += (sub - 1) * 0x8000 / sub;
5767
5768         phase += scale / (2 * sub);
5769
5770         /*
5771          * Hardware initial phase limited to [-0.5:1.5].
5772          * Since the max hardware scale factor is 3.0, we
5773          * should never actually excdeed 1.0 here.
5774          */
5775         WARN_ON(phase < -0x8000 || phase > 0x18000);
5776
5777         if (phase < 0)
5778                 phase = 0x10000 + phase;
5779         else
5780                 trip = PS_PHASE_TRIP;
5781
5782         return ((phase >> 2) & PS_PHASE_MASK) | trip;
5783 }
5784
5785 #define SKL_MIN_SRC_W 8
5786 #define SKL_MAX_SRC_W 4096
5787 #define SKL_MIN_SRC_H 8
5788 #define SKL_MAX_SRC_H 4096
5789 #define SKL_MIN_DST_W 8
5790 #define SKL_MAX_DST_W 4096
5791 #define SKL_MIN_DST_H 8
5792 #define SKL_MAX_DST_H 4096
5793 #define ICL_MAX_SRC_W 5120
5794 #define ICL_MAX_SRC_H 4096
5795 #define ICL_MAX_DST_W 5120
5796 #define ICL_MAX_DST_H 4096
5797 #define SKL_MIN_YUV_420_SRC_W 16
5798 #define SKL_MIN_YUV_420_SRC_H 16
5799
5800 static int
5801 skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
5802                   unsigned int scaler_user, int *scaler_id,
5803                   int src_w, int src_h, int dst_w, int dst_h,
5804                   const struct drm_format_info *format,
5805                   u64 modifier, bool need_scaler)
5806 {
5807         struct intel_crtc_scaler_state *scaler_state =
5808                 &crtc_state->scaler_state;
5809         struct intel_crtc *intel_crtc =
5810                 to_intel_crtc(crtc_state->uapi.crtc);
5811         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
5812         const struct drm_display_mode *adjusted_mode =
5813                 &crtc_state->hw.adjusted_mode;
5814
5815         /*
5816          * Src coordinates are already rotated by 270 degrees for
5817          * the 90/270 degree plane rotation cases (to match the
5818          * GTT mapping), hence no need to account for rotation here.
5819          */
5820         if (src_w != dst_w || src_h != dst_h)
5821                 need_scaler = true;
5822
5823         /*
5824          * Scaling/fitting not supported in IF-ID mode in GEN9+
5825          * TODO: Interlace fetch mode doesn't support YUV420 planar formats.
5826          * Once NV12 is enabled, handle it here while allocating scaler
5827          * for NV12.
5828          */
5829         if (INTEL_GEN(dev_priv) >= 9 && crtc_state->hw.enable &&
5830             need_scaler && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
5831                 DRM_DEBUG_KMS("Pipe/Plane scaling not supported with IF-ID mode\n");
5832                 return -EINVAL;
5833         }
5834
5835         /*
5836          * if plane is being disabled or scaler is no more required or force detach
5837          *  - free scaler binded to this plane/crtc
5838          *  - in order to do this, update crtc->scaler_usage
5839          *
5840          * Here scaler state in crtc_state is set free so that
5841          * scaler can be assigned to other user. Actual register
5842          * update to free the scaler is done in plane/panel-fit programming.
5843          * For this purpose crtc/plane_state->scaler_id isn't reset here.
5844          */
5845         if (force_detach || !need_scaler) {
5846                 if (*scaler_id >= 0) {
5847                         scaler_state->scaler_users &= ~(1 << scaler_user);
5848                         scaler_state->scalers[*scaler_id].in_use = 0;
5849
5850                         DRM_DEBUG_KMS("scaler_user index %u.%u: "
5851                                 "Staged freeing scaler id %d scaler_users = 0x%x\n",
5852                                 intel_crtc->pipe, scaler_user, *scaler_id,
5853                                 scaler_state->scaler_users);
5854                         *scaler_id = -1;
5855                 }
5856                 return 0;
5857         }
5858
5859         if (format && intel_format_info_is_yuv_semiplanar(format, modifier) &&
5860             (src_h < SKL_MIN_YUV_420_SRC_H || src_w < SKL_MIN_YUV_420_SRC_W)) {
5861                 DRM_DEBUG_KMS("Planar YUV: src dimensions not met\n");
5862                 return -EINVAL;
5863         }
5864
5865         /* range checks */
5866         if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H ||
5867             dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
5868             (INTEL_GEN(dev_priv) >= 11 &&
5869              (src_w > ICL_MAX_SRC_W || src_h > ICL_MAX_SRC_H ||
5870               dst_w > ICL_MAX_DST_W || dst_h > ICL_MAX_DST_H)) ||
5871             (INTEL_GEN(dev_priv) < 11 &&
5872              (src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
5873               dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H))) {
5874                 DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u "
5875                         "size is out of scaler range\n",
5876                         intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h);
5877                 return -EINVAL;
5878         }
5879
5880         /* mark this plane as a scaler user in crtc_state */
5881         scaler_state->scaler_users |= (1 << scaler_user);
5882         DRM_DEBUG_KMS("scaler_user index %u.%u: "
5883                 "staged scaling request for %ux%u->%ux%u scaler_users = 0x%x\n",
5884                 intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h,
5885                 scaler_state->scaler_users);
5886
5887         return 0;
5888 }
5889
5890 /**
5891  * skl_update_scaler_crtc - Stages update to scaler state for a given crtc.
5892  *
5893  * @state: crtc's scaler state
5894  *
5895  * Return
5896  *     0 - scaler_usage updated successfully
5897  *    error - requested scaling cannot be supported or other error condition
5898  */
5899 int skl_update_scaler_crtc(struct intel_crtc_state *state)
5900 {
5901         const struct drm_display_mode *adjusted_mode = &state->hw.adjusted_mode;
5902         bool need_scaler = false;
5903
5904         if (state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
5905                 need_scaler = true;
5906
5907         return skl_update_scaler(state, !state->hw.active, SKL_CRTC_INDEX,
5908                                  &state->scaler_state.scaler_id,
5909                                  state->pipe_src_w, state->pipe_src_h,
5910                                  adjusted_mode->crtc_hdisplay,
5911                                  adjusted_mode->crtc_vdisplay, NULL, 0,
5912                                  need_scaler);
5913 }
5914
5915 /**
5916  * skl_update_scaler_plane - Stages update to scaler state for a given plane.
5917  * @crtc_state: crtc's scaler state
5918  * @plane_state: atomic plane state to update
5919  *
5920  * Return
5921  *     0 - scaler_usage updated successfully
5922  *    error - requested scaling cannot be supported or other error condition
5923  */
5924 static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
5925                                    struct intel_plane_state *plane_state)
5926 {
5927         struct intel_plane *intel_plane =
5928                 to_intel_plane(plane_state->uapi.plane);
5929         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
5930         struct drm_framebuffer *fb = plane_state->hw.fb;
5931         int ret;
5932         bool force_detach = !fb || !plane_state->uapi.visible;
5933         bool need_scaler = false;
5934
5935         /* Pre-gen11 and SDR planes always need a scaler for planar formats. */
5936         if (!icl_is_hdr_plane(dev_priv, intel_plane->id) &&
5937             fb && intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
5938                 need_scaler = true;
5939
5940         ret = skl_update_scaler(crtc_state, force_detach,
5941                                 drm_plane_index(&intel_plane->base),
5942                                 &plane_state->scaler_id,
5943                                 drm_rect_width(&plane_state->uapi.src) >> 16,
5944                                 drm_rect_height(&plane_state->uapi.src) >> 16,
5945                                 drm_rect_width(&plane_state->uapi.dst),
5946                                 drm_rect_height(&plane_state->uapi.dst),
5947                                 fb ? fb->format : NULL,
5948                                 fb ? fb->modifier : 0,
5949                                 need_scaler);
5950
5951         if (ret || plane_state->scaler_id < 0)
5952                 return ret;
5953
5954         /* check colorkey */
5955         if (plane_state->ckey.flags) {
5956                 DRM_DEBUG_KMS("[PLANE:%d:%s] scaling with color key not allowed",
5957                               intel_plane->base.base.id,
5958                               intel_plane->base.name);
5959                 return -EINVAL;
5960         }
5961
5962         /* Check src format */
5963         switch (fb->format->format) {
5964         case DRM_FORMAT_RGB565:
5965         case DRM_FORMAT_XBGR8888:
5966         case DRM_FORMAT_XRGB8888:
5967         case DRM_FORMAT_ABGR8888:
5968         case DRM_FORMAT_ARGB8888:
5969         case DRM_FORMAT_XRGB2101010:
5970         case DRM_FORMAT_XBGR2101010:
5971         case DRM_FORMAT_ARGB2101010:
5972         case DRM_FORMAT_ABGR2101010:
5973         case DRM_FORMAT_YUYV:
5974         case DRM_FORMAT_YVYU:
5975         case DRM_FORMAT_UYVY:
5976         case DRM_FORMAT_VYUY:
5977         case DRM_FORMAT_NV12:
5978         case DRM_FORMAT_P010:
5979         case DRM_FORMAT_P012:
5980         case DRM_FORMAT_P016:
5981         case DRM_FORMAT_Y210:
5982         case DRM_FORMAT_Y212:
5983         case DRM_FORMAT_Y216:
5984         case DRM_FORMAT_XVYU2101010:
5985         case DRM_FORMAT_XVYU12_16161616:
5986         case DRM_FORMAT_XVYU16161616:
5987                 break;
5988         case DRM_FORMAT_XBGR16161616F:
5989         case DRM_FORMAT_ABGR16161616F:
5990         case DRM_FORMAT_XRGB16161616F:
5991         case DRM_FORMAT_ARGB16161616F:
5992                 if (INTEL_GEN(dev_priv) >= 11)
5993                         break;
5994                 /* fall through */
5995         default:
5996                 DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n",
5997                               intel_plane->base.base.id, intel_plane->base.name,
5998                               fb->base.id, fb->format->format);
5999                 return -EINVAL;
6000         }
6001
6002         return 0;
6003 }
6004
6005 void skylake_scaler_disable(const struct intel_crtc_state *old_crtc_state)
6006 {
6007         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
6008         int i;
6009
6010         for (i = 0; i < crtc->num_scalers; i++)
6011                 skl_detach_scaler(crtc, i);
6012 }
6013
6014 static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state)
6015 {
6016         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6017         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6018         enum pipe pipe = crtc->pipe;
6019         const struct intel_crtc_scaler_state *scaler_state =
6020                 &crtc_state->scaler_state;
6021
6022         if (crtc_state->pch_pfit.enabled) {
6023                 u16 uv_rgb_hphase, uv_rgb_vphase;
6024                 int pfit_w, pfit_h, hscale, vscale;
6025                 int id;
6026
6027                 if (WARN_ON(crtc_state->scaler_state.scaler_id < 0))
6028                         return;
6029
6030                 pfit_w = (crtc_state->pch_pfit.size >> 16) & 0xFFFF;
6031                 pfit_h = crtc_state->pch_pfit.size & 0xFFFF;
6032
6033                 hscale = (crtc_state->pipe_src_w << 16) / pfit_w;
6034                 vscale = (crtc_state->pipe_src_h << 16) / pfit_h;
6035
6036                 uv_rgb_hphase = skl_scaler_calc_phase(1, hscale, false);
6037                 uv_rgb_vphase = skl_scaler_calc_phase(1, vscale, false);
6038
6039                 id = scaler_state->scaler_id;
6040                 I915_WRITE(SKL_PS_CTRL(pipe, id), PS_SCALER_EN |
6041                         PS_FILTER_MEDIUM | scaler_state->scalers[id].mode);
6042                 I915_WRITE_FW(SKL_PS_VPHASE(pipe, id),
6043                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_vphase));
6044                 I915_WRITE_FW(SKL_PS_HPHASE(pipe, id),
6045                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_hphase));
6046                 I915_WRITE(SKL_PS_WIN_POS(pipe, id), crtc_state->pch_pfit.pos);
6047                 I915_WRITE(SKL_PS_WIN_SZ(pipe, id), crtc_state->pch_pfit.size);
6048         }
6049 }
6050
6051 static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state)
6052 {
6053         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6054         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6055         enum pipe pipe = crtc->pipe;
6056
6057         if (crtc_state->pch_pfit.enabled) {
6058                 /* Force use of hard-coded filter coefficients
6059                  * as some pre-programmed values are broken,
6060                  * e.g. x201.
6061                  */
6062                 if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv))
6063                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
6064                                                  PF_PIPE_SEL_IVB(pipe));
6065                 else
6066                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
6067                 I915_WRITE(PF_WIN_POS(pipe), crtc_state->pch_pfit.pos);
6068                 I915_WRITE(PF_WIN_SZ(pipe), crtc_state->pch_pfit.size);
6069         }
6070 }
6071
6072 void hsw_enable_ips(const struct intel_crtc_state *crtc_state)
6073 {
6074         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6075         struct drm_device *dev = crtc->base.dev;
6076         struct drm_i915_private *dev_priv = to_i915(dev);
6077
6078         if (!crtc_state->ips_enabled)
6079                 return;
6080
6081         /*
6082          * We can only enable IPS after we enable a plane and wait for a vblank
6083          * This function is called from post_plane_update, which is run after
6084          * a vblank wait.
6085          */
6086         WARN_ON(!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)));
6087
6088         if (IS_BROADWELL(dev_priv)) {
6089                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL,
6090                                                 IPS_ENABLE | IPS_PCODE_CONTROL));
6091                 /* Quoting Art Runyan: "its not safe to expect any particular
6092                  * value in IPS_CTL bit 31 after enabling IPS through the
6093                  * mailbox." Moreover, the mailbox may return a bogus state,
6094                  * so we need to just enable it and continue on.
6095                  */
6096         } else {
6097                 I915_WRITE(IPS_CTL, IPS_ENABLE);
6098                 /* The bit only becomes 1 in the next vblank, so this wait here
6099                  * is essentially intel_wait_for_vblank. If we don't have this
6100                  * and don't wait for vblanks until the end of crtc_enable, then
6101                  * the HW state readout code will complain that the expected
6102                  * IPS_CTL value is not the one we read. */
6103                 if (intel_de_wait_for_set(dev_priv, IPS_CTL, IPS_ENABLE, 50))
6104                         DRM_ERROR("Timed out waiting for IPS enable\n");
6105         }
6106 }
6107
6108 void hsw_disable_ips(const struct intel_crtc_state *crtc_state)
6109 {
6110         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6111         struct drm_device *dev = crtc->base.dev;
6112         struct drm_i915_private *dev_priv = to_i915(dev);
6113
6114         if (!crtc_state->ips_enabled)
6115                 return;
6116
6117         if (IS_BROADWELL(dev_priv)) {
6118                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
6119                 /*
6120                  * Wait for PCODE to finish disabling IPS. The BSpec specified
6121                  * 42ms timeout value leads to occasional timeouts so use 100ms
6122                  * instead.
6123                  */
6124                 if (intel_de_wait_for_clear(dev_priv, IPS_CTL, IPS_ENABLE, 100))
6125                         DRM_ERROR("Timed out waiting for IPS disable\n");
6126         } else {
6127                 I915_WRITE(IPS_CTL, 0);
6128                 POSTING_READ(IPS_CTL);
6129         }
6130
6131         /* We need to wait for a vblank before we can disable the plane. */
6132         intel_wait_for_vblank(dev_priv, crtc->pipe);
6133 }
6134
6135 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
6136 {
6137         if (intel_crtc->overlay)
6138                 (void) intel_overlay_switch_off(intel_crtc->overlay);
6139
6140         /* Let userspace switch the overlay on again. In most cases userspace
6141          * has to recompute where to put it anyway.
6142          */
6143 }
6144
6145 static bool hsw_pre_update_disable_ips(const struct intel_crtc_state *old_crtc_state,
6146                                        const struct intel_crtc_state *new_crtc_state)
6147 {
6148         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6149         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6150
6151         if (!old_crtc_state->ips_enabled)
6152                 return false;
6153
6154         if (needs_modeset(new_crtc_state))
6155                 return true;
6156
6157         /*
6158          * Workaround : Do not read or write the pipe palette/gamma data while
6159          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
6160          *
6161          * Disable IPS before we program the LUT.
6162          */
6163         if (IS_HASWELL(dev_priv) &&
6164             (new_crtc_state->uapi.color_mgmt_changed ||
6165              new_crtc_state->update_pipe) &&
6166             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
6167                 return true;
6168
6169         return !new_crtc_state->ips_enabled;
6170 }
6171
6172 static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_state,
6173                                        const struct intel_crtc_state *new_crtc_state)
6174 {
6175         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6176         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6177
6178         if (!new_crtc_state->ips_enabled)
6179                 return false;
6180
6181         if (needs_modeset(new_crtc_state))
6182                 return true;
6183
6184         /*
6185          * Workaround : Do not read or write the pipe palette/gamma data while
6186          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
6187          *
6188          * Re-enable IPS after the LUT has been programmed.
6189          */
6190         if (IS_HASWELL(dev_priv) &&
6191             (new_crtc_state->uapi.color_mgmt_changed ||
6192              new_crtc_state->update_pipe) &&
6193             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
6194                 return true;
6195
6196         /*
6197          * We can't read out IPS on broadwell, assume the worst and
6198          * forcibly enable IPS on the first fastset.
6199          */
6200         if (new_crtc_state->update_pipe &&
6201             old_crtc_state->hw.adjusted_mode.private_flags & I915_MODE_FLAG_INHERITED)
6202                 return true;
6203
6204         return !old_crtc_state->ips_enabled;
6205 }
6206
6207 static bool needs_nv12_wa(const struct intel_crtc_state *crtc_state)
6208 {
6209         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
6210
6211         if (!crtc_state->nv12_planes)
6212                 return false;
6213
6214         /* WA Display #0827: Gen9:all */
6215         if (IS_GEN(dev_priv, 9) && !IS_GEMINILAKE(dev_priv))
6216                 return true;
6217
6218         return false;
6219 }
6220
6221 static bool needs_scalerclk_wa(const struct intel_crtc_state *crtc_state)
6222 {
6223         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
6224
6225         /* Wa_2006604312:icl */
6226         if (crtc_state->scaler_state.scaler_users > 0 && IS_ICELAKE(dev_priv))
6227                 return true;
6228
6229         return false;
6230 }
6231
6232 static bool planes_enabling(const struct intel_crtc_state *old_crtc_state,
6233                             const struct intel_crtc_state *new_crtc_state)
6234 {
6235         return (!old_crtc_state->active_planes || needs_modeset(new_crtc_state)) &&
6236                 new_crtc_state->active_planes;
6237 }
6238
6239 static bool planes_disabling(const struct intel_crtc_state *old_crtc_state,
6240                              const struct intel_crtc_state *new_crtc_state)
6241 {
6242         return old_crtc_state->active_planes &&
6243                 (!new_crtc_state->active_planes || needs_modeset(new_crtc_state));
6244 }
6245
6246 static void intel_post_plane_update(struct intel_atomic_state *state,
6247                                     struct intel_crtc *crtc)
6248 {
6249         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6250         struct intel_plane *primary = to_intel_plane(crtc->base.primary);
6251         const struct intel_crtc_state *old_crtc_state =
6252                 intel_atomic_get_old_crtc_state(state, crtc);
6253         const struct intel_crtc_state *new_crtc_state =
6254                 intel_atomic_get_new_crtc_state(state, crtc);
6255         const struct intel_plane_state *new_primary_state =
6256                 intel_atomic_get_new_plane_state(state, primary);
6257         enum pipe pipe = crtc->pipe;
6258
6259         intel_frontbuffer_flip(dev_priv, new_crtc_state->fb_bits);
6260
6261         if (new_crtc_state->update_wm_post && new_crtc_state->hw.active)
6262                 intel_update_watermarks(crtc);
6263
6264         if (hsw_post_update_enable_ips(old_crtc_state, new_crtc_state))
6265                 hsw_enable_ips(new_crtc_state);
6266
6267         if (new_primary_state)
6268                 intel_fbc_post_update(crtc);
6269
6270         if (needs_nv12_wa(old_crtc_state) &&
6271             !needs_nv12_wa(new_crtc_state))
6272                 skl_wa_827(dev_priv, pipe, false);
6273
6274         if (needs_scalerclk_wa(old_crtc_state) &&
6275             !needs_scalerclk_wa(new_crtc_state))
6276                 icl_wa_scalerclkgating(dev_priv, pipe, false);
6277 }
6278
6279 static void intel_pre_plane_update(struct intel_atomic_state *state,
6280                                    struct intel_crtc *crtc)
6281 {
6282         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6283         struct intel_plane *primary = to_intel_plane(crtc->base.primary);
6284         const struct intel_crtc_state *old_crtc_state =
6285                 intel_atomic_get_old_crtc_state(state, crtc);
6286         const struct intel_crtc_state *new_crtc_state =
6287                 intel_atomic_get_new_crtc_state(state, crtc);
6288         const struct intel_plane_state *new_primary_state =
6289                 intel_atomic_get_new_plane_state(state, primary);
6290         enum pipe pipe = crtc->pipe;
6291
6292         if (hsw_pre_update_disable_ips(old_crtc_state, new_crtc_state))
6293                 hsw_disable_ips(old_crtc_state);
6294
6295         if (new_primary_state &&
6296             intel_fbc_pre_update(crtc, new_crtc_state, new_primary_state))
6297                 intel_wait_for_vblank(dev_priv, pipe);
6298
6299         /* Display WA 827 */
6300         if (!needs_nv12_wa(old_crtc_state) &&
6301             needs_nv12_wa(new_crtc_state))
6302                 skl_wa_827(dev_priv, pipe, true);
6303
6304         /* Wa_2006604312:icl */
6305         if (!needs_scalerclk_wa(old_crtc_state) &&
6306             needs_scalerclk_wa(new_crtc_state))
6307                 icl_wa_scalerclkgating(dev_priv, pipe, true);
6308
6309         /*
6310          * Vblank time updates from the shadow to live plane control register
6311          * are blocked if the memory self-refresh mode is active at that
6312          * moment. So to make sure the plane gets truly disabled, disable
6313          * first the self-refresh mode. The self-refresh enable bit in turn
6314          * will be checked/applied by the HW only at the next frame start
6315          * event which is after the vblank start event, so we need to have a
6316          * wait-for-vblank between disabling the plane and the pipe.
6317          */
6318         if (HAS_GMCH(dev_priv) && old_crtc_state->hw.active &&
6319             new_crtc_state->disable_cxsr && intel_set_memory_cxsr(dev_priv, false))
6320                 intel_wait_for_vblank(dev_priv, pipe);
6321
6322         /*
6323          * IVB workaround: must disable low power watermarks for at least
6324          * one frame before enabling scaling.  LP watermarks can be re-enabled
6325          * when scaling is disabled.
6326          *
6327          * WaCxSRDisabledForSpriteScaling:ivb
6328          */
6329         if (old_crtc_state->hw.active &&
6330             new_crtc_state->disable_lp_wm && ilk_disable_lp_wm(dev_priv))
6331                 intel_wait_for_vblank(dev_priv, pipe);
6332
6333         /*
6334          * If we're doing a modeset we don't need to do any
6335          * pre-vblank watermark programming here.
6336          */
6337         if (!needs_modeset(new_crtc_state)) {
6338                 /*
6339                  * For platforms that support atomic watermarks, program the
6340                  * 'intermediate' watermarks immediately.  On pre-gen9 platforms, these
6341                  * will be the intermediate values that are safe for both pre- and
6342                  * post- vblank; when vblank happens, the 'active' values will be set
6343                  * to the final 'target' values and we'll do this again to get the
6344                  * optimal watermarks.  For gen9+ platforms, the values we program here
6345                  * will be the final target values which will get automatically latched
6346                  * at vblank time; no further programming will be necessary.
6347                  *
6348                  * If a platform hasn't been transitioned to atomic watermarks yet,
6349                  * we'll continue to update watermarks the old way, if flags tell
6350                  * us to.
6351                  */
6352                 if (dev_priv->display.initial_watermarks)
6353                         dev_priv->display.initial_watermarks(state, crtc);
6354                 else if (new_crtc_state->update_wm_pre)
6355                         intel_update_watermarks(crtc);
6356         }
6357
6358         /*
6359          * Gen2 reports pipe underruns whenever all planes are disabled.
6360          * So disable underrun reporting before all the planes get disabled.
6361          *
6362          * We do this after .initial_watermarks() so that we have a
6363          * chance of catching underruns with the intermediate watermarks
6364          * vs. the old plane configuration.
6365          */
6366         if (IS_GEN(dev_priv, 2) && planes_disabling(old_crtc_state, new_crtc_state))
6367                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6368 }
6369
6370 static void intel_crtc_disable_planes(struct intel_atomic_state *state,
6371                                       struct intel_crtc *crtc)
6372 {
6373         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6374         const struct intel_crtc_state *new_crtc_state =
6375                 intel_atomic_get_new_crtc_state(state, crtc);
6376         unsigned int update_mask = new_crtc_state->update_planes;
6377         const struct intel_plane_state *old_plane_state;
6378         struct intel_plane *plane;
6379         unsigned fb_bits = 0;
6380         int i;
6381
6382         intel_crtc_dpms_overlay_disable(crtc);
6383
6384         for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
6385                 if (crtc->pipe != plane->pipe ||
6386                     !(update_mask & BIT(plane->id)))
6387                         continue;
6388
6389                 intel_disable_plane(plane, new_crtc_state);
6390
6391                 if (old_plane_state->uapi.visible)
6392                         fb_bits |= plane->frontbuffer_bit;
6393         }
6394
6395         intel_frontbuffer_flip(dev_priv, fb_bits);
6396 }
6397
6398 /*
6399  * intel_connector_primary_encoder - get the primary encoder for a connector
6400  * @connector: connector for which to return the encoder
6401  *
6402  * Returns the primary encoder for a connector. There is a 1:1 mapping from
6403  * all connectors to their encoder, except for DP-MST connectors which have
6404  * both a virtual and a primary encoder. These DP-MST primary encoders can be
6405  * pointed to by as many DP-MST connectors as there are pipes.
6406  */
6407 static struct intel_encoder *
6408 intel_connector_primary_encoder(struct intel_connector *connector)
6409 {
6410         struct intel_encoder *encoder;
6411
6412         if (connector->mst_port)
6413                 return &dp_to_dig_port(connector->mst_port)->base;
6414
6415         encoder = intel_attached_encoder(&connector->base);
6416         WARN_ON(!encoder);
6417
6418         return encoder;
6419 }
6420
6421 static void intel_encoders_update_prepare(struct intel_atomic_state *state)
6422 {
6423         struct drm_connector_state *new_conn_state;
6424         struct drm_connector *connector;
6425         int i;
6426
6427         for_each_new_connector_in_state(&state->base, connector, new_conn_state,
6428                                         i) {
6429                 struct intel_connector *intel_connector;
6430                 struct intel_encoder *encoder;
6431                 struct intel_crtc *crtc;
6432
6433                 if (!intel_connector_needs_modeset(state, connector))
6434                         continue;
6435
6436                 intel_connector = to_intel_connector(connector);
6437                 encoder = intel_connector_primary_encoder(intel_connector);
6438                 if (!encoder->update_prepare)
6439                         continue;
6440
6441                 crtc = new_conn_state->crtc ?
6442                         to_intel_crtc(new_conn_state->crtc) : NULL;
6443                 encoder->update_prepare(state, encoder, crtc);
6444         }
6445 }
6446
6447 static void intel_encoders_update_complete(struct intel_atomic_state *state)
6448 {
6449         struct drm_connector_state *new_conn_state;
6450         struct drm_connector *connector;
6451         int i;
6452
6453         for_each_new_connector_in_state(&state->base, connector, new_conn_state,
6454                                         i) {
6455                 struct intel_connector *intel_connector;
6456                 struct intel_encoder *encoder;
6457                 struct intel_crtc *crtc;
6458
6459                 if (!intel_connector_needs_modeset(state, connector))
6460                         continue;
6461
6462                 intel_connector = to_intel_connector(connector);
6463                 encoder = intel_connector_primary_encoder(intel_connector);
6464                 if (!encoder->update_complete)
6465                         continue;
6466
6467                 crtc = new_conn_state->crtc ?
6468                         to_intel_crtc(new_conn_state->crtc) : NULL;
6469                 encoder->update_complete(state, encoder, crtc);
6470         }
6471 }
6472
6473 static void intel_encoders_pre_pll_enable(struct intel_atomic_state *state,
6474                                           struct intel_crtc *crtc)
6475 {
6476         const struct intel_crtc_state *crtc_state =
6477                 intel_atomic_get_new_crtc_state(state, crtc);
6478         const struct drm_connector_state *conn_state;
6479         struct drm_connector *conn;
6480         int i;
6481
6482         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6483                 struct intel_encoder *encoder =
6484                         to_intel_encoder(conn_state->best_encoder);
6485
6486                 if (conn_state->crtc != &crtc->base)
6487                         continue;
6488
6489                 if (encoder->pre_pll_enable)
6490                         encoder->pre_pll_enable(encoder, crtc_state, conn_state);
6491         }
6492 }
6493
6494 static void intel_encoders_pre_enable(struct intel_atomic_state *state,
6495                                       struct intel_crtc *crtc)
6496 {
6497         const struct intel_crtc_state *crtc_state =
6498                 intel_atomic_get_new_crtc_state(state, crtc);
6499         const struct drm_connector_state *conn_state;
6500         struct drm_connector *conn;
6501         int i;
6502
6503         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6504                 struct intel_encoder *encoder =
6505                         to_intel_encoder(conn_state->best_encoder);
6506
6507                 if (conn_state->crtc != &crtc->base)
6508                         continue;
6509
6510                 if (encoder->pre_enable)
6511                         encoder->pre_enable(encoder, crtc_state, conn_state);
6512         }
6513 }
6514
6515 static void intel_encoders_enable(struct intel_atomic_state *state,
6516                                   struct intel_crtc *crtc)
6517 {
6518         const struct intel_crtc_state *crtc_state =
6519                 intel_atomic_get_new_crtc_state(state, crtc);
6520         const struct drm_connector_state *conn_state;
6521         struct drm_connector *conn;
6522         int i;
6523
6524         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6525                 struct intel_encoder *encoder =
6526                         to_intel_encoder(conn_state->best_encoder);
6527
6528                 if (conn_state->crtc != &crtc->base)
6529                         continue;
6530
6531                 if (encoder->enable)
6532                         encoder->enable(encoder, crtc_state, conn_state);
6533                 intel_opregion_notify_encoder(encoder, true);
6534         }
6535 }
6536
6537 static void intel_encoders_disable(struct intel_atomic_state *state,
6538                                    struct intel_crtc *crtc)
6539 {
6540         const struct intel_crtc_state *old_crtc_state =
6541                 intel_atomic_get_old_crtc_state(state, crtc);
6542         const struct drm_connector_state *old_conn_state;
6543         struct drm_connector *conn;
6544         int i;
6545
6546         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6547                 struct intel_encoder *encoder =
6548                         to_intel_encoder(old_conn_state->best_encoder);
6549
6550                 if (old_conn_state->crtc != &crtc->base)
6551                         continue;
6552
6553                 intel_opregion_notify_encoder(encoder, false);
6554                 if (encoder->disable)
6555                         encoder->disable(encoder, old_crtc_state, old_conn_state);
6556         }
6557 }
6558
6559 static void intel_encoders_post_disable(struct intel_atomic_state *state,
6560                                         struct intel_crtc *crtc)
6561 {
6562         const struct intel_crtc_state *old_crtc_state =
6563                 intel_atomic_get_old_crtc_state(state, crtc);
6564         const struct drm_connector_state *old_conn_state;
6565         struct drm_connector *conn;
6566         int i;
6567
6568         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6569                 struct intel_encoder *encoder =
6570                         to_intel_encoder(old_conn_state->best_encoder);
6571
6572                 if (old_conn_state->crtc != &crtc->base)
6573                         continue;
6574
6575                 if (encoder->post_disable)
6576                         encoder->post_disable(encoder, old_crtc_state, old_conn_state);
6577         }
6578 }
6579
6580 static void intel_encoders_post_pll_disable(struct intel_atomic_state *state,
6581                                             struct intel_crtc *crtc)
6582 {
6583         const struct intel_crtc_state *old_crtc_state =
6584                 intel_atomic_get_old_crtc_state(state, crtc);
6585         const struct drm_connector_state *old_conn_state;
6586         struct drm_connector *conn;
6587         int i;
6588
6589         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6590                 struct intel_encoder *encoder =
6591                         to_intel_encoder(old_conn_state->best_encoder);
6592
6593                 if (old_conn_state->crtc != &crtc->base)
6594                         continue;
6595
6596                 if (encoder->post_pll_disable)
6597                         encoder->post_pll_disable(encoder, old_crtc_state, old_conn_state);
6598         }
6599 }
6600
6601 static void intel_encoders_update_pipe(struct intel_atomic_state *state,
6602                                        struct intel_crtc *crtc)
6603 {
6604         const struct intel_crtc_state *crtc_state =
6605                 intel_atomic_get_new_crtc_state(state, crtc);
6606         const struct drm_connector_state *conn_state;
6607         struct drm_connector *conn;
6608         int i;
6609
6610         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6611                 struct intel_encoder *encoder =
6612                         to_intel_encoder(conn_state->best_encoder);
6613
6614                 if (conn_state->crtc != &crtc->base)
6615                         continue;
6616
6617                 if (encoder->update_pipe)
6618                         encoder->update_pipe(encoder, crtc_state, conn_state);
6619         }
6620 }
6621
6622 static void intel_disable_primary_plane(const struct intel_crtc_state *crtc_state)
6623 {
6624         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6625         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
6626
6627         plane->disable_plane(plane, crtc_state);
6628 }
6629
6630 static void ironlake_crtc_enable(struct intel_atomic_state *state,
6631                                  struct intel_crtc *crtc)
6632 {
6633         const struct intel_crtc_state *new_crtc_state =
6634                 intel_atomic_get_new_crtc_state(state, crtc);
6635         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6636         enum pipe pipe = crtc->pipe;
6637
6638         if (WARN_ON(crtc->active))
6639                 return;
6640
6641         /*
6642          * Sometimes spurious CPU pipe underruns happen during FDI
6643          * training, at least with VGA+HDMI cloning. Suppress them.
6644          *
6645          * On ILK we get an occasional spurious CPU pipe underruns
6646          * between eDP port A enable and vdd enable. Also PCH port
6647          * enable seems to result in the occasional CPU pipe underrun.
6648          *
6649          * Spurious PCH underruns also occur during PCH enabling.
6650          */
6651         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6652         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
6653
6654         if (new_crtc_state->has_pch_encoder)
6655                 intel_prepare_shared_dpll(new_crtc_state);
6656
6657         if (intel_crtc_has_dp_encoder(new_crtc_state))
6658                 intel_dp_set_m_n(new_crtc_state, M1_N1);
6659
6660         intel_set_pipe_timings(new_crtc_state);
6661         intel_set_pipe_src_size(new_crtc_state);
6662
6663         if (new_crtc_state->has_pch_encoder)
6664                 intel_cpu_transcoder_set_m_n(new_crtc_state,
6665                                              &new_crtc_state->fdi_m_n, NULL);
6666
6667         ironlake_set_pipeconf(new_crtc_state);
6668
6669         crtc->active = true;
6670
6671         intel_encoders_pre_enable(state, crtc);
6672
6673         if (new_crtc_state->has_pch_encoder) {
6674                 /* Note: FDI PLL enabling _must_ be done before we enable the
6675                  * cpu pipes, hence this is separate from all the other fdi/pch
6676                  * enabling. */
6677                 ironlake_fdi_pll_enable(new_crtc_state);
6678         } else {
6679                 assert_fdi_tx_disabled(dev_priv, pipe);
6680                 assert_fdi_rx_disabled(dev_priv, pipe);
6681         }
6682
6683         ironlake_pfit_enable(new_crtc_state);
6684
6685         /*
6686          * On ILK+ LUT must be loaded before the pipe is running but with
6687          * clocks enabled
6688          */
6689         intel_color_load_luts(new_crtc_state);
6690         intel_color_commit(new_crtc_state);
6691         /* update DSPCNTR to configure gamma for pipe bottom color */
6692         intel_disable_primary_plane(new_crtc_state);
6693
6694         if (dev_priv->display.initial_watermarks)
6695                 dev_priv->display.initial_watermarks(state, crtc);
6696         intel_enable_pipe(new_crtc_state);
6697
6698         if (new_crtc_state->has_pch_encoder)
6699                 ironlake_pch_enable(state, new_crtc_state);
6700
6701         intel_crtc_vblank_on(new_crtc_state);
6702
6703         intel_encoders_enable(state, crtc);
6704
6705         if (HAS_PCH_CPT(dev_priv))
6706                 cpt_verify_modeset(dev_priv, pipe);
6707
6708         /*
6709          * Must wait for vblank to avoid spurious PCH FIFO underruns.
6710          * And a second vblank wait is needed at least on ILK with
6711          * some interlaced HDMI modes. Let's do the double wait always
6712          * in case there are more corner cases we don't know about.
6713          */
6714         if (new_crtc_state->has_pch_encoder) {
6715                 intel_wait_for_vblank(dev_priv, pipe);
6716                 intel_wait_for_vblank(dev_priv, pipe);
6717         }
6718         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6719         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
6720 }
6721
6722 /* IPS only exists on ULT machines and is tied to pipe A. */
6723 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
6724 {
6725         return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A;
6726 }
6727
6728 static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv,
6729                                             enum pipe pipe, bool apply)
6730 {
6731         u32 val = I915_READ(CLKGATE_DIS_PSL(pipe));
6732         u32 mask = DPF_GATING_DIS | DPF_RAM_GATING_DIS | DPFR_GATING_DIS;
6733
6734         if (apply)
6735                 val |= mask;
6736         else
6737                 val &= ~mask;
6738
6739         I915_WRITE(CLKGATE_DIS_PSL(pipe), val);
6740 }
6741
6742 static void icl_pipe_mbus_enable(struct intel_crtc *crtc)
6743 {
6744         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6745         enum pipe pipe = crtc->pipe;
6746         u32 val;
6747
6748         val = MBUS_DBOX_A_CREDIT(2);
6749
6750         if (INTEL_GEN(dev_priv) >= 12) {
6751                 val |= MBUS_DBOX_BW_CREDIT(2);
6752                 val |= MBUS_DBOX_B_CREDIT(12);
6753         } else {
6754                 val |= MBUS_DBOX_BW_CREDIT(1);
6755                 val |= MBUS_DBOX_B_CREDIT(8);
6756         }
6757
6758         I915_WRITE(PIPE_MBUS_DBOX_CTL(pipe), val);
6759 }
6760
6761 static void hsw_set_frame_start_delay(const struct intel_crtc_state *crtc_state)
6762 {
6763         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6764         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6765         i915_reg_t reg = CHICKEN_TRANS(crtc_state->cpu_transcoder);
6766         u32 val;
6767
6768         val = I915_READ(reg);
6769         val &= ~HSW_FRAME_START_DELAY_MASK;
6770         val |= HSW_FRAME_START_DELAY(0);
6771         I915_WRITE(reg, val);
6772 }
6773
6774 static void haswell_crtc_enable(struct intel_atomic_state *state,
6775                                 struct intel_crtc *crtc)
6776 {
6777         const struct intel_crtc_state *new_crtc_state =
6778                 intel_atomic_get_new_crtc_state(state, crtc);
6779         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6780         enum pipe pipe = crtc->pipe, hsw_workaround_pipe;
6781         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
6782         bool psl_clkgate_wa;
6783
6784         if (WARN_ON(crtc->active))
6785                 return;
6786
6787         intel_encoders_pre_pll_enable(state, crtc);
6788
6789         if (new_crtc_state->shared_dpll)
6790                 intel_enable_shared_dpll(new_crtc_state);
6791
6792         intel_encoders_pre_enable(state, crtc);
6793
6794         if (intel_crtc_has_dp_encoder(new_crtc_state))
6795                 intel_dp_set_m_n(new_crtc_state, M1_N1);
6796
6797         if (!transcoder_is_dsi(cpu_transcoder))
6798                 intel_set_pipe_timings(new_crtc_state);
6799
6800         if (INTEL_GEN(dev_priv) >= 11)
6801                 icl_enable_trans_port_sync(new_crtc_state);
6802
6803         intel_set_pipe_src_size(new_crtc_state);
6804
6805         if (cpu_transcoder != TRANSCODER_EDP &&
6806             !transcoder_is_dsi(cpu_transcoder))
6807                 I915_WRITE(PIPE_MULT(cpu_transcoder),
6808                            new_crtc_state->pixel_multiplier - 1);
6809
6810         if (new_crtc_state->has_pch_encoder)
6811                 intel_cpu_transcoder_set_m_n(new_crtc_state,
6812                                              &new_crtc_state->fdi_m_n, NULL);
6813
6814         if (!transcoder_is_dsi(cpu_transcoder)) {
6815                 hsw_set_frame_start_delay(new_crtc_state);
6816                 haswell_set_pipeconf(new_crtc_state);
6817         }
6818
6819         if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
6820                 bdw_set_pipemisc(new_crtc_state);
6821
6822         crtc->active = true;
6823
6824         /* Display WA #1180: WaDisableScalarClockGating: glk, cnl */
6825         psl_clkgate_wa = (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) &&
6826                 new_crtc_state->pch_pfit.enabled;
6827         if (psl_clkgate_wa)
6828                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, true);
6829
6830         if (INTEL_GEN(dev_priv) >= 9)
6831                 skylake_pfit_enable(new_crtc_state);
6832         else
6833                 ironlake_pfit_enable(new_crtc_state);
6834
6835         /*
6836          * On ILK+ LUT must be loaded before the pipe is running but with
6837          * clocks enabled
6838          */
6839         intel_color_load_luts(new_crtc_state);
6840         intel_color_commit(new_crtc_state);
6841         /* update DSPCNTR to configure gamma/csc for pipe bottom color */
6842         if (INTEL_GEN(dev_priv) < 9)
6843                 intel_disable_primary_plane(new_crtc_state);
6844
6845         if (INTEL_GEN(dev_priv) >= 11)
6846                 icl_set_pipe_chicken(crtc);
6847
6848         if (!transcoder_is_dsi(cpu_transcoder))
6849                 intel_ddi_enable_transcoder_func(new_crtc_state);
6850
6851         if (dev_priv->display.initial_watermarks)
6852                 dev_priv->display.initial_watermarks(state, crtc);
6853
6854         if (INTEL_GEN(dev_priv) >= 11)
6855                 icl_pipe_mbus_enable(crtc);
6856
6857         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
6858         if (!transcoder_is_dsi(cpu_transcoder))
6859                 intel_enable_pipe(new_crtc_state);
6860
6861         if (new_crtc_state->has_pch_encoder)
6862                 lpt_pch_enable(state, new_crtc_state);
6863
6864         intel_crtc_vblank_on(new_crtc_state);
6865
6866         intel_encoders_enable(state, crtc);
6867
6868         if (psl_clkgate_wa) {
6869                 intel_wait_for_vblank(dev_priv, pipe);
6870                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, false);
6871         }
6872
6873         /* If we change the relative order between pipe/planes enabling, we need
6874          * to change the workaround. */
6875         hsw_workaround_pipe = new_crtc_state->hsw_workaround_pipe;
6876         if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
6877                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
6878                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
6879         }
6880 }
6881
6882 void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state)
6883 {
6884         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
6885         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6886         enum pipe pipe = crtc->pipe;
6887
6888         /* To avoid upsetting the power well on haswell only disable the pfit if
6889          * it's in use. The hw state code will make sure we get this right. */
6890         if (old_crtc_state->pch_pfit.enabled) {
6891                 I915_WRITE(PF_CTL(pipe), 0);
6892                 I915_WRITE(PF_WIN_POS(pipe), 0);
6893                 I915_WRITE(PF_WIN_SZ(pipe), 0);
6894         }
6895 }
6896
6897 static void ironlake_crtc_disable(struct intel_atomic_state *state,
6898                                   struct intel_crtc *crtc)
6899 {
6900         const struct intel_crtc_state *old_crtc_state =
6901                 intel_atomic_get_old_crtc_state(state, crtc);
6902         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6903         enum pipe pipe = crtc->pipe;
6904
6905         /*
6906          * Sometimes spurious CPU pipe underruns happen when the
6907          * pipe is already disabled, but FDI RX/TX is still enabled.
6908          * Happens at least with VGA+HDMI cloning. Suppress them.
6909          */
6910         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6911         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
6912
6913         intel_encoders_disable(state, crtc);
6914
6915         intel_crtc_vblank_off(old_crtc_state);
6916
6917         intel_disable_pipe(old_crtc_state);
6918
6919         ironlake_pfit_disable(old_crtc_state);
6920
6921         if (old_crtc_state->has_pch_encoder)
6922                 ironlake_fdi_disable(crtc);
6923
6924         intel_encoders_post_disable(state, crtc);
6925
6926         if (old_crtc_state->has_pch_encoder) {
6927                 ironlake_disable_pch_transcoder(dev_priv, pipe);
6928
6929                 if (HAS_PCH_CPT(dev_priv)) {
6930                         i915_reg_t reg;
6931                         u32 temp;
6932
6933                         /* disable TRANS_DP_CTL */
6934                         reg = TRANS_DP_CTL(pipe);
6935                         temp = I915_READ(reg);
6936                         temp &= ~(TRANS_DP_OUTPUT_ENABLE |
6937                                   TRANS_DP_PORT_SEL_MASK);
6938                         temp |= TRANS_DP_PORT_SEL_NONE;
6939                         I915_WRITE(reg, temp);
6940
6941                         /* disable DPLL_SEL */
6942                         temp = I915_READ(PCH_DPLL_SEL);
6943                         temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
6944                         I915_WRITE(PCH_DPLL_SEL, temp);
6945                 }
6946
6947                 ironlake_fdi_pll_disable(crtc);
6948         }
6949
6950         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6951         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
6952 }
6953
6954 static void haswell_crtc_disable(struct intel_atomic_state *state,
6955                                  struct intel_crtc *crtc)
6956 {
6957         /*
6958          * FIXME collapse everything to one hook.
6959          * Need care with mst->ddi interactions.
6960          */
6961         intel_encoders_disable(state, crtc);
6962         intel_encoders_post_disable(state, crtc);
6963 }
6964
6965 static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state)
6966 {
6967         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6968         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6969
6970         if (!crtc_state->gmch_pfit.control)
6971                 return;
6972
6973         /*
6974          * The panel fitter should only be adjusted whilst the pipe is disabled,
6975          * according to register description and PRM.
6976          */
6977         WARN_ON(I915_READ(PFIT_CONTROL) & PFIT_ENABLE);
6978         assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder);
6979
6980         I915_WRITE(PFIT_PGM_RATIOS, crtc_state->gmch_pfit.pgm_ratios);
6981         I915_WRITE(PFIT_CONTROL, crtc_state->gmch_pfit.control);
6982
6983         /* Border color in case we don't scale up to the full screen. Black by
6984          * default, change to something else for debugging. */
6985         I915_WRITE(BCLRPAT(crtc->pipe), 0);
6986 }
6987
6988 bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy)
6989 {
6990         if (phy == PHY_NONE)
6991                 return false;
6992
6993         if (IS_ELKHARTLAKE(dev_priv))
6994                 return phy <= PHY_C;
6995
6996         if (INTEL_GEN(dev_priv) >= 11)
6997                 return phy <= PHY_B;
6998
6999         return false;
7000 }
7001
7002 bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy)
7003 {
7004         if (INTEL_GEN(dev_priv) >= 12)
7005                 return phy >= PHY_D && phy <= PHY_I;
7006
7007         if (INTEL_GEN(dev_priv) >= 11 && !IS_ELKHARTLAKE(dev_priv))
7008                 return phy >= PHY_C && phy <= PHY_F;
7009
7010         return false;
7011 }
7012
7013 enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port)
7014 {
7015         if (IS_ELKHARTLAKE(i915) && port == PORT_D)
7016                 return PHY_A;
7017
7018         return (enum phy)port;
7019 }
7020
7021 enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port)
7022 {
7023         if (!intel_phy_is_tc(dev_priv, intel_port_to_phy(dev_priv, port)))
7024                 return PORT_TC_NONE;
7025
7026         if (INTEL_GEN(dev_priv) >= 12)
7027                 return port - PORT_D;
7028
7029         return port - PORT_C;
7030 }
7031
7032 enum intel_display_power_domain intel_port_to_power_domain(enum port port)
7033 {
7034         switch (port) {
7035         case PORT_A:
7036                 return POWER_DOMAIN_PORT_DDI_A_LANES;
7037         case PORT_B:
7038                 return POWER_DOMAIN_PORT_DDI_B_LANES;
7039         case PORT_C:
7040                 return POWER_DOMAIN_PORT_DDI_C_LANES;
7041         case PORT_D:
7042                 return POWER_DOMAIN_PORT_DDI_D_LANES;
7043         case PORT_E:
7044                 return POWER_DOMAIN_PORT_DDI_E_LANES;
7045         case PORT_F:
7046                 return POWER_DOMAIN_PORT_DDI_F_LANES;
7047         case PORT_G:
7048                 return POWER_DOMAIN_PORT_DDI_G_LANES;
7049         default:
7050                 MISSING_CASE(port);
7051                 return POWER_DOMAIN_PORT_OTHER;
7052         }
7053 }
7054
7055 enum intel_display_power_domain
7056 intel_aux_power_domain(struct intel_digital_port *dig_port)
7057 {
7058         struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
7059         enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port);
7060
7061         if (intel_phy_is_tc(dev_priv, phy) &&
7062             dig_port->tc_mode == TC_PORT_TBT_ALT) {
7063                 switch (dig_port->aux_ch) {
7064                 case AUX_CH_C:
7065                         return POWER_DOMAIN_AUX_C_TBT;
7066                 case AUX_CH_D:
7067                         return POWER_DOMAIN_AUX_D_TBT;
7068                 case AUX_CH_E:
7069                         return POWER_DOMAIN_AUX_E_TBT;
7070                 case AUX_CH_F:
7071                         return POWER_DOMAIN_AUX_F_TBT;
7072                 case AUX_CH_G:
7073                         return POWER_DOMAIN_AUX_G_TBT;
7074                 default:
7075                         MISSING_CASE(dig_port->aux_ch);
7076                         return POWER_DOMAIN_AUX_C_TBT;
7077                 }
7078         }
7079
7080         switch (dig_port->aux_ch) {
7081         case AUX_CH_A:
7082                 return POWER_DOMAIN_AUX_A;
7083         case AUX_CH_B:
7084                 return POWER_DOMAIN_AUX_B;
7085         case AUX_CH_C:
7086                 return POWER_DOMAIN_AUX_C;
7087         case AUX_CH_D:
7088                 return POWER_DOMAIN_AUX_D;
7089         case AUX_CH_E:
7090                 return POWER_DOMAIN_AUX_E;
7091         case AUX_CH_F:
7092                 return POWER_DOMAIN_AUX_F;
7093         case AUX_CH_G:
7094                 return POWER_DOMAIN_AUX_G;
7095         default:
7096                 MISSING_CASE(dig_port->aux_ch);
7097                 return POWER_DOMAIN_AUX_A;
7098         }
7099 }
7100
7101 static u64 get_crtc_power_domains(struct intel_crtc_state *crtc_state)
7102 {
7103         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7104         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7105         struct drm_encoder *encoder;
7106         enum pipe pipe = crtc->pipe;
7107         u64 mask;
7108         enum transcoder transcoder = crtc_state->cpu_transcoder;
7109
7110         if (!crtc_state->hw.active)
7111                 return 0;
7112
7113         mask = BIT_ULL(POWER_DOMAIN_PIPE(pipe));
7114         mask |= BIT_ULL(POWER_DOMAIN_TRANSCODER(transcoder));
7115         if (crtc_state->pch_pfit.enabled ||
7116             crtc_state->pch_pfit.force_thru)
7117                 mask |= BIT_ULL(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
7118
7119         drm_for_each_encoder_mask(encoder, &dev_priv->drm,
7120                                   crtc_state->uapi.encoder_mask) {
7121                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
7122
7123                 mask |= BIT_ULL(intel_encoder->power_domain);
7124         }
7125
7126         if (HAS_DDI(dev_priv) && crtc_state->has_audio)
7127                 mask |= BIT_ULL(POWER_DOMAIN_AUDIO);
7128
7129         if (crtc_state->shared_dpll)
7130                 mask |= BIT_ULL(POWER_DOMAIN_DISPLAY_CORE);
7131
7132         return mask;
7133 }
7134
7135 static u64
7136 modeset_get_crtc_power_domains(struct intel_crtc_state *crtc_state)
7137 {
7138         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7139         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7140         enum intel_display_power_domain domain;
7141         u64 domains, new_domains, old_domains;
7142
7143         old_domains = crtc->enabled_power_domains;
7144         crtc->enabled_power_domains = new_domains =
7145                 get_crtc_power_domains(crtc_state);
7146
7147         domains = new_domains & ~old_domains;
7148
7149         for_each_power_domain(domain, domains)
7150                 intel_display_power_get(dev_priv, domain);
7151
7152         return old_domains & ~new_domains;
7153 }
7154
7155 static void modeset_put_power_domains(struct drm_i915_private *dev_priv,
7156                                       u64 domains)
7157 {
7158         enum intel_display_power_domain domain;
7159
7160         for_each_power_domain(domain, domains)
7161                 intel_display_power_put_unchecked(dev_priv, domain);
7162 }
7163
7164 static void valleyview_crtc_enable(struct intel_atomic_state *state,
7165                                    struct intel_crtc *crtc)
7166 {
7167         const struct intel_crtc_state *new_crtc_state =
7168                 intel_atomic_get_new_crtc_state(state, crtc);
7169         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7170         enum pipe pipe = crtc->pipe;
7171
7172         if (WARN_ON(crtc->active))
7173                 return;
7174
7175         if (intel_crtc_has_dp_encoder(new_crtc_state))
7176                 intel_dp_set_m_n(new_crtc_state, M1_N1);
7177
7178         intel_set_pipe_timings(new_crtc_state);
7179         intel_set_pipe_src_size(new_crtc_state);
7180
7181         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
7182                 I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY);
7183                 I915_WRITE(CHV_CANVAS(pipe), 0);
7184         }
7185
7186         i9xx_set_pipeconf(new_crtc_state);
7187
7188         crtc->active = true;
7189
7190         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
7191
7192         intel_encoders_pre_pll_enable(state, crtc);
7193
7194         if (IS_CHERRYVIEW(dev_priv)) {
7195                 chv_prepare_pll(crtc, new_crtc_state);
7196                 chv_enable_pll(crtc, new_crtc_state);
7197         } else {
7198                 vlv_prepare_pll(crtc, new_crtc_state);
7199                 vlv_enable_pll(crtc, new_crtc_state);
7200         }
7201
7202         intel_encoders_pre_enable(state, crtc);
7203
7204         i9xx_pfit_enable(new_crtc_state);
7205
7206         intel_color_load_luts(new_crtc_state);
7207         intel_color_commit(new_crtc_state);
7208         /* update DSPCNTR to configure gamma for pipe bottom color */
7209         intel_disable_primary_plane(new_crtc_state);
7210
7211         dev_priv->display.initial_watermarks(state, crtc);
7212         intel_enable_pipe(new_crtc_state);
7213
7214         intel_crtc_vblank_on(new_crtc_state);
7215
7216         intel_encoders_enable(state, crtc);
7217 }
7218
7219 static void i9xx_set_pll_dividers(const struct intel_crtc_state *crtc_state)
7220 {
7221         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7222         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7223
7224         I915_WRITE(FP0(crtc->pipe), crtc_state->dpll_hw_state.fp0);
7225         I915_WRITE(FP1(crtc->pipe), crtc_state->dpll_hw_state.fp1);
7226 }
7227
7228 static void i9xx_crtc_enable(struct intel_atomic_state *state,
7229                              struct intel_crtc *crtc)
7230 {
7231         const struct intel_crtc_state *new_crtc_state =
7232                 intel_atomic_get_new_crtc_state(state, crtc);
7233         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7234         enum pipe pipe = crtc->pipe;
7235
7236         if (WARN_ON(crtc->active))
7237                 return;
7238
7239         i9xx_set_pll_dividers(new_crtc_state);
7240
7241         if (intel_crtc_has_dp_encoder(new_crtc_state))
7242                 intel_dp_set_m_n(new_crtc_state, M1_N1);
7243
7244         intel_set_pipe_timings(new_crtc_state);
7245         intel_set_pipe_src_size(new_crtc_state);
7246
7247         i9xx_set_pipeconf(new_crtc_state);
7248
7249         crtc->active = true;
7250
7251         if (!IS_GEN(dev_priv, 2))
7252                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
7253
7254         intel_encoders_pre_enable(state, crtc);
7255
7256         i9xx_enable_pll(crtc, new_crtc_state);
7257
7258         i9xx_pfit_enable(new_crtc_state);
7259
7260         intel_color_load_luts(new_crtc_state);
7261         intel_color_commit(new_crtc_state);
7262         /* update DSPCNTR to configure gamma for pipe bottom color */
7263         intel_disable_primary_plane(new_crtc_state);
7264
7265         if (dev_priv->display.initial_watermarks)
7266                 dev_priv->display.initial_watermarks(state, crtc);
7267         else
7268                 intel_update_watermarks(crtc);
7269         intel_enable_pipe(new_crtc_state);
7270
7271         intel_crtc_vblank_on(new_crtc_state);
7272
7273         intel_encoders_enable(state, crtc);
7274 }
7275
7276 static void i9xx_pfit_disable(const struct intel_crtc_state *old_crtc_state)
7277 {
7278         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
7279         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7280
7281         if (!old_crtc_state->gmch_pfit.control)
7282                 return;
7283
7284         assert_pipe_disabled(dev_priv, old_crtc_state->cpu_transcoder);
7285
7286         DRM_DEBUG_KMS("disabling pfit, current: 0x%08x\n",
7287                       I915_READ(PFIT_CONTROL));
7288         I915_WRITE(PFIT_CONTROL, 0);
7289 }
7290
7291 static void i9xx_crtc_disable(struct intel_atomic_state *state,
7292                               struct intel_crtc *crtc)
7293 {
7294         struct intel_crtc_state *old_crtc_state =
7295                 intel_atomic_get_old_crtc_state(state, crtc);
7296         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7297         enum pipe pipe = crtc->pipe;
7298
7299         /*
7300          * On gen2 planes are double buffered but the pipe isn't, so we must
7301          * wait for planes to fully turn off before disabling the pipe.
7302          */
7303         if (IS_GEN(dev_priv, 2))
7304                 intel_wait_for_vblank(dev_priv, pipe);
7305
7306         intel_encoders_disable(state, crtc);
7307
7308         intel_crtc_vblank_off(old_crtc_state);
7309
7310         intel_disable_pipe(old_crtc_state);
7311
7312         i9xx_pfit_disable(old_crtc_state);
7313
7314         intel_encoders_post_disable(state, crtc);
7315
7316         if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DSI)) {
7317                 if (IS_CHERRYVIEW(dev_priv))
7318                         chv_disable_pll(dev_priv, pipe);
7319                 else if (IS_VALLEYVIEW(dev_priv))
7320                         vlv_disable_pll(dev_priv, pipe);
7321                 else
7322                         i9xx_disable_pll(old_crtc_state);
7323         }
7324
7325         intel_encoders_post_pll_disable(state, crtc);
7326
7327         if (!IS_GEN(dev_priv, 2))
7328                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
7329
7330         if (!dev_priv->display.initial_watermarks)
7331                 intel_update_watermarks(crtc);
7332
7333         /* clock the pipe down to 640x480@60 to potentially save power */
7334         if (IS_I830(dev_priv))
7335                 i830_enable_pipe(dev_priv, pipe);
7336 }
7337
7338 static void intel_crtc_disable_noatomic(struct intel_crtc *crtc,
7339                                         struct drm_modeset_acquire_ctx *ctx)
7340 {
7341         struct intel_encoder *encoder;
7342         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7343         struct intel_bw_state *bw_state =
7344                 to_intel_bw_state(dev_priv->bw_obj.state);
7345         struct intel_crtc_state *crtc_state =
7346                 to_intel_crtc_state(crtc->base.state);
7347         enum intel_display_power_domain domain;
7348         struct intel_plane *plane;
7349         struct drm_atomic_state *state;
7350         struct intel_crtc_state *temp_crtc_state;
7351         enum pipe pipe = crtc->pipe;
7352         u64 domains;
7353         int ret;
7354
7355         if (!crtc_state->hw.active)
7356                 return;
7357
7358         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
7359                 const struct intel_plane_state *plane_state =
7360                         to_intel_plane_state(plane->base.state);
7361
7362                 if (plane_state->uapi.visible)
7363                         intel_plane_disable_noatomic(crtc, plane);
7364         }
7365
7366         state = drm_atomic_state_alloc(&dev_priv->drm);
7367         if (!state) {
7368                 DRM_DEBUG_KMS("failed to disable [CRTC:%d:%s], out of memory",
7369                               crtc->base.base.id, crtc->base.name);
7370                 return;
7371         }
7372
7373         state->acquire_ctx = ctx;
7374
7375         /* Everything's already locked, -EDEADLK can't happen. */
7376         temp_crtc_state = intel_atomic_get_crtc_state(state, crtc);
7377         ret = drm_atomic_add_affected_connectors(state, &crtc->base);
7378
7379         WARN_ON(IS_ERR(temp_crtc_state) || ret);
7380
7381         dev_priv->display.crtc_disable(to_intel_atomic_state(state), crtc);
7382
7383         drm_atomic_state_put(state);
7384
7385         DRM_DEBUG_KMS("[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
7386                       crtc->base.base.id, crtc->base.name);
7387
7388         crtc->active = false;
7389         crtc->base.enabled = false;
7390
7391         WARN_ON(drm_atomic_set_mode_for_crtc(&crtc_state->uapi, NULL) < 0);
7392         crtc_state->uapi.active = false;
7393         crtc_state->uapi.connector_mask = 0;
7394         crtc_state->uapi.encoder_mask = 0;
7395         intel_crtc_free_hw_state(crtc_state);
7396         memset(&crtc_state->hw, 0, sizeof(crtc_state->hw));
7397
7398         for_each_encoder_on_crtc(&dev_priv->drm, &crtc->base, encoder)
7399                 encoder->base.crtc = NULL;
7400
7401         intel_fbc_disable(crtc);
7402         intel_update_watermarks(crtc);
7403         intel_disable_shared_dpll(crtc_state);
7404
7405         domains = crtc->enabled_power_domains;
7406         for_each_power_domain(domain, domains)
7407                 intel_display_power_put_unchecked(dev_priv, domain);
7408         crtc->enabled_power_domains = 0;
7409
7410         dev_priv->active_pipes &= ~BIT(pipe);
7411         dev_priv->min_cdclk[pipe] = 0;
7412         dev_priv->min_voltage_level[pipe] = 0;
7413
7414         bw_state->data_rate[pipe] = 0;
7415         bw_state->num_active_planes[pipe] = 0;
7416 }
7417
7418 /*
7419  * turn all crtc's off, but do not adjust state
7420  * This has to be paired with a call to intel_modeset_setup_hw_state.
7421  */
7422 int intel_display_suspend(struct drm_device *dev)
7423 {
7424         struct drm_i915_private *dev_priv = to_i915(dev);
7425         struct drm_atomic_state *state;
7426         int ret;
7427
7428         state = drm_atomic_helper_suspend(dev);
7429         ret = PTR_ERR_OR_ZERO(state);
7430         if (ret)
7431                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
7432         else
7433                 dev_priv->modeset_restore_state = state;
7434         return ret;
7435 }
7436
7437 void intel_encoder_destroy(struct drm_encoder *encoder)
7438 {
7439         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
7440
7441         drm_encoder_cleanup(encoder);
7442         kfree(intel_encoder);
7443 }
7444
7445 /* Cross check the actual hw state with our own modeset state tracking (and it's
7446  * internal consistency). */
7447 static void intel_connector_verify_state(struct intel_crtc_state *crtc_state,
7448                                          struct drm_connector_state *conn_state)
7449 {
7450         struct intel_connector *connector = to_intel_connector(conn_state->connector);
7451
7452         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
7453                       connector->base.base.id,
7454                       connector->base.name);
7455
7456         if (connector->get_hw_state(connector)) {
7457                 struct intel_encoder *encoder = connector->encoder;
7458
7459                 I915_STATE_WARN(!crtc_state,
7460                          "connector enabled without attached crtc\n");
7461
7462                 if (!crtc_state)
7463                         return;
7464
7465                 I915_STATE_WARN(!crtc_state->hw.active,
7466                                 "connector is active, but attached crtc isn't\n");
7467
7468                 if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST)
7469                         return;
7470
7471                 I915_STATE_WARN(conn_state->best_encoder != &encoder->base,
7472                         "atomic encoder doesn't match attached encoder\n");
7473
7474                 I915_STATE_WARN(conn_state->crtc != encoder->base.crtc,
7475                         "attached encoder crtc differs from connector crtc\n");
7476         } else {
7477                 I915_STATE_WARN(crtc_state && crtc_state->hw.active,
7478                                 "attached crtc is active, but connector isn't\n");
7479                 I915_STATE_WARN(!crtc_state && conn_state->best_encoder,
7480                         "best encoder set without crtc!\n");
7481         }
7482 }
7483
7484 static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state)
7485 {
7486         if (crtc_state->hw.enable && crtc_state->has_pch_encoder)
7487                 return crtc_state->fdi_lanes;
7488
7489         return 0;
7490 }
7491
7492 static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
7493                                      struct intel_crtc_state *pipe_config)
7494 {
7495         struct drm_i915_private *dev_priv = to_i915(dev);
7496         struct drm_atomic_state *state = pipe_config->uapi.state;
7497         struct intel_crtc *other_crtc;
7498         struct intel_crtc_state *other_crtc_state;
7499
7500         DRM_DEBUG_KMS("checking fdi config on pipe %c, lanes %i\n",
7501                       pipe_name(pipe), pipe_config->fdi_lanes);
7502         if (pipe_config->fdi_lanes > 4) {
7503                 DRM_DEBUG_KMS("invalid fdi lane config on pipe %c: %i lanes\n",
7504                               pipe_name(pipe), pipe_config->fdi_lanes);
7505                 return -EINVAL;
7506         }
7507
7508         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
7509                 if (pipe_config->fdi_lanes > 2) {
7510                         DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n",
7511                                       pipe_config->fdi_lanes);
7512                         return -EINVAL;
7513                 } else {
7514                         return 0;
7515                 }
7516         }
7517
7518         if (INTEL_NUM_PIPES(dev_priv) == 2)
7519                 return 0;
7520
7521         /* Ivybridge 3 pipe is really complicated */
7522         switch (pipe) {
7523         case PIPE_A:
7524                 return 0;
7525         case PIPE_B:
7526                 if (pipe_config->fdi_lanes <= 2)
7527                         return 0;
7528
7529                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_C);
7530                 other_crtc_state =
7531                         intel_atomic_get_crtc_state(state, other_crtc);
7532                 if (IS_ERR(other_crtc_state))
7533                         return PTR_ERR(other_crtc_state);
7534
7535                 if (pipe_required_fdi_lanes(other_crtc_state) > 0) {
7536                         DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %c: %i lanes\n",
7537                                       pipe_name(pipe), pipe_config->fdi_lanes);
7538                         return -EINVAL;
7539                 }
7540                 return 0;
7541         case PIPE_C:
7542                 if (pipe_config->fdi_lanes > 2) {
7543                         DRM_DEBUG_KMS("only 2 lanes on pipe %c: required %i lanes\n",
7544                                       pipe_name(pipe), pipe_config->fdi_lanes);
7545                         return -EINVAL;
7546                 }
7547
7548                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_B);
7549                 other_crtc_state =
7550                         intel_atomic_get_crtc_state(state, other_crtc);
7551                 if (IS_ERR(other_crtc_state))
7552                         return PTR_ERR(other_crtc_state);
7553
7554                 if (pipe_required_fdi_lanes(other_crtc_state) > 2) {
7555                         DRM_DEBUG_KMS("fdi link B uses too many lanes to enable link C\n");
7556                         return -EINVAL;
7557                 }
7558                 return 0;
7559         default:
7560                 BUG();
7561         }
7562 }
7563
7564 #define RETRY 1
7565 static int ironlake_fdi_compute_config(struct intel_crtc *intel_crtc,
7566                                        struct intel_crtc_state *pipe_config)
7567 {
7568         struct drm_device *dev = intel_crtc->base.dev;
7569         const struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
7570         int lane, link_bw, fdi_dotclock, ret;
7571         bool needs_recompute = false;
7572
7573 retry:
7574         /* FDI is a binary signal running at ~2.7GHz, encoding
7575          * each output octet as 10 bits. The actual frequency
7576          * is stored as a divider into a 100MHz clock, and the
7577          * mode pixel clock is stored in units of 1KHz.
7578          * Hence the bw of each lane in terms of the mode signal
7579          * is:
7580          */
7581         link_bw = intel_fdi_link_freq(to_i915(dev), pipe_config);
7582
7583         fdi_dotclock = adjusted_mode->crtc_clock;
7584
7585         lane = ironlake_get_lanes_required(fdi_dotclock, link_bw,
7586                                            pipe_config->pipe_bpp);
7587
7588         pipe_config->fdi_lanes = lane;
7589
7590         intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock,
7591                                link_bw, &pipe_config->fdi_m_n, false, false);
7592
7593         ret = ironlake_check_fdi_lanes(dev, intel_crtc->pipe, pipe_config);
7594         if (ret == -EDEADLK)
7595                 return ret;
7596
7597         if (ret == -EINVAL && pipe_config->pipe_bpp > 6*3) {
7598                 pipe_config->pipe_bpp -= 2*3;
7599                 DRM_DEBUG_KMS("fdi link bw constraint, reducing pipe bpp to %i\n",
7600                               pipe_config->pipe_bpp);
7601                 needs_recompute = true;
7602                 pipe_config->bw_constrained = true;
7603
7604                 goto retry;
7605         }
7606
7607         if (needs_recompute)
7608                 return RETRY;
7609
7610         return ret;
7611 }
7612
7613 bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state)
7614 {
7615         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7616         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7617
7618         /* IPS only exists on ULT machines and is tied to pipe A. */
7619         if (!hsw_crtc_supports_ips(crtc))
7620                 return false;
7621
7622         if (!i915_modparams.enable_ips)
7623                 return false;
7624
7625         if (crtc_state->pipe_bpp > 24)
7626                 return false;
7627
7628         /*
7629          * We compare against max which means we must take
7630          * the increased cdclk requirement into account when
7631          * calculating the new cdclk.
7632          *
7633          * Should measure whether using a lower cdclk w/o IPS
7634          */
7635         if (IS_BROADWELL(dev_priv) &&
7636             crtc_state->pixel_rate > dev_priv->max_cdclk_freq * 95 / 100)
7637                 return false;
7638
7639         return true;
7640 }
7641
7642 static bool hsw_compute_ips_config(struct intel_crtc_state *crtc_state)
7643 {
7644         struct drm_i915_private *dev_priv =
7645                 to_i915(crtc_state->uapi.crtc->dev);
7646         struct intel_atomic_state *intel_state =
7647                 to_intel_atomic_state(crtc_state->uapi.state);
7648
7649         if (!hsw_crtc_state_ips_capable(crtc_state))
7650                 return false;
7651
7652         /*
7653          * When IPS gets enabled, the pipe CRC changes. Since IPS gets
7654          * enabled and disabled dynamically based on package C states,
7655          * user space can't make reliable use of the CRCs, so let's just
7656          * completely disable it.
7657          */
7658         if (crtc_state->crc_enabled)
7659                 return false;
7660
7661         /* IPS should be fine as long as at least one plane is enabled. */
7662         if (!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)))
7663                 return false;
7664
7665         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
7666         if (IS_BROADWELL(dev_priv) &&
7667             crtc_state->pixel_rate > intel_state->cdclk.logical.cdclk * 95 / 100)
7668                 return false;
7669
7670         return true;
7671 }
7672
7673 static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc)
7674 {
7675         const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7676
7677         /* GDG double wide on either pipe, otherwise pipe A only */
7678         return INTEL_GEN(dev_priv) < 4 &&
7679                 (crtc->pipe == PIPE_A || IS_I915G(dev_priv));
7680 }
7681
7682 static u32 ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config)
7683 {
7684         u32 pixel_rate;
7685
7686         pixel_rate = pipe_config->hw.adjusted_mode.crtc_clock;
7687
7688         /*
7689          * We only use IF-ID interlacing. If we ever use
7690          * PF-ID we'll need to adjust the pixel_rate here.
7691          */
7692
7693         if (pipe_config->pch_pfit.enabled) {
7694                 u64 pipe_w, pipe_h, pfit_w, pfit_h;
7695                 u32 pfit_size = pipe_config->pch_pfit.size;
7696
7697                 pipe_w = pipe_config->pipe_src_w;
7698                 pipe_h = pipe_config->pipe_src_h;
7699
7700                 pfit_w = (pfit_size >> 16) & 0xFFFF;
7701                 pfit_h = pfit_size & 0xFFFF;
7702                 if (pipe_w < pfit_w)
7703                         pipe_w = pfit_w;
7704                 if (pipe_h < pfit_h)
7705                         pipe_h = pfit_h;
7706
7707                 if (WARN_ON(!pfit_w || !pfit_h))
7708                         return pixel_rate;
7709
7710                 pixel_rate = div_u64(mul_u32_u32(pixel_rate, pipe_w * pipe_h),
7711                                      pfit_w * pfit_h);
7712         }
7713
7714         return pixel_rate;
7715 }
7716
7717 static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state)
7718 {
7719         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
7720
7721         if (HAS_GMCH(dev_priv))
7722                 /* FIXME calculate proper pipe pixel rate for GMCH pfit */
7723                 crtc_state->pixel_rate =
7724                         crtc_state->hw.adjusted_mode.crtc_clock;
7725         else
7726                 crtc_state->pixel_rate =
7727                         ilk_pipe_pixel_rate(crtc_state);
7728 }
7729
7730 static int intel_crtc_compute_config(struct intel_crtc *crtc,
7731                                      struct intel_crtc_state *pipe_config)
7732 {
7733         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7734         const struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
7735         int clock_limit = dev_priv->max_dotclk_freq;
7736
7737         if (INTEL_GEN(dev_priv) < 4) {
7738                 clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
7739
7740                 /*
7741                  * Enable double wide mode when the dot clock
7742                  * is > 90% of the (display) core speed.
7743                  */
7744                 if (intel_crtc_supports_double_wide(crtc) &&
7745                     adjusted_mode->crtc_clock > clock_limit) {
7746                         clock_limit = dev_priv->max_dotclk_freq;
7747                         pipe_config->double_wide = true;
7748                 }
7749         }
7750
7751         if (adjusted_mode->crtc_clock > clock_limit) {
7752                 DRM_DEBUG_KMS("requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
7753                               adjusted_mode->crtc_clock, clock_limit,
7754                               yesno(pipe_config->double_wide));
7755                 return -EINVAL;
7756         }
7757
7758         if ((pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
7759              pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) &&
7760              pipe_config->hw.ctm) {
7761                 /*
7762                  * There is only one pipe CSC unit per pipe, and we need that
7763                  * for output conversion from RGB->YCBCR. So if CTM is already
7764                  * applied we can't support YCBCR420 output.
7765                  */
7766                 DRM_DEBUG_KMS("YCBCR420 and CTM together are not possible\n");
7767                 return -EINVAL;
7768         }
7769
7770         /*
7771          * Pipe horizontal size must be even in:
7772          * - DVO ganged mode
7773          * - LVDS dual channel mode
7774          * - Double wide pipe
7775          */
7776         if (pipe_config->pipe_src_w & 1) {
7777                 if (pipe_config->double_wide) {
7778                         DRM_DEBUG_KMS("Odd pipe source width not supported with double wide pipe\n");
7779                         return -EINVAL;
7780                 }
7781
7782                 if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_LVDS) &&
7783                     intel_is_dual_link_lvds(dev_priv)) {
7784                         DRM_DEBUG_KMS("Odd pipe source width not supported with dual link LVDS\n");
7785                         return -EINVAL;
7786                 }
7787         }
7788
7789         /* Cantiga+ cannot handle modes with a hsync front porch of 0.
7790          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
7791          */
7792         if ((INTEL_GEN(dev_priv) > 4 || IS_G4X(dev_priv)) &&
7793                 adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay)
7794                 return -EINVAL;
7795
7796         intel_crtc_compute_pixel_rate(pipe_config);
7797
7798         if (pipe_config->has_pch_encoder)
7799                 return ironlake_fdi_compute_config(crtc, pipe_config);
7800
7801         return 0;
7802 }
7803
7804 static void
7805 intel_reduce_m_n_ratio(u32 *num, u32 *den)
7806 {
7807         while (*num > DATA_LINK_M_N_MASK ||
7808                *den > DATA_LINK_M_N_MASK) {
7809                 *num >>= 1;
7810                 *den >>= 1;
7811         }
7812 }
7813
7814 static void compute_m_n(unsigned int m, unsigned int n,
7815                         u32 *ret_m, u32 *ret_n,
7816                         bool constant_n)
7817 {
7818         /*
7819          * Several DP dongles in particular seem to be fussy about
7820          * too large link M/N values. Give N value as 0x8000 that
7821          * should be acceptable by specific devices. 0x8000 is the
7822          * specified fixed N value for asynchronous clock mode,
7823          * which the devices expect also in synchronous clock mode.
7824          */
7825         if (constant_n)
7826                 *ret_n = 0x8000;
7827         else
7828                 *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
7829
7830         *ret_m = div_u64(mul_u32_u32(m, *ret_n), n);
7831         intel_reduce_m_n_ratio(ret_m, ret_n);
7832 }
7833
7834 void
7835 intel_link_compute_m_n(u16 bits_per_pixel, int nlanes,
7836                        int pixel_clock, int link_clock,
7837                        struct intel_link_m_n *m_n,
7838                        bool constant_n, bool fec_enable)
7839 {
7840         u32 data_clock = bits_per_pixel * pixel_clock;
7841
7842         if (fec_enable)
7843                 data_clock = intel_dp_mode_to_fec_clock(data_clock);
7844
7845         m_n->tu = 64;
7846         compute_m_n(data_clock,
7847                     link_clock * nlanes * 8,
7848                     &m_n->gmch_m, &m_n->gmch_n,
7849                     constant_n);
7850
7851         compute_m_n(pixel_clock, link_clock,
7852                     &m_n->link_m, &m_n->link_n,
7853                     constant_n);
7854 }
7855
7856 static void intel_panel_sanitize_ssc(struct drm_i915_private *dev_priv)
7857 {
7858         /*
7859          * There may be no VBT; and if the BIOS enabled SSC we can
7860          * just keep using it to avoid unnecessary flicker.  Whereas if the
7861          * BIOS isn't using it, don't assume it will work even if the VBT
7862          * indicates as much.
7863          */
7864         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
7865                 bool bios_lvds_use_ssc = I915_READ(PCH_DREF_CONTROL) &
7866                         DREF_SSC1_ENABLE;
7867
7868                 if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) {
7869                         DRM_DEBUG_KMS("SSC %s by BIOS, overriding VBT which says %s\n",
7870                                       enableddisabled(bios_lvds_use_ssc),
7871                                       enableddisabled(dev_priv->vbt.lvds_use_ssc));
7872                         dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc;
7873                 }
7874         }
7875 }
7876
7877 static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
7878 {
7879         if (i915_modparams.panel_use_ssc >= 0)
7880                 return i915_modparams.panel_use_ssc != 0;
7881         return dev_priv->vbt.lvds_use_ssc
7882                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
7883 }
7884
7885 static u32 pnv_dpll_compute_fp(struct dpll *dpll)
7886 {
7887         return (1 << dpll->n) << 16 | dpll->m2;
7888 }
7889
7890 static u32 i9xx_dpll_compute_fp(struct dpll *dpll)
7891 {
7892         return dpll->n << 16 | dpll->m1 << 8 | dpll->m2;
7893 }
7894
7895 static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
7896                                      struct intel_crtc_state *crtc_state,
7897                                      struct dpll *reduced_clock)
7898 {
7899         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7900         u32 fp, fp2 = 0;
7901
7902         if (IS_PINEVIEW(dev_priv)) {
7903                 fp = pnv_dpll_compute_fp(&crtc_state->dpll);
7904                 if (reduced_clock)
7905                         fp2 = pnv_dpll_compute_fp(reduced_clock);
7906         } else {
7907                 fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
7908                 if (reduced_clock)
7909                         fp2 = i9xx_dpll_compute_fp(reduced_clock);
7910         }
7911
7912         crtc_state->dpll_hw_state.fp0 = fp;
7913
7914         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7915             reduced_clock) {
7916                 crtc_state->dpll_hw_state.fp1 = fp2;
7917         } else {
7918                 crtc_state->dpll_hw_state.fp1 = fp;
7919         }
7920 }
7921
7922 static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe
7923                 pipe)
7924 {
7925         u32 reg_val;
7926
7927         /*
7928          * PLLB opamp always calibrates to max value of 0x3f, force enable it
7929          * and set it to a reasonable value instead.
7930          */
7931         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7932         reg_val &= 0xffffff00;
7933         reg_val |= 0x00000030;
7934         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7935
7936         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7937         reg_val &= 0x00ffffff;
7938         reg_val |= 0x8c000000;
7939         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7940
7941         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7942         reg_val &= 0xffffff00;
7943         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7944
7945         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7946         reg_val &= 0x00ffffff;
7947         reg_val |= 0xb0000000;
7948         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7949 }
7950
7951 static void intel_pch_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
7952                                          const struct intel_link_m_n *m_n)
7953 {
7954         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7955         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7956         enum pipe pipe = crtc->pipe;
7957
7958         I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
7959         I915_WRITE(PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
7960         I915_WRITE(PCH_TRANS_LINK_M1(pipe), m_n->link_m);
7961         I915_WRITE(PCH_TRANS_LINK_N1(pipe), m_n->link_n);
7962 }
7963
7964 static bool transcoder_has_m2_n2(struct drm_i915_private *dev_priv,
7965                                  enum transcoder transcoder)
7966 {
7967         if (IS_HASWELL(dev_priv))
7968                 return transcoder == TRANSCODER_EDP;
7969
7970         /*
7971          * Strictly speaking some registers are available before
7972          * gen7, but we only support DRRS on gen7+
7973          */
7974         return IS_GEN(dev_priv, 7) || IS_CHERRYVIEW(dev_priv);
7975 }
7976
7977 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
7978                                          const struct intel_link_m_n *m_n,
7979                                          const struct intel_link_m_n *m2_n2)
7980 {
7981         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7982         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7983         enum pipe pipe = crtc->pipe;
7984         enum transcoder transcoder = crtc_state->cpu_transcoder;
7985
7986         if (INTEL_GEN(dev_priv) >= 5) {
7987                 I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m);
7988                 I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
7989                 I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
7990                 I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n);
7991                 /*
7992                  *  M2_N2 registers are set only if DRRS is supported
7993                  * (to make sure the registers are not unnecessarily accessed).
7994                  */
7995                 if (m2_n2 && crtc_state->has_drrs &&
7996                     transcoder_has_m2_n2(dev_priv, transcoder)) {
7997                         I915_WRITE(PIPE_DATA_M2(transcoder),
7998                                         TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
7999                         I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n);
8000                         I915_WRITE(PIPE_LINK_M2(transcoder), m2_n2->link_m);
8001                         I915_WRITE(PIPE_LINK_N2(transcoder), m2_n2->link_n);
8002                 }
8003         } else {
8004                 I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
8005                 I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
8006                 I915_WRITE(PIPE_LINK_M_G4X(pipe), m_n->link_m);
8007                 I915_WRITE(PIPE_LINK_N_G4X(pipe), m_n->link_n);
8008         }
8009 }
8010
8011 void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, enum link_m_n_set m_n)
8012 {
8013         const struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL;
8014
8015         if (m_n == M1_N1) {
8016                 dp_m_n = &crtc_state->dp_m_n;
8017                 dp_m2_n2 = &crtc_state->dp_m2_n2;
8018         } else if (m_n == M2_N2) {
8019
8020                 /*
8021                  * M2_N2 registers are not supported. Hence m2_n2 divider value
8022                  * needs to be programmed into M1_N1.
8023                  */
8024                 dp_m_n = &crtc_state->dp_m2_n2;
8025         } else {
8026                 DRM_ERROR("Unsupported divider value\n");
8027                 return;
8028         }
8029
8030         if (crtc_state->has_pch_encoder)
8031                 intel_pch_transcoder_set_m_n(crtc_state, &crtc_state->dp_m_n);
8032         else
8033                 intel_cpu_transcoder_set_m_n(crtc_state, dp_m_n, dp_m2_n2);
8034 }
8035
8036 static void vlv_compute_dpll(struct intel_crtc *crtc,
8037                              struct intel_crtc_state *pipe_config)
8038 {
8039         pipe_config->dpll_hw_state.dpll = DPLL_INTEGRATED_REF_CLK_VLV |
8040                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
8041         if (crtc->pipe != PIPE_A)
8042                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
8043
8044         /* DPLL not used with DSI, but still need the rest set up */
8045         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
8046                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE |
8047                         DPLL_EXT_BUFFER_ENABLE_VLV;
8048
8049         pipe_config->dpll_hw_state.dpll_md =
8050                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
8051 }
8052
8053 static void chv_compute_dpll(struct intel_crtc *crtc,
8054                              struct intel_crtc_state *pipe_config)
8055 {
8056         pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV |
8057                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
8058         if (crtc->pipe != PIPE_A)
8059                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
8060
8061         /* DPLL not used with DSI, but still need the rest set up */
8062         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
8063                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE;
8064
8065         pipe_config->dpll_hw_state.dpll_md =
8066                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
8067 }
8068
8069 static void vlv_prepare_pll(struct intel_crtc *crtc,
8070                             const struct intel_crtc_state *pipe_config)
8071 {
8072         struct drm_device *dev = crtc->base.dev;
8073         struct drm_i915_private *dev_priv = to_i915(dev);
8074         enum pipe pipe = crtc->pipe;
8075         u32 mdiv;
8076         u32 bestn, bestm1, bestm2, bestp1, bestp2;
8077         u32 coreclk, reg_val;
8078
8079         /* Enable Refclk */
8080         I915_WRITE(DPLL(pipe),
8081                    pipe_config->dpll_hw_state.dpll &
8082                    ~(DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV));
8083
8084         /* No need to actually set up the DPLL with DSI */
8085         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8086                 return;
8087
8088         vlv_dpio_get(dev_priv);
8089
8090         bestn = pipe_config->dpll.n;
8091         bestm1 = pipe_config->dpll.m1;
8092         bestm2 = pipe_config->dpll.m2;
8093         bestp1 = pipe_config->dpll.p1;
8094         bestp2 = pipe_config->dpll.p2;
8095
8096         /* See eDP HDMI DPIO driver vbios notes doc */
8097
8098         /* PLL B needs special handling */
8099         if (pipe == PIPE_B)
8100                 vlv_pllb_recal_opamp(dev_priv, pipe);
8101
8102         /* Set up Tx target for periodic Rcomp update */
8103         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9_BCAST, 0x0100000f);
8104
8105         /* Disable target IRef on PLL */
8106         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW8(pipe));
8107         reg_val &= 0x00ffffff;
8108         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW8(pipe), reg_val);
8109
8110         /* Disable fast lock */
8111         vlv_dpio_write(dev_priv, pipe, VLV_CMN_DW0, 0x610);
8112
8113         /* Set idtafcrecal before PLL is enabled */
8114         mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
8115         mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT));
8116         mdiv |= ((bestn << DPIO_N_SHIFT));
8117         mdiv |= (1 << DPIO_K_SHIFT);
8118
8119         /*
8120          * Post divider depends on pixel clock rate, DAC vs digital (and LVDS,
8121          * but we don't support that).
8122          * Note: don't use the DAC post divider as it seems unstable.
8123          */
8124         mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
8125         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
8126
8127         mdiv |= DPIO_ENABLE_CALIBRATION;
8128         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
8129
8130         /* Set HBR and RBR LPF coefficients */
8131         if (pipe_config->port_clock == 162000 ||
8132             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_ANALOG) ||
8133             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI))
8134                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
8135                                  0x009f0003);
8136         else
8137                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
8138                                  0x00d0000f);
8139
8140         if (intel_crtc_has_dp_encoder(pipe_config)) {
8141                 /* Use SSC source */
8142                 if (pipe == PIPE_A)
8143                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
8144                                          0x0df40000);
8145                 else
8146                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
8147                                          0x0df70000);
8148         } else { /* HDMI or VGA */
8149                 /* Use bend source */
8150                 if (pipe == PIPE_A)
8151                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
8152                                          0x0df70000);
8153                 else
8154                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
8155                                          0x0df40000);
8156         }
8157
8158         coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe));
8159         coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
8160         if (intel_crtc_has_dp_encoder(pipe_config))
8161                 coreclk |= 0x01000000;
8162         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk);
8163
8164         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000);
8165
8166         vlv_dpio_put(dev_priv);
8167 }
8168
8169 static void chv_prepare_pll(struct intel_crtc *crtc,
8170                             const struct intel_crtc_state *pipe_config)
8171 {
8172         struct drm_device *dev = crtc->base.dev;
8173         struct drm_i915_private *dev_priv = to_i915(dev);
8174         enum pipe pipe = crtc->pipe;
8175         enum dpio_channel port = vlv_pipe_to_channel(pipe);
8176         u32 loopfilter, tribuf_calcntr;
8177         u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
8178         u32 dpio_val;
8179         int vco;
8180
8181         /* Enable Refclk and SSC */
8182         I915_WRITE(DPLL(pipe),
8183                    pipe_config->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE);
8184
8185         /* No need to actually set up the DPLL with DSI */
8186         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8187                 return;
8188
8189         bestn = pipe_config->dpll.n;
8190         bestm2_frac = pipe_config->dpll.m2 & 0x3fffff;
8191         bestm1 = pipe_config->dpll.m1;
8192         bestm2 = pipe_config->dpll.m2 >> 22;
8193         bestp1 = pipe_config->dpll.p1;
8194         bestp2 = pipe_config->dpll.p2;
8195         vco = pipe_config->dpll.vco;
8196         dpio_val = 0;
8197         loopfilter = 0;
8198
8199         vlv_dpio_get(dev_priv);
8200
8201         /* p1 and p2 divider */
8202         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port),
8203                         5 << DPIO_CHV_S1_DIV_SHIFT |
8204                         bestp1 << DPIO_CHV_P1_DIV_SHIFT |
8205                         bestp2 << DPIO_CHV_P2_DIV_SHIFT |
8206                         1 << DPIO_CHV_K_DIV_SHIFT);
8207
8208         /* Feedback post-divider - m2 */
8209         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2);
8210
8211         /* Feedback refclk divider - n and m1 */
8212         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port),
8213                         DPIO_CHV_M1_DIV_BY_2 |
8214                         1 << DPIO_CHV_N_DIV_SHIFT);
8215
8216         /* M2 fraction division */
8217         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
8218
8219         /* M2 fraction division enable */
8220         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
8221         dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
8222         dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT);
8223         if (bestm2_frac)
8224                 dpio_val |= DPIO_CHV_FRAC_DIV_EN;
8225         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), dpio_val);
8226
8227         /* Program digital lock detect threshold */
8228         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW9(port));
8229         dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
8230                                         DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
8231         dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT);
8232         if (!bestm2_frac)
8233                 dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
8234         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW9(port), dpio_val);
8235
8236         /* Loop filter */
8237         if (vco == 5400000) {
8238                 loopfilter |= (0x3 << DPIO_CHV_PROP_COEFF_SHIFT);
8239                 loopfilter |= (0x8 << DPIO_CHV_INT_COEFF_SHIFT);
8240                 loopfilter |= (0x1 << DPIO_CHV_GAIN_CTRL_SHIFT);
8241                 tribuf_calcntr = 0x9;
8242         } else if (vco <= 6200000) {
8243                 loopfilter |= (0x5 << DPIO_CHV_PROP_COEFF_SHIFT);
8244                 loopfilter |= (0xB << DPIO_CHV_INT_COEFF_SHIFT);
8245                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
8246                 tribuf_calcntr = 0x9;
8247         } else if (vco <= 6480000) {
8248                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
8249                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
8250                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
8251                 tribuf_calcntr = 0x8;
8252         } else {
8253                 /* Not supported. Apply the same limits as in the max case */
8254                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
8255                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
8256                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
8257                 tribuf_calcntr = 0;
8258         }
8259         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter);
8260
8261         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW8(port));
8262         dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
8263         dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT);
8264         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW8(port), dpio_val);
8265
8266         /* AFC Recal */
8267         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port),
8268                         vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) |
8269                         DPIO_AFC_RECAL);
8270
8271         vlv_dpio_put(dev_priv);
8272 }
8273
8274 /**
8275  * vlv_force_pll_on - forcibly enable just the PLL
8276  * @dev_priv: i915 private structure
8277  * @pipe: pipe PLL to enable
8278  * @dpll: PLL configuration
8279  *
8280  * Enable the PLL for @pipe using the supplied @dpll config. To be used
8281  * in cases where we need the PLL enabled even when @pipe is not going to
8282  * be enabled.
8283  */
8284 int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe,
8285                      const struct dpll *dpll)
8286 {
8287         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
8288         struct intel_crtc_state *pipe_config;
8289
8290         pipe_config = intel_crtc_state_alloc(crtc);
8291         if (!pipe_config)
8292                 return -ENOMEM;
8293
8294         pipe_config->cpu_transcoder = (enum transcoder)pipe;
8295         pipe_config->pixel_multiplier = 1;
8296         pipe_config->dpll = *dpll;
8297
8298         if (IS_CHERRYVIEW(dev_priv)) {
8299                 chv_compute_dpll(crtc, pipe_config);
8300                 chv_prepare_pll(crtc, pipe_config);
8301                 chv_enable_pll(crtc, pipe_config);
8302         } else {
8303                 vlv_compute_dpll(crtc, pipe_config);
8304                 vlv_prepare_pll(crtc, pipe_config);
8305                 vlv_enable_pll(crtc, pipe_config);
8306         }
8307
8308         kfree(pipe_config);
8309
8310         return 0;
8311 }
8312
8313 /**
8314  * vlv_force_pll_off - forcibly disable just the PLL
8315  * @dev_priv: i915 private structure
8316  * @pipe: pipe PLL to disable
8317  *
8318  * Disable the PLL for @pipe. To be used in cases where we need
8319  * the PLL enabled even when @pipe is not going to be enabled.
8320  */
8321 void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe)
8322 {
8323         if (IS_CHERRYVIEW(dev_priv))
8324                 chv_disable_pll(dev_priv, pipe);
8325         else
8326                 vlv_disable_pll(dev_priv, pipe);
8327 }
8328
8329 static void i9xx_compute_dpll(struct intel_crtc *crtc,
8330                               struct intel_crtc_state *crtc_state,
8331                               struct dpll *reduced_clock)
8332 {
8333         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8334         u32 dpll;
8335         struct dpll *clock = &crtc_state->dpll;
8336
8337         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
8338
8339         dpll = DPLL_VGA_MODE_DIS;
8340
8341         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
8342                 dpll |= DPLLB_MODE_LVDS;
8343         else
8344                 dpll |= DPLLB_MODE_DAC_SERIAL;
8345
8346         if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
8347             IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
8348                 dpll |= (crtc_state->pixel_multiplier - 1)
8349                         << SDVO_MULTIPLIER_SHIFT_HIRES;
8350         }
8351
8352         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
8353             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
8354                 dpll |= DPLL_SDVO_HIGH_SPEED;
8355
8356         if (intel_crtc_has_dp_encoder(crtc_state))
8357                 dpll |= DPLL_SDVO_HIGH_SPEED;
8358
8359         /* compute bitmask from p1 value */
8360         if (IS_PINEVIEW(dev_priv))
8361                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
8362         else {
8363                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8364                 if (IS_G4X(dev_priv) && reduced_clock)
8365                         dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
8366         }
8367         switch (clock->p2) {
8368         case 5:
8369                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
8370                 break;
8371         case 7:
8372                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
8373                 break;
8374         case 10:
8375                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
8376                 break;
8377         case 14:
8378                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
8379                 break;
8380         }
8381         if (INTEL_GEN(dev_priv) >= 4)
8382                 dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
8383
8384         if (crtc_state->sdvo_tv_clock)
8385                 dpll |= PLL_REF_INPUT_TVCLKINBC;
8386         else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8387                  intel_panel_use_ssc(dev_priv))
8388                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8389         else
8390                 dpll |= PLL_REF_INPUT_DREFCLK;
8391
8392         dpll |= DPLL_VCO_ENABLE;
8393         crtc_state->dpll_hw_state.dpll = dpll;
8394
8395         if (INTEL_GEN(dev_priv) >= 4) {
8396                 u32 dpll_md = (crtc_state->pixel_multiplier - 1)
8397                         << DPLL_MD_UDI_MULTIPLIER_SHIFT;
8398                 crtc_state->dpll_hw_state.dpll_md = dpll_md;
8399         }
8400 }
8401
8402 static void i8xx_compute_dpll(struct intel_crtc *crtc,
8403                               struct intel_crtc_state *crtc_state,
8404                               struct dpll *reduced_clock)
8405 {
8406         struct drm_device *dev = crtc->base.dev;
8407         struct drm_i915_private *dev_priv = to_i915(dev);
8408         u32 dpll;
8409         struct dpll *clock = &crtc_state->dpll;
8410
8411         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
8412
8413         dpll = DPLL_VGA_MODE_DIS;
8414
8415         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8416                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8417         } else {
8418                 if (clock->p1 == 2)
8419                         dpll |= PLL_P1_DIVIDE_BY_TWO;
8420                 else
8421                         dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8422                 if (clock->p2 == 4)
8423                         dpll |= PLL_P2_DIVIDE_BY_4;
8424         }
8425
8426         /*
8427          * Bspec:
8428          * "[Almador Errata}: For the correct operation of the muxed DVO pins
8429          *  (GDEVSELB/I2Cdata, GIRDBY/I2CClk) and (GFRAMEB/DVI_Data,
8430          *  GTRDYB/DVI_Clk): Bit 31 (DPLL VCO Enable) and Bit 30 (2X Clock
8431          *  Enable) must be set to “1” in both the DPLL A Control Register
8432          *  (06014h-06017h) and DPLL B Control Register (06018h-0601Bh)."
8433          *
8434          * For simplicity We simply keep both bits always enabled in
8435          * both DPLLS. The spec says we should disable the DVO 2X clock
8436          * when not needed, but this seems to work fine in practice.
8437          */
8438         if (IS_I830(dev_priv) ||
8439             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO))
8440                 dpll |= DPLL_DVO_2X_MODE;
8441
8442         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8443             intel_panel_use_ssc(dev_priv))
8444                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8445         else
8446                 dpll |= PLL_REF_INPUT_DREFCLK;
8447
8448         dpll |= DPLL_VCO_ENABLE;
8449         crtc_state->dpll_hw_state.dpll = dpll;
8450 }
8451
8452 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state)
8453 {
8454         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8455         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8456         enum pipe pipe = crtc->pipe;
8457         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
8458         const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
8459         u32 crtc_vtotal, crtc_vblank_end;
8460         int vsyncshift = 0;
8461
8462         /* We need to be careful not to changed the adjusted mode, for otherwise
8463          * the hw state checker will get angry at the mismatch. */
8464         crtc_vtotal = adjusted_mode->crtc_vtotal;
8465         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
8466
8467         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
8468                 /* the chip adds 2 halflines automatically */
8469                 crtc_vtotal -= 1;
8470                 crtc_vblank_end -= 1;
8471
8472                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
8473                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
8474                 else
8475                         vsyncshift = adjusted_mode->crtc_hsync_start -
8476                                 adjusted_mode->crtc_htotal / 2;
8477                 if (vsyncshift < 0)
8478                         vsyncshift += adjusted_mode->crtc_htotal;
8479         }
8480
8481         if (INTEL_GEN(dev_priv) > 3)
8482                 I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift);
8483
8484         I915_WRITE(HTOTAL(cpu_transcoder),
8485                    (adjusted_mode->crtc_hdisplay - 1) |
8486                    ((adjusted_mode->crtc_htotal - 1) << 16));
8487         I915_WRITE(HBLANK(cpu_transcoder),
8488                    (adjusted_mode->crtc_hblank_start - 1) |
8489                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
8490         I915_WRITE(HSYNC(cpu_transcoder),
8491                    (adjusted_mode->crtc_hsync_start - 1) |
8492                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
8493
8494         I915_WRITE(VTOTAL(cpu_transcoder),
8495                    (adjusted_mode->crtc_vdisplay - 1) |
8496                    ((crtc_vtotal - 1) << 16));
8497         I915_WRITE(VBLANK(cpu_transcoder),
8498                    (adjusted_mode->crtc_vblank_start - 1) |
8499                    ((crtc_vblank_end - 1) << 16));
8500         I915_WRITE(VSYNC(cpu_transcoder),
8501                    (adjusted_mode->crtc_vsync_start - 1) |
8502                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
8503
8504         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
8505          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
8506          * documented on the DDI_FUNC_CTL register description, EDP Input Select
8507          * bits. */
8508         if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP &&
8509             (pipe == PIPE_B || pipe == PIPE_C))
8510                 I915_WRITE(VTOTAL(pipe), I915_READ(VTOTAL(cpu_transcoder)));
8511
8512 }
8513
8514 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state)
8515 {
8516         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8517         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8518         enum pipe pipe = crtc->pipe;
8519
8520         /* pipesrc controls the size that is scaled from, which should
8521          * always be the user's requested size.
8522          */
8523         I915_WRITE(PIPESRC(pipe),
8524                    ((crtc_state->pipe_src_w - 1) << 16) |
8525                    (crtc_state->pipe_src_h - 1));
8526 }
8527
8528 static bool intel_pipe_is_interlaced(const struct intel_crtc_state *crtc_state)
8529 {
8530         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
8531         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
8532
8533         if (IS_GEN(dev_priv, 2))
8534                 return false;
8535
8536         if (INTEL_GEN(dev_priv) >= 9 ||
8537             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
8538                 return I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK_HSW;
8539         else
8540                 return I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK;
8541 }
8542
8543 static void intel_get_pipe_timings(struct intel_crtc *crtc,
8544                                    struct intel_crtc_state *pipe_config)
8545 {
8546         struct drm_device *dev = crtc->base.dev;
8547         struct drm_i915_private *dev_priv = to_i915(dev);
8548         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
8549         u32 tmp;
8550
8551         tmp = I915_READ(HTOTAL(cpu_transcoder));
8552         pipe_config->hw.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
8553         pipe_config->hw.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
8554
8555         if (!transcoder_is_dsi(cpu_transcoder)) {
8556                 tmp = I915_READ(HBLANK(cpu_transcoder));
8557                 pipe_config->hw.adjusted_mode.crtc_hblank_start =
8558                                                         (tmp & 0xffff) + 1;
8559                 pipe_config->hw.adjusted_mode.crtc_hblank_end =
8560                                                 ((tmp >> 16) & 0xffff) + 1;
8561         }
8562         tmp = I915_READ(HSYNC(cpu_transcoder));
8563         pipe_config->hw.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
8564         pipe_config->hw.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
8565
8566         tmp = I915_READ(VTOTAL(cpu_transcoder));
8567         pipe_config->hw.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
8568         pipe_config->hw.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
8569
8570         if (!transcoder_is_dsi(cpu_transcoder)) {
8571                 tmp = I915_READ(VBLANK(cpu_transcoder));
8572                 pipe_config->hw.adjusted_mode.crtc_vblank_start =
8573                                                         (tmp & 0xffff) + 1;
8574                 pipe_config->hw.adjusted_mode.crtc_vblank_end =
8575                                                 ((tmp >> 16) & 0xffff) + 1;
8576         }
8577         tmp = I915_READ(VSYNC(cpu_transcoder));
8578         pipe_config->hw.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
8579         pipe_config->hw.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
8580
8581         if (intel_pipe_is_interlaced(pipe_config)) {
8582                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
8583                 pipe_config->hw.adjusted_mode.crtc_vtotal += 1;
8584                 pipe_config->hw.adjusted_mode.crtc_vblank_end += 1;
8585         }
8586 }
8587
8588 static void intel_get_pipe_src_size(struct intel_crtc *crtc,
8589                                     struct intel_crtc_state *pipe_config)
8590 {
8591         struct drm_device *dev = crtc->base.dev;
8592         struct drm_i915_private *dev_priv = to_i915(dev);
8593         u32 tmp;
8594
8595         tmp = I915_READ(PIPESRC(crtc->pipe));
8596         pipe_config->pipe_src_h = (tmp & 0xffff) + 1;
8597         pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1;
8598
8599         pipe_config->hw.mode.vdisplay = pipe_config->pipe_src_h;
8600         pipe_config->hw.mode.hdisplay = pipe_config->pipe_src_w;
8601 }
8602
8603 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
8604                                  struct intel_crtc_state *pipe_config)
8605 {
8606         mode->hdisplay = pipe_config->hw.adjusted_mode.crtc_hdisplay;
8607         mode->htotal = pipe_config->hw.adjusted_mode.crtc_htotal;
8608         mode->hsync_start = pipe_config->hw.adjusted_mode.crtc_hsync_start;
8609         mode->hsync_end = pipe_config->hw.adjusted_mode.crtc_hsync_end;
8610
8611         mode->vdisplay = pipe_config->hw.adjusted_mode.crtc_vdisplay;
8612         mode->vtotal = pipe_config->hw.adjusted_mode.crtc_vtotal;
8613         mode->vsync_start = pipe_config->hw.adjusted_mode.crtc_vsync_start;
8614         mode->vsync_end = pipe_config->hw.adjusted_mode.crtc_vsync_end;
8615
8616         mode->flags = pipe_config->hw.adjusted_mode.flags;
8617         mode->type = DRM_MODE_TYPE_DRIVER;
8618
8619         mode->clock = pipe_config->hw.adjusted_mode.crtc_clock;
8620
8621         mode->hsync = drm_mode_hsync(mode);
8622         mode->vrefresh = drm_mode_vrefresh(mode);
8623         drm_mode_set_name(mode);
8624 }
8625
8626 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
8627 {
8628         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8629         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8630         u32 pipeconf;
8631
8632         pipeconf = 0;
8633
8634         /* we keep both pipes enabled on 830 */
8635         if (IS_I830(dev_priv))
8636                 pipeconf |= I915_READ(PIPECONF(crtc->pipe)) & PIPECONF_ENABLE;
8637
8638         if (crtc_state->double_wide)
8639                 pipeconf |= PIPECONF_DOUBLE_WIDE;
8640
8641         /* only g4x and later have fancy bpc/dither controls */
8642         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
8643             IS_CHERRYVIEW(dev_priv)) {
8644                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
8645                 if (crtc_state->dither && crtc_state->pipe_bpp != 30)
8646                         pipeconf |= PIPECONF_DITHER_EN |
8647                                     PIPECONF_DITHER_TYPE_SP;
8648
8649                 switch (crtc_state->pipe_bpp) {
8650                 case 18:
8651                         pipeconf |= PIPECONF_6BPC;
8652                         break;
8653                 case 24:
8654                         pipeconf |= PIPECONF_8BPC;
8655                         break;
8656                 case 30:
8657                         pipeconf |= PIPECONF_10BPC;
8658                         break;
8659                 default:
8660                         /* Case prevented by intel_choose_pipe_bpp_dither. */
8661                         BUG();
8662                 }
8663         }
8664
8665         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
8666                 if (INTEL_GEN(dev_priv) < 4 ||
8667                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
8668                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
8669                 else
8670                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
8671         } else {
8672                 pipeconf |= PIPECONF_PROGRESSIVE;
8673         }
8674
8675         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
8676              crtc_state->limited_color_range)
8677                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
8678
8679         pipeconf |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
8680
8681         pipeconf |= PIPECONF_FRAME_START_DELAY(0);
8682
8683         I915_WRITE(PIPECONF(crtc->pipe), pipeconf);
8684         POSTING_READ(PIPECONF(crtc->pipe));
8685 }
8686
8687 static int i8xx_crtc_compute_clock(struct intel_crtc *crtc,
8688                                    struct intel_crtc_state *crtc_state)
8689 {
8690         struct drm_device *dev = crtc->base.dev;
8691         struct drm_i915_private *dev_priv = to_i915(dev);
8692         const struct intel_limit *limit;
8693         int refclk = 48000;
8694
8695         memset(&crtc_state->dpll_hw_state, 0,
8696                sizeof(crtc_state->dpll_hw_state));
8697
8698         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8699                 if (intel_panel_use_ssc(dev_priv)) {
8700                         refclk = dev_priv->vbt.lvds_ssc_freq;
8701                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8702                 }
8703
8704                 limit = &intel_limits_i8xx_lvds;
8705         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO)) {
8706                 limit = &intel_limits_i8xx_dvo;
8707         } else {
8708                 limit = &intel_limits_i8xx_dac;
8709         }
8710
8711         if (!crtc_state->clock_set &&
8712             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8713                                  refclk, NULL, &crtc_state->dpll)) {
8714                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8715                 return -EINVAL;
8716         }
8717
8718         i8xx_compute_dpll(crtc, crtc_state, NULL);
8719
8720         return 0;
8721 }
8722
8723 static int g4x_crtc_compute_clock(struct intel_crtc *crtc,
8724                                   struct intel_crtc_state *crtc_state)
8725 {
8726         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8727         const struct intel_limit *limit;
8728         int refclk = 96000;
8729
8730         memset(&crtc_state->dpll_hw_state, 0,
8731                sizeof(crtc_state->dpll_hw_state));
8732
8733         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8734                 if (intel_panel_use_ssc(dev_priv)) {
8735                         refclk = dev_priv->vbt.lvds_ssc_freq;
8736                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8737                 }
8738
8739                 if (intel_is_dual_link_lvds(dev_priv))
8740                         limit = &intel_limits_g4x_dual_channel_lvds;
8741                 else
8742                         limit = &intel_limits_g4x_single_channel_lvds;
8743         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) ||
8744                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
8745                 limit = &intel_limits_g4x_hdmi;
8746         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) {
8747                 limit = &intel_limits_g4x_sdvo;
8748         } else {
8749                 /* The option is for other outputs */
8750                 limit = &intel_limits_i9xx_sdvo;
8751         }
8752
8753         if (!crtc_state->clock_set &&
8754             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8755                                 refclk, NULL, &crtc_state->dpll)) {
8756                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8757                 return -EINVAL;
8758         }
8759
8760         i9xx_compute_dpll(crtc, crtc_state, NULL);
8761
8762         return 0;
8763 }
8764
8765 static int pnv_crtc_compute_clock(struct intel_crtc *crtc,
8766                                   struct intel_crtc_state *crtc_state)
8767 {
8768         struct drm_device *dev = crtc->base.dev;
8769         struct drm_i915_private *dev_priv = to_i915(dev);
8770         const struct intel_limit *limit;
8771         int refclk = 96000;
8772
8773         memset(&crtc_state->dpll_hw_state, 0,
8774                sizeof(crtc_state->dpll_hw_state));
8775
8776         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8777                 if (intel_panel_use_ssc(dev_priv)) {
8778                         refclk = dev_priv->vbt.lvds_ssc_freq;
8779                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8780                 }
8781
8782                 limit = &intel_limits_pineview_lvds;
8783         } else {
8784                 limit = &intel_limits_pineview_sdvo;
8785         }
8786
8787         if (!crtc_state->clock_set &&
8788             !pnv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8789                                 refclk, NULL, &crtc_state->dpll)) {
8790                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8791                 return -EINVAL;
8792         }
8793
8794         i9xx_compute_dpll(crtc, crtc_state, NULL);
8795
8796         return 0;
8797 }
8798
8799 static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
8800                                    struct intel_crtc_state *crtc_state)
8801 {
8802         struct drm_device *dev = crtc->base.dev;
8803         struct drm_i915_private *dev_priv = to_i915(dev);
8804         const struct intel_limit *limit;
8805         int refclk = 96000;
8806
8807         memset(&crtc_state->dpll_hw_state, 0,
8808                sizeof(crtc_state->dpll_hw_state));
8809
8810         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8811                 if (intel_panel_use_ssc(dev_priv)) {
8812                         refclk = dev_priv->vbt.lvds_ssc_freq;
8813                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8814                 }
8815
8816                 limit = &intel_limits_i9xx_lvds;
8817         } else {
8818                 limit = &intel_limits_i9xx_sdvo;
8819         }
8820
8821         if (!crtc_state->clock_set &&
8822             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8823                                  refclk, NULL, &crtc_state->dpll)) {
8824                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8825                 return -EINVAL;
8826         }
8827
8828         i9xx_compute_dpll(crtc, crtc_state, NULL);
8829
8830         return 0;
8831 }
8832
8833 static int chv_crtc_compute_clock(struct intel_crtc *crtc,
8834                                   struct intel_crtc_state *crtc_state)
8835 {
8836         int refclk = 100000;
8837         const struct intel_limit *limit = &intel_limits_chv;
8838
8839         memset(&crtc_state->dpll_hw_state, 0,
8840                sizeof(crtc_state->dpll_hw_state));
8841
8842         if (!crtc_state->clock_set &&
8843             !chv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8844                                 refclk, NULL, &crtc_state->dpll)) {
8845                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8846                 return -EINVAL;
8847         }
8848
8849         chv_compute_dpll(crtc, crtc_state);
8850
8851         return 0;
8852 }
8853
8854 static int vlv_crtc_compute_clock(struct intel_crtc *crtc,
8855                                   struct intel_crtc_state *crtc_state)
8856 {
8857         int refclk = 100000;
8858         const struct intel_limit *limit = &intel_limits_vlv;
8859
8860         memset(&crtc_state->dpll_hw_state, 0,
8861                sizeof(crtc_state->dpll_hw_state));
8862
8863         if (!crtc_state->clock_set &&
8864             !vlv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8865                                 refclk, NULL, &crtc_state->dpll)) {
8866                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8867                 return -EINVAL;
8868         }
8869
8870         vlv_compute_dpll(crtc, crtc_state);
8871
8872         return 0;
8873 }
8874
8875 static bool i9xx_has_pfit(struct drm_i915_private *dev_priv)
8876 {
8877         if (IS_I830(dev_priv))
8878                 return false;
8879
8880         return INTEL_GEN(dev_priv) >= 4 ||
8881                 IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
8882 }
8883
8884 static void i9xx_get_pfit_config(struct intel_crtc *crtc,
8885                                  struct intel_crtc_state *pipe_config)
8886 {
8887         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8888         u32 tmp;
8889
8890         if (!i9xx_has_pfit(dev_priv))
8891                 return;
8892
8893         tmp = I915_READ(PFIT_CONTROL);
8894         if (!(tmp & PFIT_ENABLE))
8895                 return;
8896
8897         /* Check whether the pfit is attached to our pipe. */
8898         if (INTEL_GEN(dev_priv) < 4) {
8899                 if (crtc->pipe != PIPE_B)
8900                         return;
8901         } else {
8902                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
8903                         return;
8904         }
8905
8906         pipe_config->gmch_pfit.control = tmp;
8907         pipe_config->gmch_pfit.pgm_ratios = I915_READ(PFIT_PGM_RATIOS);
8908 }
8909
8910 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
8911                                struct intel_crtc_state *pipe_config)
8912 {
8913         struct drm_device *dev = crtc->base.dev;
8914         struct drm_i915_private *dev_priv = to_i915(dev);
8915         enum pipe pipe = crtc->pipe;
8916         struct dpll clock;
8917         u32 mdiv;
8918         int refclk = 100000;
8919
8920         /* In case of DSI, DPLL will not be used */
8921         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8922                 return;
8923
8924         vlv_dpio_get(dev_priv);
8925         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
8926         vlv_dpio_put(dev_priv);
8927
8928         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
8929         clock.m2 = mdiv & DPIO_M2DIV_MASK;
8930         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
8931         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
8932         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
8933
8934         pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
8935 }
8936
8937 static void
8938 i9xx_get_initial_plane_config(struct intel_crtc *crtc,
8939                               struct intel_initial_plane_config *plane_config)
8940 {
8941         struct drm_device *dev = crtc->base.dev;
8942         struct drm_i915_private *dev_priv = to_i915(dev);
8943         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
8944         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
8945         enum pipe pipe;
8946         u32 val, base, offset;
8947         int fourcc, pixel_format;
8948         unsigned int aligned_height;
8949         struct drm_framebuffer *fb;
8950         struct intel_framebuffer *intel_fb;
8951
8952         if (!plane->get_hw_state(plane, &pipe))
8953                 return;
8954
8955         WARN_ON(pipe != crtc->pipe);
8956
8957         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
8958         if (!intel_fb) {
8959                 DRM_DEBUG_KMS("failed to alloc fb\n");
8960                 return;
8961         }
8962
8963         fb = &intel_fb->base;
8964
8965         fb->dev = dev;
8966
8967         val = I915_READ(DSPCNTR(i9xx_plane));
8968
8969         if (INTEL_GEN(dev_priv) >= 4) {
8970                 if (val & DISPPLANE_TILED) {
8971                         plane_config->tiling = I915_TILING_X;
8972                         fb->modifier = I915_FORMAT_MOD_X_TILED;
8973                 }
8974
8975                 if (val & DISPPLANE_ROTATE_180)
8976                         plane_config->rotation = DRM_MODE_ROTATE_180;
8977         }
8978
8979         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B &&
8980             val & DISPPLANE_MIRROR)
8981                 plane_config->rotation |= DRM_MODE_REFLECT_X;
8982
8983         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
8984         fourcc = i9xx_format_to_fourcc(pixel_format);
8985         fb->format = drm_format_info(fourcc);
8986
8987         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
8988                 offset = I915_READ(DSPOFFSET(i9xx_plane));
8989                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
8990         } else if (INTEL_GEN(dev_priv) >= 4) {
8991                 if (plane_config->tiling)
8992                         offset = I915_READ(DSPTILEOFF(i9xx_plane));
8993                 else
8994                         offset = I915_READ(DSPLINOFF(i9xx_plane));
8995                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
8996         } else {
8997                 base = I915_READ(DSPADDR(i9xx_plane));
8998         }
8999         plane_config->base = base;
9000
9001         val = I915_READ(PIPESRC(pipe));
9002         fb->width = ((val >> 16) & 0xfff) + 1;
9003         fb->height = ((val >> 0) & 0xfff) + 1;
9004
9005         val = I915_READ(DSPSTRIDE(i9xx_plane));
9006         fb->pitches[0] = val & 0xffffffc0;
9007
9008         aligned_height = intel_fb_align_height(fb, 0, fb->height);
9009
9010         plane_config->size = fb->pitches[0] * aligned_height;
9011
9012         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
9013                       crtc->base.name, plane->base.name, fb->width, fb->height,
9014                       fb->format->cpp[0] * 8, base, fb->pitches[0],
9015                       plane_config->size);
9016
9017         plane_config->fb = intel_fb;
9018 }
9019
9020 static void chv_crtc_clock_get(struct intel_crtc *crtc,
9021                                struct intel_crtc_state *pipe_config)
9022 {
9023         struct drm_device *dev = crtc->base.dev;
9024         struct drm_i915_private *dev_priv = to_i915(dev);
9025         enum pipe pipe = crtc->pipe;
9026         enum dpio_channel port = vlv_pipe_to_channel(pipe);
9027         struct dpll clock;
9028         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
9029         int refclk = 100000;
9030
9031         /* In case of DSI, DPLL will not be used */
9032         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
9033                 return;
9034
9035         vlv_dpio_get(dev_priv);
9036         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
9037         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
9038         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
9039         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
9040         pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
9041         vlv_dpio_put(dev_priv);
9042
9043         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
9044         clock.m2 = (pll_dw0 & 0xff) << 22;
9045         if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
9046                 clock.m2 |= pll_dw2 & 0x3fffff;
9047         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
9048         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
9049         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
9050
9051         pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
9052 }
9053
9054 static enum intel_output_format
9055 bdw_get_pipemisc_output_format(struct intel_crtc *crtc)
9056 {
9057         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9058         u32 tmp;
9059
9060         tmp = I915_READ(PIPEMISC(crtc->pipe));
9061
9062         if (tmp & PIPEMISC_YUV420_ENABLE) {
9063                 /* We support 4:2:0 in full blend mode only */
9064                 WARN_ON((tmp & PIPEMISC_YUV420_MODE_FULL_BLEND) == 0);
9065
9066                 return INTEL_OUTPUT_FORMAT_YCBCR420;
9067         } else if (tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV) {
9068                 return INTEL_OUTPUT_FORMAT_YCBCR444;
9069         } else {
9070                 return INTEL_OUTPUT_FORMAT_RGB;
9071         }
9072 }
9073
9074 static void i9xx_get_pipe_color_config(struct intel_crtc_state *crtc_state)
9075 {
9076         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9077         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
9078         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9079         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
9080         u32 tmp;
9081
9082         tmp = I915_READ(DSPCNTR(i9xx_plane));
9083
9084         if (tmp & DISPPLANE_GAMMA_ENABLE)
9085                 crtc_state->gamma_enable = true;
9086
9087         if (!HAS_GMCH(dev_priv) &&
9088             tmp & DISPPLANE_PIPE_CSC_ENABLE)
9089                 crtc_state->csc_enable = true;
9090 }
9091
9092 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
9093                                  struct intel_crtc_state *pipe_config)
9094 {
9095         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9096         enum intel_display_power_domain power_domain;
9097         intel_wakeref_t wakeref;
9098         u32 tmp;
9099         bool ret;
9100
9101         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
9102         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
9103         if (!wakeref)
9104                 return false;
9105
9106         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
9107         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9108         pipe_config->shared_dpll = NULL;
9109         pipe_config->master_transcoder = INVALID_TRANSCODER;
9110
9111         ret = false;
9112
9113         tmp = I915_READ(PIPECONF(crtc->pipe));
9114         if (!(tmp & PIPECONF_ENABLE))
9115                 goto out;
9116
9117         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
9118             IS_CHERRYVIEW(dev_priv)) {
9119                 switch (tmp & PIPECONF_BPC_MASK) {
9120                 case PIPECONF_6BPC:
9121                         pipe_config->pipe_bpp = 18;
9122                         break;
9123                 case PIPECONF_8BPC:
9124                         pipe_config->pipe_bpp = 24;
9125                         break;
9126                 case PIPECONF_10BPC:
9127                         pipe_config->pipe_bpp = 30;
9128                         break;
9129                 default:
9130                         break;
9131                 }
9132         }
9133
9134         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
9135             (tmp & PIPECONF_COLOR_RANGE_SELECT))
9136                 pipe_config->limited_color_range = true;
9137
9138         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_I9XX) >>
9139                 PIPECONF_GAMMA_MODE_SHIFT;
9140
9141         if (IS_CHERRYVIEW(dev_priv))
9142                 pipe_config->cgm_mode = I915_READ(CGM_PIPE_MODE(crtc->pipe));
9143
9144         i9xx_get_pipe_color_config(pipe_config);
9145         intel_color_get_config(pipe_config);
9146
9147         if (INTEL_GEN(dev_priv) < 4)
9148                 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
9149
9150         intel_get_pipe_timings(crtc, pipe_config);
9151         intel_get_pipe_src_size(crtc, pipe_config);
9152
9153         i9xx_get_pfit_config(crtc, pipe_config);
9154
9155         if (INTEL_GEN(dev_priv) >= 4) {
9156                 /* No way to read it out on pipes B and C */
9157                 if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
9158                         tmp = dev_priv->chv_dpll_md[crtc->pipe];
9159                 else
9160                         tmp = I915_READ(DPLL_MD(crtc->pipe));
9161                 pipe_config->pixel_multiplier =
9162                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
9163                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
9164                 pipe_config->dpll_hw_state.dpll_md = tmp;
9165         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
9166                    IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
9167                 tmp = I915_READ(DPLL(crtc->pipe));
9168                 pipe_config->pixel_multiplier =
9169                         ((tmp & SDVO_MULTIPLIER_MASK)
9170                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
9171         } else {
9172                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
9173                  * port and will be fixed up in the encoder->get_config
9174                  * function. */
9175                 pipe_config->pixel_multiplier = 1;
9176         }
9177         pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
9178         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
9179                 pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
9180                 pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe));
9181         } else {
9182                 /* Mask out read-only status bits. */
9183                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
9184                                                      DPLL_PORTC_READY_MASK |
9185                                                      DPLL_PORTB_READY_MASK);
9186         }
9187
9188         if (IS_CHERRYVIEW(dev_priv))
9189                 chv_crtc_clock_get(crtc, pipe_config);
9190         else if (IS_VALLEYVIEW(dev_priv))
9191                 vlv_crtc_clock_get(crtc, pipe_config);
9192         else
9193                 i9xx_crtc_clock_get(crtc, pipe_config);
9194
9195         /*
9196          * Normally the dotclock is filled in by the encoder .get_config()
9197          * but in case the pipe is enabled w/o any ports we need a sane
9198          * default.
9199          */
9200         pipe_config->hw.adjusted_mode.crtc_clock =
9201                 pipe_config->port_clock / pipe_config->pixel_multiplier;
9202
9203         ret = true;
9204
9205 out:
9206         intel_display_power_put(dev_priv, power_domain, wakeref);
9207
9208         return ret;
9209 }
9210
9211 static void ironlake_init_pch_refclk(struct drm_i915_private *dev_priv)
9212 {
9213         struct intel_encoder *encoder;
9214         int i;
9215         u32 val, final;
9216         bool has_lvds = false;
9217         bool has_cpu_edp = false;
9218         bool has_panel = false;
9219         bool has_ck505 = false;
9220         bool can_ssc = false;
9221         bool using_ssc_source = false;
9222
9223         /* We need to take the global config into account */
9224         for_each_intel_encoder(&dev_priv->drm, encoder) {
9225                 switch (encoder->type) {
9226                 case INTEL_OUTPUT_LVDS:
9227                         has_panel = true;
9228                         has_lvds = true;
9229                         break;
9230                 case INTEL_OUTPUT_EDP:
9231                         has_panel = true;
9232                         if (encoder->port == PORT_A)
9233                                 has_cpu_edp = true;
9234                         break;
9235                 default:
9236                         break;
9237                 }
9238         }
9239
9240         if (HAS_PCH_IBX(dev_priv)) {
9241                 has_ck505 = dev_priv->vbt.display_clock_mode;
9242                 can_ssc = has_ck505;
9243         } else {
9244                 has_ck505 = false;
9245                 can_ssc = true;
9246         }
9247
9248         /* Check if any DPLLs are using the SSC source */
9249         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
9250                 u32 temp = I915_READ(PCH_DPLL(i));
9251
9252                 if (!(temp & DPLL_VCO_ENABLE))
9253                         continue;
9254
9255                 if ((temp & PLL_REF_INPUT_MASK) ==
9256                     PLLB_REF_INPUT_SPREADSPECTRUMIN) {
9257                         using_ssc_source = true;
9258                         break;
9259                 }
9260         }
9261
9262         DRM_DEBUG_KMS("has_panel %d has_lvds %d has_ck505 %d using_ssc_source %d\n",
9263                       has_panel, has_lvds, has_ck505, using_ssc_source);
9264
9265         /* Ironlake: try to setup display ref clock before DPLL
9266          * enabling. This is only under driver's control after
9267          * PCH B stepping, previous chipset stepping should be
9268          * ignoring this setting.
9269          */
9270         val = I915_READ(PCH_DREF_CONTROL);
9271
9272         /* As we must carefully and slowly disable/enable each source in turn,
9273          * compute the final state we want first and check if we need to
9274          * make any changes at all.
9275          */
9276         final = val;
9277         final &= ~DREF_NONSPREAD_SOURCE_MASK;
9278         if (has_ck505)
9279                 final |= DREF_NONSPREAD_CK505_ENABLE;
9280         else
9281                 final |= DREF_NONSPREAD_SOURCE_ENABLE;
9282
9283         final &= ~DREF_SSC_SOURCE_MASK;
9284         final &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9285         final &= ~DREF_SSC1_ENABLE;
9286
9287         if (has_panel) {
9288                 final |= DREF_SSC_SOURCE_ENABLE;
9289
9290                 if (intel_panel_use_ssc(dev_priv) && can_ssc)
9291                         final |= DREF_SSC1_ENABLE;
9292
9293                 if (has_cpu_edp) {
9294                         if (intel_panel_use_ssc(dev_priv) && can_ssc)
9295                                 final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
9296                         else
9297                                 final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
9298                 } else
9299                         final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9300         } else if (using_ssc_source) {
9301                 final |= DREF_SSC_SOURCE_ENABLE;
9302                 final |= DREF_SSC1_ENABLE;
9303         }
9304
9305         if (final == val)
9306                 return;
9307
9308         /* Always enable nonspread source */
9309         val &= ~DREF_NONSPREAD_SOURCE_MASK;
9310
9311         if (has_ck505)
9312                 val |= DREF_NONSPREAD_CK505_ENABLE;
9313         else
9314                 val |= DREF_NONSPREAD_SOURCE_ENABLE;
9315
9316         if (has_panel) {
9317                 val &= ~DREF_SSC_SOURCE_MASK;
9318                 val |= DREF_SSC_SOURCE_ENABLE;
9319
9320                 /* SSC must be turned on before enabling the CPU output  */
9321                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
9322                         DRM_DEBUG_KMS("Using SSC on panel\n");
9323                         val |= DREF_SSC1_ENABLE;
9324                 } else
9325                         val &= ~DREF_SSC1_ENABLE;
9326
9327                 /* Get SSC going before enabling the outputs */
9328                 I915_WRITE(PCH_DREF_CONTROL, val);
9329                 POSTING_READ(PCH_DREF_CONTROL);
9330                 udelay(200);
9331
9332                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9333
9334                 /* Enable CPU source on CPU attached eDP */
9335                 if (has_cpu_edp) {
9336                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
9337                                 DRM_DEBUG_KMS("Using SSC on eDP\n");
9338                                 val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
9339                         } else
9340                                 val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
9341                 } else
9342                         val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9343
9344                 I915_WRITE(PCH_DREF_CONTROL, val);
9345                 POSTING_READ(PCH_DREF_CONTROL);
9346                 udelay(200);
9347         } else {
9348                 DRM_DEBUG_KMS("Disabling CPU source output\n");
9349
9350                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9351
9352                 /* Turn off CPU output */
9353                 val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9354
9355                 I915_WRITE(PCH_DREF_CONTROL, val);
9356                 POSTING_READ(PCH_DREF_CONTROL);
9357                 udelay(200);
9358
9359                 if (!using_ssc_source) {
9360                         DRM_DEBUG_KMS("Disabling SSC source\n");
9361
9362                         /* Turn off the SSC source */
9363                         val &= ~DREF_SSC_SOURCE_MASK;
9364                         val |= DREF_SSC_SOURCE_DISABLE;
9365
9366                         /* Turn off SSC1 */
9367                         val &= ~DREF_SSC1_ENABLE;
9368
9369                         I915_WRITE(PCH_DREF_CONTROL, val);
9370                         POSTING_READ(PCH_DREF_CONTROL);
9371                         udelay(200);
9372                 }
9373         }
9374
9375         BUG_ON(val != final);
9376 }
9377
9378 static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv)
9379 {
9380         u32 tmp;
9381
9382         tmp = I915_READ(SOUTH_CHICKEN2);
9383         tmp |= FDI_MPHY_IOSFSB_RESET_CTL;
9384         I915_WRITE(SOUTH_CHICKEN2, tmp);
9385
9386         if (wait_for_us(I915_READ(SOUTH_CHICKEN2) &
9387                         FDI_MPHY_IOSFSB_RESET_STATUS, 100))
9388                 DRM_ERROR("FDI mPHY reset assert timeout\n");
9389
9390         tmp = I915_READ(SOUTH_CHICKEN2);
9391         tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL;
9392         I915_WRITE(SOUTH_CHICKEN2, tmp);
9393
9394         if (wait_for_us((I915_READ(SOUTH_CHICKEN2) &
9395                          FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100))
9396                 DRM_ERROR("FDI mPHY reset de-assert timeout\n");
9397 }
9398
9399 /* WaMPhyProgramming:hsw */
9400 static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv)
9401 {
9402         u32 tmp;
9403
9404         tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY);
9405         tmp &= ~(0xFF << 24);
9406         tmp |= (0x12 << 24);
9407         intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY);
9408
9409         tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY);
9410         tmp |= (1 << 11);
9411         intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY);
9412
9413         tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY);
9414         tmp |= (1 << 11);
9415         intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY);
9416
9417         tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY);
9418         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
9419         intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY);
9420
9421         tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY);
9422         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
9423         intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY);
9424
9425         tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY);
9426         tmp &= ~(7 << 13);
9427         tmp |= (5 << 13);
9428         intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY);
9429
9430         tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY);
9431         tmp &= ~(7 << 13);
9432         tmp |= (5 << 13);
9433         intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY);
9434
9435         tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY);
9436         tmp &= ~0xFF;
9437         tmp |= 0x1C;
9438         intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY);
9439
9440         tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY);
9441         tmp &= ~0xFF;
9442         tmp |= 0x1C;
9443         intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY);
9444
9445         tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY);
9446         tmp &= ~(0xFF << 16);
9447         tmp |= (0x1C << 16);
9448         intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY);
9449
9450         tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY);
9451         tmp &= ~(0xFF << 16);
9452         tmp |= (0x1C << 16);
9453         intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY);
9454
9455         tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY);
9456         tmp |= (1 << 27);
9457         intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY);
9458
9459         tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY);
9460         tmp |= (1 << 27);
9461         intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY);
9462
9463         tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY);
9464         tmp &= ~(0xF << 28);
9465         tmp |= (4 << 28);
9466         intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY);
9467
9468         tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY);
9469         tmp &= ~(0xF << 28);
9470         tmp |= (4 << 28);
9471         intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY);
9472 }
9473
9474 /* Implements 3 different sequences from BSpec chapter "Display iCLK
9475  * Programming" based on the parameters passed:
9476  * - Sequence to enable CLKOUT_DP
9477  * - Sequence to enable CLKOUT_DP without spread
9478  * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O
9479  */
9480 static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv,
9481                                  bool with_spread, bool with_fdi)
9482 {
9483         u32 reg, tmp;
9484
9485         if (WARN(with_fdi && !with_spread, "FDI requires downspread\n"))
9486                 with_spread = true;
9487         if (WARN(HAS_PCH_LPT_LP(dev_priv) &&
9488             with_fdi, "LP PCH doesn't have FDI\n"))
9489                 with_fdi = false;
9490
9491         mutex_lock(&dev_priv->sb_lock);
9492
9493         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9494         tmp &= ~SBI_SSCCTL_DISABLE;
9495         tmp |= SBI_SSCCTL_PATHALT;
9496         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9497
9498         udelay(24);
9499
9500         if (with_spread) {
9501                 tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9502                 tmp &= ~SBI_SSCCTL_PATHALT;
9503                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9504
9505                 if (with_fdi) {
9506                         lpt_reset_fdi_mphy(dev_priv);
9507                         lpt_program_fdi_mphy(dev_priv);
9508                 }
9509         }
9510
9511         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
9512         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
9513         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
9514         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
9515
9516         mutex_unlock(&dev_priv->sb_lock);
9517 }
9518
9519 /* Sequence to disable CLKOUT_DP */
9520 void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv)
9521 {
9522         u32 reg, tmp;
9523
9524         mutex_lock(&dev_priv->sb_lock);
9525
9526         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
9527         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
9528         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
9529         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
9530
9531         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9532         if (!(tmp & SBI_SSCCTL_DISABLE)) {
9533                 if (!(tmp & SBI_SSCCTL_PATHALT)) {
9534                         tmp |= SBI_SSCCTL_PATHALT;
9535                         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9536                         udelay(32);
9537                 }
9538                 tmp |= SBI_SSCCTL_DISABLE;
9539                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9540         }
9541
9542         mutex_unlock(&dev_priv->sb_lock);
9543 }
9544
9545 #define BEND_IDX(steps) ((50 + (steps)) / 5)
9546
9547 static const u16 sscdivintphase[] = {
9548         [BEND_IDX( 50)] = 0x3B23,
9549         [BEND_IDX( 45)] = 0x3B23,
9550         [BEND_IDX( 40)] = 0x3C23,
9551         [BEND_IDX( 35)] = 0x3C23,
9552         [BEND_IDX( 30)] = 0x3D23,
9553         [BEND_IDX( 25)] = 0x3D23,
9554         [BEND_IDX( 20)] = 0x3E23,
9555         [BEND_IDX( 15)] = 0x3E23,
9556         [BEND_IDX( 10)] = 0x3F23,
9557         [BEND_IDX(  5)] = 0x3F23,
9558         [BEND_IDX(  0)] = 0x0025,
9559         [BEND_IDX( -5)] = 0x0025,
9560         [BEND_IDX(-10)] = 0x0125,
9561         [BEND_IDX(-15)] = 0x0125,
9562         [BEND_IDX(-20)] = 0x0225,
9563         [BEND_IDX(-25)] = 0x0225,
9564         [BEND_IDX(-30)] = 0x0325,
9565         [BEND_IDX(-35)] = 0x0325,
9566         [BEND_IDX(-40)] = 0x0425,
9567         [BEND_IDX(-45)] = 0x0425,
9568         [BEND_IDX(-50)] = 0x0525,
9569 };
9570
9571 /*
9572  * Bend CLKOUT_DP
9573  * steps -50 to 50 inclusive, in steps of 5
9574  * < 0 slow down the clock, > 0 speed up the clock, 0 == no bend (135MHz)
9575  * change in clock period = -(steps / 10) * 5.787 ps
9576  */
9577 static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps)
9578 {
9579         u32 tmp;
9580         int idx = BEND_IDX(steps);
9581
9582         if (WARN_ON(steps % 5 != 0))
9583                 return;
9584
9585         if (WARN_ON(idx >= ARRAY_SIZE(sscdivintphase)))
9586                 return;
9587
9588         mutex_lock(&dev_priv->sb_lock);
9589
9590         if (steps % 10 != 0)
9591                 tmp = 0xAAAAAAAB;
9592         else
9593                 tmp = 0x00000000;
9594         intel_sbi_write(dev_priv, SBI_SSCDITHPHASE, tmp, SBI_ICLK);
9595
9596         tmp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE, SBI_ICLK);
9597         tmp &= 0xffff0000;
9598         tmp |= sscdivintphase[idx];
9599         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE, tmp, SBI_ICLK);
9600
9601         mutex_unlock(&dev_priv->sb_lock);
9602 }
9603
9604 #undef BEND_IDX
9605
9606 static bool spll_uses_pch_ssc(struct drm_i915_private *dev_priv)
9607 {
9608         u32 fuse_strap = I915_READ(FUSE_STRAP);
9609         u32 ctl = I915_READ(SPLL_CTL);
9610
9611         if ((ctl & SPLL_PLL_ENABLE) == 0)
9612                 return false;
9613
9614         if ((ctl & SPLL_REF_MASK) == SPLL_REF_MUXED_SSC &&
9615             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
9616                 return true;
9617
9618         if (IS_BROADWELL(dev_priv) &&
9619             (ctl & SPLL_REF_MASK) == SPLL_REF_PCH_SSC_BDW)
9620                 return true;
9621
9622         return false;
9623 }
9624
9625 static bool wrpll_uses_pch_ssc(struct drm_i915_private *dev_priv,
9626                                enum intel_dpll_id id)
9627 {
9628         u32 fuse_strap = I915_READ(FUSE_STRAP);
9629         u32 ctl = I915_READ(WRPLL_CTL(id));
9630
9631         if ((ctl & WRPLL_PLL_ENABLE) == 0)
9632                 return false;
9633
9634         if ((ctl & WRPLL_REF_MASK) == WRPLL_REF_PCH_SSC)
9635                 return true;
9636
9637         if ((IS_BROADWELL(dev_priv) || IS_HSW_ULT(dev_priv)) &&
9638             (ctl & WRPLL_REF_MASK) == WRPLL_REF_MUXED_SSC_BDW &&
9639             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
9640                 return true;
9641
9642         return false;
9643 }
9644
9645 static void lpt_init_pch_refclk(struct drm_i915_private *dev_priv)
9646 {
9647         struct intel_encoder *encoder;
9648         bool has_fdi = false;
9649
9650         for_each_intel_encoder(&dev_priv->drm, encoder) {
9651                 switch (encoder->type) {
9652                 case INTEL_OUTPUT_ANALOG:
9653                         has_fdi = true;
9654                         break;
9655                 default:
9656                         break;
9657                 }
9658         }
9659
9660         /*
9661          * The BIOS may have decided to use the PCH SSC
9662          * reference so we must not disable it until the
9663          * relevant PLLs have stopped relying on it. We'll
9664          * just leave the PCH SSC reference enabled in case
9665          * any active PLL is using it. It will get disabled
9666          * after runtime suspend if we don't have FDI.
9667          *
9668          * TODO: Move the whole reference clock handling
9669          * to the modeset sequence proper so that we can
9670          * actually enable/disable/reconfigure these things
9671          * safely. To do that we need to introduce a real
9672          * clock hierarchy. That would also allow us to do
9673          * clock bending finally.
9674          */
9675         dev_priv->pch_ssc_use = 0;
9676
9677         if (spll_uses_pch_ssc(dev_priv)) {
9678                 DRM_DEBUG_KMS("SPLL using PCH SSC\n");
9679                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_SPLL);
9680         }
9681
9682         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL1)) {
9683                 DRM_DEBUG_KMS("WRPLL1 using PCH SSC\n");
9684                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_WRPLL1);
9685         }
9686
9687         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL2)) {
9688                 DRM_DEBUG_KMS("WRPLL2 using PCH SSC\n");
9689                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_WRPLL2);
9690         }
9691
9692         if (dev_priv->pch_ssc_use)
9693                 return;
9694
9695         if (has_fdi) {
9696                 lpt_bend_clkout_dp(dev_priv, 0);
9697                 lpt_enable_clkout_dp(dev_priv, true, true);
9698         } else {
9699                 lpt_disable_clkout_dp(dev_priv);
9700         }
9701 }
9702
9703 /*
9704  * Initialize reference clocks when the driver loads
9705  */
9706 void intel_init_pch_refclk(struct drm_i915_private *dev_priv)
9707 {
9708         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
9709                 ironlake_init_pch_refclk(dev_priv);
9710         else if (HAS_PCH_LPT(dev_priv))
9711                 lpt_init_pch_refclk(dev_priv);
9712 }
9713
9714 static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state)
9715 {
9716         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9717         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9718         enum pipe pipe = crtc->pipe;
9719         u32 val;
9720
9721         val = 0;
9722
9723         switch (crtc_state->pipe_bpp) {
9724         case 18:
9725                 val |= PIPECONF_6BPC;
9726                 break;
9727         case 24:
9728                 val |= PIPECONF_8BPC;
9729                 break;
9730         case 30:
9731                 val |= PIPECONF_10BPC;
9732                 break;
9733         case 36:
9734                 val |= PIPECONF_12BPC;
9735                 break;
9736         default:
9737                 /* Case prevented by intel_choose_pipe_bpp_dither. */
9738                 BUG();
9739         }
9740
9741         if (crtc_state->dither)
9742                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
9743
9744         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
9745                 val |= PIPECONF_INTERLACED_ILK;
9746         else
9747                 val |= PIPECONF_PROGRESSIVE;
9748
9749         /*
9750          * This would end up with an odd purple hue over
9751          * the entire display. Make sure we don't do it.
9752          */
9753         WARN_ON(crtc_state->limited_color_range &&
9754                 crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB);
9755
9756         if (crtc_state->limited_color_range)
9757                 val |= PIPECONF_COLOR_RANGE_SELECT;
9758
9759         if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
9760                 val |= PIPECONF_OUTPUT_COLORSPACE_YUV709;
9761
9762         val |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
9763
9764         val |= PIPECONF_FRAME_START_DELAY(0);
9765
9766         I915_WRITE(PIPECONF(pipe), val);
9767         POSTING_READ(PIPECONF(pipe));
9768 }
9769
9770 static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state)
9771 {
9772         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9773         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9774         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
9775         u32 val = 0;
9776
9777         if (IS_HASWELL(dev_priv) && crtc_state->dither)
9778                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
9779
9780         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
9781                 val |= PIPECONF_INTERLACED_ILK;
9782         else
9783                 val |= PIPECONF_PROGRESSIVE;
9784
9785         if (IS_HASWELL(dev_priv) &&
9786             crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
9787                 val |= PIPECONF_OUTPUT_COLORSPACE_YUV_HSW;
9788
9789         I915_WRITE(PIPECONF(cpu_transcoder), val);
9790         POSTING_READ(PIPECONF(cpu_transcoder));
9791 }
9792
9793 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state)
9794 {
9795         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9796         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9797         u32 val = 0;
9798
9799         switch (crtc_state->pipe_bpp) {
9800         case 18:
9801                 val |= PIPEMISC_DITHER_6_BPC;
9802                 break;
9803         case 24:
9804                 val |= PIPEMISC_DITHER_8_BPC;
9805                 break;
9806         case 30:
9807                 val |= PIPEMISC_DITHER_10_BPC;
9808                 break;
9809         case 36:
9810                 val |= PIPEMISC_DITHER_12_BPC;
9811                 break;
9812         default:
9813                 MISSING_CASE(crtc_state->pipe_bpp);
9814                 break;
9815         }
9816
9817         if (crtc_state->dither)
9818                 val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
9819
9820         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
9821             crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444)
9822                 val |= PIPEMISC_OUTPUT_COLORSPACE_YUV;
9823
9824         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
9825                 val |= PIPEMISC_YUV420_ENABLE |
9826                         PIPEMISC_YUV420_MODE_FULL_BLEND;
9827
9828         if (INTEL_GEN(dev_priv) >= 11 &&
9829             (crtc_state->active_planes & ~(icl_hdr_plane_mask() |
9830                                            BIT(PLANE_CURSOR))) == 0)
9831                 val |= PIPEMISC_HDR_MODE_PRECISION;
9832
9833         I915_WRITE(PIPEMISC(crtc->pipe), val);
9834 }
9835
9836 int bdw_get_pipemisc_bpp(struct intel_crtc *crtc)
9837 {
9838         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9839         u32 tmp;
9840
9841         tmp = I915_READ(PIPEMISC(crtc->pipe));
9842
9843         switch (tmp & PIPEMISC_DITHER_BPC_MASK) {
9844         case PIPEMISC_DITHER_6_BPC:
9845                 return 18;
9846         case PIPEMISC_DITHER_8_BPC:
9847                 return 24;
9848         case PIPEMISC_DITHER_10_BPC:
9849                 return 30;
9850         case PIPEMISC_DITHER_12_BPC:
9851                 return 36;
9852         default:
9853                 MISSING_CASE(tmp);
9854                 return 0;
9855         }
9856 }
9857
9858 int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp)
9859 {
9860         /*
9861          * Account for spread spectrum to avoid
9862          * oversubscribing the link. Max center spread
9863          * is 2.5%; use 5% for safety's sake.
9864          */
9865         u32 bps = target_clock * bpp * 21 / 20;
9866         return DIV_ROUND_UP(bps, link_bw * 8);
9867 }
9868
9869 static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor)
9870 {
9871         return i9xx_dpll_compute_m(dpll) < factor * dpll->n;
9872 }
9873
9874 static void ironlake_compute_dpll(struct intel_crtc *crtc,
9875                                   struct intel_crtc_state *crtc_state,
9876                                   struct dpll *reduced_clock)
9877 {
9878         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9879         u32 dpll, fp, fp2;
9880         int factor;
9881
9882         /* Enable autotuning of the PLL clock (if permissible) */
9883         factor = 21;
9884         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
9885                 if ((intel_panel_use_ssc(dev_priv) &&
9886                      dev_priv->vbt.lvds_ssc_freq == 100000) ||
9887                     (HAS_PCH_IBX(dev_priv) &&
9888                      intel_is_dual_link_lvds(dev_priv)))
9889                         factor = 25;
9890         } else if (crtc_state->sdvo_tv_clock) {
9891                 factor = 20;
9892         }
9893
9894         fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
9895
9896         if (ironlake_needs_fb_cb_tune(&crtc_state->dpll, factor))
9897                 fp |= FP_CB_TUNE;
9898
9899         if (reduced_clock) {
9900                 fp2 = i9xx_dpll_compute_fp(reduced_clock);
9901
9902                 if (reduced_clock->m < factor * reduced_clock->n)
9903                         fp2 |= FP_CB_TUNE;
9904         } else {
9905                 fp2 = fp;
9906         }
9907
9908         dpll = 0;
9909
9910         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
9911                 dpll |= DPLLB_MODE_LVDS;
9912         else
9913                 dpll |= DPLLB_MODE_DAC_SERIAL;
9914
9915         dpll |= (crtc_state->pixel_multiplier - 1)
9916                 << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
9917
9918         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
9919             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
9920                 dpll |= DPLL_SDVO_HIGH_SPEED;
9921
9922         if (intel_crtc_has_dp_encoder(crtc_state))
9923                 dpll |= DPLL_SDVO_HIGH_SPEED;
9924
9925         /*
9926          * The high speed IO clock is only really required for
9927          * SDVO/HDMI/DP, but we also enable it for CRT to make it
9928          * possible to share the DPLL between CRT and HDMI. Enabling
9929          * the clock needlessly does no real harm, except use up a
9930          * bit of power potentially.
9931          *
9932          * We'll limit this to IVB with 3 pipes, since it has only two
9933          * DPLLs and so DPLL sharing is the only way to get three pipes
9934          * driving PCH ports at the same time. On SNB we could do this,
9935          * and potentially avoid enabling the second DPLL, but it's not
9936          * clear if it''s a win or loss power wise. No point in doing
9937          * this on ILK at all since it has a fixed DPLL<->pipe mapping.
9938          */
9939         if (INTEL_NUM_PIPES(dev_priv) == 3 &&
9940             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG))
9941                 dpll |= DPLL_SDVO_HIGH_SPEED;
9942
9943         /* compute bitmask from p1 value */
9944         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
9945         /* also FPA1 */
9946         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
9947
9948         switch (crtc_state->dpll.p2) {
9949         case 5:
9950                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
9951                 break;
9952         case 7:
9953                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
9954                 break;
9955         case 10:
9956                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
9957                 break;
9958         case 14:
9959                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
9960                 break;
9961         }
9962
9963         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
9964             intel_panel_use_ssc(dev_priv))
9965                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
9966         else
9967                 dpll |= PLL_REF_INPUT_DREFCLK;
9968
9969         dpll |= DPLL_VCO_ENABLE;
9970
9971         crtc_state->dpll_hw_state.dpll = dpll;
9972         crtc_state->dpll_hw_state.fp0 = fp;
9973         crtc_state->dpll_hw_state.fp1 = fp2;
9974 }
9975
9976 static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
9977                                        struct intel_crtc_state *crtc_state)
9978 {
9979         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9980         struct intel_atomic_state *state =
9981                 to_intel_atomic_state(crtc_state->uapi.state);
9982         const struct intel_limit *limit;
9983         int refclk = 120000;
9984
9985         memset(&crtc_state->dpll_hw_state, 0,
9986                sizeof(crtc_state->dpll_hw_state));
9987
9988         /* CPU eDP is the only output that doesn't need a PCH PLL of its own. */
9989         if (!crtc_state->has_pch_encoder)
9990                 return 0;
9991
9992         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
9993                 if (intel_panel_use_ssc(dev_priv)) {
9994                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n",
9995                                       dev_priv->vbt.lvds_ssc_freq);
9996                         refclk = dev_priv->vbt.lvds_ssc_freq;
9997                 }
9998
9999                 if (intel_is_dual_link_lvds(dev_priv)) {
10000                         if (refclk == 100000)
10001                                 limit = &intel_limits_ironlake_dual_lvds_100m;
10002                         else
10003                                 limit = &intel_limits_ironlake_dual_lvds;
10004                 } else {
10005                         if (refclk == 100000)
10006                                 limit = &intel_limits_ironlake_single_lvds_100m;
10007                         else
10008                                 limit = &intel_limits_ironlake_single_lvds;
10009                 }
10010         } else {
10011                 limit = &intel_limits_ironlake_dac;
10012         }
10013
10014         if (!crtc_state->clock_set &&
10015             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
10016                                 refclk, NULL, &crtc_state->dpll)) {
10017                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
10018                 return -EINVAL;
10019         }
10020
10021         ironlake_compute_dpll(crtc, crtc_state, NULL);
10022
10023         if (!intel_reserve_shared_dplls(state, crtc, NULL)) {
10024                 DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
10025                               pipe_name(crtc->pipe));
10026                 return -EINVAL;
10027         }
10028
10029         return 0;
10030 }
10031
10032 static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
10033                                          struct intel_link_m_n *m_n)
10034 {
10035         struct drm_device *dev = crtc->base.dev;
10036         struct drm_i915_private *dev_priv = to_i915(dev);
10037         enum pipe pipe = crtc->pipe;
10038
10039         m_n->link_m = I915_READ(PCH_TRANS_LINK_M1(pipe));
10040         m_n->link_n = I915_READ(PCH_TRANS_LINK_N1(pipe));
10041         m_n->gmch_m = I915_READ(PCH_TRANS_DATA_M1(pipe))
10042                 & ~TU_SIZE_MASK;
10043         m_n->gmch_n = I915_READ(PCH_TRANS_DATA_N1(pipe));
10044         m_n->tu = ((I915_READ(PCH_TRANS_DATA_M1(pipe))
10045                     & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
10046 }
10047
10048 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
10049                                          enum transcoder transcoder,
10050                                          struct intel_link_m_n *m_n,
10051                                          struct intel_link_m_n *m2_n2)
10052 {
10053         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10054         enum pipe pipe = crtc->pipe;
10055
10056         if (INTEL_GEN(dev_priv) >= 5) {
10057                 m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder));
10058                 m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder));
10059                 m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder))
10060                         & ~TU_SIZE_MASK;
10061                 m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder));
10062                 m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder))
10063                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
10064
10065                 if (m2_n2 && transcoder_has_m2_n2(dev_priv, transcoder)) {
10066                         m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder));
10067                         m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder));
10068                         m2_n2->gmch_m = I915_READ(PIPE_DATA_M2(transcoder))
10069                                         & ~TU_SIZE_MASK;
10070                         m2_n2->gmch_n = I915_READ(PIPE_DATA_N2(transcoder));
10071                         m2_n2->tu = ((I915_READ(PIPE_DATA_M2(transcoder))
10072                                         & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
10073                 }
10074         } else {
10075                 m_n->link_m = I915_READ(PIPE_LINK_M_G4X(pipe));
10076                 m_n->link_n = I915_READ(PIPE_LINK_N_G4X(pipe));
10077                 m_n->gmch_m = I915_READ(PIPE_DATA_M_G4X(pipe))
10078                         & ~TU_SIZE_MASK;
10079                 m_n->gmch_n = I915_READ(PIPE_DATA_N_G4X(pipe));
10080                 m_n->tu = ((I915_READ(PIPE_DATA_M_G4X(pipe))
10081                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
10082         }
10083 }
10084
10085 void intel_dp_get_m_n(struct intel_crtc *crtc,
10086                       struct intel_crtc_state *pipe_config)
10087 {
10088         if (pipe_config->has_pch_encoder)
10089                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
10090         else
10091                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
10092                                              &pipe_config->dp_m_n,
10093                                              &pipe_config->dp_m2_n2);
10094 }
10095
10096 static void ironlake_get_fdi_m_n_config(struct intel_crtc *crtc,
10097                                         struct intel_crtc_state *pipe_config)
10098 {
10099         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
10100                                      &pipe_config->fdi_m_n, NULL);
10101 }
10102
10103 static void skylake_get_pfit_config(struct intel_crtc *crtc,
10104                                     struct intel_crtc_state *pipe_config)
10105 {
10106         struct drm_device *dev = crtc->base.dev;
10107         struct drm_i915_private *dev_priv = to_i915(dev);
10108         struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state;
10109         u32 ps_ctrl = 0;
10110         int id = -1;
10111         int i;
10112
10113         /* find scaler attached to this pipe */
10114         for (i = 0; i < crtc->num_scalers; i++) {
10115                 ps_ctrl = I915_READ(SKL_PS_CTRL(crtc->pipe, i));
10116                 if (ps_ctrl & PS_SCALER_EN && !(ps_ctrl & PS_PLANE_SEL_MASK)) {
10117                         id = i;
10118                         pipe_config->pch_pfit.enabled = true;
10119                         pipe_config->pch_pfit.pos = I915_READ(SKL_PS_WIN_POS(crtc->pipe, i));
10120                         pipe_config->pch_pfit.size = I915_READ(SKL_PS_WIN_SZ(crtc->pipe, i));
10121                         scaler_state->scalers[i].in_use = true;
10122                         break;
10123                 }
10124         }
10125
10126         scaler_state->scaler_id = id;
10127         if (id >= 0) {
10128                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
10129         } else {
10130                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
10131         }
10132 }
10133
10134 static void
10135 skylake_get_initial_plane_config(struct intel_crtc *crtc,
10136                                  struct intel_initial_plane_config *plane_config)
10137 {
10138         struct drm_device *dev = crtc->base.dev;
10139         struct drm_i915_private *dev_priv = to_i915(dev);
10140         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
10141         enum plane_id plane_id = plane->id;
10142         enum pipe pipe;
10143         u32 val, base, offset, stride_mult, tiling, alpha;
10144         int fourcc, pixel_format;
10145         unsigned int aligned_height;
10146         struct drm_framebuffer *fb;
10147         struct intel_framebuffer *intel_fb;
10148
10149         if (!plane->get_hw_state(plane, &pipe))
10150                 return;
10151
10152         WARN_ON(pipe != crtc->pipe);
10153
10154         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
10155         if (!intel_fb) {
10156                 DRM_DEBUG_KMS("failed to alloc fb\n");
10157                 return;
10158         }
10159
10160         fb = &intel_fb->base;
10161
10162         fb->dev = dev;
10163
10164         val = I915_READ(PLANE_CTL(pipe, plane_id));
10165
10166         if (INTEL_GEN(dev_priv) >= 11)
10167                 pixel_format = val & ICL_PLANE_CTL_FORMAT_MASK;
10168         else
10169                 pixel_format = val & PLANE_CTL_FORMAT_MASK;
10170
10171         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
10172                 alpha = I915_READ(PLANE_COLOR_CTL(pipe, plane_id));
10173                 alpha &= PLANE_COLOR_ALPHA_MASK;
10174         } else {
10175                 alpha = val & PLANE_CTL_ALPHA_MASK;
10176         }
10177
10178         fourcc = skl_format_to_fourcc(pixel_format,
10179                                       val & PLANE_CTL_ORDER_RGBX, alpha);
10180         fb->format = drm_format_info(fourcc);
10181
10182         tiling = val & PLANE_CTL_TILED_MASK;
10183         switch (tiling) {
10184         case PLANE_CTL_TILED_LINEAR:
10185                 fb->modifier = DRM_FORMAT_MOD_LINEAR;
10186                 break;
10187         case PLANE_CTL_TILED_X:
10188                 plane_config->tiling = I915_TILING_X;
10189                 fb->modifier = I915_FORMAT_MOD_X_TILED;
10190                 break;
10191         case PLANE_CTL_TILED_Y:
10192                 plane_config->tiling = I915_TILING_Y;
10193                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
10194                         fb->modifier = INTEL_GEN(dev_priv) >= 12 ?
10195                                 I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS :
10196                                 I915_FORMAT_MOD_Y_TILED_CCS;
10197                 else
10198                         fb->modifier = I915_FORMAT_MOD_Y_TILED;
10199                 break;
10200         case PLANE_CTL_TILED_YF:
10201                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
10202                         fb->modifier = I915_FORMAT_MOD_Yf_TILED_CCS;
10203                 else
10204                         fb->modifier = I915_FORMAT_MOD_Yf_TILED;
10205                 break;
10206         default:
10207                 MISSING_CASE(tiling);
10208                 goto error;
10209         }
10210
10211         /*
10212          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
10213          * while i915 HW rotation is clockwise, thats why this swapping.
10214          */
10215         switch (val & PLANE_CTL_ROTATE_MASK) {
10216         case PLANE_CTL_ROTATE_0:
10217                 plane_config->rotation = DRM_MODE_ROTATE_0;
10218                 break;
10219         case PLANE_CTL_ROTATE_90:
10220                 plane_config->rotation = DRM_MODE_ROTATE_270;
10221                 break;
10222         case PLANE_CTL_ROTATE_180:
10223                 plane_config->rotation = DRM_MODE_ROTATE_180;
10224                 break;
10225         case PLANE_CTL_ROTATE_270:
10226                 plane_config->rotation = DRM_MODE_ROTATE_90;
10227                 break;
10228         }
10229
10230         if (INTEL_GEN(dev_priv) >= 10 &&
10231             val & PLANE_CTL_FLIP_HORIZONTAL)
10232                 plane_config->rotation |= DRM_MODE_REFLECT_X;
10233
10234         base = I915_READ(PLANE_SURF(pipe, plane_id)) & 0xfffff000;
10235         plane_config->base = base;
10236
10237         offset = I915_READ(PLANE_OFFSET(pipe, plane_id));
10238
10239         val = I915_READ(PLANE_SIZE(pipe, plane_id));
10240         fb->height = ((val >> 16) & 0xffff) + 1;
10241         fb->width = ((val >> 0) & 0xffff) + 1;
10242
10243         val = I915_READ(PLANE_STRIDE(pipe, plane_id));
10244         stride_mult = skl_plane_stride_mult(fb, 0, DRM_MODE_ROTATE_0);
10245         fb->pitches[0] = (val & 0x3ff) * stride_mult;
10246
10247         aligned_height = intel_fb_align_height(fb, 0, fb->height);
10248
10249         plane_config->size = fb->pitches[0] * aligned_height;
10250
10251         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
10252                       crtc->base.name, plane->base.name, fb->width, fb->height,
10253                       fb->format->cpp[0] * 8, base, fb->pitches[0],
10254                       plane_config->size);
10255
10256         plane_config->fb = intel_fb;
10257         return;
10258
10259 error:
10260         kfree(intel_fb);
10261 }
10262
10263 static void ironlake_get_pfit_config(struct intel_crtc *crtc,
10264                                      struct intel_crtc_state *pipe_config)
10265 {
10266         struct drm_device *dev = crtc->base.dev;
10267         struct drm_i915_private *dev_priv = to_i915(dev);
10268         u32 tmp;
10269
10270         tmp = I915_READ(PF_CTL(crtc->pipe));
10271
10272         if (tmp & PF_ENABLE) {
10273                 pipe_config->pch_pfit.enabled = true;
10274                 pipe_config->pch_pfit.pos = I915_READ(PF_WIN_POS(crtc->pipe));
10275                 pipe_config->pch_pfit.size = I915_READ(PF_WIN_SZ(crtc->pipe));
10276
10277                 /* We currently do not free assignements of panel fitters on
10278                  * ivb/hsw (since we don't use the higher upscaling modes which
10279                  * differentiates them) so just WARN about this case for now. */
10280                 if (IS_GEN(dev_priv, 7)) {
10281                         WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) !=
10282                                 PF_PIPE_SEL_IVB(crtc->pipe));
10283                 }
10284         }
10285 }
10286
10287 static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
10288                                      struct intel_crtc_state *pipe_config)
10289 {
10290         struct drm_device *dev = crtc->base.dev;
10291         struct drm_i915_private *dev_priv = to_i915(dev);
10292         enum intel_display_power_domain power_domain;
10293         intel_wakeref_t wakeref;
10294         u32 tmp;
10295         bool ret;
10296
10297         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
10298         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
10299         if (!wakeref)
10300                 return false;
10301
10302         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
10303         pipe_config->shared_dpll = NULL;
10304         pipe_config->master_transcoder = INVALID_TRANSCODER;
10305
10306         ret = false;
10307         tmp = I915_READ(PIPECONF(crtc->pipe));
10308         if (!(tmp & PIPECONF_ENABLE))
10309                 goto out;
10310
10311         switch (tmp & PIPECONF_BPC_MASK) {
10312         case PIPECONF_6BPC:
10313                 pipe_config->pipe_bpp = 18;
10314                 break;
10315         case PIPECONF_8BPC:
10316                 pipe_config->pipe_bpp = 24;
10317                 break;
10318         case PIPECONF_10BPC:
10319                 pipe_config->pipe_bpp = 30;
10320                 break;
10321         case PIPECONF_12BPC:
10322                 pipe_config->pipe_bpp = 36;
10323                 break;
10324         default:
10325                 break;
10326         }
10327
10328         if (tmp & PIPECONF_COLOR_RANGE_SELECT)
10329                 pipe_config->limited_color_range = true;
10330
10331         switch (tmp & PIPECONF_OUTPUT_COLORSPACE_MASK) {
10332         case PIPECONF_OUTPUT_COLORSPACE_YUV601:
10333         case PIPECONF_OUTPUT_COLORSPACE_YUV709:
10334                 pipe_config->output_format = INTEL_OUTPUT_FORMAT_YCBCR444;
10335                 break;
10336         default:
10337                 pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
10338                 break;
10339         }
10340
10341         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_ILK) >>
10342                 PIPECONF_GAMMA_MODE_SHIFT;
10343
10344         pipe_config->csc_mode = I915_READ(PIPE_CSC_MODE(crtc->pipe));
10345
10346         i9xx_get_pipe_color_config(pipe_config);
10347         intel_color_get_config(pipe_config);
10348
10349         if (I915_READ(PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
10350                 struct intel_shared_dpll *pll;
10351                 enum intel_dpll_id pll_id;
10352
10353                 pipe_config->has_pch_encoder = true;
10354
10355                 tmp = I915_READ(FDI_RX_CTL(crtc->pipe));
10356                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
10357                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
10358
10359                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
10360
10361                 if (HAS_PCH_IBX(dev_priv)) {
10362                         /*
10363                          * The pipe->pch transcoder and pch transcoder->pll
10364                          * mapping is fixed.
10365                          */
10366                         pll_id = (enum intel_dpll_id) crtc->pipe;
10367                 } else {
10368                         tmp = I915_READ(PCH_DPLL_SEL);
10369                         if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
10370                                 pll_id = DPLL_ID_PCH_PLL_B;
10371                         else
10372                                 pll_id= DPLL_ID_PCH_PLL_A;
10373                 }
10374
10375                 pipe_config->shared_dpll =
10376                         intel_get_shared_dpll_by_id(dev_priv, pll_id);
10377                 pll = pipe_config->shared_dpll;
10378
10379                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
10380                                                 &pipe_config->dpll_hw_state));
10381
10382                 tmp = pipe_config->dpll_hw_state.dpll;
10383                 pipe_config->pixel_multiplier =
10384                         ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
10385                          >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
10386
10387                 ironlake_pch_clock_get(crtc, pipe_config);
10388         } else {
10389                 pipe_config->pixel_multiplier = 1;
10390         }
10391
10392         intel_get_pipe_timings(crtc, pipe_config);
10393         intel_get_pipe_src_size(crtc, pipe_config);
10394
10395         ironlake_get_pfit_config(crtc, pipe_config);
10396
10397         ret = true;
10398
10399 out:
10400         intel_display_power_put(dev_priv, power_domain, wakeref);
10401
10402         return ret;
10403 }
10404 static int haswell_crtc_compute_clock(struct intel_crtc *crtc,
10405                                       struct intel_crtc_state *crtc_state)
10406 {
10407         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10408         struct intel_atomic_state *state =
10409                 to_intel_atomic_state(crtc_state->uapi.state);
10410
10411         if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) ||
10412             INTEL_GEN(dev_priv) >= 11) {
10413                 struct intel_encoder *encoder =
10414                         intel_get_crtc_new_encoder(state, crtc_state);
10415
10416                 if (!intel_reserve_shared_dplls(state, crtc, encoder)) {
10417                         DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
10418                                       pipe_name(crtc->pipe));
10419                         return -EINVAL;
10420                 }
10421         }
10422
10423         return 0;
10424 }
10425
10426 static void cannonlake_get_ddi_pll(struct drm_i915_private *dev_priv,
10427                                    enum port port,
10428                                    struct intel_crtc_state *pipe_config)
10429 {
10430         enum intel_dpll_id id;
10431         u32 temp;
10432
10433         temp = I915_READ(DPCLKA_CFGCR0) & DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
10434         id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port);
10435
10436         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL2))
10437                 return;
10438
10439         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10440 }
10441
10442 static void icelake_get_ddi_pll(struct drm_i915_private *dev_priv,
10443                                 enum port port,
10444                                 struct intel_crtc_state *pipe_config)
10445 {
10446         enum phy phy = intel_port_to_phy(dev_priv, port);
10447         enum icl_port_dpll_id port_dpll_id;
10448         enum intel_dpll_id id;
10449         u32 temp;
10450
10451         if (intel_phy_is_combo(dev_priv, phy)) {
10452                 temp = I915_READ(ICL_DPCLKA_CFGCR0) &
10453                         ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy);
10454                 id = temp >> ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy);
10455                 port_dpll_id = ICL_PORT_DPLL_DEFAULT;
10456         } else if (intel_phy_is_tc(dev_priv, phy)) {
10457                 u32 clk_sel = I915_READ(DDI_CLK_SEL(port)) & DDI_CLK_SEL_MASK;
10458
10459                 if (clk_sel == DDI_CLK_SEL_MG) {
10460                         id = icl_tc_port_to_pll_id(intel_port_to_tc(dev_priv,
10461                                                                     port));
10462                         port_dpll_id = ICL_PORT_DPLL_MG_PHY;
10463                 } else {
10464                         WARN_ON(clk_sel < DDI_CLK_SEL_TBT_162);
10465                         id = DPLL_ID_ICL_TBTPLL;
10466                         port_dpll_id = ICL_PORT_DPLL_DEFAULT;
10467                 }
10468         } else {
10469                 WARN(1, "Invalid port %x\n", port);
10470                 return;
10471         }
10472
10473         pipe_config->icl_port_dplls[port_dpll_id].pll =
10474                 intel_get_shared_dpll_by_id(dev_priv, id);
10475
10476         icl_set_active_port_dpll(pipe_config, port_dpll_id);
10477 }
10478
10479 static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv,
10480                                 enum port port,
10481                                 struct intel_crtc_state *pipe_config)
10482 {
10483         enum intel_dpll_id id;
10484
10485         switch (port) {
10486         case PORT_A:
10487                 id = DPLL_ID_SKL_DPLL0;
10488                 break;
10489         case PORT_B:
10490                 id = DPLL_ID_SKL_DPLL1;
10491                 break;
10492         case PORT_C:
10493                 id = DPLL_ID_SKL_DPLL2;
10494                 break;
10495         default:
10496                 DRM_ERROR("Incorrect port type\n");
10497                 return;
10498         }
10499
10500         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10501 }
10502
10503 static void skylake_get_ddi_pll(struct drm_i915_private *dev_priv,
10504                                 enum port port,
10505                                 struct intel_crtc_state *pipe_config)
10506 {
10507         enum intel_dpll_id id;
10508         u32 temp;
10509
10510         temp = I915_READ(DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port);
10511         id = temp >> (port * 3 + 1);
10512
10513         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL3))
10514                 return;
10515
10516         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10517 }
10518
10519 static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
10520                                 enum port port,
10521                                 struct intel_crtc_state *pipe_config)
10522 {
10523         enum intel_dpll_id id;
10524         u32 ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
10525
10526         switch (ddi_pll_sel) {
10527         case PORT_CLK_SEL_WRPLL1:
10528                 id = DPLL_ID_WRPLL1;
10529                 break;
10530         case PORT_CLK_SEL_WRPLL2:
10531                 id = DPLL_ID_WRPLL2;
10532                 break;
10533         case PORT_CLK_SEL_SPLL:
10534                 id = DPLL_ID_SPLL;
10535                 break;
10536         case PORT_CLK_SEL_LCPLL_810:
10537                 id = DPLL_ID_LCPLL_810;
10538                 break;
10539         case PORT_CLK_SEL_LCPLL_1350:
10540                 id = DPLL_ID_LCPLL_1350;
10541                 break;
10542         case PORT_CLK_SEL_LCPLL_2700:
10543                 id = DPLL_ID_LCPLL_2700;
10544                 break;
10545         default:
10546                 MISSING_CASE(ddi_pll_sel);
10547                 /* fall through */
10548         case PORT_CLK_SEL_NONE:
10549                 return;
10550         }
10551
10552         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10553 }
10554
10555 static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
10556                                      struct intel_crtc_state *pipe_config,
10557                                      u64 *power_domain_mask,
10558                                      intel_wakeref_t *wakerefs)
10559 {
10560         struct drm_device *dev = crtc->base.dev;
10561         struct drm_i915_private *dev_priv = to_i915(dev);
10562         enum intel_display_power_domain power_domain;
10563         unsigned long panel_transcoder_mask = 0;
10564         unsigned long enabled_panel_transcoders = 0;
10565         enum transcoder panel_transcoder;
10566         intel_wakeref_t wf;
10567         u32 tmp;
10568
10569         if (INTEL_GEN(dev_priv) >= 11)
10570                 panel_transcoder_mask |=
10571                         BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1);
10572
10573         if (HAS_TRANSCODER_EDP(dev_priv))
10574                 panel_transcoder_mask |= BIT(TRANSCODER_EDP);
10575
10576         /*
10577          * The pipe->transcoder mapping is fixed with the exception of the eDP
10578          * and DSI transcoders handled below.
10579          */
10580         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
10581
10582         /*
10583          * XXX: Do intel_display_power_get_if_enabled before reading this (for
10584          * consistency and less surprising code; it's in always on power).
10585          */
10586         for_each_set_bit(panel_transcoder,
10587                          &panel_transcoder_mask,
10588                          ARRAY_SIZE(INTEL_INFO(dev_priv)->trans_offsets)) {
10589                 bool force_thru = false;
10590                 enum pipe trans_pipe;
10591
10592                 tmp = I915_READ(TRANS_DDI_FUNC_CTL(panel_transcoder));
10593                 if (!(tmp & TRANS_DDI_FUNC_ENABLE))
10594                         continue;
10595
10596                 /*
10597                  * Log all enabled ones, only use the first one.
10598                  *
10599                  * FIXME: This won't work for two separate DSI displays.
10600                  */
10601                 enabled_panel_transcoders |= BIT(panel_transcoder);
10602                 if (enabled_panel_transcoders != BIT(panel_transcoder))
10603                         continue;
10604
10605                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
10606                 default:
10607                         WARN(1, "unknown pipe linked to transcoder %s\n",
10608                              transcoder_name(panel_transcoder));
10609                         /* fall through */
10610                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
10611                         force_thru = true;
10612                         /* fall through */
10613                 case TRANS_DDI_EDP_INPUT_A_ON:
10614                         trans_pipe = PIPE_A;
10615                         break;
10616                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
10617                         trans_pipe = PIPE_B;
10618                         break;
10619                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
10620                         trans_pipe = PIPE_C;
10621                         break;
10622                 case TRANS_DDI_EDP_INPUT_D_ONOFF:
10623                         trans_pipe = PIPE_D;
10624                         break;
10625                 }
10626
10627                 if (trans_pipe == crtc->pipe) {
10628                         pipe_config->cpu_transcoder = panel_transcoder;
10629                         pipe_config->pch_pfit.force_thru = force_thru;
10630                 }
10631         }
10632
10633         /*
10634          * Valid combos: none, eDP, DSI0, DSI1, DSI0+DSI1
10635          */
10636         WARN_ON((enabled_panel_transcoders & BIT(TRANSCODER_EDP)) &&
10637                 enabled_panel_transcoders != BIT(TRANSCODER_EDP));
10638
10639         power_domain = POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder);
10640         WARN_ON(*power_domain_mask & BIT_ULL(power_domain));
10641
10642         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10643         if (!wf)
10644                 return false;
10645
10646         wakerefs[power_domain] = wf;
10647         *power_domain_mask |= BIT_ULL(power_domain);
10648
10649         tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
10650
10651         return tmp & PIPECONF_ENABLE;
10652 }
10653
10654 static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
10655                                          struct intel_crtc_state *pipe_config,
10656                                          u64 *power_domain_mask,
10657                                          intel_wakeref_t *wakerefs)
10658 {
10659         struct drm_device *dev = crtc->base.dev;
10660         struct drm_i915_private *dev_priv = to_i915(dev);
10661         enum intel_display_power_domain power_domain;
10662         enum transcoder cpu_transcoder;
10663         intel_wakeref_t wf;
10664         enum port port;
10665         u32 tmp;
10666
10667         for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) {
10668                 if (port == PORT_A)
10669                         cpu_transcoder = TRANSCODER_DSI_A;
10670                 else
10671                         cpu_transcoder = TRANSCODER_DSI_C;
10672
10673                 power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
10674                 WARN_ON(*power_domain_mask & BIT_ULL(power_domain));
10675
10676                 wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10677                 if (!wf)
10678                         continue;
10679
10680                 wakerefs[power_domain] = wf;
10681                 *power_domain_mask |= BIT_ULL(power_domain);
10682
10683                 /*
10684                  * The PLL needs to be enabled with a valid divider
10685                  * configuration, otherwise accessing DSI registers will hang
10686                  * the machine. See BSpec North Display Engine
10687                  * registers/MIPI[BXT]. We can break out here early, since we
10688                  * need the same DSI PLL to be enabled for both DSI ports.
10689                  */
10690                 if (!bxt_dsi_pll_is_enabled(dev_priv))
10691                         break;
10692
10693                 /* XXX: this works for video mode only */
10694                 tmp = I915_READ(BXT_MIPI_PORT_CTRL(port));
10695                 if (!(tmp & DPI_ENABLE))
10696                         continue;
10697
10698                 tmp = I915_READ(MIPI_CTRL(port));
10699                 if ((tmp & BXT_PIPE_SELECT_MASK) != BXT_PIPE_SELECT(crtc->pipe))
10700                         continue;
10701
10702                 pipe_config->cpu_transcoder = cpu_transcoder;
10703                 break;
10704         }
10705
10706         return transcoder_is_dsi(pipe_config->cpu_transcoder);
10707 }
10708
10709 static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
10710                                        struct intel_crtc_state *pipe_config)
10711 {
10712         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10713         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
10714         struct intel_shared_dpll *pll;
10715         enum port port;
10716         u32 tmp;
10717
10718         if (transcoder_is_dsi(cpu_transcoder)) {
10719                 port = (cpu_transcoder == TRANSCODER_DSI_A) ?
10720                                                 PORT_A : PORT_B;
10721         } else {
10722                 tmp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
10723                 if (INTEL_GEN(dev_priv) >= 12)
10724                         port = TGL_TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp);
10725                 else
10726                         port = TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp);
10727         }
10728
10729         if (INTEL_GEN(dev_priv) >= 11)
10730                 icelake_get_ddi_pll(dev_priv, port, pipe_config);
10731         else if (IS_CANNONLAKE(dev_priv))
10732                 cannonlake_get_ddi_pll(dev_priv, port, pipe_config);
10733         else if (IS_GEN9_BC(dev_priv))
10734                 skylake_get_ddi_pll(dev_priv, port, pipe_config);
10735         else if (IS_GEN9_LP(dev_priv))
10736                 bxt_get_ddi_pll(dev_priv, port, pipe_config);
10737         else
10738                 haswell_get_ddi_pll(dev_priv, port, pipe_config);
10739
10740         pll = pipe_config->shared_dpll;
10741         if (pll) {
10742                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
10743                                                 &pipe_config->dpll_hw_state));
10744         }
10745
10746         /*
10747          * Haswell has only FDI/PCH transcoder A. It is which is connected to
10748          * DDI E. So just check whether this pipe is wired to DDI E and whether
10749          * the PCH transcoder is on.
10750          */
10751         if (INTEL_GEN(dev_priv) < 9 &&
10752             (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) {
10753                 pipe_config->has_pch_encoder = true;
10754
10755                 tmp = I915_READ(FDI_RX_CTL(PIPE_A));
10756                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
10757                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
10758
10759                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
10760         }
10761 }
10762
10763 static enum transcoder transcoder_master_readout(struct drm_i915_private *dev_priv,
10764                                                  enum transcoder cpu_transcoder)
10765 {
10766         u32 trans_port_sync, master_select;
10767
10768         trans_port_sync = I915_READ(TRANS_DDI_FUNC_CTL2(cpu_transcoder));
10769
10770         if ((trans_port_sync & PORT_SYNC_MODE_ENABLE) == 0)
10771                 return INVALID_TRANSCODER;
10772
10773         master_select = trans_port_sync &
10774                         PORT_SYNC_MODE_MASTER_SELECT_MASK;
10775         if (master_select == 0)
10776                 return TRANSCODER_EDP;
10777         else
10778                 return master_select - 1;
10779 }
10780
10781 static void icelake_get_trans_port_sync_config(struct intel_crtc_state *crtc_state)
10782 {
10783         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
10784         u32 transcoders;
10785         enum transcoder cpu_transcoder;
10786
10787         crtc_state->master_transcoder = transcoder_master_readout(dev_priv,
10788                                                                   crtc_state->cpu_transcoder);
10789
10790         transcoders = BIT(TRANSCODER_A) |
10791                 BIT(TRANSCODER_B) |
10792                 BIT(TRANSCODER_C) |
10793                 BIT(TRANSCODER_D);
10794         for_each_cpu_transcoder_masked(dev_priv, cpu_transcoder, transcoders) {
10795                 enum intel_display_power_domain power_domain;
10796                 intel_wakeref_t trans_wakeref;
10797
10798                 power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
10799                 trans_wakeref = intel_display_power_get_if_enabled(dev_priv,
10800                                                                    power_domain);
10801
10802                 if (!trans_wakeref)
10803                         continue;
10804
10805                 if (transcoder_master_readout(dev_priv, cpu_transcoder) ==
10806                     crtc_state->cpu_transcoder)
10807                         crtc_state->sync_mode_slaves_mask |= BIT(cpu_transcoder);
10808
10809                 intel_display_power_put(dev_priv, power_domain, trans_wakeref);
10810         }
10811
10812         WARN_ON(crtc_state->master_transcoder != INVALID_TRANSCODER &&
10813                 crtc_state->sync_mode_slaves_mask);
10814 }
10815
10816 static bool haswell_get_pipe_config(struct intel_crtc *crtc,
10817                                     struct intel_crtc_state *pipe_config)
10818 {
10819         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10820         intel_wakeref_t wakerefs[POWER_DOMAIN_NUM], wf;
10821         enum intel_display_power_domain power_domain;
10822         u64 power_domain_mask;
10823         bool active;
10824
10825         pipe_config->master_transcoder = INVALID_TRANSCODER;
10826
10827         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
10828         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10829         if (!wf)
10830                 return false;
10831
10832         wakerefs[power_domain] = wf;
10833         power_domain_mask = BIT_ULL(power_domain);
10834
10835         pipe_config->shared_dpll = NULL;
10836
10837         active = hsw_get_transcoder_state(crtc, pipe_config,
10838                                           &power_domain_mask, wakerefs);
10839
10840         if (IS_GEN9_LP(dev_priv) &&
10841             bxt_get_dsi_transcoder_state(crtc, pipe_config,
10842                                          &power_domain_mask, wakerefs)) {
10843                 WARN_ON(active);
10844                 active = true;
10845         }
10846
10847         if (!active)
10848                 goto out;
10849
10850         if (!transcoder_is_dsi(pipe_config->cpu_transcoder) ||
10851             INTEL_GEN(dev_priv) >= 11) {
10852                 haswell_get_ddi_port_state(crtc, pipe_config);
10853                 intel_get_pipe_timings(crtc, pipe_config);
10854         }
10855
10856         intel_get_pipe_src_size(crtc, pipe_config);
10857
10858         if (IS_HASWELL(dev_priv)) {
10859                 u32 tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
10860
10861                 if (tmp & PIPECONF_OUTPUT_COLORSPACE_YUV_HSW)
10862                         pipe_config->output_format = INTEL_OUTPUT_FORMAT_YCBCR444;
10863                 else
10864                         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
10865         } else {
10866                 pipe_config->output_format =
10867                         bdw_get_pipemisc_output_format(crtc);
10868
10869                 /*
10870                  * Currently there is no interface defined to
10871                  * check user preference between RGB/YCBCR444
10872                  * or YCBCR420. So the only possible case for
10873                  * YCBCR444 usage is driving YCBCR420 output
10874                  * with LSPCON, when pipe is configured for
10875                  * YCBCR444 output and LSPCON takes care of
10876                  * downsampling it.
10877                  */
10878                 pipe_config->lspcon_downsampling =
10879                         pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR444;
10880         }
10881
10882         pipe_config->gamma_mode = I915_READ(GAMMA_MODE(crtc->pipe));
10883
10884         pipe_config->csc_mode = I915_READ(PIPE_CSC_MODE(crtc->pipe));
10885
10886         if (INTEL_GEN(dev_priv) >= 9) {
10887                 u32 tmp = I915_READ(SKL_BOTTOM_COLOR(crtc->pipe));
10888
10889                 if (tmp & SKL_BOTTOM_COLOR_GAMMA_ENABLE)
10890                         pipe_config->gamma_enable = true;
10891
10892                 if (tmp & SKL_BOTTOM_COLOR_CSC_ENABLE)
10893                         pipe_config->csc_enable = true;
10894         } else {
10895                 i9xx_get_pipe_color_config(pipe_config);
10896         }
10897
10898         intel_color_get_config(pipe_config);
10899
10900         power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
10901         WARN_ON(power_domain_mask & BIT_ULL(power_domain));
10902
10903         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10904         if (wf) {
10905                 wakerefs[power_domain] = wf;
10906                 power_domain_mask |= BIT_ULL(power_domain);
10907
10908                 if (INTEL_GEN(dev_priv) >= 9)
10909                         skylake_get_pfit_config(crtc, pipe_config);
10910                 else
10911                         ironlake_get_pfit_config(crtc, pipe_config);
10912         }
10913
10914         if (hsw_crtc_supports_ips(crtc)) {
10915                 if (IS_HASWELL(dev_priv))
10916                         pipe_config->ips_enabled = I915_READ(IPS_CTL) & IPS_ENABLE;
10917                 else {
10918                         /*
10919                          * We cannot readout IPS state on broadwell, set to
10920                          * true so we can set it to a defined state on first
10921                          * commit.
10922                          */
10923                         pipe_config->ips_enabled = true;
10924                 }
10925         }
10926
10927         if (pipe_config->cpu_transcoder != TRANSCODER_EDP &&
10928             !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
10929                 pipe_config->pixel_multiplier =
10930                         I915_READ(PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
10931         } else {
10932                 pipe_config->pixel_multiplier = 1;
10933         }
10934
10935         if (INTEL_GEN(dev_priv) >= 11 &&
10936             !transcoder_is_dsi(pipe_config->cpu_transcoder))
10937                 icelake_get_trans_port_sync_config(pipe_config);
10938
10939 out:
10940         for_each_power_domain(power_domain, power_domain_mask)
10941                 intel_display_power_put(dev_priv,
10942                                         power_domain, wakerefs[power_domain]);
10943
10944         return active;
10945 }
10946
10947 static u32 intel_cursor_base(const struct intel_plane_state *plane_state)
10948 {
10949         struct drm_i915_private *dev_priv =
10950                 to_i915(plane_state->uapi.plane->dev);
10951         const struct drm_framebuffer *fb = plane_state->hw.fb;
10952         const struct drm_i915_gem_object *obj = intel_fb_obj(fb);
10953         u32 base;
10954
10955         if (INTEL_INFO(dev_priv)->display.cursor_needs_physical)
10956                 base = obj->phys_handle->busaddr;
10957         else
10958                 base = intel_plane_ggtt_offset(plane_state);
10959
10960         return base + plane_state->color_plane[0].offset;
10961 }
10962
10963 static u32 intel_cursor_position(const struct intel_plane_state *plane_state)
10964 {
10965         int x = plane_state->uapi.dst.x1;
10966         int y = plane_state->uapi.dst.y1;
10967         u32 pos = 0;
10968
10969         if (x < 0) {
10970                 pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
10971                 x = -x;
10972         }
10973         pos |= x << CURSOR_X_SHIFT;
10974
10975         if (y < 0) {
10976                 pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
10977                 y = -y;
10978         }
10979         pos |= y << CURSOR_Y_SHIFT;
10980
10981         return pos;
10982 }
10983
10984 static bool intel_cursor_size_ok(const struct intel_plane_state *plane_state)
10985 {
10986         const struct drm_mode_config *config =
10987                 &plane_state->uapi.plane->dev->mode_config;
10988         int width = drm_rect_width(&plane_state->uapi.dst);
10989         int height = drm_rect_height(&plane_state->uapi.dst);
10990
10991         return width > 0 && width <= config->cursor_width &&
10992                 height > 0 && height <= config->cursor_height;
10993 }
10994
10995 static int intel_cursor_check_surface(struct intel_plane_state *plane_state)
10996 {
10997         struct drm_i915_private *dev_priv =
10998                 to_i915(plane_state->uapi.plane->dev);
10999         unsigned int rotation = plane_state->hw.rotation;
11000         int src_x, src_y;
11001         u32 offset;
11002         int ret;
11003
11004         ret = intel_plane_compute_gtt(plane_state);
11005         if (ret)
11006                 return ret;
11007
11008         if (!plane_state->uapi.visible)
11009                 return 0;
11010
11011         src_x = plane_state->uapi.src.x1 >> 16;
11012         src_y = plane_state->uapi.src.y1 >> 16;
11013
11014         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
11015         offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
11016                                                     plane_state, 0);
11017
11018         if (src_x != 0 || src_y != 0) {
11019                 DRM_DEBUG_KMS("Arbitrary cursor panning not supported\n");
11020                 return -EINVAL;
11021         }
11022
11023         /*
11024          * Put the final coordinates back so that the src
11025          * coordinate checks will see the right values.
11026          */
11027         drm_rect_translate_to(&plane_state->uapi.src,
11028                               src_x << 16, src_y << 16);
11029
11030         /* ILK+ do this automagically in hardware */
11031         if (HAS_GMCH(dev_priv) && rotation & DRM_MODE_ROTATE_180) {
11032                 const struct drm_framebuffer *fb = plane_state->hw.fb;
11033                 int src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
11034                 int src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
11035
11036                 offset += (src_h * src_w - 1) * fb->format->cpp[0];
11037         }
11038
11039         plane_state->color_plane[0].offset = offset;
11040         plane_state->color_plane[0].x = src_x;
11041         plane_state->color_plane[0].y = src_y;
11042
11043         return 0;
11044 }
11045
11046 static int intel_check_cursor(struct intel_crtc_state *crtc_state,
11047                               struct intel_plane_state *plane_state)
11048 {
11049         const struct drm_framebuffer *fb = plane_state->hw.fb;
11050         int ret;
11051
11052         if (fb && fb->modifier != DRM_FORMAT_MOD_LINEAR) {
11053                 DRM_DEBUG_KMS("cursor cannot be tiled\n");
11054                 return -EINVAL;
11055         }
11056
11057         ret = drm_atomic_helper_check_plane_state(&plane_state->uapi,
11058                                                   &crtc_state->uapi,
11059                                                   DRM_PLANE_HELPER_NO_SCALING,
11060                                                   DRM_PLANE_HELPER_NO_SCALING,
11061                                                   true, true);
11062         if (ret)
11063                 return ret;
11064
11065         /* Use the unclipped src/dst rectangles, which we program to hw */
11066         plane_state->uapi.src = drm_plane_state_src(&plane_state->uapi);
11067         plane_state->uapi.dst = drm_plane_state_dest(&plane_state->uapi);
11068
11069         ret = intel_cursor_check_surface(plane_state);
11070         if (ret)
11071                 return ret;
11072
11073         if (!plane_state->uapi.visible)
11074                 return 0;
11075
11076         ret = intel_plane_check_src_coordinates(plane_state);
11077         if (ret)
11078                 return ret;
11079
11080         return 0;
11081 }
11082
11083 static unsigned int
11084 i845_cursor_max_stride(struct intel_plane *plane,
11085                        u32 pixel_format, u64 modifier,
11086                        unsigned int rotation)
11087 {
11088         return 2048;
11089 }
11090
11091 static u32 i845_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state)
11092 {
11093         u32 cntl = 0;
11094
11095         if (crtc_state->gamma_enable)
11096                 cntl |= CURSOR_GAMMA_ENABLE;
11097
11098         return cntl;
11099 }
11100
11101 static u32 i845_cursor_ctl(const struct intel_crtc_state *crtc_state,
11102                            const struct intel_plane_state *plane_state)
11103 {
11104         return CURSOR_ENABLE |
11105                 CURSOR_FORMAT_ARGB |
11106                 CURSOR_STRIDE(plane_state->color_plane[0].stride);
11107 }
11108
11109 static bool i845_cursor_size_ok(const struct intel_plane_state *plane_state)
11110 {
11111         int width = drm_rect_width(&plane_state->uapi.dst);
11112
11113         /*
11114          * 845g/865g are only limited by the width of their cursors,
11115          * the height is arbitrary up to the precision of the register.
11116          */
11117         return intel_cursor_size_ok(plane_state) && IS_ALIGNED(width, 64);
11118 }
11119
11120 static int i845_check_cursor(struct intel_crtc_state *crtc_state,
11121                              struct intel_plane_state *plane_state)
11122 {
11123         const struct drm_framebuffer *fb = plane_state->hw.fb;
11124         int ret;
11125
11126         ret = intel_check_cursor(crtc_state, plane_state);
11127         if (ret)
11128                 return ret;
11129
11130         /* if we want to turn off the cursor ignore width and height */
11131         if (!fb)
11132                 return 0;
11133
11134         /* Check for which cursor types we support */
11135         if (!i845_cursor_size_ok(plane_state)) {
11136                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
11137                           drm_rect_width(&plane_state->uapi.dst),
11138                           drm_rect_height(&plane_state->uapi.dst));
11139                 return -EINVAL;
11140         }
11141
11142         WARN_ON(plane_state->uapi.visible &&
11143                 plane_state->color_plane[0].stride != fb->pitches[0]);
11144
11145         switch (fb->pitches[0]) {
11146         case 256:
11147         case 512:
11148         case 1024:
11149         case 2048:
11150                 break;
11151         default:
11152                 DRM_DEBUG_KMS("Invalid cursor stride (%u)\n",
11153                               fb->pitches[0]);
11154                 return -EINVAL;
11155         }
11156
11157         plane_state->ctl = i845_cursor_ctl(crtc_state, plane_state);
11158
11159         return 0;
11160 }
11161
11162 static void i845_update_cursor(struct intel_plane *plane,
11163                                const struct intel_crtc_state *crtc_state,
11164                                const struct intel_plane_state *plane_state)
11165 {
11166         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11167         u32 cntl = 0, base = 0, pos = 0, size = 0;
11168         unsigned long irqflags;
11169
11170         if (plane_state && plane_state->uapi.visible) {
11171                 unsigned int width = drm_rect_width(&plane_state->uapi.dst);
11172                 unsigned int height = drm_rect_height(&plane_state->uapi.dst);
11173
11174                 cntl = plane_state->ctl |
11175                         i845_cursor_ctl_crtc(crtc_state);
11176
11177                 size = (height << 12) | width;
11178
11179                 base = intel_cursor_base(plane_state);
11180                 pos = intel_cursor_position(plane_state);
11181         }
11182
11183         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
11184
11185         /* On these chipsets we can only modify the base/size/stride
11186          * whilst the cursor is disabled.
11187          */
11188         if (plane->cursor.base != base ||
11189             plane->cursor.size != size ||
11190             plane->cursor.cntl != cntl) {
11191                 I915_WRITE_FW(CURCNTR(PIPE_A), 0);
11192                 I915_WRITE_FW(CURBASE(PIPE_A), base);
11193                 I915_WRITE_FW(CURSIZE, size);
11194                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
11195                 I915_WRITE_FW(CURCNTR(PIPE_A), cntl);
11196
11197                 plane->cursor.base = base;
11198                 plane->cursor.size = size;
11199                 plane->cursor.cntl = cntl;
11200         } else {
11201                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
11202         }
11203
11204         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
11205 }
11206
11207 static void i845_disable_cursor(struct intel_plane *plane,
11208                                 const struct intel_crtc_state *crtc_state)
11209 {
11210         i845_update_cursor(plane, crtc_state, NULL);
11211 }
11212
11213 static bool i845_cursor_get_hw_state(struct intel_plane *plane,
11214                                      enum pipe *pipe)
11215 {
11216         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11217         enum intel_display_power_domain power_domain;
11218         intel_wakeref_t wakeref;
11219         bool ret;
11220
11221         power_domain = POWER_DOMAIN_PIPE(PIPE_A);
11222         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
11223         if (!wakeref)
11224                 return false;
11225
11226         ret = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
11227
11228         *pipe = PIPE_A;
11229
11230         intel_display_power_put(dev_priv, power_domain, wakeref);
11231
11232         return ret;
11233 }
11234
11235 static unsigned int
11236 i9xx_cursor_max_stride(struct intel_plane *plane,
11237                        u32 pixel_format, u64 modifier,
11238                        unsigned int rotation)
11239 {
11240         return plane->base.dev->mode_config.cursor_width * 4;
11241 }
11242
11243 static u32 i9xx_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state)
11244 {
11245         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
11246         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11247         u32 cntl = 0;
11248
11249         if (INTEL_GEN(dev_priv) >= 11)
11250                 return cntl;
11251
11252         if (crtc_state->gamma_enable)
11253                 cntl = MCURSOR_GAMMA_ENABLE;
11254
11255         if (crtc_state->csc_enable)
11256                 cntl |= MCURSOR_PIPE_CSC_ENABLE;
11257
11258         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
11259                 cntl |= MCURSOR_PIPE_SELECT(crtc->pipe);
11260
11261         return cntl;
11262 }
11263
11264 static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state,
11265                            const struct intel_plane_state *plane_state)
11266 {
11267         struct drm_i915_private *dev_priv =
11268                 to_i915(plane_state->uapi.plane->dev);
11269         u32 cntl = 0;
11270
11271         if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
11272                 cntl |= MCURSOR_TRICKLE_FEED_DISABLE;
11273
11274         switch (drm_rect_width(&plane_state->uapi.dst)) {
11275         case 64:
11276                 cntl |= MCURSOR_MODE_64_ARGB_AX;
11277                 break;
11278         case 128:
11279                 cntl |= MCURSOR_MODE_128_ARGB_AX;
11280                 break;
11281         case 256:
11282                 cntl |= MCURSOR_MODE_256_ARGB_AX;
11283                 break;
11284         default:
11285                 MISSING_CASE(drm_rect_width(&plane_state->uapi.dst));
11286                 return 0;
11287         }
11288
11289         if (plane_state->hw.rotation & DRM_MODE_ROTATE_180)
11290                 cntl |= MCURSOR_ROTATE_180;
11291
11292         return cntl;
11293 }
11294
11295 static bool i9xx_cursor_size_ok(const struct intel_plane_state *plane_state)
11296 {
11297         struct drm_i915_private *dev_priv =
11298                 to_i915(plane_state->uapi.plane->dev);
11299         int width = drm_rect_width(&plane_state->uapi.dst);
11300         int height = drm_rect_height(&plane_state->uapi.dst);
11301
11302         if (!intel_cursor_size_ok(plane_state))
11303                 return false;
11304
11305         /* Cursor width is limited to a few power-of-two sizes */
11306         switch (width) {
11307         case 256:
11308         case 128:
11309         case 64:
11310                 break;
11311         default:
11312                 return false;
11313         }
11314
11315         /*
11316          * IVB+ have CUR_FBC_CTL which allows an arbitrary cursor
11317          * height from 8 lines up to the cursor width, when the
11318          * cursor is not rotated. Everything else requires square
11319          * cursors.
11320          */
11321         if (HAS_CUR_FBC(dev_priv) &&
11322             plane_state->hw.rotation & DRM_MODE_ROTATE_0) {
11323                 if (height < 8 || height > width)
11324                         return false;
11325         } else {
11326                 if (height != width)
11327                         return false;
11328         }
11329
11330         return true;
11331 }
11332
11333 static int i9xx_check_cursor(struct intel_crtc_state *crtc_state,
11334                              struct intel_plane_state *plane_state)
11335 {
11336         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
11337         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11338         const struct drm_framebuffer *fb = plane_state->hw.fb;
11339         enum pipe pipe = plane->pipe;
11340         int ret;
11341
11342         ret = intel_check_cursor(crtc_state, plane_state);
11343         if (ret)
11344                 return ret;
11345
11346         /* if we want to turn off the cursor ignore width and height */
11347         if (!fb)
11348                 return 0;
11349
11350         /* Check for which cursor types we support */
11351         if (!i9xx_cursor_size_ok(plane_state)) {
11352                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
11353                           drm_rect_width(&plane_state->uapi.dst),
11354                           drm_rect_height(&plane_state->uapi.dst));
11355                 return -EINVAL;
11356         }
11357
11358         WARN_ON(plane_state->uapi.visible &&
11359                 plane_state->color_plane[0].stride != fb->pitches[0]);
11360
11361         if (fb->pitches[0] !=
11362             drm_rect_width(&plane_state->uapi.dst) * fb->format->cpp[0]) {
11363                 DRM_DEBUG_KMS("Invalid cursor stride (%u) (cursor width %d)\n",
11364                               fb->pitches[0],
11365                               drm_rect_width(&plane_state->uapi.dst));
11366                 return -EINVAL;
11367         }
11368
11369         /*
11370          * There's something wrong with the cursor on CHV pipe C.
11371          * If it straddles the left edge of the screen then
11372          * moving it away from the edge or disabling it often
11373          * results in a pipe underrun, and often that can lead to
11374          * dead pipe (constant underrun reported, and it scans
11375          * out just a solid color). To recover from that, the
11376          * display power well must be turned off and on again.
11377          * Refuse the put the cursor into that compromised position.
11378          */
11379         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_C &&
11380             plane_state->uapi.visible && plane_state->uapi.dst.x1 < 0) {
11381                 DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
11382                 return -EINVAL;
11383         }
11384
11385         plane_state->ctl = i9xx_cursor_ctl(crtc_state, plane_state);
11386
11387         return 0;
11388 }
11389
11390 static void i9xx_update_cursor(struct intel_plane *plane,
11391                                const struct intel_crtc_state *crtc_state,
11392                                const struct intel_plane_state *plane_state)
11393 {
11394         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11395         enum pipe pipe = plane->pipe;
11396         u32 cntl = 0, base = 0, pos = 0, fbc_ctl = 0;
11397         unsigned long irqflags;
11398
11399         if (plane_state && plane_state->uapi.visible) {
11400                 unsigned width = drm_rect_width(&plane_state->uapi.dst);
11401                 unsigned height = drm_rect_height(&plane_state->uapi.dst);
11402
11403                 cntl = plane_state->ctl |
11404                         i9xx_cursor_ctl_crtc(crtc_state);
11405
11406                 if (width != height)
11407                         fbc_ctl = CUR_FBC_CTL_EN | (height - 1);
11408
11409                 base = intel_cursor_base(plane_state);
11410                 pos = intel_cursor_position(plane_state);
11411         }
11412
11413         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
11414
11415         /*
11416          * On some platforms writing CURCNTR first will also
11417          * cause CURPOS to be armed by the CURBASE write.
11418          * Without the CURCNTR write the CURPOS write would
11419          * arm itself. Thus we always update CURCNTR before
11420          * CURPOS.
11421          *
11422          * On other platforms CURPOS always requires the
11423          * CURBASE write to arm the update. Additonally
11424          * a write to any of the cursor register will cancel
11425          * an already armed cursor update. Thus leaving out
11426          * the CURBASE write after CURPOS could lead to a
11427          * cursor that doesn't appear to move, or even change
11428          * shape. Thus we always write CURBASE.
11429          *
11430          * The other registers are armed by by the CURBASE write
11431          * except when the plane is getting enabled at which time
11432          * the CURCNTR write arms the update.
11433          */
11434
11435         if (INTEL_GEN(dev_priv) >= 9)
11436                 skl_write_cursor_wm(plane, crtc_state);
11437
11438         if (plane->cursor.base != base ||
11439             plane->cursor.size != fbc_ctl ||
11440             plane->cursor.cntl != cntl) {
11441                 if (HAS_CUR_FBC(dev_priv))
11442                         I915_WRITE_FW(CUR_FBC_CTL(pipe), fbc_ctl);
11443                 I915_WRITE_FW(CURCNTR(pipe), cntl);
11444                 I915_WRITE_FW(CURPOS(pipe), pos);
11445                 I915_WRITE_FW(CURBASE(pipe), base);
11446
11447                 plane->cursor.base = base;
11448                 plane->cursor.size = fbc_ctl;
11449                 plane->cursor.cntl = cntl;
11450         } else {
11451                 I915_WRITE_FW(CURPOS(pipe), pos);
11452                 I915_WRITE_FW(CURBASE(pipe), base);
11453         }
11454
11455         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
11456 }
11457
11458 static void i9xx_disable_cursor(struct intel_plane *plane,
11459                                 const struct intel_crtc_state *crtc_state)
11460 {
11461         i9xx_update_cursor(plane, crtc_state, NULL);
11462 }
11463
11464 static bool i9xx_cursor_get_hw_state(struct intel_plane *plane,
11465                                      enum pipe *pipe)
11466 {
11467         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11468         enum intel_display_power_domain power_domain;
11469         intel_wakeref_t wakeref;
11470         bool ret;
11471         u32 val;
11472
11473         /*
11474          * Not 100% correct for planes that can move between pipes,
11475          * but that's only the case for gen2-3 which don't have any
11476          * display power wells.
11477          */
11478         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
11479         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
11480         if (!wakeref)
11481                 return false;
11482
11483         val = I915_READ(CURCNTR(plane->pipe));
11484
11485         ret = val & MCURSOR_MODE;
11486
11487         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
11488                 *pipe = plane->pipe;
11489         else
11490                 *pipe = (val & MCURSOR_PIPE_SELECT_MASK) >>
11491                         MCURSOR_PIPE_SELECT_SHIFT;
11492
11493         intel_display_power_put(dev_priv, power_domain, wakeref);
11494
11495         return ret;
11496 }
11497
11498 /* VESA 640x480x72Hz mode to set on the pipe */
11499 static const struct drm_display_mode load_detect_mode = {
11500         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
11501                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
11502 };
11503
11504 struct drm_framebuffer *
11505 intel_framebuffer_create(struct drm_i915_gem_object *obj,
11506                          struct drm_mode_fb_cmd2 *mode_cmd)
11507 {
11508         struct intel_framebuffer *intel_fb;
11509         int ret;
11510
11511         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
11512         if (!intel_fb)
11513                 return ERR_PTR(-ENOMEM);
11514
11515         ret = intel_framebuffer_init(intel_fb, obj, mode_cmd);
11516         if (ret)
11517                 goto err;
11518
11519         return &intel_fb->base;
11520
11521 err:
11522         kfree(intel_fb);
11523         return ERR_PTR(ret);
11524 }
11525
11526 static int intel_modeset_disable_planes(struct drm_atomic_state *state,
11527                                         struct drm_crtc *crtc)
11528 {
11529         struct drm_plane *plane;
11530         struct drm_plane_state *plane_state;
11531         int ret, i;
11532
11533         ret = drm_atomic_add_affected_planes(state, crtc);
11534         if (ret)
11535                 return ret;
11536
11537         for_each_new_plane_in_state(state, plane, plane_state, i) {
11538                 if (plane_state->crtc != crtc)
11539                         continue;
11540
11541                 ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
11542                 if (ret)
11543                         return ret;
11544
11545                 drm_atomic_set_fb_for_plane(plane_state, NULL);
11546         }
11547
11548         return 0;
11549 }
11550
11551 int intel_get_load_detect_pipe(struct drm_connector *connector,
11552                                struct intel_load_detect_pipe *old,
11553                                struct drm_modeset_acquire_ctx *ctx)
11554 {
11555         struct intel_crtc *intel_crtc;
11556         struct intel_encoder *intel_encoder =
11557                 intel_attached_encoder(connector);
11558         struct drm_crtc *possible_crtc;
11559         struct drm_encoder *encoder = &intel_encoder->base;
11560         struct drm_crtc *crtc = NULL;
11561         struct drm_device *dev = encoder->dev;
11562         struct drm_i915_private *dev_priv = to_i915(dev);
11563         struct drm_mode_config *config = &dev->mode_config;
11564         struct drm_atomic_state *state = NULL, *restore_state = NULL;
11565         struct drm_connector_state *connector_state;
11566         struct intel_crtc_state *crtc_state;
11567         int ret, i = -1;
11568
11569         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
11570                       connector->base.id, connector->name,
11571                       encoder->base.id, encoder->name);
11572
11573         old->restore_state = NULL;
11574
11575         WARN_ON(!drm_modeset_is_locked(&config->connection_mutex));
11576
11577         /*
11578          * Algorithm gets a little messy:
11579          *
11580          *   - if the connector already has an assigned crtc, use it (but make
11581          *     sure it's on first)
11582          *
11583          *   - try to find the first unused crtc that can drive this connector,
11584          *     and use that if we find one
11585          */
11586
11587         /* See if we already have a CRTC for this connector */
11588         if (connector->state->crtc) {
11589                 crtc = connector->state->crtc;
11590
11591                 ret = drm_modeset_lock(&crtc->mutex, ctx);
11592                 if (ret)
11593                         goto fail;
11594
11595                 /* Make sure the crtc and connector are running */
11596                 goto found;
11597         }
11598
11599         /* Find an unused one (if possible) */
11600         for_each_crtc(dev, possible_crtc) {
11601                 i++;
11602                 if (!(encoder->possible_crtcs & (1 << i)))
11603                         continue;
11604
11605                 ret = drm_modeset_lock(&possible_crtc->mutex, ctx);
11606                 if (ret)
11607                         goto fail;
11608
11609                 if (possible_crtc->state->enable) {
11610                         drm_modeset_unlock(&possible_crtc->mutex);
11611                         continue;
11612                 }
11613
11614                 crtc = possible_crtc;
11615                 break;
11616         }
11617
11618         /*
11619          * If we didn't find an unused CRTC, don't use any.
11620          */
11621         if (!crtc) {
11622                 DRM_DEBUG_KMS("no pipe available for load-detect\n");
11623                 ret = -ENODEV;
11624                 goto fail;
11625         }
11626
11627 found:
11628         intel_crtc = to_intel_crtc(crtc);
11629
11630         state = drm_atomic_state_alloc(dev);
11631         restore_state = drm_atomic_state_alloc(dev);
11632         if (!state || !restore_state) {
11633                 ret = -ENOMEM;
11634                 goto fail;
11635         }
11636
11637         state->acquire_ctx = ctx;
11638         restore_state->acquire_ctx = ctx;
11639
11640         connector_state = drm_atomic_get_connector_state(state, connector);
11641         if (IS_ERR(connector_state)) {
11642                 ret = PTR_ERR(connector_state);
11643                 goto fail;
11644         }
11645
11646         ret = drm_atomic_set_crtc_for_connector(connector_state, crtc);
11647         if (ret)
11648                 goto fail;
11649
11650         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
11651         if (IS_ERR(crtc_state)) {
11652                 ret = PTR_ERR(crtc_state);
11653                 goto fail;
11654         }
11655
11656         crtc_state->uapi.active = true;
11657
11658         ret = drm_atomic_set_mode_for_crtc(&crtc_state->uapi,
11659                                            &load_detect_mode);
11660         if (ret)
11661                 goto fail;
11662
11663         ret = intel_modeset_disable_planes(state, crtc);
11664         if (ret)
11665                 goto fail;
11666
11667         ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector));
11668         if (!ret)
11669                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, crtc));
11670         if (!ret)
11671                 ret = drm_atomic_add_affected_planes(restore_state, crtc);
11672         if (ret) {
11673                 DRM_DEBUG_KMS("Failed to create a copy of old state to restore: %i\n", ret);
11674                 goto fail;
11675         }
11676
11677         ret = drm_atomic_commit(state);
11678         if (ret) {
11679                 DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
11680                 goto fail;
11681         }
11682
11683         old->restore_state = restore_state;
11684         drm_atomic_state_put(state);
11685
11686         /* let the connector get through one full cycle before testing */
11687         intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
11688         return true;
11689
11690 fail:
11691         if (state) {
11692                 drm_atomic_state_put(state);
11693                 state = NULL;
11694         }
11695         if (restore_state) {
11696                 drm_atomic_state_put(restore_state);
11697                 restore_state = NULL;
11698         }
11699
11700         if (ret == -EDEADLK)
11701                 return ret;
11702
11703         return false;
11704 }
11705
11706 void intel_release_load_detect_pipe(struct drm_connector *connector,
11707                                     struct intel_load_detect_pipe *old,
11708                                     struct drm_modeset_acquire_ctx *ctx)
11709 {
11710         struct intel_encoder *intel_encoder =
11711                 intel_attached_encoder(connector);
11712         struct drm_encoder *encoder = &intel_encoder->base;
11713         struct drm_atomic_state *state = old->restore_state;
11714         int ret;
11715
11716         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
11717                       connector->base.id, connector->name,
11718                       encoder->base.id, encoder->name);
11719
11720         if (!state)
11721                 return;
11722
11723         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
11724         if (ret)
11725                 DRM_DEBUG_KMS("Couldn't release load detect pipe: %i\n", ret);
11726         drm_atomic_state_put(state);
11727 }
11728
11729 static int i9xx_pll_refclk(struct drm_device *dev,
11730                            const struct intel_crtc_state *pipe_config)
11731 {
11732         struct drm_i915_private *dev_priv = to_i915(dev);
11733         u32 dpll = pipe_config->dpll_hw_state.dpll;
11734
11735         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
11736                 return dev_priv->vbt.lvds_ssc_freq;
11737         else if (HAS_PCH_SPLIT(dev_priv))
11738                 return 120000;
11739         else if (!IS_GEN(dev_priv, 2))
11740                 return 96000;
11741         else
11742                 return 48000;
11743 }
11744
11745 /* Returns the clock of the currently programmed mode of the given pipe. */
11746 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
11747                                 struct intel_crtc_state *pipe_config)
11748 {
11749         struct drm_device *dev = crtc->base.dev;
11750         struct drm_i915_private *dev_priv = to_i915(dev);
11751         enum pipe pipe = crtc->pipe;
11752         u32 dpll = pipe_config->dpll_hw_state.dpll;
11753         u32 fp;
11754         struct dpll clock;
11755         int port_clock;
11756         int refclk = i9xx_pll_refclk(dev, pipe_config);
11757
11758         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
11759                 fp = pipe_config->dpll_hw_state.fp0;
11760         else
11761                 fp = pipe_config->dpll_hw_state.fp1;
11762
11763         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
11764         if (IS_PINEVIEW(dev_priv)) {
11765                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
11766                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
11767         } else {
11768                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
11769                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
11770         }
11771
11772         if (!IS_GEN(dev_priv, 2)) {
11773                 if (IS_PINEVIEW(dev_priv))
11774                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
11775                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
11776                 else
11777                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
11778                                DPLL_FPA01_P1_POST_DIV_SHIFT);
11779
11780                 switch (dpll & DPLL_MODE_MASK) {
11781                 case DPLLB_MODE_DAC_SERIAL:
11782                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
11783                                 5 : 10;
11784                         break;
11785                 case DPLLB_MODE_LVDS:
11786                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
11787                                 7 : 14;
11788                         break;
11789                 default:
11790                         DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed "
11791                                   "mode\n", (int)(dpll & DPLL_MODE_MASK));
11792                         return;
11793                 }
11794
11795                 if (IS_PINEVIEW(dev_priv))
11796                         port_clock = pnv_calc_dpll_params(refclk, &clock);
11797                 else
11798                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
11799         } else {
11800                 u32 lvds = IS_I830(dev_priv) ? 0 : I915_READ(LVDS);
11801                 bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN);
11802
11803                 if (is_lvds) {
11804                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
11805                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
11806
11807                         if (lvds & LVDS_CLKB_POWER_UP)
11808                                 clock.p2 = 7;
11809                         else
11810                                 clock.p2 = 14;
11811                 } else {
11812                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
11813                                 clock.p1 = 2;
11814                         else {
11815                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
11816                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
11817                         }
11818                         if (dpll & PLL_P2_DIVIDE_BY_4)
11819                                 clock.p2 = 4;
11820                         else
11821                                 clock.p2 = 2;
11822                 }
11823
11824                 port_clock = i9xx_calc_dpll_params(refclk, &clock);
11825         }
11826
11827         /*
11828          * This value includes pixel_multiplier. We will use
11829          * port_clock to compute adjusted_mode.crtc_clock in the
11830          * encoder's get_config() function.
11831          */
11832         pipe_config->port_clock = port_clock;
11833 }
11834
11835 int intel_dotclock_calculate(int link_freq,
11836                              const struct intel_link_m_n *m_n)
11837 {
11838         /*
11839          * The calculation for the data clock is:
11840          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
11841          * But we want to avoid losing precison if possible, so:
11842          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
11843          *
11844          * and the link clock is simpler:
11845          * link_clock = (m * link_clock) / n
11846          */
11847
11848         if (!m_n->link_n)
11849                 return 0;
11850
11851         return div_u64(mul_u32_u32(m_n->link_m, link_freq), m_n->link_n);
11852 }
11853
11854 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
11855                                    struct intel_crtc_state *pipe_config)
11856 {
11857         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11858
11859         /* read out port_clock from the DPLL */
11860         i9xx_crtc_clock_get(crtc, pipe_config);
11861
11862         /*
11863          * In case there is an active pipe without active ports,
11864          * we may need some idea for the dotclock anyway.
11865          * Calculate one based on the FDI configuration.
11866          */
11867         pipe_config->hw.adjusted_mode.crtc_clock =
11868                 intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
11869                                          &pipe_config->fdi_m_n);
11870 }
11871
11872 static void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
11873                                    struct intel_crtc *crtc)
11874 {
11875         memset(crtc_state, 0, sizeof(*crtc_state));
11876
11877         __drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
11878
11879         crtc_state->cpu_transcoder = INVALID_TRANSCODER;
11880         crtc_state->master_transcoder = INVALID_TRANSCODER;
11881         crtc_state->hsw_workaround_pipe = INVALID_PIPE;
11882         crtc_state->output_format = INTEL_OUTPUT_FORMAT_INVALID;
11883         crtc_state->scaler_state.scaler_id = -1;
11884         crtc_state->mst_master_transcoder = INVALID_TRANSCODER;
11885 }
11886
11887 static struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
11888 {
11889         struct intel_crtc_state *crtc_state;
11890
11891         crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
11892
11893         if (crtc_state)
11894                 intel_crtc_state_reset(crtc_state, crtc);
11895
11896         return crtc_state;
11897 }
11898
11899 /* Returns the currently programmed mode of the given encoder. */
11900 struct drm_display_mode *
11901 intel_encoder_current_mode(struct intel_encoder *encoder)
11902 {
11903         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
11904         struct intel_crtc_state *crtc_state;
11905         struct drm_display_mode *mode;
11906         struct intel_crtc *crtc;
11907         enum pipe pipe;
11908
11909         if (!encoder->get_hw_state(encoder, &pipe))
11910                 return NULL;
11911
11912         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
11913
11914         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
11915         if (!mode)
11916                 return NULL;
11917
11918         crtc_state = intel_crtc_state_alloc(crtc);
11919         if (!crtc_state) {
11920                 kfree(mode);
11921                 return NULL;
11922         }
11923
11924         if (!dev_priv->display.get_pipe_config(crtc, crtc_state)) {
11925                 kfree(crtc_state);
11926                 kfree(mode);
11927                 return NULL;
11928         }
11929
11930         encoder->get_config(encoder, crtc_state);
11931
11932         intel_mode_from_pipe_config(mode, crtc_state);
11933
11934         kfree(crtc_state);
11935
11936         return mode;
11937 }
11938
11939 static void intel_crtc_destroy(struct drm_crtc *crtc)
11940 {
11941         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11942
11943         drm_crtc_cleanup(crtc);
11944         kfree(intel_crtc);
11945 }
11946
11947 /**
11948  * intel_wm_need_update - Check whether watermarks need updating
11949  * @cur: current plane state
11950  * @new: new plane state
11951  *
11952  * Check current plane state versus the new one to determine whether
11953  * watermarks need to be recalculated.
11954  *
11955  * Returns true or false.
11956  */
11957 static bool intel_wm_need_update(const struct intel_plane_state *cur,
11958                                  struct intel_plane_state *new)
11959 {
11960         /* Update watermarks on tiling or size changes. */
11961         if (new->uapi.visible != cur->uapi.visible)
11962                 return true;
11963
11964         if (!cur->hw.fb || !new->hw.fb)
11965                 return false;
11966
11967         if (cur->hw.fb->modifier != new->hw.fb->modifier ||
11968             cur->hw.rotation != new->hw.rotation ||
11969             drm_rect_width(&new->uapi.src) != drm_rect_width(&cur->uapi.src) ||
11970             drm_rect_height(&new->uapi.src) != drm_rect_height(&cur->uapi.src) ||
11971             drm_rect_width(&new->uapi.dst) != drm_rect_width(&cur->uapi.dst) ||
11972             drm_rect_height(&new->uapi.dst) != drm_rect_height(&cur->uapi.dst))
11973                 return true;
11974
11975         return false;
11976 }
11977
11978 static bool needs_scaling(const struct intel_plane_state *state)
11979 {
11980         int src_w = drm_rect_width(&state->uapi.src) >> 16;
11981         int src_h = drm_rect_height(&state->uapi.src) >> 16;
11982         int dst_w = drm_rect_width(&state->uapi.dst);
11983         int dst_h = drm_rect_height(&state->uapi.dst);
11984
11985         return (src_w != dst_w || src_h != dst_h);
11986 }
11987
11988 int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state,
11989                                     struct intel_crtc_state *crtc_state,
11990                                     const struct intel_plane_state *old_plane_state,
11991                                     struct intel_plane_state *plane_state)
11992 {
11993         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
11994         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
11995         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11996         bool mode_changed = needs_modeset(crtc_state);
11997         bool was_crtc_enabled = old_crtc_state->hw.active;
11998         bool is_crtc_enabled = crtc_state->hw.active;
11999         bool turn_off, turn_on, visible, was_visible;
12000         int ret;
12001
12002         if (INTEL_GEN(dev_priv) >= 9 && plane->id != PLANE_CURSOR) {
12003                 ret = skl_update_scaler_plane(crtc_state, plane_state);
12004                 if (ret)
12005                         return ret;
12006         }
12007
12008         was_visible = old_plane_state->uapi.visible;
12009         visible = plane_state->uapi.visible;
12010
12011         if (!was_crtc_enabled && WARN_ON(was_visible))
12012                 was_visible = false;
12013
12014         /*
12015          * Visibility is calculated as if the crtc was on, but
12016          * after scaler setup everything depends on it being off
12017          * when the crtc isn't active.
12018          *
12019          * FIXME this is wrong for watermarks. Watermarks should also
12020          * be computed as if the pipe would be active. Perhaps move
12021          * per-plane wm computation to the .check_plane() hook, and
12022          * only combine the results from all planes in the current place?
12023          */
12024         if (!is_crtc_enabled) {
12025                 plane_state->uapi.visible = visible = false;
12026                 crtc_state->active_planes &= ~BIT(plane->id);
12027                 crtc_state->data_rate[plane->id] = 0;
12028                 crtc_state->min_cdclk[plane->id] = 0;
12029         }
12030
12031         if (!was_visible && !visible)
12032                 return 0;
12033
12034         turn_off = was_visible && (!visible || mode_changed);
12035         turn_on = visible && (!was_visible || mode_changed);
12036
12037         DRM_DEBUG_ATOMIC("[CRTC:%d:%s] with [PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n",
12038                          crtc->base.base.id, crtc->base.name,
12039                          plane->base.base.id, plane->base.name,
12040                          was_visible, visible,
12041                          turn_off, turn_on, mode_changed);
12042
12043         if (turn_on) {
12044                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
12045                         crtc_state->update_wm_pre = true;
12046
12047                 /* must disable cxsr around plane enable/disable */
12048                 if (plane->id != PLANE_CURSOR)
12049                         crtc_state->disable_cxsr = true;
12050         } else if (turn_off) {
12051                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
12052                         crtc_state->update_wm_post = true;
12053
12054                 /* must disable cxsr around plane enable/disable */
12055                 if (plane->id != PLANE_CURSOR)
12056                         crtc_state->disable_cxsr = true;
12057         } else if (intel_wm_need_update(old_plane_state, plane_state)) {
12058                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) {
12059                         /* FIXME bollocks */
12060                         crtc_state->update_wm_pre = true;
12061                         crtc_state->update_wm_post = true;
12062                 }
12063         }
12064
12065         if (visible || was_visible)
12066                 crtc_state->fb_bits |= plane->frontbuffer_bit;
12067
12068         /*
12069          * ILK/SNB DVSACNTR/Sprite Enable
12070          * IVB SPR_CTL/Sprite Enable
12071          * "When in Self Refresh Big FIFO mode, a write to enable the
12072          *  plane will be internally buffered and delayed while Big FIFO
12073          *  mode is exiting."
12074          *
12075          * Which means that enabling the sprite can take an extra frame
12076          * when we start in big FIFO mode (LP1+). Thus we need to drop
12077          * down to LP0 and wait for vblank in order to make sure the
12078          * sprite gets enabled on the next vblank after the register write.
12079          * Doing otherwise would risk enabling the sprite one frame after
12080          * we've already signalled flip completion. We can resume LP1+
12081          * once the sprite has been enabled.
12082          *
12083          *
12084          * WaCxSRDisabledForSpriteScaling:ivb
12085          * IVB SPR_SCALE/Scaling Enable
12086          * "Low Power watermarks must be disabled for at least one
12087          *  frame before enabling sprite scaling, and kept disabled
12088          *  until sprite scaling is disabled."
12089          *
12090          * ILK/SNB DVSASCALE/Scaling Enable
12091          * "When in Self Refresh Big FIFO mode, scaling enable will be
12092          *  masked off while Big FIFO mode is exiting."
12093          *
12094          * Despite the w/a only being listed for IVB we assume that
12095          * the ILK/SNB note has similar ramifications, hence we apply
12096          * the w/a on all three platforms.
12097          *
12098          * With experimental results seems this is needed also for primary
12099          * plane, not only sprite plane.
12100          */
12101         if (plane->id != PLANE_CURSOR &&
12102             (IS_GEN_RANGE(dev_priv, 5, 6) ||
12103              IS_IVYBRIDGE(dev_priv)) &&
12104             (turn_on || (!needs_scaling(old_plane_state) &&
12105                          needs_scaling(plane_state))))
12106                 crtc_state->disable_lp_wm = true;
12107
12108         return 0;
12109 }
12110
12111 static bool encoders_cloneable(const struct intel_encoder *a,
12112                                const struct intel_encoder *b)
12113 {
12114         /* masks could be asymmetric, so check both ways */
12115         return a == b || (a->cloneable & (1 << b->type) &&
12116                           b->cloneable & (1 << a->type));
12117 }
12118
12119 static bool check_single_encoder_cloning(struct drm_atomic_state *state,
12120                                          struct intel_crtc *crtc,
12121                                          struct intel_encoder *encoder)
12122 {
12123         struct intel_encoder *source_encoder;
12124         struct drm_connector *connector;
12125         struct drm_connector_state *connector_state;
12126         int i;
12127
12128         for_each_new_connector_in_state(state, connector, connector_state, i) {
12129                 if (connector_state->crtc != &crtc->base)
12130                         continue;
12131
12132                 source_encoder =
12133                         to_intel_encoder(connector_state->best_encoder);
12134                 if (!encoders_cloneable(encoder, source_encoder))
12135                         return false;
12136         }
12137
12138         return true;
12139 }
12140
12141 static int icl_add_linked_planes(struct intel_atomic_state *state)
12142 {
12143         struct intel_plane *plane, *linked;
12144         struct intel_plane_state *plane_state, *linked_plane_state;
12145         int i;
12146
12147         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12148                 linked = plane_state->planar_linked_plane;
12149
12150                 if (!linked)
12151                         continue;
12152
12153                 linked_plane_state = intel_atomic_get_plane_state(state, linked);
12154                 if (IS_ERR(linked_plane_state))
12155                         return PTR_ERR(linked_plane_state);
12156
12157                 WARN_ON(linked_plane_state->planar_linked_plane != plane);
12158                 WARN_ON(linked_plane_state->planar_slave == plane_state->planar_slave);
12159         }
12160
12161         return 0;
12162 }
12163
12164 static int icl_check_nv12_planes(struct intel_crtc_state *crtc_state)
12165 {
12166         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
12167         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12168         struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->uapi.state);
12169         struct intel_plane *plane, *linked;
12170         struct intel_plane_state *plane_state;
12171         int i;
12172
12173         if (INTEL_GEN(dev_priv) < 11)
12174                 return 0;
12175
12176         /*
12177          * Destroy all old plane links and make the slave plane invisible
12178          * in the crtc_state->active_planes mask.
12179          */
12180         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12181                 if (plane->pipe != crtc->pipe || !plane_state->planar_linked_plane)
12182                         continue;
12183
12184                 plane_state->planar_linked_plane = NULL;
12185                 if (plane_state->planar_slave && !plane_state->uapi.visible) {
12186                         crtc_state->active_planes &= ~BIT(plane->id);
12187                         crtc_state->update_planes |= BIT(plane->id);
12188                 }
12189
12190                 plane_state->planar_slave = false;
12191         }
12192
12193         if (!crtc_state->nv12_planes)
12194                 return 0;
12195
12196         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12197                 struct intel_plane_state *linked_state = NULL;
12198
12199                 if (plane->pipe != crtc->pipe ||
12200                     !(crtc_state->nv12_planes & BIT(plane->id)))
12201                         continue;
12202
12203                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, linked) {
12204                         if (!icl_is_nv12_y_plane(linked->id))
12205                                 continue;
12206
12207                         if (crtc_state->active_planes & BIT(linked->id))
12208                                 continue;
12209
12210                         linked_state = intel_atomic_get_plane_state(state, linked);
12211                         if (IS_ERR(linked_state))
12212                                 return PTR_ERR(linked_state);
12213
12214                         break;
12215                 }
12216
12217                 if (!linked_state) {
12218                         DRM_DEBUG_KMS("Need %d free Y planes for planar YUV\n",
12219                                       hweight8(crtc_state->nv12_planes));
12220
12221                         return -EINVAL;
12222                 }
12223
12224                 plane_state->planar_linked_plane = linked;
12225
12226                 linked_state->planar_slave = true;
12227                 linked_state->planar_linked_plane = plane;
12228                 crtc_state->active_planes |= BIT(linked->id);
12229                 crtc_state->update_planes |= BIT(linked->id);
12230                 DRM_DEBUG_KMS("Using %s as Y plane for %s\n", linked->base.name, plane->base.name);
12231
12232                 /* Copy parameters to slave plane */
12233                 linked_state->ctl = plane_state->ctl | PLANE_CTL_YUV420_Y_PLANE;
12234                 linked_state->color_ctl = plane_state->color_ctl;
12235                 memcpy(linked_state->color_plane, plane_state->color_plane,
12236                        sizeof(linked_state->color_plane));
12237
12238                 intel_plane_copy_uapi_to_hw_state(linked_state, plane_state);
12239                 linked_state->uapi.src = plane_state->uapi.src;
12240                 linked_state->uapi.dst = plane_state->uapi.dst;
12241
12242                 if (icl_is_hdr_plane(dev_priv, plane->id)) {
12243                         if (linked->id == PLANE_SPRITE5)
12244                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_7;
12245                         else if (linked->id == PLANE_SPRITE4)
12246                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_6;
12247                         else
12248                                 MISSING_CASE(linked->id);
12249                 }
12250         }
12251
12252         return 0;
12253 }
12254
12255 static bool c8_planes_changed(const struct intel_crtc_state *new_crtc_state)
12256 {
12257         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
12258         struct intel_atomic_state *state =
12259                 to_intel_atomic_state(new_crtc_state->uapi.state);
12260         const struct intel_crtc_state *old_crtc_state =
12261                 intel_atomic_get_old_crtc_state(state, crtc);
12262
12263         return !old_crtc_state->c8_planes != !new_crtc_state->c8_planes;
12264 }
12265
12266 static int icl_add_sync_mode_crtcs(struct intel_crtc_state *crtc_state)
12267 {
12268         struct drm_crtc *crtc = crtc_state->uapi.crtc;
12269         struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->uapi.state);
12270         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
12271         struct drm_connector *master_connector, *connector;
12272         struct drm_connector_state *connector_state;
12273         struct drm_connector_list_iter conn_iter;
12274         struct drm_crtc *master_crtc = NULL;
12275         struct drm_crtc_state *master_crtc_state;
12276         struct intel_crtc_state *master_pipe_config;
12277         int i, tile_group_id;
12278
12279         if (INTEL_GEN(dev_priv) < 11)
12280                 return 0;
12281
12282         /*
12283          * In case of tiled displays there could be one or more slaves but there is
12284          * only one master. Lets make the CRTC used by the connector corresponding
12285          * to the last horizonal and last vertical tile a master/genlock CRTC.
12286          * All the other CRTCs corresponding to other tiles of the same Tile group
12287          * are the slave CRTCs and hold a pointer to their genlock CRTC.
12288          */
12289         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
12290                 if (connector_state->crtc != crtc)
12291                         continue;
12292                 if (!connector->has_tile)
12293                         continue;
12294                 if (crtc_state->hw.mode.hdisplay != connector->tile_h_size ||
12295                     crtc_state->hw.mode.vdisplay != connector->tile_v_size)
12296                         return 0;
12297                 if (connector->tile_h_loc == connector->num_h_tile - 1 &&
12298                     connector->tile_v_loc == connector->num_v_tile - 1)
12299                         continue;
12300                 crtc_state->sync_mode_slaves_mask = 0;
12301                 tile_group_id = connector->tile_group->id;
12302                 drm_connector_list_iter_begin(&dev_priv->drm, &conn_iter);
12303                 drm_for_each_connector_iter(master_connector, &conn_iter) {
12304                         struct drm_connector_state *master_conn_state = NULL;
12305
12306                         if (!master_connector->has_tile)
12307                                 continue;
12308                         if (master_connector->tile_h_loc != master_connector->num_h_tile - 1 ||
12309                             master_connector->tile_v_loc != master_connector->num_v_tile - 1)
12310                                 continue;
12311                         if (master_connector->tile_group->id != tile_group_id)
12312                                 continue;
12313
12314                         master_conn_state = drm_atomic_get_connector_state(&state->base,
12315                                                                            master_connector);
12316                         if (IS_ERR(master_conn_state)) {
12317                                 drm_connector_list_iter_end(&conn_iter);
12318                                 return PTR_ERR(master_conn_state);
12319                         }
12320                         if (master_conn_state->crtc) {
12321                                 master_crtc = master_conn_state->crtc;
12322                                 break;
12323                         }
12324                 }
12325                 drm_connector_list_iter_end(&conn_iter);
12326
12327                 if (!master_crtc) {
12328                         DRM_DEBUG_KMS("Could not find Master CRTC for Slave CRTC %d\n",
12329                                       connector_state->crtc->base.id);
12330                         return -EINVAL;
12331                 }
12332
12333                 master_crtc_state = drm_atomic_get_crtc_state(&state->base,
12334                                                               master_crtc);
12335                 if (IS_ERR(master_crtc_state))
12336                         return PTR_ERR(master_crtc_state);
12337
12338                 master_pipe_config = to_intel_crtc_state(master_crtc_state);
12339                 crtc_state->master_transcoder = master_pipe_config->cpu_transcoder;
12340                 master_pipe_config->sync_mode_slaves_mask |=
12341                         BIT(crtc_state->cpu_transcoder);
12342                 DRM_DEBUG_KMS("Master Transcoder = %s added for Slave CRTC = %d, slave transcoder bitmask = %d\n",
12343                               transcoder_name(crtc_state->master_transcoder),
12344                               crtc_state->uapi.crtc->base.id,
12345                               master_pipe_config->sync_mode_slaves_mask);
12346         }
12347
12348         return 0;
12349 }
12350
12351 static int intel_crtc_atomic_check(struct intel_atomic_state *state,
12352                                    struct intel_crtc *crtc)
12353 {
12354         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12355         struct intel_crtc_state *crtc_state =
12356                 intel_atomic_get_new_crtc_state(state, crtc);
12357         bool mode_changed = needs_modeset(crtc_state);
12358         int ret;
12359
12360         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv) &&
12361             mode_changed && !crtc_state->hw.active)
12362                 crtc_state->update_wm_post = true;
12363
12364         if (mode_changed && crtc_state->hw.enable &&
12365             dev_priv->display.crtc_compute_clock &&
12366             !WARN_ON(crtc_state->shared_dpll)) {
12367                 ret = dev_priv->display.crtc_compute_clock(crtc, crtc_state);
12368                 if (ret)
12369                         return ret;
12370         }
12371
12372         /*
12373          * May need to update pipe gamma enable bits
12374          * when C8 planes are getting enabled/disabled.
12375          */
12376         if (c8_planes_changed(crtc_state))
12377                 crtc_state->uapi.color_mgmt_changed = true;
12378
12379         if (mode_changed || crtc_state->update_pipe ||
12380             crtc_state->uapi.color_mgmt_changed) {
12381                 ret = intel_color_check(crtc_state);
12382                 if (ret)
12383                         return ret;
12384         }
12385
12386         ret = 0;
12387         if (dev_priv->display.compute_pipe_wm) {
12388                 ret = dev_priv->display.compute_pipe_wm(crtc_state);
12389                 if (ret) {
12390                         DRM_DEBUG_KMS("Target pipe watermarks are invalid\n");
12391                         return ret;
12392                 }
12393         }
12394
12395         if (dev_priv->display.compute_intermediate_wm) {
12396                 if (WARN_ON(!dev_priv->display.compute_pipe_wm))
12397                         return 0;
12398
12399                 /*
12400                  * Calculate 'intermediate' watermarks that satisfy both the
12401                  * old state and the new state.  We can program these
12402                  * immediately.
12403                  */
12404                 ret = dev_priv->display.compute_intermediate_wm(crtc_state);
12405                 if (ret) {
12406                         DRM_DEBUG_KMS("No valid intermediate pipe watermarks are possible\n");
12407                         return ret;
12408                 }
12409         }
12410
12411         if (INTEL_GEN(dev_priv) >= 9) {
12412                 if (mode_changed || crtc_state->update_pipe)
12413                         ret = skl_update_scaler_crtc(crtc_state);
12414                 if (!ret)
12415                         ret = intel_atomic_setup_scalers(dev_priv, crtc,
12416                                                          crtc_state);
12417         }
12418
12419         if (HAS_IPS(dev_priv))
12420                 crtc_state->ips_enabled = hsw_compute_ips_config(crtc_state);
12421
12422         return ret;
12423 }
12424
12425 static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
12426 {
12427         struct intel_connector *connector;
12428         struct drm_connector_list_iter conn_iter;
12429
12430         drm_connector_list_iter_begin(dev, &conn_iter);
12431         for_each_intel_connector_iter(connector, &conn_iter) {
12432                 if (connector->base.state->crtc)
12433                         drm_connector_put(&connector->base);
12434
12435                 if (connector->base.encoder) {
12436                         connector->base.state->best_encoder =
12437                                 connector->base.encoder;
12438                         connector->base.state->crtc =
12439                                 connector->base.encoder->crtc;
12440
12441                         drm_connector_get(&connector->base);
12442                 } else {
12443                         connector->base.state->best_encoder = NULL;
12444                         connector->base.state->crtc = NULL;
12445                 }
12446         }
12447         drm_connector_list_iter_end(&conn_iter);
12448 }
12449
12450 static int
12451 compute_sink_pipe_bpp(const struct drm_connector_state *conn_state,
12452                       struct intel_crtc_state *pipe_config)
12453 {
12454         struct drm_connector *connector = conn_state->connector;
12455         const struct drm_display_info *info = &connector->display_info;
12456         int bpp;
12457
12458         switch (conn_state->max_bpc) {
12459         case 6 ... 7:
12460                 bpp = 6 * 3;
12461                 break;
12462         case 8 ... 9:
12463                 bpp = 8 * 3;
12464                 break;
12465         case 10 ... 11:
12466                 bpp = 10 * 3;
12467                 break;
12468         case 12:
12469                 bpp = 12 * 3;
12470                 break;
12471         default:
12472                 return -EINVAL;
12473         }
12474
12475         if (bpp < pipe_config->pipe_bpp) {
12476                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Limiting display bpp to %d instead of "
12477                               "EDID bpp %d, requested bpp %d, max platform bpp %d\n",
12478                               connector->base.id, connector->name,
12479                               bpp, 3 * info->bpc, 3 * conn_state->max_requested_bpc,
12480                               pipe_config->pipe_bpp);
12481
12482                 pipe_config->pipe_bpp = bpp;
12483         }
12484
12485         return 0;
12486 }
12487
12488 static int
12489 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
12490                           struct intel_crtc_state *pipe_config)
12491 {
12492         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12493         struct drm_atomic_state *state = pipe_config->uapi.state;
12494         struct drm_connector *connector;
12495         struct drm_connector_state *connector_state;
12496         int bpp, i;
12497
12498         if ((IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
12499             IS_CHERRYVIEW(dev_priv)))
12500                 bpp = 10*3;
12501         else if (INTEL_GEN(dev_priv) >= 5)
12502                 bpp = 12*3;
12503         else
12504                 bpp = 8*3;
12505
12506         pipe_config->pipe_bpp = bpp;
12507
12508         /* Clamp display bpp to connector max bpp */
12509         for_each_new_connector_in_state(state, connector, connector_state, i) {
12510                 int ret;
12511
12512                 if (connector_state->crtc != &crtc->base)
12513                         continue;
12514
12515                 ret = compute_sink_pipe_bpp(connector_state, pipe_config);
12516                 if (ret)
12517                         return ret;
12518         }
12519
12520         return 0;
12521 }
12522
12523 static void intel_dump_crtc_timings(const struct drm_display_mode *mode)
12524 {
12525         DRM_DEBUG_KMS("crtc timings: %d %d %d %d %d %d %d %d %d, "
12526                       "type: 0x%x flags: 0x%x\n",
12527                       mode->crtc_clock,
12528                       mode->crtc_hdisplay, mode->crtc_hsync_start,
12529                       mode->crtc_hsync_end, mode->crtc_htotal,
12530                       mode->crtc_vdisplay, mode->crtc_vsync_start,
12531                       mode->crtc_vsync_end, mode->crtc_vtotal,
12532                       mode->type, mode->flags);
12533 }
12534
12535 static inline void
12536 intel_dump_m_n_config(const struct intel_crtc_state *pipe_config,
12537                       const char *id, unsigned int lane_count,
12538                       const struct intel_link_m_n *m_n)
12539 {
12540         DRM_DEBUG_KMS("%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
12541                       id, lane_count,
12542                       m_n->gmch_m, m_n->gmch_n,
12543                       m_n->link_m, m_n->link_n, m_n->tu);
12544 }
12545
12546 static void
12547 intel_dump_infoframe(struct drm_i915_private *dev_priv,
12548                      const union hdmi_infoframe *frame)
12549 {
12550         if ((drm_debug & DRM_UT_KMS) == 0)
12551                 return;
12552
12553         hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, frame);
12554 }
12555
12556 #define OUTPUT_TYPE(x) [INTEL_OUTPUT_ ## x] = #x
12557
12558 static const char * const output_type_str[] = {
12559         OUTPUT_TYPE(UNUSED),
12560         OUTPUT_TYPE(ANALOG),
12561         OUTPUT_TYPE(DVO),
12562         OUTPUT_TYPE(SDVO),
12563         OUTPUT_TYPE(LVDS),
12564         OUTPUT_TYPE(TVOUT),
12565         OUTPUT_TYPE(HDMI),
12566         OUTPUT_TYPE(DP),
12567         OUTPUT_TYPE(EDP),
12568         OUTPUT_TYPE(DSI),
12569         OUTPUT_TYPE(DDI),
12570         OUTPUT_TYPE(DP_MST),
12571 };
12572
12573 #undef OUTPUT_TYPE
12574
12575 static void snprintf_output_types(char *buf, size_t len,
12576                                   unsigned int output_types)
12577 {
12578         char *str = buf;
12579         int i;
12580
12581         str[0] = '\0';
12582
12583         for (i = 0; i < ARRAY_SIZE(output_type_str); i++) {
12584                 int r;
12585
12586                 if ((output_types & BIT(i)) == 0)
12587                         continue;
12588
12589                 r = snprintf(str, len, "%s%s",
12590                              str != buf ? "," : "", output_type_str[i]);
12591                 if (r >= len)
12592                         break;
12593                 str += r;
12594                 len -= r;
12595
12596                 output_types &= ~BIT(i);
12597         }
12598
12599         WARN_ON_ONCE(output_types != 0);
12600 }
12601
12602 static const char * const output_format_str[] = {
12603         [INTEL_OUTPUT_FORMAT_INVALID] = "Invalid",
12604         [INTEL_OUTPUT_FORMAT_RGB] = "RGB",
12605         [INTEL_OUTPUT_FORMAT_YCBCR420] = "YCBCR4:2:0",
12606         [INTEL_OUTPUT_FORMAT_YCBCR444] = "YCBCR4:4:4",
12607 };
12608
12609 static const char *output_formats(enum intel_output_format format)
12610 {
12611         if (format >= ARRAY_SIZE(output_format_str))
12612                 format = INTEL_OUTPUT_FORMAT_INVALID;
12613         return output_format_str[format];
12614 }
12615
12616 static void intel_dump_plane_state(const struct intel_plane_state *plane_state)
12617 {
12618         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
12619         const struct drm_framebuffer *fb = plane_state->hw.fb;
12620         struct drm_format_name_buf format_name;
12621
12622         if (!fb) {
12623                 DRM_DEBUG_KMS("[PLANE:%d:%s] fb: [NOFB], visible: %s\n",
12624                               plane->base.base.id, plane->base.name,
12625                               yesno(plane_state->uapi.visible));
12626                 return;
12627         }
12628
12629         DRM_DEBUG_KMS("[PLANE:%d:%s] fb: [FB:%d] %ux%u format = %s, visible: %s\n",
12630                       plane->base.base.id, plane->base.name,
12631                       fb->base.id, fb->width, fb->height,
12632                       drm_get_format_name(fb->format->format, &format_name),
12633                       yesno(plane_state->uapi.visible));
12634         DRM_DEBUG_KMS("\trotation: 0x%x, scaler: %d\n",
12635                       plane_state->hw.rotation, plane_state->scaler_id);
12636         if (plane_state->uapi.visible)
12637                 DRM_DEBUG_KMS("\tsrc: " DRM_RECT_FP_FMT " dst: " DRM_RECT_FMT "\n",
12638                               DRM_RECT_FP_ARG(&plane_state->uapi.src),
12639                               DRM_RECT_ARG(&plane_state->uapi.dst));
12640 }
12641
12642 static void intel_dump_pipe_config(const struct intel_crtc_state *pipe_config,
12643                                    struct intel_atomic_state *state,
12644                                    const char *context)
12645 {
12646         struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
12647         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12648         const struct intel_plane_state *plane_state;
12649         struct intel_plane *plane;
12650         char buf[64];
12651         int i;
12652
12653         DRM_DEBUG_KMS("[CRTC:%d:%s] enable: %s %s\n",
12654                       crtc->base.base.id, crtc->base.name,
12655                       yesno(pipe_config->hw.enable), context);
12656
12657         if (!pipe_config->hw.enable)
12658                 goto dump_planes;
12659
12660         snprintf_output_types(buf, sizeof(buf), pipe_config->output_types);
12661         DRM_DEBUG_KMS("active: %s, output_types: %s (0x%x), output format: %s\n",
12662                       yesno(pipe_config->hw.active),
12663                       buf, pipe_config->output_types,
12664                       output_formats(pipe_config->output_format));
12665
12666         DRM_DEBUG_KMS("cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
12667                       transcoder_name(pipe_config->cpu_transcoder),
12668                       pipe_config->pipe_bpp, pipe_config->dither);
12669
12670         if (pipe_config->has_pch_encoder)
12671                 intel_dump_m_n_config(pipe_config, "fdi",
12672                                       pipe_config->fdi_lanes,
12673                                       &pipe_config->fdi_m_n);
12674
12675         if (intel_crtc_has_dp_encoder(pipe_config)) {
12676                 intel_dump_m_n_config(pipe_config, "dp m_n",
12677                                 pipe_config->lane_count, &pipe_config->dp_m_n);
12678                 if (pipe_config->has_drrs)
12679                         intel_dump_m_n_config(pipe_config, "dp m2_n2",
12680                                               pipe_config->lane_count,
12681                                               &pipe_config->dp_m2_n2);
12682         }
12683
12684         DRM_DEBUG_KMS("audio: %i, infoframes: %i, infoframes enabled: 0x%x\n",
12685                       pipe_config->has_audio, pipe_config->has_infoframe,
12686                       pipe_config->infoframes.enable);
12687
12688         if (pipe_config->infoframes.enable &
12689             intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GENERAL_CONTROL))
12690                 DRM_DEBUG_KMS("GCP: 0x%x\n", pipe_config->infoframes.gcp);
12691         if (pipe_config->infoframes.enable &
12692             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_AVI))
12693                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.avi);
12694         if (pipe_config->infoframes.enable &
12695             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_SPD))
12696                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.spd);
12697         if (pipe_config->infoframes.enable &
12698             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_VENDOR))
12699                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.hdmi);
12700
12701         DRM_DEBUG_KMS("requested mode:\n");
12702         drm_mode_debug_printmodeline(&pipe_config->hw.mode);
12703         DRM_DEBUG_KMS("adjusted mode:\n");
12704         drm_mode_debug_printmodeline(&pipe_config->hw.adjusted_mode);
12705         intel_dump_crtc_timings(&pipe_config->hw.adjusted_mode);
12706         DRM_DEBUG_KMS("port clock: %d, pipe src size: %dx%d, pixel rate %d\n",
12707                       pipe_config->port_clock,
12708                       pipe_config->pipe_src_w, pipe_config->pipe_src_h,
12709                       pipe_config->pixel_rate);
12710
12711         if (INTEL_GEN(dev_priv) >= 9)
12712                 DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
12713                               crtc->num_scalers,
12714                               pipe_config->scaler_state.scaler_users,
12715                               pipe_config->scaler_state.scaler_id);
12716
12717         if (HAS_GMCH(dev_priv))
12718                 DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
12719                               pipe_config->gmch_pfit.control,
12720                               pipe_config->gmch_pfit.pgm_ratios,
12721                               pipe_config->gmch_pfit.lvds_border_bits);
12722         else
12723                 DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s, force thru: %s\n",
12724                               pipe_config->pch_pfit.pos,
12725                               pipe_config->pch_pfit.size,
12726                               enableddisabled(pipe_config->pch_pfit.enabled),
12727                               yesno(pipe_config->pch_pfit.force_thru));
12728
12729         DRM_DEBUG_KMS("ips: %i, double wide: %i\n",
12730                       pipe_config->ips_enabled, pipe_config->double_wide);
12731
12732         intel_dpll_dump_hw_state(dev_priv, &pipe_config->dpll_hw_state);
12733
12734         if (IS_CHERRYVIEW(dev_priv))
12735                 DRM_DEBUG_KMS("cgm_mode: 0x%x gamma_mode: 0x%x gamma_enable: %d csc_enable: %d\n",
12736                               pipe_config->cgm_mode, pipe_config->gamma_mode,
12737                               pipe_config->gamma_enable, pipe_config->csc_enable);
12738         else
12739                 DRM_DEBUG_KMS("csc_mode: 0x%x gamma_mode: 0x%x gamma_enable: %d csc_enable: %d\n",
12740                               pipe_config->csc_mode, pipe_config->gamma_mode,
12741                               pipe_config->gamma_enable, pipe_config->csc_enable);
12742
12743         DRM_DEBUG_KMS("MST master transcoder: %s\n",
12744                       transcoder_name(pipe_config->mst_master_transcoder));
12745
12746 dump_planes:
12747         if (!state)
12748                 return;
12749
12750         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12751                 if (plane->pipe == crtc->pipe)
12752                         intel_dump_plane_state(plane_state);
12753         }
12754 }
12755
12756 static bool check_digital_port_conflicts(struct intel_atomic_state *state)
12757 {
12758         struct drm_device *dev = state->base.dev;
12759         struct drm_connector *connector;
12760         struct drm_connector_list_iter conn_iter;
12761         unsigned int used_ports = 0;
12762         unsigned int used_mst_ports = 0;
12763         bool ret = true;
12764
12765         /*
12766          * We're going to peek into connector->state,
12767          * hence connection_mutex must be held.
12768          */
12769         drm_modeset_lock_assert_held(&dev->mode_config.connection_mutex);
12770
12771         /*
12772          * Walk the connector list instead of the encoder
12773          * list to detect the problem on ddi platforms
12774          * where there's just one encoder per digital port.
12775          */
12776         drm_connector_list_iter_begin(dev, &conn_iter);
12777         drm_for_each_connector_iter(connector, &conn_iter) {
12778                 struct drm_connector_state *connector_state;
12779                 struct intel_encoder *encoder;
12780
12781                 connector_state =
12782                         drm_atomic_get_new_connector_state(&state->base,
12783                                                            connector);
12784                 if (!connector_state)
12785                         connector_state = connector->state;
12786
12787                 if (!connector_state->best_encoder)
12788                         continue;
12789
12790                 encoder = to_intel_encoder(connector_state->best_encoder);
12791
12792                 WARN_ON(!connector_state->crtc);
12793
12794                 switch (encoder->type) {
12795                         unsigned int port_mask;
12796                 case INTEL_OUTPUT_DDI:
12797                         if (WARN_ON(!HAS_DDI(to_i915(dev))))
12798                                 break;
12799                         /* else, fall through */
12800                 case INTEL_OUTPUT_DP:
12801                 case INTEL_OUTPUT_HDMI:
12802                 case INTEL_OUTPUT_EDP:
12803                         port_mask = 1 << encoder->port;
12804
12805                         /* the same port mustn't appear more than once */
12806                         if (used_ports & port_mask)
12807                                 ret = false;
12808
12809                         used_ports |= port_mask;
12810                         break;
12811                 case INTEL_OUTPUT_DP_MST:
12812                         used_mst_ports |=
12813                                 1 << encoder->port;
12814                         break;
12815                 default:
12816                         break;
12817                 }
12818         }
12819         drm_connector_list_iter_end(&conn_iter);
12820
12821         /* can't mix MST and SST/HDMI on the same port */
12822         if (used_ports & used_mst_ports)
12823                 return false;
12824
12825         return ret;
12826 }
12827
12828 static void
12829 intel_crtc_copy_uapi_to_hw_state_nomodeset(struct intel_crtc_state *crtc_state)
12830 {
12831         intel_crtc_copy_color_blobs(crtc_state);
12832 }
12833
12834 static void
12835 intel_crtc_copy_uapi_to_hw_state(struct intel_crtc_state *crtc_state)
12836 {
12837         crtc_state->hw.enable = crtc_state->uapi.enable;
12838         crtc_state->hw.active = crtc_state->uapi.active;
12839         crtc_state->hw.mode = crtc_state->uapi.mode;
12840         crtc_state->hw.adjusted_mode = crtc_state->uapi.adjusted_mode;
12841         intel_crtc_copy_uapi_to_hw_state_nomodeset(crtc_state);
12842 }
12843
12844 static void intel_crtc_copy_hw_to_uapi_state(struct intel_crtc_state *crtc_state)
12845 {
12846         crtc_state->uapi.enable = crtc_state->hw.enable;
12847         crtc_state->uapi.active = crtc_state->hw.active;
12848         WARN_ON(drm_atomic_set_mode_for_crtc(&crtc_state->uapi, &crtc_state->hw.mode) < 0);
12849
12850         crtc_state->uapi.adjusted_mode = crtc_state->hw.adjusted_mode;
12851
12852         /* copy color blobs to uapi */
12853         drm_property_replace_blob(&crtc_state->uapi.degamma_lut,
12854                                   crtc_state->hw.degamma_lut);
12855         drm_property_replace_blob(&crtc_state->uapi.gamma_lut,
12856                                   crtc_state->hw.gamma_lut);
12857         drm_property_replace_blob(&crtc_state->uapi.ctm,
12858                                   crtc_state->hw.ctm);
12859 }
12860
12861 static int
12862 intel_crtc_prepare_cleared_state(struct intel_crtc_state *crtc_state)
12863 {
12864         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
12865         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12866         struct intel_crtc_state *saved_state;
12867
12868         saved_state = intel_crtc_state_alloc(crtc);
12869         if (!saved_state)
12870                 return -ENOMEM;
12871
12872         /* free the old crtc_state->hw members */
12873         intel_crtc_free_hw_state(crtc_state);
12874
12875         /* FIXME: before the switch to atomic started, a new pipe_config was
12876          * kzalloc'd. Code that depends on any field being zero should be
12877          * fixed, so that the crtc_state can be safely duplicated. For now,
12878          * only fields that are know to not cause problems are preserved. */
12879
12880         saved_state->uapi = crtc_state->uapi;
12881         saved_state->scaler_state = crtc_state->scaler_state;
12882         saved_state->shared_dpll = crtc_state->shared_dpll;
12883         saved_state->dpll_hw_state = crtc_state->dpll_hw_state;
12884         memcpy(saved_state->icl_port_dplls, crtc_state->icl_port_dplls,
12885                sizeof(saved_state->icl_port_dplls));
12886         saved_state->crc_enabled = crtc_state->crc_enabled;
12887         if (IS_G4X(dev_priv) ||
12888             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
12889                 saved_state->wm = crtc_state->wm;
12890         /*
12891          * Save the slave bitmask which gets filled for master crtc state during
12892          * slave atomic check call.
12893          */
12894         if (is_trans_port_sync_master(crtc_state))
12895                 saved_state->sync_mode_slaves_mask =
12896                         crtc_state->sync_mode_slaves_mask;
12897
12898         memcpy(crtc_state, saved_state, sizeof(*crtc_state));
12899         kfree(saved_state);
12900
12901         intel_crtc_copy_uapi_to_hw_state(crtc_state);
12902
12903         return 0;
12904 }
12905
12906 static int
12907 intel_modeset_pipe_config(struct intel_crtc_state *pipe_config)
12908 {
12909         struct drm_crtc *crtc = pipe_config->uapi.crtc;
12910         struct drm_atomic_state *state = pipe_config->uapi.state;
12911         struct intel_encoder *encoder;
12912         struct drm_connector *connector;
12913         struct drm_connector_state *connector_state;
12914         int base_bpp, ret;
12915         int i;
12916         bool retry = true;
12917
12918         pipe_config->cpu_transcoder =
12919                 (enum transcoder) to_intel_crtc(crtc)->pipe;
12920
12921         /*
12922          * Sanitize sync polarity flags based on requested ones. If neither
12923          * positive or negative polarity is requested, treat this as meaning
12924          * negative polarity.
12925          */
12926         if (!(pipe_config->hw.adjusted_mode.flags &
12927               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
12928                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
12929
12930         if (!(pipe_config->hw.adjusted_mode.flags &
12931               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
12932                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
12933
12934         ret = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
12935                                         pipe_config);
12936         if (ret)
12937                 return ret;
12938
12939         base_bpp = pipe_config->pipe_bpp;
12940
12941         /*
12942          * Determine the real pipe dimensions. Note that stereo modes can
12943          * increase the actual pipe size due to the frame doubling and
12944          * insertion of additional space for blanks between the frame. This
12945          * is stored in the crtc timings. We use the requested mode to do this
12946          * computation to clearly distinguish it from the adjusted mode, which
12947          * can be changed by the connectors in the below retry loop.
12948          */
12949         drm_mode_get_hv_timing(&pipe_config->hw.mode,
12950                                &pipe_config->pipe_src_w,
12951                                &pipe_config->pipe_src_h);
12952
12953         for_each_new_connector_in_state(state, connector, connector_state, i) {
12954                 if (connector_state->crtc != crtc)
12955                         continue;
12956
12957                 encoder = to_intel_encoder(connector_state->best_encoder);
12958
12959                 if (!check_single_encoder_cloning(state, to_intel_crtc(crtc), encoder)) {
12960                         DRM_DEBUG_KMS("rejecting invalid cloning configuration\n");
12961                         return -EINVAL;
12962                 }
12963
12964                 /*
12965                  * Determine output_types before calling the .compute_config()
12966                  * hooks so that the hooks can use this information safely.
12967                  */
12968                 if (encoder->compute_output_type)
12969                         pipe_config->output_types |=
12970                                 BIT(encoder->compute_output_type(encoder, pipe_config,
12971                                                                  connector_state));
12972                 else
12973                         pipe_config->output_types |= BIT(encoder->type);
12974         }
12975
12976 encoder_retry:
12977         /* Ensure the port clock defaults are reset when retrying. */
12978         pipe_config->port_clock = 0;
12979         pipe_config->pixel_multiplier = 1;
12980
12981         /* Fill in default crtc timings, allow encoders to overwrite them. */
12982         drm_mode_set_crtcinfo(&pipe_config->hw.adjusted_mode,
12983                               CRTC_STEREO_DOUBLE);
12984
12985         /* Set the crtc_state defaults for trans_port_sync */
12986         pipe_config->master_transcoder = INVALID_TRANSCODER;
12987         ret = icl_add_sync_mode_crtcs(pipe_config);
12988         if (ret) {
12989                 DRM_DEBUG_KMS("Cannot assign Sync Mode CRTCs: %d\n",
12990                               ret);
12991                 return ret;
12992         }
12993
12994         /* Pass our mode to the connectors and the CRTC to give them a chance to
12995          * adjust it according to limitations or connector properties, and also
12996          * a chance to reject the mode entirely.
12997          */
12998         for_each_new_connector_in_state(state, connector, connector_state, i) {
12999                 if (connector_state->crtc != crtc)
13000                         continue;
13001
13002                 encoder = to_intel_encoder(connector_state->best_encoder);
13003                 ret = encoder->compute_config(encoder, pipe_config,
13004                                               connector_state);
13005                 if (ret < 0) {
13006                         if (ret != -EDEADLK)
13007                                 DRM_DEBUG_KMS("Encoder config failure: %d\n",
13008                                               ret);
13009                         return ret;
13010                 }
13011         }
13012
13013         /* Set default port clock if not overwritten by the encoder. Needs to be
13014          * done afterwards in case the encoder adjusts the mode. */
13015         if (!pipe_config->port_clock)
13016                 pipe_config->port_clock = pipe_config->hw.adjusted_mode.crtc_clock
13017                         * pipe_config->pixel_multiplier;
13018
13019         ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
13020         if (ret == -EDEADLK)
13021                 return ret;
13022         if (ret < 0) {
13023                 DRM_DEBUG_KMS("CRTC fixup failed\n");
13024                 return ret;
13025         }
13026
13027         if (ret == RETRY) {
13028                 if (WARN(!retry, "loop in pipe configuration computation\n"))
13029                         return -EINVAL;
13030
13031                 DRM_DEBUG_KMS("CRTC bw constrained, retrying\n");
13032                 retry = false;
13033                 goto encoder_retry;
13034         }
13035
13036         /* Dithering seems to not pass-through bits correctly when it should, so
13037          * only enable it on 6bpc panels and when its not a compliance
13038          * test requesting 6bpc video pattern.
13039          */
13040         pipe_config->dither = (pipe_config->pipe_bpp == 6*3) &&
13041                 !pipe_config->dither_force_disable;
13042         DRM_DEBUG_KMS("hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
13043                       base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
13044
13045         /*
13046          * Make drm_calc_timestamping_constants in
13047          * drm_atomic_helper_update_legacy_modeset_state() happy
13048          */
13049         pipe_config->uapi.adjusted_mode = pipe_config->hw.adjusted_mode;
13050
13051         return 0;
13052 }
13053
13054 bool intel_fuzzy_clock_check(int clock1, int clock2)
13055 {
13056         int diff;
13057
13058         if (clock1 == clock2)
13059                 return true;
13060
13061         if (!clock1 || !clock2)
13062                 return false;
13063
13064         diff = abs(clock1 - clock2);
13065
13066         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
13067                 return true;
13068
13069         return false;
13070 }
13071
13072 static bool
13073 intel_compare_m_n(unsigned int m, unsigned int n,
13074                   unsigned int m2, unsigned int n2,
13075                   bool exact)
13076 {
13077         if (m == m2 && n == n2)
13078                 return true;
13079
13080         if (exact || !m || !n || !m2 || !n2)
13081                 return false;
13082
13083         BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
13084
13085         if (n > n2) {
13086                 while (n > n2) {
13087                         m2 <<= 1;
13088                         n2 <<= 1;
13089                 }
13090         } else if (n < n2) {
13091                 while (n < n2) {
13092                         m <<= 1;
13093                         n <<= 1;
13094                 }
13095         }
13096
13097         if (n != n2)
13098                 return false;
13099
13100         return intel_fuzzy_clock_check(m, m2);
13101 }
13102
13103 static bool
13104 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
13105                        const struct intel_link_m_n *m2_n2,
13106                        bool exact)
13107 {
13108         return m_n->tu == m2_n2->tu &&
13109                 intel_compare_m_n(m_n->gmch_m, m_n->gmch_n,
13110                                   m2_n2->gmch_m, m2_n2->gmch_n, exact) &&
13111                 intel_compare_m_n(m_n->link_m, m_n->link_n,
13112                                   m2_n2->link_m, m2_n2->link_n, exact);
13113 }
13114
13115 static bool
13116 intel_compare_infoframe(const union hdmi_infoframe *a,
13117                         const union hdmi_infoframe *b)
13118 {
13119         return memcmp(a, b, sizeof(*a)) == 0;
13120 }
13121
13122 static void
13123 pipe_config_infoframe_mismatch(struct drm_i915_private *dev_priv,
13124                                bool fastset, const char *name,
13125                                const union hdmi_infoframe *a,
13126                                const union hdmi_infoframe *b)
13127 {
13128         if (fastset) {
13129                 if ((drm_debug & DRM_UT_KMS) == 0)
13130                         return;
13131
13132                 DRM_DEBUG_KMS("fastset mismatch in %s infoframe\n", name);
13133                 DRM_DEBUG_KMS("expected:\n");
13134                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, a);
13135                 DRM_DEBUG_KMS("found:\n");
13136                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, b);
13137         } else {
13138                 DRM_ERROR("mismatch in %s infoframe\n", name);
13139                 DRM_ERROR("expected:\n");
13140                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, a);
13141                 DRM_ERROR("found:\n");
13142                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, b);
13143         }
13144 }
13145
13146 static void __printf(4, 5)
13147 pipe_config_mismatch(bool fastset, const struct intel_crtc *crtc,
13148                      const char *name, const char *format, ...)
13149 {
13150         struct va_format vaf;
13151         va_list args;
13152
13153         va_start(args, format);
13154         vaf.fmt = format;
13155         vaf.va = &args;
13156
13157         if (fastset)
13158                 DRM_DEBUG_KMS("[CRTC:%d:%s] fastset mismatch in %s %pV\n",
13159                               crtc->base.base.id, crtc->base.name, name, &vaf);
13160         else
13161                 DRM_ERROR("[CRTC:%d:%s] mismatch in %s %pV\n",
13162                           crtc->base.base.id, crtc->base.name, name, &vaf);
13163
13164         va_end(args);
13165 }
13166
13167 static bool fastboot_enabled(struct drm_i915_private *dev_priv)
13168 {
13169         if (i915_modparams.fastboot != -1)
13170                 return i915_modparams.fastboot;
13171
13172         /* Enable fastboot by default on Skylake and newer */
13173         if (INTEL_GEN(dev_priv) >= 9)
13174                 return true;
13175
13176         /* Enable fastboot by default on VLV and CHV */
13177         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
13178                 return true;
13179
13180         /* Disabled by default on all others */
13181         return false;
13182 }
13183
13184 static bool
13185 intel_pipe_config_compare(const struct intel_crtc_state *current_config,
13186                           const struct intel_crtc_state *pipe_config,
13187                           bool fastset)
13188 {
13189         struct drm_i915_private *dev_priv = to_i915(current_config->uapi.crtc->dev);
13190         struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
13191         bool ret = true;
13192         u32 bp_gamma = 0;
13193         bool fixup_inherited = fastset &&
13194                 (current_config->hw.mode.private_flags & I915_MODE_FLAG_INHERITED) &&
13195                 !(pipe_config->hw.mode.private_flags & I915_MODE_FLAG_INHERITED);
13196
13197         if (fixup_inherited && !fastboot_enabled(dev_priv)) {
13198                 DRM_DEBUG_KMS("initial modeset and fastboot not set\n");
13199                 ret = false;
13200         }
13201
13202 #define PIPE_CONF_CHECK_X(name) do { \
13203         if (current_config->name != pipe_config->name) { \
13204                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13205                                      "(expected 0x%08x, found 0x%08x)", \
13206                                      current_config->name, \
13207                                      pipe_config->name); \
13208                 ret = false; \
13209         } \
13210 } while (0)
13211
13212 #define PIPE_CONF_CHECK_I(name) do { \
13213         if (current_config->name != pipe_config->name) { \
13214                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13215                                      "(expected %i, found %i)", \
13216                                      current_config->name, \
13217                                      pipe_config->name); \
13218                 ret = false; \
13219         } \
13220 } while (0)
13221
13222 #define PIPE_CONF_CHECK_BOOL(name) do { \
13223         if (current_config->name != pipe_config->name) { \
13224                 pipe_config_mismatch(fastset, crtc,  __stringify(name), \
13225                                      "(expected %s, found %s)", \
13226                                      yesno(current_config->name), \
13227                                      yesno(pipe_config->name)); \
13228                 ret = false; \
13229         } \
13230 } while (0)
13231
13232 /*
13233  * Checks state where we only read out the enabling, but not the entire
13234  * state itself (like full infoframes or ELD for audio). These states
13235  * require a full modeset on bootup to fix up.
13236  */
13237 #define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) do { \
13238         if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \
13239                 PIPE_CONF_CHECK_BOOL(name); \
13240         } else { \
13241                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13242                                      "unable to verify whether state matches exactly, forcing modeset (expected %s, found %s)", \
13243                                      yesno(current_config->name), \
13244                                      yesno(pipe_config->name)); \
13245                 ret = false; \
13246         } \
13247 } while (0)
13248
13249 #define PIPE_CONF_CHECK_P(name) do { \
13250         if (current_config->name != pipe_config->name) { \
13251                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13252                                      "(expected %p, found %p)", \
13253                                      current_config->name, \
13254                                      pipe_config->name); \
13255                 ret = false; \
13256         } \
13257 } while (0)
13258
13259 #define PIPE_CONF_CHECK_M_N(name) do { \
13260         if (!intel_compare_link_m_n(&current_config->name, \
13261                                     &pipe_config->name,\
13262                                     !fastset)) { \
13263                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13264                                      "(expected tu %i gmch %i/%i link %i/%i, " \
13265                                      "found tu %i, gmch %i/%i link %i/%i)", \
13266                                      current_config->name.tu, \
13267                                      current_config->name.gmch_m, \
13268                                      current_config->name.gmch_n, \
13269                                      current_config->name.link_m, \
13270                                      current_config->name.link_n, \
13271                                      pipe_config->name.tu, \
13272                                      pipe_config->name.gmch_m, \
13273                                      pipe_config->name.gmch_n, \
13274                                      pipe_config->name.link_m, \
13275                                      pipe_config->name.link_n); \
13276                 ret = false; \
13277         } \
13278 } while (0)
13279
13280 /* This is required for BDW+ where there is only one set of registers for
13281  * switching between high and low RR.
13282  * This macro can be used whenever a comparison has to be made between one
13283  * hw state and multiple sw state variables.
13284  */
13285 #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) do { \
13286         if (!intel_compare_link_m_n(&current_config->name, \
13287                                     &pipe_config->name, !fastset) && \
13288             !intel_compare_link_m_n(&current_config->alt_name, \
13289                                     &pipe_config->name, !fastset)) { \
13290                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13291                                      "(expected tu %i gmch %i/%i link %i/%i, " \
13292                                      "or tu %i gmch %i/%i link %i/%i, " \
13293                                      "found tu %i, gmch %i/%i link %i/%i)", \
13294                                      current_config->name.tu, \
13295                                      current_config->name.gmch_m, \
13296                                      current_config->name.gmch_n, \
13297                                      current_config->name.link_m, \
13298                                      current_config->name.link_n, \
13299                                      current_config->alt_name.tu, \
13300                                      current_config->alt_name.gmch_m, \
13301                                      current_config->alt_name.gmch_n, \
13302                                      current_config->alt_name.link_m, \
13303                                      current_config->alt_name.link_n, \
13304                                      pipe_config->name.tu, \
13305                                      pipe_config->name.gmch_m, \
13306                                      pipe_config->name.gmch_n, \
13307                                      pipe_config->name.link_m, \
13308                                      pipe_config->name.link_n); \
13309                 ret = false; \
13310         } \
13311 } while (0)
13312
13313 #define PIPE_CONF_CHECK_FLAGS(name, mask) do { \
13314         if ((current_config->name ^ pipe_config->name) & (mask)) { \
13315                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13316                                      "(%x) (expected %i, found %i)", \
13317                                      (mask), \
13318                                      current_config->name & (mask), \
13319                                      pipe_config->name & (mask)); \
13320                 ret = false; \
13321         } \
13322 } while (0)
13323
13324 #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) do { \
13325         if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
13326                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13327                                      "(expected %i, found %i)", \
13328                                      current_config->name, \
13329                                      pipe_config->name); \
13330                 ret = false; \
13331         } \
13332 } while (0)
13333
13334 #define PIPE_CONF_CHECK_INFOFRAME(name) do { \
13335         if (!intel_compare_infoframe(&current_config->infoframes.name, \
13336                                      &pipe_config->infoframes.name)) { \
13337                 pipe_config_infoframe_mismatch(dev_priv, fastset, __stringify(name), \
13338                                                &current_config->infoframes.name, \
13339                                                &pipe_config->infoframes.name); \
13340                 ret = false; \
13341         } \
13342 } while (0)
13343
13344 #define PIPE_CONF_CHECK_COLOR_LUT(name1, name2, bit_precision) do { \
13345         if (current_config->name1 != pipe_config->name1) { \
13346                 pipe_config_mismatch(fastset, crtc, __stringify(name1), \
13347                                 "(expected %i, found %i, won't compare lut values)", \
13348                                 current_config->name1, \
13349                                 pipe_config->name1); \
13350                 ret = false;\
13351         } else { \
13352                 if (!intel_color_lut_equal(current_config->name2, \
13353                                         pipe_config->name2, pipe_config->name1, \
13354                                         bit_precision)) { \
13355                         pipe_config_mismatch(fastset, crtc, __stringify(name2), \
13356                                         "hw_state doesn't match sw_state"); \
13357                         ret = false; \
13358                 } \
13359         } \
13360 } while (0)
13361
13362 #define PIPE_CONF_QUIRK(quirk) \
13363         ((current_config->quirks | pipe_config->quirks) & (quirk))
13364
13365         PIPE_CONF_CHECK_I(cpu_transcoder);
13366
13367         PIPE_CONF_CHECK_BOOL(has_pch_encoder);
13368         PIPE_CONF_CHECK_I(fdi_lanes);
13369         PIPE_CONF_CHECK_M_N(fdi_m_n);
13370
13371         PIPE_CONF_CHECK_I(lane_count);
13372         PIPE_CONF_CHECK_X(lane_lat_optim_mask);
13373
13374         if (INTEL_GEN(dev_priv) < 8) {
13375                 PIPE_CONF_CHECK_M_N(dp_m_n);
13376
13377                 if (current_config->has_drrs)
13378                         PIPE_CONF_CHECK_M_N(dp_m2_n2);
13379         } else
13380                 PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
13381
13382         PIPE_CONF_CHECK_X(output_types);
13383
13384         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hdisplay);
13385         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_htotal);
13386         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hblank_start);
13387         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hblank_end);
13388         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hsync_start);
13389         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hsync_end);
13390
13391         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vdisplay);
13392         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vtotal);
13393         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vblank_start);
13394         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vblank_end);
13395         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vsync_start);
13396         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vsync_end);
13397
13398         PIPE_CONF_CHECK_I(pixel_multiplier);
13399         PIPE_CONF_CHECK_I(output_format);
13400         PIPE_CONF_CHECK_I(dc3co_exitline);
13401         PIPE_CONF_CHECK_BOOL(has_hdmi_sink);
13402         if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
13403             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
13404                 PIPE_CONF_CHECK_BOOL(limited_color_range);
13405
13406         PIPE_CONF_CHECK_BOOL(hdmi_scrambling);
13407         PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio);
13408         PIPE_CONF_CHECK_BOOL(has_infoframe);
13409         PIPE_CONF_CHECK_BOOL(fec_enable);
13410
13411         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio);
13412
13413         PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13414                               DRM_MODE_FLAG_INTERLACE);
13415
13416         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
13417                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13418                                       DRM_MODE_FLAG_PHSYNC);
13419                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13420                                       DRM_MODE_FLAG_NHSYNC);
13421                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13422                                       DRM_MODE_FLAG_PVSYNC);
13423                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13424                                       DRM_MODE_FLAG_NVSYNC);
13425         }
13426
13427         PIPE_CONF_CHECK_X(gmch_pfit.control);
13428         /* pfit ratios are autocomputed by the hw on gen4+ */
13429         if (INTEL_GEN(dev_priv) < 4)
13430                 PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios);
13431         PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
13432
13433         /*
13434          * Changing the EDP transcoder input mux
13435          * (A_ONOFF vs. A_ON) requires a full modeset.
13436          */
13437         PIPE_CONF_CHECK_BOOL(pch_pfit.force_thru);
13438
13439         if (!fastset) {
13440                 PIPE_CONF_CHECK_I(pipe_src_w);
13441                 PIPE_CONF_CHECK_I(pipe_src_h);
13442
13443                 PIPE_CONF_CHECK_BOOL(pch_pfit.enabled);
13444                 if (current_config->pch_pfit.enabled) {
13445                         PIPE_CONF_CHECK_X(pch_pfit.pos);
13446                         PIPE_CONF_CHECK_X(pch_pfit.size);
13447                 }
13448
13449                 PIPE_CONF_CHECK_I(scaler_state.scaler_id);
13450                 PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate);
13451
13452                 PIPE_CONF_CHECK_X(gamma_mode);
13453                 if (IS_CHERRYVIEW(dev_priv))
13454                         PIPE_CONF_CHECK_X(cgm_mode);
13455                 else
13456                         PIPE_CONF_CHECK_X(csc_mode);
13457                 PIPE_CONF_CHECK_BOOL(gamma_enable);
13458                 PIPE_CONF_CHECK_BOOL(csc_enable);
13459
13460                 bp_gamma = intel_color_get_gamma_bit_precision(pipe_config);
13461                 if (bp_gamma)
13462                         PIPE_CONF_CHECK_COLOR_LUT(gamma_mode, hw.gamma_lut, bp_gamma);
13463
13464         }
13465
13466         PIPE_CONF_CHECK_BOOL(double_wide);
13467
13468         PIPE_CONF_CHECK_P(shared_dpll);
13469         PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
13470         PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
13471         PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
13472         PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
13473         PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
13474         PIPE_CONF_CHECK_X(dpll_hw_state.spll);
13475         PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
13476         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
13477         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
13478         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0);
13479         PIPE_CONF_CHECK_X(dpll_hw_state.ebb0);
13480         PIPE_CONF_CHECK_X(dpll_hw_state.ebb4);
13481         PIPE_CONF_CHECK_X(dpll_hw_state.pll0);
13482         PIPE_CONF_CHECK_X(dpll_hw_state.pll1);
13483         PIPE_CONF_CHECK_X(dpll_hw_state.pll2);
13484         PIPE_CONF_CHECK_X(dpll_hw_state.pll3);
13485         PIPE_CONF_CHECK_X(dpll_hw_state.pll6);
13486         PIPE_CONF_CHECK_X(dpll_hw_state.pll8);
13487         PIPE_CONF_CHECK_X(dpll_hw_state.pll9);
13488         PIPE_CONF_CHECK_X(dpll_hw_state.pll10);
13489         PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12);
13490         PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl);
13491         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1);
13492         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl);
13493         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0);
13494         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1);
13495         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf);
13496         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock);
13497         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc);
13498         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias);
13499         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias);
13500
13501         PIPE_CONF_CHECK_X(dsi_pll.ctrl);
13502         PIPE_CONF_CHECK_X(dsi_pll.div);
13503
13504         if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5)
13505                 PIPE_CONF_CHECK_I(pipe_bpp);
13506
13507         PIPE_CONF_CHECK_CLOCK_FUZZY(hw.adjusted_mode.crtc_clock);
13508         PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
13509
13510         PIPE_CONF_CHECK_I(min_voltage_level);
13511
13512         PIPE_CONF_CHECK_X(infoframes.enable);
13513         PIPE_CONF_CHECK_X(infoframes.gcp);
13514         PIPE_CONF_CHECK_INFOFRAME(avi);
13515         PIPE_CONF_CHECK_INFOFRAME(spd);
13516         PIPE_CONF_CHECK_INFOFRAME(hdmi);
13517         PIPE_CONF_CHECK_INFOFRAME(drm);
13518
13519         PIPE_CONF_CHECK_I(sync_mode_slaves_mask);
13520         PIPE_CONF_CHECK_I(master_transcoder);
13521
13522         PIPE_CONF_CHECK_I(dsc.compression_enable);
13523         PIPE_CONF_CHECK_I(dsc.dsc_split);
13524         PIPE_CONF_CHECK_I(dsc.compressed_bpp);
13525
13526         PIPE_CONF_CHECK_I(mst_master_transcoder);
13527
13528 #undef PIPE_CONF_CHECK_X
13529 #undef PIPE_CONF_CHECK_I
13530 #undef PIPE_CONF_CHECK_BOOL
13531 #undef PIPE_CONF_CHECK_BOOL_INCOMPLETE
13532 #undef PIPE_CONF_CHECK_P
13533 #undef PIPE_CONF_CHECK_FLAGS
13534 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
13535 #undef PIPE_CONF_CHECK_COLOR_LUT
13536 #undef PIPE_CONF_QUIRK
13537
13538         return ret;
13539 }
13540
13541 static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv,
13542                                            const struct intel_crtc_state *pipe_config)
13543 {
13544         if (pipe_config->has_pch_encoder) {
13545                 int fdi_dotclock = intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
13546                                                             &pipe_config->fdi_m_n);
13547                 int dotclock = pipe_config->hw.adjusted_mode.crtc_clock;
13548
13549                 /*
13550                  * FDI already provided one idea for the dotclock.
13551                  * Yell if the encoder disagrees.
13552                  */
13553                 WARN(!intel_fuzzy_clock_check(fdi_dotclock, dotclock),
13554                      "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
13555                      fdi_dotclock, dotclock);
13556         }
13557 }
13558
13559 static void verify_wm_state(struct intel_crtc *crtc,
13560                             struct intel_crtc_state *new_crtc_state)
13561 {
13562         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13563         struct skl_hw_state {
13564                 struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
13565                 struct skl_ddb_entry ddb_uv[I915_MAX_PLANES];
13566                 struct skl_ddb_allocation ddb;
13567                 struct skl_pipe_wm wm;
13568         } *hw;
13569         struct skl_ddb_allocation *sw_ddb;
13570         struct skl_pipe_wm *sw_wm;
13571         struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
13572         const enum pipe pipe = crtc->pipe;
13573         int plane, level, max_level = ilk_wm_max_level(dev_priv);
13574
13575         if (INTEL_GEN(dev_priv) < 9 || !new_crtc_state->hw.active)
13576                 return;
13577
13578         hw = kzalloc(sizeof(*hw), GFP_KERNEL);
13579         if (!hw)
13580                 return;
13581
13582         skl_pipe_wm_get_hw_state(crtc, &hw->wm);
13583         sw_wm = &new_crtc_state->wm.skl.optimal;
13584
13585         skl_pipe_ddb_get_hw_state(crtc, hw->ddb_y, hw->ddb_uv);
13586
13587         skl_ddb_get_hw_state(dev_priv, &hw->ddb);
13588         sw_ddb = &dev_priv->wm.skl_hw.ddb;
13589
13590         if (INTEL_GEN(dev_priv) >= 11 &&
13591             hw->ddb.enabled_slices != sw_ddb->enabled_slices)
13592                 DRM_ERROR("mismatch in DBUF Slices (expected %u, got %u)\n",
13593                           sw_ddb->enabled_slices,
13594                           hw->ddb.enabled_slices);
13595
13596         /* planes */
13597         for_each_universal_plane(dev_priv, pipe, plane) {
13598                 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
13599
13600                 hw_plane_wm = &hw->wm.planes[plane];
13601                 sw_plane_wm = &sw_wm->planes[plane];
13602
13603                 /* Watermarks */
13604                 for (level = 0; level <= max_level; level++) {
13605                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
13606                                                 &sw_plane_wm->wm[level]))
13607                                 continue;
13608
13609                         DRM_ERROR("mismatch in WM pipe %c plane %d level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
13610                                   pipe_name(pipe), plane + 1, level,
13611                                   sw_plane_wm->wm[level].plane_en,
13612                                   sw_plane_wm->wm[level].plane_res_b,
13613                                   sw_plane_wm->wm[level].plane_res_l,
13614                                   hw_plane_wm->wm[level].plane_en,
13615                                   hw_plane_wm->wm[level].plane_res_b,
13616                                   hw_plane_wm->wm[level].plane_res_l);
13617                 }
13618
13619                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
13620                                          &sw_plane_wm->trans_wm)) {
13621                         DRM_ERROR("mismatch in trans WM pipe %c plane %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
13622                                   pipe_name(pipe), plane + 1,
13623                                   sw_plane_wm->trans_wm.plane_en,
13624                                   sw_plane_wm->trans_wm.plane_res_b,
13625                                   sw_plane_wm->trans_wm.plane_res_l,
13626                                   hw_plane_wm->trans_wm.plane_en,
13627                                   hw_plane_wm->trans_wm.plane_res_b,
13628                                   hw_plane_wm->trans_wm.plane_res_l);
13629                 }
13630
13631                 /* DDB */
13632                 hw_ddb_entry = &hw->ddb_y[plane];
13633                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[plane];
13634
13635                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
13636                         DRM_ERROR("mismatch in DDB state pipe %c plane %d (expected (%u,%u), found (%u,%u))\n",
13637                                   pipe_name(pipe), plane + 1,
13638                                   sw_ddb_entry->start, sw_ddb_entry->end,
13639                                   hw_ddb_entry->start, hw_ddb_entry->end);
13640                 }
13641         }
13642
13643         /*
13644          * cursor
13645          * If the cursor plane isn't active, we may not have updated it's ddb
13646          * allocation. In that case since the ddb allocation will be updated
13647          * once the plane becomes visible, we can skip this check
13648          */
13649         if (1) {
13650                 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
13651
13652                 hw_plane_wm = &hw->wm.planes[PLANE_CURSOR];
13653                 sw_plane_wm = &sw_wm->planes[PLANE_CURSOR];
13654
13655                 /* Watermarks */
13656                 for (level = 0; level <= max_level; level++) {
13657                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
13658                                                 &sw_plane_wm->wm[level]))
13659                                 continue;
13660
13661                         DRM_ERROR("mismatch in WM pipe %c cursor level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
13662                                   pipe_name(pipe), level,
13663                                   sw_plane_wm->wm[level].plane_en,
13664                                   sw_plane_wm->wm[level].plane_res_b,
13665                                   sw_plane_wm->wm[level].plane_res_l,
13666                                   hw_plane_wm->wm[level].plane_en,
13667                                   hw_plane_wm->wm[level].plane_res_b,
13668                                   hw_plane_wm->wm[level].plane_res_l);
13669                 }
13670
13671                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
13672                                          &sw_plane_wm->trans_wm)) {
13673                         DRM_ERROR("mismatch in trans WM pipe %c cursor (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
13674                                   pipe_name(pipe),
13675                                   sw_plane_wm->trans_wm.plane_en,
13676                                   sw_plane_wm->trans_wm.plane_res_b,
13677                                   sw_plane_wm->trans_wm.plane_res_l,
13678                                   hw_plane_wm->trans_wm.plane_en,
13679                                   hw_plane_wm->trans_wm.plane_res_b,
13680                                   hw_plane_wm->trans_wm.plane_res_l);
13681                 }
13682
13683                 /* DDB */
13684                 hw_ddb_entry = &hw->ddb_y[PLANE_CURSOR];
13685                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR];
13686
13687                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
13688                         DRM_ERROR("mismatch in DDB state pipe %c cursor (expected (%u,%u), found (%u,%u))\n",
13689                                   pipe_name(pipe),
13690                                   sw_ddb_entry->start, sw_ddb_entry->end,
13691                                   hw_ddb_entry->start, hw_ddb_entry->end);
13692                 }
13693         }
13694
13695         kfree(hw);
13696 }
13697
13698 static void
13699 verify_connector_state(struct intel_atomic_state *state,
13700                        struct intel_crtc *crtc)
13701 {
13702         struct drm_connector *connector;
13703         struct drm_connector_state *new_conn_state;
13704         int i;
13705
13706         for_each_new_connector_in_state(&state->base, connector, new_conn_state, i) {
13707                 struct drm_encoder *encoder = connector->encoder;
13708                 struct intel_crtc_state *crtc_state = NULL;
13709
13710                 if (new_conn_state->crtc != &crtc->base)
13711                         continue;
13712
13713                 if (crtc)
13714                         crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
13715
13716                 intel_connector_verify_state(crtc_state, new_conn_state);
13717
13718                 I915_STATE_WARN(new_conn_state->best_encoder != encoder,
13719                      "connector's atomic encoder doesn't match legacy encoder\n");
13720         }
13721 }
13722
13723 static void
13724 verify_encoder_state(struct drm_i915_private *dev_priv, struct intel_atomic_state *state)
13725 {
13726         struct intel_encoder *encoder;
13727         struct drm_connector *connector;
13728         struct drm_connector_state *old_conn_state, *new_conn_state;
13729         int i;
13730
13731         for_each_intel_encoder(&dev_priv->drm, encoder) {
13732                 bool enabled = false, found = false;
13733                 enum pipe pipe;
13734
13735                 DRM_DEBUG_KMS("[ENCODER:%d:%s]\n",
13736                               encoder->base.base.id,
13737                               encoder->base.name);
13738
13739                 for_each_oldnew_connector_in_state(&state->base, connector, old_conn_state,
13740                                                    new_conn_state, i) {
13741                         if (old_conn_state->best_encoder == &encoder->base)
13742                                 found = true;
13743
13744                         if (new_conn_state->best_encoder != &encoder->base)
13745                                 continue;
13746                         found = enabled = true;
13747
13748                         I915_STATE_WARN(new_conn_state->crtc !=
13749                                         encoder->base.crtc,
13750                              "connector's crtc doesn't match encoder crtc\n");
13751                 }
13752
13753                 if (!found)
13754                         continue;
13755
13756                 I915_STATE_WARN(!!encoder->base.crtc != enabled,
13757                      "encoder's enabled state mismatch "
13758                      "(expected %i, found %i)\n",
13759                      !!encoder->base.crtc, enabled);
13760
13761                 if (!encoder->base.crtc) {
13762                         bool active;
13763
13764                         active = encoder->get_hw_state(encoder, &pipe);
13765                         I915_STATE_WARN(active,
13766                              "encoder detached but still enabled on pipe %c.\n",
13767                              pipe_name(pipe));
13768                 }
13769         }
13770 }
13771
13772 static void
13773 verify_crtc_state(struct intel_crtc *crtc,
13774                   struct intel_crtc_state *old_crtc_state,
13775                   struct intel_crtc_state *new_crtc_state)
13776 {
13777         struct drm_device *dev = crtc->base.dev;
13778         struct drm_i915_private *dev_priv = to_i915(dev);
13779         struct intel_encoder *encoder;
13780         struct intel_crtc_state *pipe_config = old_crtc_state;
13781         struct drm_atomic_state *state = old_crtc_state->uapi.state;
13782         bool active;
13783
13784         __drm_atomic_helper_crtc_destroy_state(&old_crtc_state->uapi);
13785         intel_crtc_free_hw_state(old_crtc_state);
13786         intel_crtc_state_reset(old_crtc_state, crtc);
13787         old_crtc_state->uapi.state = state;
13788
13789         DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.base.id, crtc->base.name);
13790
13791         active = dev_priv->display.get_pipe_config(crtc, pipe_config);
13792
13793         /* we keep both pipes enabled on 830 */
13794         if (IS_I830(dev_priv))
13795                 active = new_crtc_state->hw.active;
13796
13797         I915_STATE_WARN(new_crtc_state->hw.active != active,
13798                         "crtc active state doesn't match with hw state "
13799                         "(expected %i, found %i)\n",
13800                         new_crtc_state->hw.active, active);
13801
13802         I915_STATE_WARN(crtc->active != new_crtc_state->hw.active,
13803                         "transitional active state does not match atomic hw state "
13804                         "(expected %i, found %i)\n",
13805                         new_crtc_state->hw.active, crtc->active);
13806
13807         for_each_encoder_on_crtc(dev, &crtc->base, encoder) {
13808                 enum pipe pipe;
13809
13810                 active = encoder->get_hw_state(encoder, &pipe);
13811                 I915_STATE_WARN(active != new_crtc_state->hw.active,
13812                                 "[ENCODER:%i] active %i with crtc active %i\n",
13813                                 encoder->base.base.id, active,
13814                                 new_crtc_state->hw.active);
13815
13816                 I915_STATE_WARN(active && crtc->pipe != pipe,
13817                                 "Encoder connected to wrong pipe %c\n",
13818                                 pipe_name(pipe));
13819
13820                 if (active)
13821                         encoder->get_config(encoder, pipe_config);
13822         }
13823
13824         intel_crtc_compute_pixel_rate(pipe_config);
13825
13826         if (!new_crtc_state->hw.active)
13827                 return;
13828
13829         intel_pipe_config_sanity_check(dev_priv, pipe_config);
13830
13831         if (!intel_pipe_config_compare(new_crtc_state,
13832                                        pipe_config, false)) {
13833                 I915_STATE_WARN(1, "pipe state doesn't match!\n");
13834                 intel_dump_pipe_config(pipe_config, NULL, "[hw state]");
13835                 intel_dump_pipe_config(new_crtc_state, NULL, "[sw state]");
13836         }
13837 }
13838
13839 static void
13840 intel_verify_planes(struct intel_atomic_state *state)
13841 {
13842         struct intel_plane *plane;
13843         const struct intel_plane_state *plane_state;
13844         int i;
13845
13846         for_each_new_intel_plane_in_state(state, plane,
13847                                           plane_state, i)
13848                 assert_plane(plane, plane_state->planar_slave ||
13849                              plane_state->uapi.visible);
13850 }
13851
13852 static void
13853 verify_single_dpll_state(struct drm_i915_private *dev_priv,
13854                          struct intel_shared_dpll *pll,
13855                          struct intel_crtc *crtc,
13856                          struct intel_crtc_state *new_crtc_state)
13857 {
13858         struct intel_dpll_hw_state dpll_hw_state;
13859         unsigned int crtc_mask;
13860         bool active;
13861
13862         memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
13863
13864         DRM_DEBUG_KMS("%s\n", pll->info->name);
13865
13866         active = pll->info->funcs->get_hw_state(dev_priv, pll, &dpll_hw_state);
13867
13868         if (!(pll->info->flags & INTEL_DPLL_ALWAYS_ON)) {
13869                 I915_STATE_WARN(!pll->on && pll->active_mask,
13870                      "pll in active use but not on in sw tracking\n");
13871                 I915_STATE_WARN(pll->on && !pll->active_mask,
13872                      "pll is on but not used by any active crtc\n");
13873                 I915_STATE_WARN(pll->on != active,
13874                      "pll on state mismatch (expected %i, found %i)\n",
13875                      pll->on, active);
13876         }
13877
13878         if (!crtc) {
13879                 I915_STATE_WARN(pll->active_mask & ~pll->state.crtc_mask,
13880                                 "more active pll users than references: %x vs %x\n",
13881                                 pll->active_mask, pll->state.crtc_mask);
13882
13883                 return;
13884         }
13885
13886         crtc_mask = drm_crtc_mask(&crtc->base);
13887
13888         if (new_crtc_state->hw.active)
13889                 I915_STATE_WARN(!(pll->active_mask & crtc_mask),
13890                                 "pll active mismatch (expected pipe %c in active mask 0x%02x)\n",
13891                                 pipe_name(drm_crtc_index(&crtc->base)), pll->active_mask);
13892         else
13893                 I915_STATE_WARN(pll->active_mask & crtc_mask,
13894                                 "pll active mismatch (didn't expect pipe %c in active mask 0x%02x)\n",
13895                                 pipe_name(drm_crtc_index(&crtc->base)), pll->active_mask);
13896
13897         I915_STATE_WARN(!(pll->state.crtc_mask & crtc_mask),
13898                         "pll enabled crtcs mismatch (expected 0x%x in 0x%02x)\n",
13899                         crtc_mask, pll->state.crtc_mask);
13900
13901         I915_STATE_WARN(pll->on && memcmp(&pll->state.hw_state,
13902                                           &dpll_hw_state,
13903                                           sizeof(dpll_hw_state)),
13904                         "pll hw state mismatch\n");
13905 }
13906
13907 static void
13908 verify_shared_dpll_state(struct intel_crtc *crtc,
13909                          struct intel_crtc_state *old_crtc_state,
13910                          struct intel_crtc_state *new_crtc_state)
13911 {
13912         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13913
13914         if (new_crtc_state->shared_dpll)
13915                 verify_single_dpll_state(dev_priv, new_crtc_state->shared_dpll, crtc, new_crtc_state);
13916
13917         if (old_crtc_state->shared_dpll &&
13918             old_crtc_state->shared_dpll != new_crtc_state->shared_dpll) {
13919                 unsigned int crtc_mask = drm_crtc_mask(&crtc->base);
13920                 struct intel_shared_dpll *pll = old_crtc_state->shared_dpll;
13921
13922                 I915_STATE_WARN(pll->active_mask & crtc_mask,
13923                                 "pll active mismatch (didn't expect pipe %c in active mask)\n",
13924                                 pipe_name(drm_crtc_index(&crtc->base)));
13925                 I915_STATE_WARN(pll->state.crtc_mask & crtc_mask,
13926                                 "pll enabled crtcs mismatch (found %x in enabled mask)\n",
13927                                 pipe_name(drm_crtc_index(&crtc->base)));
13928         }
13929 }
13930
13931 static void
13932 intel_modeset_verify_crtc(struct intel_crtc *crtc,
13933                           struct intel_atomic_state *state,
13934                           struct intel_crtc_state *old_crtc_state,
13935                           struct intel_crtc_state *new_crtc_state)
13936 {
13937         if (!needs_modeset(new_crtc_state) && !new_crtc_state->update_pipe)
13938                 return;
13939
13940         verify_wm_state(crtc, new_crtc_state);
13941         verify_connector_state(state, crtc);
13942         verify_crtc_state(crtc, old_crtc_state, new_crtc_state);
13943         verify_shared_dpll_state(crtc, old_crtc_state, new_crtc_state);
13944 }
13945
13946 static void
13947 verify_disabled_dpll_state(struct drm_i915_private *dev_priv)
13948 {
13949         int i;
13950
13951         for (i = 0; i < dev_priv->num_shared_dpll; i++)
13952                 verify_single_dpll_state(dev_priv, &dev_priv->shared_dplls[i], NULL, NULL);
13953 }
13954
13955 static void
13956 intel_modeset_verify_disabled(struct drm_i915_private *dev_priv,
13957                               struct intel_atomic_state *state)
13958 {
13959         verify_encoder_state(dev_priv, state);
13960         verify_connector_state(state, NULL);
13961         verify_disabled_dpll_state(dev_priv);
13962 }
13963
13964 static void
13965 intel_crtc_update_active_timings(const struct intel_crtc_state *crtc_state)
13966 {
13967         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
13968         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13969         const struct drm_display_mode *adjusted_mode =
13970                 &crtc_state->hw.adjusted_mode;
13971
13972         drm_calc_timestamping_constants(&crtc->base, adjusted_mode);
13973
13974         /*
13975          * The scanline counter increments at the leading edge of hsync.
13976          *
13977          * On most platforms it starts counting from vtotal-1 on the
13978          * first active line. That means the scanline counter value is
13979          * always one less than what we would expect. Ie. just after
13980          * start of vblank, which also occurs at start of hsync (on the
13981          * last active line), the scanline counter will read vblank_start-1.
13982          *
13983          * On gen2 the scanline counter starts counting from 1 instead
13984          * of vtotal-1, so we have to subtract one (or rather add vtotal-1
13985          * to keep the value positive), instead of adding one.
13986          *
13987          * On HSW+ the behaviour of the scanline counter depends on the output
13988          * type. For DP ports it behaves like most other platforms, but on HDMI
13989          * there's an extra 1 line difference. So we need to add two instead of
13990          * one to the value.
13991          *
13992          * On VLV/CHV DSI the scanline counter would appear to increment
13993          * approx. 1/3 of a scanline before start of vblank. Unfortunately
13994          * that means we can't tell whether we're in vblank or not while
13995          * we're on that particular line. We must still set scanline_offset
13996          * to 1 so that the vblank timestamps come out correct when we query
13997          * the scanline counter from within the vblank interrupt handler.
13998          * However if queried just before the start of vblank we'll get an
13999          * answer that's slightly in the future.
14000          */
14001         if (IS_GEN(dev_priv, 2)) {
14002                 int vtotal;
14003
14004                 vtotal = adjusted_mode->crtc_vtotal;
14005                 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
14006                         vtotal /= 2;
14007
14008                 crtc->scanline_offset = vtotal - 1;
14009         } else if (HAS_DDI(dev_priv) &&
14010                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
14011                 crtc->scanline_offset = 2;
14012         } else {
14013                 crtc->scanline_offset = 1;
14014         }
14015 }
14016
14017 static void intel_modeset_clear_plls(struct intel_atomic_state *state)
14018 {
14019         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14020         struct intel_crtc_state *new_crtc_state;
14021         struct intel_crtc *crtc;
14022         int i;
14023
14024         if (!dev_priv->display.crtc_compute_clock)
14025                 return;
14026
14027         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
14028                 if (!needs_modeset(new_crtc_state))
14029                         continue;
14030
14031                 intel_release_shared_dplls(state, crtc);
14032         }
14033 }
14034
14035 /*
14036  * This implements the workaround described in the "notes" section of the mode
14037  * set sequence documentation. When going from no pipes or single pipe to
14038  * multiple pipes, and planes are enabled after the pipe, we need to wait at
14039  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
14040  */
14041 static int haswell_mode_set_planes_workaround(struct intel_atomic_state *state)
14042 {
14043         struct intel_crtc_state *crtc_state;
14044         struct intel_crtc *crtc;
14045         struct intel_crtc_state *first_crtc_state = NULL;
14046         struct intel_crtc_state *other_crtc_state = NULL;
14047         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
14048         int i;
14049
14050         /* look at all crtc's that are going to be enabled in during modeset */
14051         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
14052                 if (!crtc_state->hw.active ||
14053                     !needs_modeset(crtc_state))
14054                         continue;
14055
14056                 if (first_crtc_state) {
14057                         other_crtc_state = crtc_state;
14058                         break;
14059                 } else {
14060                         first_crtc_state = crtc_state;
14061                         first_pipe = crtc->pipe;
14062                 }
14063         }
14064
14065         /* No workaround needed? */
14066         if (!first_crtc_state)
14067                 return 0;
14068
14069         /* w/a possibly needed, check how many crtc's are already enabled. */
14070         for_each_intel_crtc(state->base.dev, crtc) {
14071                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
14072                 if (IS_ERR(crtc_state))
14073                         return PTR_ERR(crtc_state);
14074
14075                 crtc_state->hsw_workaround_pipe = INVALID_PIPE;
14076
14077                 if (!crtc_state->hw.active ||
14078                     needs_modeset(crtc_state))
14079                         continue;
14080
14081                 /* 2 or more enabled crtcs means no need for w/a */
14082                 if (enabled_pipe != INVALID_PIPE)
14083                         return 0;
14084
14085                 enabled_pipe = crtc->pipe;
14086         }
14087
14088         if (enabled_pipe != INVALID_PIPE)
14089                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
14090         else if (other_crtc_state)
14091                 other_crtc_state->hsw_workaround_pipe = first_pipe;
14092
14093         return 0;
14094 }
14095
14096 static int intel_modeset_checks(struct intel_atomic_state *state)
14097 {
14098         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14099         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14100         struct intel_crtc *crtc;
14101         int ret, i;
14102
14103         /* keep the current setting */
14104         if (!state->cdclk.force_min_cdclk_changed)
14105                 state->cdclk.force_min_cdclk = dev_priv->cdclk.force_min_cdclk;
14106
14107         state->modeset = true;
14108         state->active_pipes = dev_priv->active_pipes;
14109         state->cdclk.logical = dev_priv->cdclk.logical;
14110         state->cdclk.actual = dev_priv->cdclk.actual;
14111
14112         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14113                                             new_crtc_state, i) {
14114                 if (new_crtc_state->hw.active)
14115                         state->active_pipes |= BIT(crtc->pipe);
14116                 else
14117                         state->active_pipes &= ~BIT(crtc->pipe);
14118
14119                 if (old_crtc_state->hw.active != new_crtc_state->hw.active)
14120                         state->active_pipe_changes |= BIT(crtc->pipe);
14121         }
14122
14123         if (state->active_pipe_changes) {
14124                 ret = intel_atomic_lock_global_state(state);
14125                 if (ret)
14126                         return ret;
14127         }
14128
14129         ret = intel_modeset_calc_cdclk(state);
14130         if (ret)
14131                 return ret;
14132
14133         intel_modeset_clear_plls(state);
14134
14135         if (IS_HASWELL(dev_priv))
14136                 return haswell_mode_set_planes_workaround(state);
14137
14138         return 0;
14139 }
14140
14141 /*
14142  * Handle calculation of various watermark data at the end of the atomic check
14143  * phase.  The code here should be run after the per-crtc and per-plane 'check'
14144  * handlers to ensure that all derived state has been updated.
14145  */
14146 static int calc_watermark_data(struct intel_atomic_state *state)
14147 {
14148         struct drm_device *dev = state->base.dev;
14149         struct drm_i915_private *dev_priv = to_i915(dev);
14150
14151         /* Is there platform-specific watermark information to calculate? */
14152         if (dev_priv->display.compute_global_watermarks)
14153                 return dev_priv->display.compute_global_watermarks(state);
14154
14155         return 0;
14156 }
14157
14158 static void intel_crtc_check_fastset(const struct intel_crtc_state *old_crtc_state,
14159                                      struct intel_crtc_state *new_crtc_state)
14160 {
14161         if (!intel_pipe_config_compare(old_crtc_state, new_crtc_state, true))
14162                 return;
14163
14164         new_crtc_state->uapi.mode_changed = false;
14165         new_crtc_state->update_pipe = true;
14166 }
14167
14168 static void intel_crtc_copy_fastset(const struct intel_crtc_state *old_crtc_state,
14169                                     struct intel_crtc_state *new_crtc_state)
14170 {
14171         /*
14172          * If we're not doing the full modeset we want to
14173          * keep the current M/N values as they may be
14174          * sufficiently different to the computed values
14175          * to cause problems.
14176          *
14177          * FIXME: should really copy more fuzzy state here
14178          */
14179         new_crtc_state->fdi_m_n = old_crtc_state->fdi_m_n;
14180         new_crtc_state->dp_m_n = old_crtc_state->dp_m_n;
14181         new_crtc_state->dp_m2_n2 = old_crtc_state->dp_m2_n2;
14182         new_crtc_state->has_drrs = old_crtc_state->has_drrs;
14183 }
14184
14185 static int intel_crtc_add_planes_to_state(struct intel_atomic_state *state,
14186                                           struct intel_crtc *crtc,
14187                                           u8 plane_ids_mask)
14188 {
14189         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14190         struct intel_plane *plane;
14191
14192         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
14193                 struct intel_plane_state *plane_state;
14194
14195                 if ((plane_ids_mask & BIT(plane->id)) == 0)
14196                         continue;
14197
14198                 plane_state = intel_atomic_get_plane_state(state, plane);
14199                 if (IS_ERR(plane_state))
14200                         return PTR_ERR(plane_state);
14201         }
14202
14203         return 0;
14204 }
14205
14206 static bool active_planes_affects_min_cdclk(struct drm_i915_private *dev_priv)
14207 {
14208         /* See {hsw,vlv,ivb}_plane_ratio() */
14209         return IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv) ||
14210                 IS_CHERRYVIEW(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
14211                 IS_IVYBRIDGE(dev_priv);
14212 }
14213
14214 static int intel_atomic_check_planes(struct intel_atomic_state *state,
14215                                      bool *need_modeset)
14216 {
14217         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14218         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14219         struct intel_plane_state *plane_state;
14220         struct intel_plane *plane;
14221         struct intel_crtc *crtc;
14222         int i, ret;
14223
14224         ret = icl_add_linked_planes(state);
14225         if (ret)
14226                 return ret;
14227
14228         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
14229                 ret = intel_plane_atomic_check(state, plane);
14230                 if (ret) {
14231                         DRM_DEBUG_ATOMIC("[PLANE:%d:%s] atomic driver check failed\n",
14232                                          plane->base.base.id, plane->base.name);
14233                         return ret;
14234                 }
14235         }
14236
14237         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14238                                             new_crtc_state, i) {
14239                 u8 old_active_planes, new_active_planes;
14240
14241                 ret = icl_check_nv12_planes(new_crtc_state);
14242                 if (ret)
14243                         return ret;
14244
14245                 /*
14246                  * On some platforms the number of active planes affects
14247                  * the planes' minimum cdclk calculation. Add such planes
14248                  * to the state before we compute the minimum cdclk.
14249                  */
14250                 if (!active_planes_affects_min_cdclk(dev_priv))
14251                         continue;
14252
14253                 old_active_planes = old_crtc_state->active_planes & ~BIT(PLANE_CURSOR);
14254                 new_active_planes = new_crtc_state->active_planes & ~BIT(PLANE_CURSOR);
14255
14256                 if (hweight8(old_active_planes) == hweight8(new_active_planes))
14257                         continue;
14258
14259                 ret = intel_crtc_add_planes_to_state(state, crtc, new_active_planes);
14260                 if (ret)
14261                         return ret;
14262         }
14263
14264         /*
14265          * active_planes bitmask has been updated, and potentially
14266          * affected planes are part of the state. We can now
14267          * compute the minimum cdclk for each plane.
14268          */
14269         for_each_new_intel_plane_in_state(state, plane, plane_state, i)
14270                 *need_modeset |= intel_plane_calc_min_cdclk(state, plane);
14271
14272         return 0;
14273 }
14274
14275 static int intel_atomic_check_crtcs(struct intel_atomic_state *state)
14276 {
14277         struct intel_crtc_state *crtc_state;
14278         struct intel_crtc *crtc;
14279         int i;
14280
14281         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
14282                 int ret = intel_crtc_atomic_check(state, crtc);
14283                 if (ret) {
14284                         DRM_DEBUG_ATOMIC("[CRTC:%d:%s] atomic driver check failed\n",
14285                                          crtc->base.base.id, crtc->base.name);
14286                         return ret;
14287                 }
14288         }
14289
14290         return 0;
14291 }
14292
14293 /**
14294  * intel_atomic_check - validate state object
14295  * @dev: drm device
14296  * @_state: state to validate
14297  */
14298 static int intel_atomic_check(struct drm_device *dev,
14299                               struct drm_atomic_state *_state)
14300 {
14301         struct drm_i915_private *dev_priv = to_i915(dev);
14302         struct intel_atomic_state *state = to_intel_atomic_state(_state);
14303         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14304         struct intel_crtc *crtc;
14305         int ret, i;
14306         bool any_ms = false;
14307
14308         /* Catch I915_MODE_FLAG_INHERITED */
14309         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14310                                             new_crtc_state, i) {
14311                 if (new_crtc_state->hw.mode.private_flags !=
14312                     old_crtc_state->hw.mode.private_flags)
14313                         new_crtc_state->uapi.mode_changed = true;
14314         }
14315
14316         ret = drm_atomic_helper_check_modeset(dev, &state->base);
14317         if (ret)
14318                 goto fail;
14319
14320         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14321                                             new_crtc_state, i) {
14322                 if (!needs_modeset(new_crtc_state)) {
14323                         /* Light copy */
14324                         intel_crtc_copy_uapi_to_hw_state_nomodeset(new_crtc_state);
14325
14326                         continue;
14327                 }
14328
14329                 if (!new_crtc_state->uapi.enable) {
14330                         intel_crtc_copy_uapi_to_hw_state(new_crtc_state);
14331                         continue;
14332                 }
14333
14334                 ret = intel_crtc_prepare_cleared_state(new_crtc_state);
14335                 if (ret)
14336                         goto fail;
14337
14338                 ret = intel_modeset_pipe_config(new_crtc_state);
14339                 if (ret)
14340                         goto fail;
14341
14342                 intel_crtc_check_fastset(old_crtc_state, new_crtc_state);
14343         }
14344
14345         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14346                                             new_crtc_state, i) {
14347                 if (needs_modeset(new_crtc_state)) {
14348                         any_ms = true;
14349                         continue;
14350                 }
14351
14352                 if (!new_crtc_state->update_pipe)
14353                         continue;
14354
14355                 intel_crtc_copy_fastset(old_crtc_state, new_crtc_state);
14356         }
14357
14358         if (any_ms && !check_digital_port_conflicts(state)) {
14359                 DRM_DEBUG_KMS("rejecting conflicting digital port configuration\n");
14360                 ret = EINVAL;
14361                 goto fail;
14362         }
14363
14364         ret = drm_dp_mst_atomic_check(&state->base);
14365         if (ret)
14366                 goto fail;
14367
14368         any_ms |= state->cdclk.force_min_cdclk_changed;
14369
14370         ret = intel_atomic_check_planes(state, &any_ms);
14371         if (ret)
14372                 goto fail;
14373
14374         if (any_ms) {
14375                 ret = intel_modeset_checks(state);
14376                 if (ret)
14377                         goto fail;
14378         } else {
14379                 state->cdclk.logical = dev_priv->cdclk.logical;
14380         }
14381
14382         ret = intel_atomic_check_crtcs(state);
14383         if (ret)
14384                 goto fail;
14385
14386         intel_fbc_choose_crtc(dev_priv, state);
14387         ret = calc_watermark_data(state);
14388         if (ret)
14389                 goto fail;
14390
14391         ret = intel_bw_atomic_check(state);
14392         if (ret)
14393                 goto fail;
14394
14395         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14396                                             new_crtc_state, i) {
14397                 if (!needs_modeset(new_crtc_state) &&
14398                     !new_crtc_state->update_pipe)
14399                         continue;
14400
14401                 intel_dump_pipe_config(new_crtc_state, state,
14402                                        needs_modeset(new_crtc_state) ?
14403                                        "[modeset]" : "[fastset]");
14404         }
14405
14406         return 0;
14407
14408  fail:
14409         if (ret == -EDEADLK)
14410                 return ret;
14411
14412         /*
14413          * FIXME would probably be nice to know which crtc specifically
14414          * caused the failure, in cases where we can pinpoint it.
14415          */
14416         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14417                                             new_crtc_state, i)
14418                 intel_dump_pipe_config(new_crtc_state, state, "[failed]");
14419
14420         return ret;
14421 }
14422
14423 static int intel_atomic_prepare_commit(struct intel_atomic_state *state)
14424 {
14425         return drm_atomic_helper_prepare_planes(state->base.dev,
14426                                                 &state->base);
14427 }
14428
14429 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
14430 {
14431         struct drm_device *dev = crtc->base.dev;
14432         struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)];
14433
14434         if (!vblank->max_vblank_count)
14435                 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
14436
14437         return crtc->base.funcs->get_vblank_counter(&crtc->base);
14438 }
14439
14440 void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
14441                                   struct intel_crtc_state *crtc_state)
14442 {
14443         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14444
14445         if (!IS_GEN(dev_priv, 2) || crtc_state->active_planes)
14446                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
14447
14448         if (crtc_state->has_pch_encoder) {
14449                 enum pipe pch_transcoder =
14450                         intel_crtc_pch_transcoder(crtc);
14451
14452                 intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
14453         }
14454 }
14455
14456 static void intel_pipe_fastset(const struct intel_crtc_state *old_crtc_state,
14457                                const struct intel_crtc_state *new_crtc_state)
14458 {
14459         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
14460         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14461
14462         /*
14463          * Update pipe size and adjust fitter if needed: the reason for this is
14464          * that in compute_mode_changes we check the native mode (not the pfit
14465          * mode) to see if we can flip rather than do a full mode set. In the
14466          * fastboot case, we'll flip, but if we don't update the pipesrc and
14467          * pfit state, we'll end up with a big fb scanned out into the wrong
14468          * sized surface.
14469          */
14470         intel_set_pipe_src_size(new_crtc_state);
14471
14472         /* on skylake this is done by detaching scalers */
14473         if (INTEL_GEN(dev_priv) >= 9) {
14474                 skl_detach_scalers(new_crtc_state);
14475
14476                 if (new_crtc_state->pch_pfit.enabled)
14477                         skylake_pfit_enable(new_crtc_state);
14478         } else if (HAS_PCH_SPLIT(dev_priv)) {
14479                 if (new_crtc_state->pch_pfit.enabled)
14480                         ironlake_pfit_enable(new_crtc_state);
14481                 else if (old_crtc_state->pch_pfit.enabled)
14482                         ironlake_pfit_disable(old_crtc_state);
14483         }
14484
14485         if (INTEL_GEN(dev_priv) >= 11)
14486                 icl_set_pipe_chicken(crtc);
14487 }
14488
14489 static void commit_pipe_config(struct intel_atomic_state *state,
14490                                struct intel_crtc_state *old_crtc_state,
14491                                struct intel_crtc_state *new_crtc_state)
14492 {
14493         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
14494         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14495         bool modeset = needs_modeset(new_crtc_state);
14496
14497         /*
14498          * During modesets pipe configuration was programmed as the
14499          * CRTC was enabled.
14500          */
14501         if (!modeset) {
14502                 if (new_crtc_state->uapi.color_mgmt_changed ||
14503                     new_crtc_state->update_pipe)
14504                         intel_color_commit(new_crtc_state);
14505
14506                 if (INTEL_GEN(dev_priv) >= 9)
14507                         skl_detach_scalers(new_crtc_state);
14508
14509                 if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
14510                         bdw_set_pipemisc(new_crtc_state);
14511
14512                 if (new_crtc_state->update_pipe)
14513                         intel_pipe_fastset(old_crtc_state, new_crtc_state);
14514         }
14515
14516         if (dev_priv->display.atomic_update_watermarks)
14517                 dev_priv->display.atomic_update_watermarks(state, crtc);
14518 }
14519
14520 static void intel_update_crtc(struct intel_crtc *crtc,
14521                               struct intel_atomic_state *state,
14522                               struct intel_crtc_state *old_crtc_state,
14523                               struct intel_crtc_state *new_crtc_state)
14524 {
14525         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14526         bool modeset = needs_modeset(new_crtc_state);
14527         struct intel_plane_state *new_plane_state =
14528                 intel_atomic_get_new_plane_state(state,
14529                                                  to_intel_plane(crtc->base.primary));
14530
14531         if (modeset) {
14532                 intel_crtc_update_active_timings(new_crtc_state);
14533
14534                 dev_priv->display.crtc_enable(state, crtc);
14535
14536                 /* vblanks work again, re-enable pipe CRC. */
14537                 intel_crtc_enable_pipe_crc(crtc);
14538         } else {
14539                 if (new_crtc_state->preload_luts &&
14540                     (new_crtc_state->uapi.color_mgmt_changed ||
14541                      new_crtc_state->update_pipe))
14542                         intel_color_load_luts(new_crtc_state);
14543
14544                 intel_pre_plane_update(state, crtc);
14545
14546                 if (new_crtc_state->update_pipe)
14547                         intel_encoders_update_pipe(state, crtc);
14548         }
14549
14550         if (new_crtc_state->update_pipe && !new_crtc_state->enable_fbc)
14551                 intel_fbc_disable(crtc);
14552         else if (new_plane_state)
14553                 intel_fbc_enable(crtc, new_crtc_state, new_plane_state);
14554
14555         /* Perform vblank evasion around commit operation */
14556         intel_pipe_update_start(new_crtc_state);
14557
14558         commit_pipe_config(state, old_crtc_state, new_crtc_state);
14559
14560         if (INTEL_GEN(dev_priv) >= 9)
14561                 skl_update_planes_on_crtc(state, crtc);
14562         else
14563                 i9xx_update_planes_on_crtc(state, crtc);
14564
14565         intel_pipe_update_end(new_crtc_state);
14566
14567         /*
14568          * We usually enable FIFO underrun interrupts as part of the
14569          * CRTC enable sequence during modesets.  But when we inherit a
14570          * valid pipe configuration from the BIOS we need to take care
14571          * of enabling them on the CRTC's first fastset.
14572          */
14573         if (new_crtc_state->update_pipe && !modeset &&
14574             old_crtc_state->hw.mode.private_flags & I915_MODE_FLAG_INHERITED)
14575                 intel_crtc_arm_fifo_underrun(crtc, new_crtc_state);
14576 }
14577
14578 static struct intel_crtc *intel_get_slave_crtc(const struct intel_crtc_state *new_crtc_state)
14579 {
14580         struct drm_i915_private *dev_priv = to_i915(new_crtc_state->uapi.crtc->dev);
14581         enum transcoder slave_transcoder;
14582
14583         WARN_ON(!is_power_of_2(new_crtc_state->sync_mode_slaves_mask));
14584
14585         slave_transcoder = ffs(new_crtc_state->sync_mode_slaves_mask) - 1;
14586         return intel_get_crtc_for_pipe(dev_priv,
14587                                        (enum pipe)slave_transcoder);
14588 }
14589
14590 static void intel_old_crtc_state_disables(struct intel_atomic_state *state,
14591                                           struct intel_crtc_state *old_crtc_state,
14592                                           struct intel_crtc_state *new_crtc_state,
14593                                           struct intel_crtc *crtc)
14594 {
14595         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14596
14597         intel_crtc_disable_planes(state, crtc);
14598
14599         /*
14600          * We need to disable pipe CRC before disabling the pipe,
14601          * or we race against vblank off.
14602          */
14603         intel_crtc_disable_pipe_crc(crtc);
14604
14605         dev_priv->display.crtc_disable(state, crtc);
14606         crtc->active = false;
14607         intel_fbc_disable(crtc);
14608         intel_disable_shared_dpll(old_crtc_state);
14609
14610         /* FIXME unify this for all platforms */
14611         if (!new_crtc_state->hw.active &&
14612             !HAS_GMCH(dev_priv) &&
14613             dev_priv->display.initial_watermarks)
14614                 dev_priv->display.initial_watermarks(state, crtc);
14615 }
14616
14617 static void intel_commit_modeset_disables(struct intel_atomic_state *state)
14618 {
14619         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
14620         struct intel_crtc *crtc;
14621         u32 handled = 0;
14622         int i;
14623
14624         /* Only disable port sync and MST slaves */
14625         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14626                                             new_crtc_state, i) {
14627                 if (!needs_modeset(new_crtc_state))
14628                         continue;
14629
14630                 if (!old_crtc_state->hw.active)
14631                         continue;
14632
14633                 /* In case of Transcoder port Sync master slave CRTCs can be
14634                  * assigned in any order and we need to make sure that
14635                  * slave CRTCs are disabled first and then master CRTC since
14636                  * Slave vblanks are masked till Master Vblanks.
14637                  */
14638                 if (!is_trans_port_sync_slave(old_crtc_state) &&
14639                     !intel_dp_mst_is_slave_trans(old_crtc_state))
14640                         continue;
14641
14642                 intel_pre_plane_update(state, crtc);
14643                 intel_old_crtc_state_disables(state, old_crtc_state,
14644                                               new_crtc_state, crtc);
14645                 handled |= BIT(crtc->pipe);
14646         }
14647
14648         /* Disable everything else left on */
14649         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14650                                             new_crtc_state, i) {
14651                 if (!needs_modeset(new_crtc_state) ||
14652                     (handled & BIT(crtc->pipe)))
14653                         continue;
14654
14655                 intel_pre_plane_update(state, crtc);
14656                 if (old_crtc_state->hw.active)
14657                         intel_old_crtc_state_disables(state, old_crtc_state,
14658                                                       new_crtc_state, crtc);
14659         }
14660 }
14661
14662 static void intel_commit_modeset_enables(struct intel_atomic_state *state)
14663 {
14664         struct intel_crtc *crtc;
14665         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14666         int i;
14667
14668         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
14669                 if (!new_crtc_state->hw.active)
14670                         continue;
14671
14672                 intel_update_crtc(crtc, state, old_crtc_state,
14673                                   new_crtc_state);
14674         }
14675 }
14676
14677 static void intel_crtc_enable_trans_port_sync(struct intel_crtc *crtc,
14678                                               struct intel_atomic_state *state,
14679                                               struct intel_crtc_state *new_crtc_state)
14680 {
14681         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14682
14683         intel_crtc_update_active_timings(new_crtc_state);
14684         dev_priv->display.crtc_enable(state, crtc);
14685         intel_crtc_enable_pipe_crc(crtc);
14686 }
14687
14688 static void intel_set_dp_tp_ctl_normal(struct intel_crtc *crtc,
14689                                        struct intel_atomic_state *state)
14690 {
14691         struct drm_connector *uninitialized_var(conn);
14692         struct drm_connector_state *conn_state;
14693         struct intel_dp *intel_dp;
14694         int i;
14695
14696         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
14697                 if (conn_state->crtc == &crtc->base)
14698                         break;
14699         }
14700         intel_dp = enc_to_intel_dp(&intel_attached_encoder(conn)->base);
14701         intel_dp_stop_link_train(intel_dp);
14702 }
14703
14704 static void intel_post_crtc_enable_updates(struct intel_crtc *crtc,
14705                                            struct intel_atomic_state *state)
14706 {
14707         struct intel_crtc_state *new_crtc_state =
14708                 intel_atomic_get_new_crtc_state(state, crtc);
14709         struct intel_crtc_state *old_crtc_state =
14710                 intel_atomic_get_old_crtc_state(state, crtc);
14711         struct intel_plane_state *new_plane_state =
14712                 intel_atomic_get_new_plane_state(state,
14713                                                  to_intel_plane(crtc->base.primary));
14714         bool modeset = needs_modeset(new_crtc_state);
14715
14716         if (new_crtc_state->update_pipe && !new_crtc_state->enable_fbc)
14717                 intel_fbc_disable(crtc);
14718         else if (new_plane_state)
14719                 intel_fbc_enable(crtc, new_crtc_state, new_plane_state);
14720
14721         /* Perform vblank evasion around commit operation */
14722         intel_pipe_update_start(new_crtc_state);
14723         commit_pipe_config(state, old_crtc_state, new_crtc_state);
14724         skl_update_planes_on_crtc(state, crtc);
14725         intel_pipe_update_end(new_crtc_state);
14726
14727         /*
14728          * We usually enable FIFO underrun interrupts as part of the
14729          * CRTC enable sequence during modesets.  But when we inherit a
14730          * valid pipe configuration from the BIOS we need to take care
14731          * of enabling them on the CRTC's first fastset.
14732          */
14733         if (new_crtc_state->update_pipe && !modeset &&
14734             old_crtc_state->hw.mode.private_flags & I915_MODE_FLAG_INHERITED)
14735                 intel_crtc_arm_fifo_underrun(crtc, new_crtc_state);
14736 }
14737
14738 static void intel_update_trans_port_sync_crtcs(struct intel_crtc *crtc,
14739                                                struct intel_atomic_state *state,
14740                                                struct intel_crtc_state *old_crtc_state,
14741                                                struct intel_crtc_state *new_crtc_state)
14742 {
14743         struct intel_crtc *slave_crtc = intel_get_slave_crtc(new_crtc_state);
14744         struct intel_crtc_state *new_slave_crtc_state =
14745                 intel_atomic_get_new_crtc_state(state, slave_crtc);
14746         struct intel_crtc_state *old_slave_crtc_state =
14747                 intel_atomic_get_old_crtc_state(state, slave_crtc);
14748
14749         WARN_ON(!slave_crtc || !new_slave_crtc_state ||
14750                 !old_slave_crtc_state);
14751
14752         DRM_DEBUG_KMS("Updating Transcoder Port Sync Master CRTC = %d %s and Slave CRTC %d %s\n",
14753                       crtc->base.base.id, crtc->base.name, slave_crtc->base.base.id,
14754                       slave_crtc->base.name);
14755
14756         /* Enable seq for slave with with DP_TP_CTL left Idle until the
14757          * master is ready
14758          */
14759         intel_crtc_enable_trans_port_sync(slave_crtc,
14760                                           state,
14761                                           new_slave_crtc_state);
14762
14763         /* Enable seq for master with with DP_TP_CTL left Idle */
14764         intel_crtc_enable_trans_port_sync(crtc,
14765                                           state,
14766                                           new_crtc_state);
14767
14768         /* Set Slave's DP_TP_CTL to Normal */
14769         intel_set_dp_tp_ctl_normal(slave_crtc,
14770                                    state);
14771
14772         /* Set Master's DP_TP_CTL To Normal */
14773         usleep_range(200, 400);
14774         intel_set_dp_tp_ctl_normal(crtc,
14775                                    state);
14776
14777         /* Now do the post crtc enable for all master and slaves */
14778         intel_post_crtc_enable_updates(slave_crtc,
14779                                        state);
14780         intel_post_crtc_enable_updates(crtc,
14781                                        state);
14782 }
14783
14784 static void skl_commit_modeset_enables(struct intel_atomic_state *state)
14785 {
14786         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14787         struct intel_crtc *crtc;
14788         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14789         u8 hw_enabled_slices = dev_priv->wm.skl_hw.ddb.enabled_slices;
14790         u8 required_slices = state->wm_results.ddb.enabled_slices;
14791         struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
14792         const u8 num_pipes = INTEL_NUM_PIPES(dev_priv);
14793         u8 update_pipes = 0, modeset_pipes = 0;
14794         int i;
14795
14796         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
14797                 if (!new_crtc_state->hw.active)
14798                         continue;
14799
14800                 /* ignore allocations for crtc's that have been turned off. */
14801                 if (!needs_modeset(new_crtc_state)) {
14802                         entries[i] = old_crtc_state->wm.skl.ddb;
14803                         update_pipes |= BIT(crtc->pipe);
14804                 } else {
14805                         modeset_pipes |= BIT(crtc->pipe);
14806                 }
14807         }
14808
14809         /* If 2nd DBuf slice required, enable it here */
14810         if (INTEL_GEN(dev_priv) >= 11 && required_slices > hw_enabled_slices)
14811                 icl_dbuf_slices_update(dev_priv, required_slices);
14812
14813         /*
14814          * Whenever the number of active pipes changes, we need to make sure we
14815          * update the pipes in the right order so that their ddb allocations
14816          * never overlap with each other between CRTC updates. Otherwise we'll
14817          * cause pipe underruns and other bad stuff.
14818          *
14819          * So first lets enable all pipes that do not need a fullmodeset as
14820          * those don't have any external dependency.
14821          */
14822         while (update_pipes) {
14823                 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14824                                                     new_crtc_state, i) {
14825                         enum pipe pipe = crtc->pipe;
14826
14827                         if ((update_pipes & BIT(pipe)) == 0)
14828                                 continue;
14829
14830                         if (skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
14831                                                         entries, num_pipes, i))
14832                                 continue;
14833
14834                         entries[i] = new_crtc_state->wm.skl.ddb;
14835                         update_pipes &= ~BIT(pipe);
14836
14837                         intel_update_crtc(crtc, state, old_crtc_state,
14838                                           new_crtc_state);
14839
14840                         /*
14841                          * If this is an already active pipe, it's DDB changed,
14842                          * and this isn't the last pipe that needs updating
14843                          * then we need to wait for a vblank to pass for the
14844                          * new ddb allocation to take effect.
14845                          */
14846                         if (!skl_ddb_entry_equal(&new_crtc_state->wm.skl.ddb,
14847                                                  &old_crtc_state->wm.skl.ddb) &&
14848                             (update_pipes | modeset_pipes))
14849                                 intel_wait_for_vblank(dev_priv, pipe);
14850                 }
14851         }
14852
14853         /*
14854          * Enable all pipes that needs a modeset and do not depends on other
14855          * pipes
14856          */
14857         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14858                                             new_crtc_state, i) {
14859                 enum pipe pipe = crtc->pipe;
14860
14861                 if ((modeset_pipes & BIT(pipe)) == 0)
14862                         continue;
14863
14864                 if (intel_dp_mst_is_slave_trans(new_crtc_state) ||
14865                     is_trans_port_sync_slave(new_crtc_state))
14866                         continue;
14867
14868                 WARN_ON(skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
14869                                                     entries, num_pipes, i));
14870
14871                 entries[i] = new_crtc_state->wm.skl.ddb;
14872                 modeset_pipes &= ~BIT(pipe);
14873
14874                 if (is_trans_port_sync_mode(new_crtc_state)) {
14875                         struct intel_crtc *slave_crtc;
14876
14877                         intel_update_trans_port_sync_crtcs(crtc, state,
14878                                                            old_crtc_state,
14879                                                            new_crtc_state);
14880
14881                         slave_crtc = intel_get_slave_crtc(new_crtc_state);
14882                         /* TODO: update entries[] of slave */
14883                         modeset_pipes &= ~BIT(slave_crtc->pipe);
14884
14885                 } else {
14886                         intel_update_crtc(crtc, state, old_crtc_state,
14887                                           new_crtc_state);
14888                 }
14889         }
14890
14891         /*
14892          * Finally enable all pipes that needs a modeset and depends on
14893          * other pipes, right now it is only MST slaves as both port sync slave
14894          * and master are enabled together
14895          */
14896         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14897                                             new_crtc_state, i) {
14898                 enum pipe pipe = crtc->pipe;
14899
14900                 if ((modeset_pipes & BIT(pipe)) == 0)
14901                         continue;
14902
14903                 WARN_ON(skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
14904                                                     entries, num_pipes, i));
14905
14906                 entries[i] = new_crtc_state->wm.skl.ddb;
14907                 modeset_pipes &= ~BIT(pipe);
14908
14909                 intel_update_crtc(crtc, state, old_crtc_state, new_crtc_state);
14910         }
14911
14912         WARN_ON(modeset_pipes);
14913
14914         /* If 2nd DBuf slice is no more required disable it */
14915         if (INTEL_GEN(dev_priv) >= 11 && required_slices < hw_enabled_slices)
14916                 icl_dbuf_slices_update(dev_priv, required_slices);
14917 }
14918
14919 static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv)
14920 {
14921         struct intel_atomic_state *state, *next;
14922         struct llist_node *freed;
14923
14924         freed = llist_del_all(&dev_priv->atomic_helper.free_list);
14925         llist_for_each_entry_safe(state, next, freed, freed)
14926                 drm_atomic_state_put(&state->base);
14927 }
14928
14929 static void intel_atomic_helper_free_state_worker(struct work_struct *work)
14930 {
14931         struct drm_i915_private *dev_priv =
14932                 container_of(work, typeof(*dev_priv), atomic_helper.free_work);
14933
14934         intel_atomic_helper_free_state(dev_priv);
14935 }
14936
14937 static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state)
14938 {
14939         struct wait_queue_entry wait_fence, wait_reset;
14940         struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev);
14941
14942         init_wait_entry(&wait_fence, 0);
14943         init_wait_entry(&wait_reset, 0);
14944         for (;;) {
14945                 prepare_to_wait(&intel_state->commit_ready.wait,
14946                                 &wait_fence, TASK_UNINTERRUPTIBLE);
14947                 prepare_to_wait(bit_waitqueue(&dev_priv->gt.reset.flags,
14948                                               I915_RESET_MODESET),
14949                                 &wait_reset, TASK_UNINTERRUPTIBLE);
14950
14951
14952                 if (i915_sw_fence_done(&intel_state->commit_ready) ||
14953                     test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags))
14954                         break;
14955
14956                 schedule();
14957         }
14958         finish_wait(&intel_state->commit_ready.wait, &wait_fence);
14959         finish_wait(bit_waitqueue(&dev_priv->gt.reset.flags,
14960                                   I915_RESET_MODESET),
14961                     &wait_reset);
14962 }
14963
14964 static void intel_atomic_cleanup_work(struct work_struct *work)
14965 {
14966         struct drm_atomic_state *state =
14967                 container_of(work, struct drm_atomic_state, commit_work);
14968         struct drm_i915_private *i915 = to_i915(state->dev);
14969
14970         drm_atomic_helper_cleanup_planes(&i915->drm, state);
14971         drm_atomic_helper_commit_cleanup_done(state);
14972         drm_atomic_state_put(state);
14973
14974         intel_atomic_helper_free_state(i915);
14975 }
14976
14977 static void intel_atomic_commit_tail(struct intel_atomic_state *state)
14978 {
14979         struct drm_device *dev = state->base.dev;
14980         struct drm_i915_private *dev_priv = to_i915(dev);
14981         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
14982         struct intel_crtc *crtc;
14983         u64 put_domains[I915_MAX_PIPES] = {};
14984         intel_wakeref_t wakeref = 0;
14985         int i;
14986
14987         intel_atomic_commit_fence_wait(state);
14988
14989         drm_atomic_helper_wait_for_dependencies(&state->base);
14990
14991         if (state->modeset)
14992                 wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
14993
14994         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14995                                             new_crtc_state, i) {
14996                 if (needs_modeset(new_crtc_state) ||
14997                     new_crtc_state->update_pipe) {
14998
14999                         put_domains[crtc->pipe] =
15000                                 modeset_get_crtc_power_domains(new_crtc_state);
15001                 }
15002         }
15003
15004         intel_commit_modeset_disables(state);
15005
15006         /* FIXME: Eventually get rid of our crtc->config pointer */
15007         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
15008                 crtc->config = new_crtc_state;
15009
15010         if (state->modeset) {
15011                 drm_atomic_helper_update_legacy_modeset_state(dev, &state->base);
15012
15013                 intel_set_cdclk_pre_plane_update(dev_priv,
15014                                                  &state->cdclk.actual,
15015                                                  &dev_priv->cdclk.actual,
15016                                                  state->cdclk.pipe);
15017
15018                 /*
15019                  * SKL workaround: bspec recommends we disable the SAGV when we
15020                  * have more then one pipe enabled
15021                  */
15022                 if (!intel_can_enable_sagv(state))
15023                         intel_disable_sagv(dev_priv);
15024
15025                 intel_modeset_verify_disabled(dev_priv, state);
15026         }
15027
15028         /* Complete the events for pipes that have now been disabled */
15029         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
15030                 bool modeset = needs_modeset(new_crtc_state);
15031
15032                 /* Complete events for now disable pipes here. */
15033                 if (modeset && !new_crtc_state->hw.active && new_crtc_state->uapi.event) {
15034                         spin_lock_irq(&dev->event_lock);
15035                         drm_crtc_send_vblank_event(&crtc->base,
15036                                                    new_crtc_state->uapi.event);
15037                         spin_unlock_irq(&dev->event_lock);
15038
15039                         new_crtc_state->uapi.event = NULL;
15040                 }
15041         }
15042
15043         if (state->modeset)
15044                 intel_encoders_update_prepare(state);
15045
15046         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
15047         dev_priv->display.commit_modeset_enables(state);
15048
15049         if (state->modeset) {
15050                 intel_encoders_update_complete(state);
15051
15052                 intel_set_cdclk_post_plane_update(dev_priv,
15053                                                   &state->cdclk.actual,
15054                                                   &dev_priv->cdclk.actual,
15055                                                   state->cdclk.pipe);
15056         }
15057
15058         /* FIXME: We should call drm_atomic_helper_commit_hw_done() here
15059          * already, but still need the state for the delayed optimization. To
15060          * fix this:
15061          * - wrap the optimization/post_plane_update stuff into a per-crtc work.
15062          * - schedule that vblank worker _before_ calling hw_done
15063          * - at the start of commit_tail, cancel it _synchrously
15064          * - switch over to the vblank wait helper in the core after that since
15065          *   we don't need out special handling any more.
15066          */
15067         drm_atomic_helper_wait_for_flip_done(dev, &state->base);
15068
15069         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
15070                 if (new_crtc_state->hw.active &&
15071                     !needs_modeset(new_crtc_state) &&
15072                     !new_crtc_state->preload_luts &&
15073                     (new_crtc_state->uapi.color_mgmt_changed ||
15074                      new_crtc_state->update_pipe))
15075                         intel_color_load_luts(new_crtc_state);
15076         }
15077
15078         /*
15079          * Now that the vblank has passed, we can go ahead and program the
15080          * optimal watermarks on platforms that need two-step watermark
15081          * programming.
15082          *
15083          * TODO: Move this (and other cleanup) to an async worker eventually.
15084          */
15085         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
15086                                             new_crtc_state, i) {
15087                 /*
15088                  * Gen2 reports pipe underruns whenever all planes are disabled.
15089                  * So re-enable underrun reporting after some planes get enabled.
15090                  *
15091                  * We do this before .optimize_watermarks() so that we have a
15092                  * chance of catching underruns with the intermediate watermarks
15093                  * vs. the new plane configuration.
15094                  */
15095                 if (IS_GEN(dev_priv, 2) && planes_enabling(old_crtc_state, new_crtc_state))
15096                         intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
15097
15098                 if (dev_priv->display.optimize_watermarks)
15099                         dev_priv->display.optimize_watermarks(state, crtc);
15100         }
15101
15102         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
15103                 intel_post_plane_update(state, crtc);
15104
15105                 if (put_domains[i])
15106                         modeset_put_power_domains(dev_priv, put_domains[i]);
15107
15108                 intel_modeset_verify_crtc(crtc, state, old_crtc_state, new_crtc_state);
15109         }
15110
15111         /* Underruns don't always raise interrupts, so check manually */
15112         intel_check_cpu_fifo_underruns(dev_priv);
15113         intel_check_pch_fifo_underruns(dev_priv);
15114
15115         if (state->modeset)
15116                 intel_verify_planes(state);
15117
15118         if (state->modeset && intel_can_enable_sagv(state))
15119                 intel_enable_sagv(dev_priv);
15120
15121         drm_atomic_helper_commit_hw_done(&state->base);
15122
15123         if (state->modeset) {
15124                 /* As one of the primary mmio accessors, KMS has a high
15125                  * likelihood of triggering bugs in unclaimed access. After we
15126                  * finish modesetting, see if an error has been flagged, and if
15127                  * so enable debugging for the next modeset - and hope we catch
15128                  * the culprit.
15129                  */
15130                 intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore);
15131                 intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET, wakeref);
15132         }
15133         intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
15134
15135         /*
15136          * Defer the cleanup of the old state to a separate worker to not
15137          * impede the current task (userspace for blocking modesets) that
15138          * are executed inline. For out-of-line asynchronous modesets/flips,
15139          * deferring to a new worker seems overkill, but we would place a
15140          * schedule point (cond_resched()) here anyway to keep latencies
15141          * down.
15142          */
15143         INIT_WORK(&state->base.commit_work, intel_atomic_cleanup_work);
15144         queue_work(system_highpri_wq, &state->base.commit_work);
15145 }
15146
15147 static void intel_atomic_commit_work(struct work_struct *work)
15148 {
15149         struct intel_atomic_state *state =
15150                 container_of(work, struct intel_atomic_state, base.commit_work);
15151
15152         intel_atomic_commit_tail(state);
15153 }
15154
15155 static int __i915_sw_fence_call
15156 intel_atomic_commit_ready(struct i915_sw_fence *fence,
15157                           enum i915_sw_fence_notify notify)
15158 {
15159         struct intel_atomic_state *state =
15160                 container_of(fence, struct intel_atomic_state, commit_ready);
15161
15162         switch (notify) {
15163         case FENCE_COMPLETE:
15164                 /* we do blocking waits in the worker, nothing to do here */
15165                 break;
15166         case FENCE_FREE:
15167                 {
15168                         struct intel_atomic_helper *helper =
15169                                 &to_i915(state->base.dev)->atomic_helper;
15170
15171                         if (llist_add(&state->freed, &helper->free_list))
15172                                 schedule_work(&helper->free_work);
15173                         break;
15174                 }
15175         }
15176
15177         return NOTIFY_DONE;
15178 }
15179
15180 static void intel_atomic_track_fbs(struct intel_atomic_state *state)
15181 {
15182         struct intel_plane_state *old_plane_state, *new_plane_state;
15183         struct intel_plane *plane;
15184         int i;
15185
15186         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
15187                                              new_plane_state, i)
15188                 intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->hw.fb),
15189                                         to_intel_frontbuffer(new_plane_state->hw.fb),
15190                                         plane->frontbuffer_bit);
15191 }
15192
15193 static void assert_global_state_locked(struct drm_i915_private *dev_priv)
15194 {
15195         struct intel_crtc *crtc;
15196
15197         for_each_intel_crtc(&dev_priv->drm, crtc)
15198                 drm_modeset_lock_assert_held(&crtc->base.mutex);
15199 }
15200
15201 static int intel_atomic_commit(struct drm_device *dev,
15202                                struct drm_atomic_state *_state,
15203                                bool nonblock)
15204 {
15205         struct intel_atomic_state *state = to_intel_atomic_state(_state);
15206         struct drm_i915_private *dev_priv = to_i915(dev);
15207         int ret = 0;
15208
15209         state->wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
15210
15211         drm_atomic_state_get(&state->base);
15212         i915_sw_fence_init(&state->commit_ready,
15213                            intel_atomic_commit_ready);
15214
15215         /*
15216          * The intel_legacy_cursor_update() fast path takes care
15217          * of avoiding the vblank waits for simple cursor
15218          * movement and flips. For cursor on/off and size changes,
15219          * we want to perform the vblank waits so that watermark
15220          * updates happen during the correct frames. Gen9+ have
15221          * double buffered watermarks and so shouldn't need this.
15222          *
15223          * Unset state->legacy_cursor_update before the call to
15224          * drm_atomic_helper_setup_commit() because otherwise
15225          * drm_atomic_helper_wait_for_flip_done() is a noop and
15226          * we get FIFO underruns because we didn't wait
15227          * for vblank.
15228          *
15229          * FIXME doing watermarks and fb cleanup from a vblank worker
15230          * (assuming we had any) would solve these problems.
15231          */
15232         if (INTEL_GEN(dev_priv) < 9 && state->base.legacy_cursor_update) {
15233                 struct intel_crtc_state *new_crtc_state;
15234                 struct intel_crtc *crtc;
15235                 int i;
15236
15237                 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
15238                         if (new_crtc_state->wm.need_postvbl_update ||
15239                             new_crtc_state->update_wm_post)
15240                                 state->base.legacy_cursor_update = false;
15241         }
15242
15243         ret = intel_atomic_prepare_commit(state);
15244         if (ret) {
15245                 DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret);
15246                 i915_sw_fence_commit(&state->commit_ready);
15247                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
15248                 return ret;
15249         }
15250
15251         ret = drm_atomic_helper_setup_commit(&state->base, nonblock);
15252         if (!ret)
15253                 ret = drm_atomic_helper_swap_state(&state->base, true);
15254
15255         if (ret) {
15256                 i915_sw_fence_commit(&state->commit_ready);
15257
15258                 drm_atomic_helper_cleanup_planes(dev, &state->base);
15259                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
15260                 return ret;
15261         }
15262         dev_priv->wm.distrust_bios_wm = false;
15263         intel_shared_dpll_swap_state(state);
15264         intel_atomic_track_fbs(state);
15265
15266         if (state->global_state_changed) {
15267                 assert_global_state_locked(dev_priv);
15268
15269                 memcpy(dev_priv->min_cdclk, state->min_cdclk,
15270                        sizeof(state->min_cdclk));
15271                 memcpy(dev_priv->min_voltage_level, state->min_voltage_level,
15272                        sizeof(state->min_voltage_level));
15273                 dev_priv->active_pipes = state->active_pipes;
15274                 dev_priv->cdclk.force_min_cdclk = state->cdclk.force_min_cdclk;
15275
15276                 intel_cdclk_swap_state(state);
15277         }
15278
15279         drm_atomic_state_get(&state->base);
15280         INIT_WORK(&state->base.commit_work, intel_atomic_commit_work);
15281
15282         i915_sw_fence_commit(&state->commit_ready);
15283         if (nonblock && state->modeset) {
15284                 queue_work(dev_priv->modeset_wq, &state->base.commit_work);
15285         } else if (nonblock) {
15286                 queue_work(dev_priv->flip_wq, &state->base.commit_work);
15287         } else {
15288                 if (state->modeset)
15289                         flush_workqueue(dev_priv->modeset_wq);
15290                 intel_atomic_commit_tail(state);
15291         }
15292
15293         return 0;
15294 }
15295
15296 struct wait_rps_boost {
15297         struct wait_queue_entry wait;
15298
15299         struct drm_crtc *crtc;
15300         struct i915_request *request;
15301 };
15302
15303 static int do_rps_boost(struct wait_queue_entry *_wait,
15304                         unsigned mode, int sync, void *key)
15305 {
15306         struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait);
15307         struct i915_request *rq = wait->request;
15308
15309         /*
15310          * If we missed the vblank, but the request is already running it
15311          * is reasonable to assume that it will complete before the next
15312          * vblank without our intervention, so leave RPS alone.
15313          */
15314         if (!i915_request_started(rq))
15315                 intel_rps_boost(rq);
15316         i915_request_put(rq);
15317
15318         drm_crtc_vblank_put(wait->crtc);
15319
15320         list_del(&wait->wait.entry);
15321         kfree(wait);
15322         return 1;
15323 }
15324
15325 static void add_rps_boost_after_vblank(struct drm_crtc *crtc,
15326                                        struct dma_fence *fence)
15327 {
15328         struct wait_rps_boost *wait;
15329
15330         if (!dma_fence_is_i915(fence))
15331                 return;
15332
15333         if (INTEL_GEN(to_i915(crtc->dev)) < 6)
15334                 return;
15335
15336         if (drm_crtc_vblank_get(crtc))
15337                 return;
15338
15339         wait = kmalloc(sizeof(*wait), GFP_KERNEL);
15340         if (!wait) {
15341                 drm_crtc_vblank_put(crtc);
15342                 return;
15343         }
15344
15345         wait->request = to_request(dma_fence_get(fence));
15346         wait->crtc = crtc;
15347
15348         wait->wait.func = do_rps_boost;
15349         wait->wait.flags = 0;
15350
15351         add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait);
15352 }
15353
15354 static int intel_plane_pin_fb(struct intel_plane_state *plane_state)
15355 {
15356         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
15357         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
15358         struct drm_framebuffer *fb = plane_state->hw.fb;
15359         struct i915_vma *vma;
15360
15361         if (plane->id == PLANE_CURSOR &&
15362             INTEL_INFO(dev_priv)->display.cursor_needs_physical) {
15363                 struct drm_i915_gem_object *obj = intel_fb_obj(fb);
15364                 const int align = intel_cursor_alignment(dev_priv);
15365                 int err;
15366
15367                 err = i915_gem_object_attach_phys(obj, align);
15368                 if (err)
15369                         return err;
15370         }
15371
15372         vma = intel_pin_and_fence_fb_obj(fb,
15373                                          &plane_state->view,
15374                                          intel_plane_uses_fence(plane_state),
15375                                          &plane_state->flags);
15376         if (IS_ERR(vma))
15377                 return PTR_ERR(vma);
15378
15379         plane_state->vma = vma;
15380
15381         return 0;
15382 }
15383
15384 static void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
15385 {
15386         struct i915_vma *vma;
15387
15388         vma = fetch_and_zero(&old_plane_state->vma);
15389         if (vma)
15390                 intel_unpin_fb_vma(vma, old_plane_state->flags);
15391 }
15392
15393 static void fb_obj_bump_render_priority(struct drm_i915_gem_object *obj)
15394 {
15395         struct i915_sched_attr attr = {
15396                 .priority = I915_USER_PRIORITY(I915_PRIORITY_DISPLAY),
15397         };
15398
15399         i915_gem_object_wait_priority(obj, 0, &attr);
15400 }
15401
15402 /**
15403  * intel_prepare_plane_fb - Prepare fb for usage on plane
15404  * @plane: drm plane to prepare for
15405  * @_new_plane_state: the plane state being prepared
15406  *
15407  * Prepares a framebuffer for usage on a display plane.  Generally this
15408  * involves pinning the underlying object and updating the frontbuffer tracking
15409  * bits.  Some older platforms need special physical address handling for
15410  * cursor planes.
15411  *
15412  * Returns 0 on success, negative error code on failure.
15413  */
15414 int
15415 intel_prepare_plane_fb(struct drm_plane *plane,
15416                        struct drm_plane_state *_new_plane_state)
15417 {
15418         struct intel_plane_state *new_plane_state =
15419                 to_intel_plane_state(_new_plane_state);
15420         struct intel_atomic_state *intel_state =
15421                 to_intel_atomic_state(new_plane_state->uapi.state);
15422         struct drm_i915_private *dev_priv = to_i915(plane->dev);
15423         struct drm_framebuffer *fb = new_plane_state->hw.fb;
15424         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
15425         struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb);
15426         int ret;
15427
15428         if (old_obj) {
15429                 struct intel_crtc_state *crtc_state =
15430                         intel_atomic_get_new_crtc_state(intel_state,
15431                                                         to_intel_crtc(plane->state->crtc));
15432
15433                 /* Big Hammer, we also need to ensure that any pending
15434                  * MI_WAIT_FOR_EVENT inside a user batch buffer on the
15435                  * current scanout is retired before unpinning the old
15436                  * framebuffer. Note that we rely on userspace rendering
15437                  * into the buffer attached to the pipe they are waiting
15438                  * on. If not, userspace generates a GPU hang with IPEHR
15439                  * point to the MI_WAIT_FOR_EVENT.
15440                  *
15441                  * This should only fail upon a hung GPU, in which case we
15442                  * can safely continue.
15443                  */
15444                 if (needs_modeset(crtc_state)) {
15445                         ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
15446                                                               old_obj->base.resv, NULL,
15447                                                               false, 0,
15448                                                               GFP_KERNEL);
15449                         if (ret < 0)
15450                                 return ret;
15451                 }
15452         }
15453
15454         if (new_plane_state->uapi.fence) { /* explicit fencing */
15455                 ret = i915_sw_fence_await_dma_fence(&intel_state->commit_ready,
15456                                                     new_plane_state->uapi.fence,
15457                                                     I915_FENCE_TIMEOUT,
15458                                                     GFP_KERNEL);
15459                 if (ret < 0)
15460                         return ret;
15461         }
15462
15463         if (!obj)
15464                 return 0;
15465
15466         ret = i915_gem_object_pin_pages(obj);
15467         if (ret)
15468                 return ret;
15469
15470         ret = intel_plane_pin_fb(new_plane_state);
15471
15472         i915_gem_object_unpin_pages(obj);
15473         if (ret)
15474                 return ret;
15475
15476         fb_obj_bump_render_priority(obj);
15477         i915_gem_object_flush_frontbuffer(obj, ORIGIN_DIRTYFB);
15478
15479         if (!new_plane_state->uapi.fence) { /* implicit fencing */
15480                 struct dma_fence *fence;
15481
15482                 ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
15483                                                       obj->base.resv, NULL,
15484                                                       false, I915_FENCE_TIMEOUT,
15485                                                       GFP_KERNEL);
15486                 if (ret < 0)
15487                         return ret;
15488
15489                 fence = dma_resv_get_excl_rcu(obj->base.resv);
15490                 if (fence) {
15491                         add_rps_boost_after_vblank(new_plane_state->hw.crtc,
15492                                                    fence);
15493                         dma_fence_put(fence);
15494                 }
15495         } else {
15496                 add_rps_boost_after_vblank(new_plane_state->hw.crtc,
15497                                            new_plane_state->uapi.fence);
15498         }
15499
15500         /*
15501          * We declare pageflips to be interactive and so merit a small bias
15502          * towards upclocking to deliver the frame on time. By only changing
15503          * the RPS thresholds to sample more regularly and aim for higher
15504          * clocks we can hopefully deliver low power workloads (like kodi)
15505          * that are not quite steady state without resorting to forcing
15506          * maximum clocks following a vblank miss (see do_rps_boost()).
15507          */
15508         if (!intel_state->rps_interactive) {
15509                 intel_rps_mark_interactive(&dev_priv->gt.rps, true);
15510                 intel_state->rps_interactive = true;
15511         }
15512
15513         return 0;
15514 }
15515
15516 /**
15517  * intel_cleanup_plane_fb - Cleans up an fb after plane use
15518  * @plane: drm plane to clean up for
15519  * @_old_plane_state: the state from the previous modeset
15520  *
15521  * Cleans up a framebuffer that has just been removed from a plane.
15522  */
15523 void
15524 intel_cleanup_plane_fb(struct drm_plane *plane,
15525                        struct drm_plane_state *_old_plane_state)
15526 {
15527         struct intel_plane_state *old_plane_state =
15528                 to_intel_plane_state(_old_plane_state);
15529         struct intel_atomic_state *intel_state =
15530                 to_intel_atomic_state(old_plane_state->uapi.state);
15531         struct drm_i915_private *dev_priv = to_i915(plane->dev);
15532
15533         if (intel_state->rps_interactive) {
15534                 intel_rps_mark_interactive(&dev_priv->gt.rps, false);
15535                 intel_state->rps_interactive = false;
15536         }
15537
15538         /* Should only be called after a successful intel_prepare_plane_fb()! */
15539         intel_plane_unpin_fb(old_plane_state);
15540 }
15541
15542 /**
15543  * intel_plane_destroy - destroy a plane
15544  * @plane: plane to destroy
15545  *
15546  * Common destruction function for all types of planes (primary, cursor,
15547  * sprite).
15548  */
15549 void intel_plane_destroy(struct drm_plane *plane)
15550 {
15551         drm_plane_cleanup(plane);
15552         kfree(to_intel_plane(plane));
15553 }
15554
15555 static bool i8xx_plane_format_mod_supported(struct drm_plane *_plane,
15556                                             u32 format, u64 modifier)
15557 {
15558         switch (modifier) {
15559         case DRM_FORMAT_MOD_LINEAR:
15560         case I915_FORMAT_MOD_X_TILED:
15561                 break;
15562         default:
15563                 return false;
15564         }
15565
15566         switch (format) {
15567         case DRM_FORMAT_C8:
15568         case DRM_FORMAT_RGB565:
15569         case DRM_FORMAT_XRGB1555:
15570         case DRM_FORMAT_XRGB8888:
15571                 return modifier == DRM_FORMAT_MOD_LINEAR ||
15572                         modifier == I915_FORMAT_MOD_X_TILED;
15573         default:
15574                 return false;
15575         }
15576 }
15577
15578 static bool i965_plane_format_mod_supported(struct drm_plane *_plane,
15579                                             u32 format, u64 modifier)
15580 {
15581         switch (modifier) {
15582         case DRM_FORMAT_MOD_LINEAR:
15583         case I915_FORMAT_MOD_X_TILED:
15584                 break;
15585         default:
15586                 return false;
15587         }
15588
15589         switch (format) {
15590         case DRM_FORMAT_C8:
15591         case DRM_FORMAT_RGB565:
15592         case DRM_FORMAT_XRGB8888:
15593         case DRM_FORMAT_XBGR8888:
15594         case DRM_FORMAT_ARGB8888:
15595         case DRM_FORMAT_ABGR8888:
15596         case DRM_FORMAT_XRGB2101010:
15597         case DRM_FORMAT_XBGR2101010:
15598         case DRM_FORMAT_ARGB2101010:
15599         case DRM_FORMAT_ABGR2101010:
15600         case DRM_FORMAT_XBGR16161616F:
15601                 return modifier == DRM_FORMAT_MOD_LINEAR ||
15602                         modifier == I915_FORMAT_MOD_X_TILED;
15603         default:
15604                 return false;
15605         }
15606 }
15607
15608 static bool intel_cursor_format_mod_supported(struct drm_plane *_plane,
15609                                               u32 format, u64 modifier)
15610 {
15611         return modifier == DRM_FORMAT_MOD_LINEAR &&
15612                 format == DRM_FORMAT_ARGB8888;
15613 }
15614
15615 static const struct drm_plane_funcs i965_plane_funcs = {
15616         .update_plane = drm_atomic_helper_update_plane,
15617         .disable_plane = drm_atomic_helper_disable_plane,
15618         .destroy = intel_plane_destroy,
15619         .atomic_duplicate_state = intel_plane_duplicate_state,
15620         .atomic_destroy_state = intel_plane_destroy_state,
15621         .format_mod_supported = i965_plane_format_mod_supported,
15622 };
15623
15624 static const struct drm_plane_funcs i8xx_plane_funcs = {
15625         .update_plane = drm_atomic_helper_update_plane,
15626         .disable_plane = drm_atomic_helper_disable_plane,
15627         .destroy = intel_plane_destroy,
15628         .atomic_duplicate_state = intel_plane_duplicate_state,
15629         .atomic_destroy_state = intel_plane_destroy_state,
15630         .format_mod_supported = i8xx_plane_format_mod_supported,
15631 };
15632
15633 static int
15634 intel_legacy_cursor_update(struct drm_plane *_plane,
15635                            struct drm_crtc *_crtc,
15636                            struct drm_framebuffer *fb,
15637                            int crtc_x, int crtc_y,
15638                            unsigned int crtc_w, unsigned int crtc_h,
15639                            u32 src_x, u32 src_y,
15640                            u32 src_w, u32 src_h,
15641                            struct drm_modeset_acquire_ctx *ctx)
15642 {
15643         struct intel_plane *plane = to_intel_plane(_plane);
15644         struct intel_crtc *crtc = to_intel_crtc(_crtc);
15645         struct intel_plane_state *old_plane_state =
15646                 to_intel_plane_state(plane->base.state);
15647         struct intel_plane_state *new_plane_state;
15648         struct intel_crtc_state *crtc_state =
15649                 to_intel_crtc_state(crtc->base.state);
15650         struct intel_crtc_state *new_crtc_state;
15651         int ret;
15652
15653         /*
15654          * When crtc is inactive or there is a modeset pending,
15655          * wait for it to complete in the slowpath
15656          */
15657         if (!crtc_state->hw.active || needs_modeset(crtc_state) ||
15658             crtc_state->update_pipe)
15659                 goto slow;
15660
15661         /*
15662          * Don't do an async update if there is an outstanding commit modifying
15663          * the plane.  This prevents our async update's changes from getting
15664          * overridden by a previous synchronous update's state.
15665          */
15666         if (old_plane_state->uapi.commit &&
15667             !try_wait_for_completion(&old_plane_state->uapi.commit->hw_done))
15668                 goto slow;
15669
15670         /*
15671          * If any parameters change that may affect watermarks,
15672          * take the slowpath. Only changing fb or position should be
15673          * in the fastpath.
15674          */
15675         if (old_plane_state->uapi.crtc != &crtc->base ||
15676             old_plane_state->uapi.src_w != src_w ||
15677             old_plane_state->uapi.src_h != src_h ||
15678             old_plane_state->uapi.crtc_w != crtc_w ||
15679             old_plane_state->uapi.crtc_h != crtc_h ||
15680             !old_plane_state->uapi.fb != !fb)
15681                 goto slow;
15682
15683         new_plane_state = to_intel_plane_state(intel_plane_duplicate_state(&plane->base));
15684         if (!new_plane_state)
15685                 return -ENOMEM;
15686
15687         new_crtc_state = to_intel_crtc_state(intel_crtc_duplicate_state(&crtc->base));
15688         if (!new_crtc_state) {
15689                 ret = -ENOMEM;
15690                 goto out_free;
15691         }
15692
15693         drm_atomic_set_fb_for_plane(&new_plane_state->uapi, fb);
15694
15695         new_plane_state->uapi.src_x = src_x;
15696         new_plane_state->uapi.src_y = src_y;
15697         new_plane_state->uapi.src_w = src_w;
15698         new_plane_state->uapi.src_h = src_h;
15699         new_plane_state->uapi.crtc_x = crtc_x;
15700         new_plane_state->uapi.crtc_y = crtc_y;
15701         new_plane_state->uapi.crtc_w = crtc_w;
15702         new_plane_state->uapi.crtc_h = crtc_h;
15703
15704         ret = intel_plane_atomic_check_with_state(crtc_state, new_crtc_state,
15705                                                   old_plane_state, new_plane_state);
15706         if (ret)
15707                 goto out_free;
15708
15709         ret = intel_plane_pin_fb(new_plane_state);
15710         if (ret)
15711                 goto out_free;
15712
15713         intel_frontbuffer_flush(to_intel_frontbuffer(new_plane_state->hw.fb),
15714                                 ORIGIN_FLIP);
15715         intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->hw.fb),
15716                                 to_intel_frontbuffer(new_plane_state->hw.fb),
15717                                 plane->frontbuffer_bit);
15718
15719         /* Swap plane state */
15720         plane->base.state = &new_plane_state->uapi;
15721
15722         /*
15723          * We cannot swap crtc_state as it may be in use by an atomic commit or
15724          * page flip that's running simultaneously. If we swap crtc_state and
15725          * destroy the old state, we will cause a use-after-free there.
15726          *
15727          * Only update active_planes, which is needed for our internal
15728          * bookkeeping. Either value will do the right thing when updating
15729          * planes atomically. If the cursor was part of the atomic update then
15730          * we would have taken the slowpath.
15731          */
15732         crtc_state->active_planes = new_crtc_state->active_planes;
15733
15734         if (new_plane_state->uapi.visible)
15735                 intel_update_plane(plane, crtc_state, new_plane_state);
15736         else
15737                 intel_disable_plane(plane, crtc_state);
15738
15739         intel_plane_unpin_fb(old_plane_state);
15740
15741 out_free:
15742         if (new_crtc_state)
15743                 intel_crtc_destroy_state(&crtc->base, &new_crtc_state->uapi);
15744         if (ret)
15745                 intel_plane_destroy_state(&plane->base, &new_plane_state->uapi);
15746         else
15747                 intel_plane_destroy_state(&plane->base, &old_plane_state->uapi);
15748         return ret;
15749
15750 slow:
15751         return drm_atomic_helper_update_plane(&plane->base, &crtc->base, fb,
15752                                               crtc_x, crtc_y, crtc_w, crtc_h,
15753                                               src_x, src_y, src_w, src_h, ctx);
15754 }
15755
15756 static const struct drm_plane_funcs intel_cursor_plane_funcs = {
15757         .update_plane = intel_legacy_cursor_update,
15758         .disable_plane = drm_atomic_helper_disable_plane,
15759         .destroy = intel_plane_destroy,
15760         .atomic_duplicate_state = intel_plane_duplicate_state,
15761         .atomic_destroy_state = intel_plane_destroy_state,
15762         .format_mod_supported = intel_cursor_format_mod_supported,
15763 };
15764
15765 static bool i9xx_plane_has_fbc(struct drm_i915_private *dev_priv,
15766                                enum i9xx_plane_id i9xx_plane)
15767 {
15768         if (!HAS_FBC(dev_priv))
15769                 return false;
15770
15771         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
15772                 return i9xx_plane == PLANE_A; /* tied to pipe A */
15773         else if (IS_IVYBRIDGE(dev_priv))
15774                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B ||
15775                         i9xx_plane == PLANE_C;
15776         else if (INTEL_GEN(dev_priv) >= 4)
15777                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B;
15778         else
15779                 return i9xx_plane == PLANE_A;
15780 }
15781
15782 static struct intel_plane *
15783 intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
15784 {
15785         struct intel_plane *plane;
15786         const struct drm_plane_funcs *plane_funcs;
15787         unsigned int supported_rotations;
15788         unsigned int possible_crtcs;
15789         const u32 *formats;
15790         int num_formats;
15791         int ret, zpos;
15792
15793         if (INTEL_GEN(dev_priv) >= 9)
15794                 return skl_universal_plane_create(dev_priv, pipe,
15795                                                   PLANE_PRIMARY);
15796
15797         plane = intel_plane_alloc();
15798         if (IS_ERR(plane))
15799                 return plane;
15800
15801         plane->pipe = pipe;
15802         /*
15803          * On gen2/3 only plane A can do FBC, but the panel fitter and LVDS
15804          * port is hooked to pipe B. Hence we want plane A feeding pipe B.
15805          */
15806         if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4)
15807                 plane->i9xx_plane = (enum i9xx_plane_id) !pipe;
15808         else
15809                 plane->i9xx_plane = (enum i9xx_plane_id) pipe;
15810         plane->id = PLANE_PRIMARY;
15811         plane->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, plane->id);
15812
15813         plane->has_fbc = i9xx_plane_has_fbc(dev_priv, plane->i9xx_plane);
15814         if (plane->has_fbc) {
15815                 struct intel_fbc *fbc = &dev_priv->fbc;
15816
15817                 fbc->possible_framebuffer_bits |= plane->frontbuffer_bit;
15818         }
15819
15820         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
15821                 formats = vlv_primary_formats;
15822                 num_formats = ARRAY_SIZE(vlv_primary_formats);
15823         } else if (INTEL_GEN(dev_priv) >= 4) {
15824                 /*
15825                  * WaFP16GammaEnabling:ivb
15826                  * "Workaround : When using the 64-bit format, the plane
15827                  *  output on each color channel has one quarter amplitude.
15828                  *  It can be brought up to full amplitude by using pipe
15829                  *  gamma correction or pipe color space conversion to
15830                  *  multiply the plane output by four."
15831                  *
15832                  * There is no dedicated plane gamma for the primary plane,
15833                  * and using the pipe gamma/csc could conflict with other
15834                  * planes, so we choose not to expose fp16 on IVB primary
15835                  * planes. HSW primary planes no longer have this problem.
15836                  */
15837                 if (IS_IVYBRIDGE(dev_priv)) {
15838                         formats = ivb_primary_formats;
15839                         num_formats = ARRAY_SIZE(ivb_primary_formats);
15840                 } else {
15841                         formats = i965_primary_formats;
15842                         num_formats = ARRAY_SIZE(i965_primary_formats);
15843                 }
15844         } else {
15845                 formats = i8xx_primary_formats;
15846                 num_formats = ARRAY_SIZE(i8xx_primary_formats);
15847         }
15848
15849         if (INTEL_GEN(dev_priv) >= 4)
15850                 plane_funcs = &i965_plane_funcs;
15851         else
15852                 plane_funcs = &i8xx_plane_funcs;
15853
15854         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
15855                 plane->min_cdclk = vlv_plane_min_cdclk;
15856         else if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
15857                 plane->min_cdclk = hsw_plane_min_cdclk;
15858         else if (IS_IVYBRIDGE(dev_priv))
15859                 plane->min_cdclk = ivb_plane_min_cdclk;
15860         else
15861                 plane->min_cdclk = i9xx_plane_min_cdclk;
15862
15863         plane->max_stride = i9xx_plane_max_stride;
15864         plane->update_plane = i9xx_update_plane;
15865         plane->disable_plane = i9xx_disable_plane;
15866         plane->get_hw_state = i9xx_plane_get_hw_state;
15867         plane->check_plane = i9xx_plane_check;
15868
15869         possible_crtcs = BIT(pipe);
15870
15871         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
15872                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
15873                                                possible_crtcs, plane_funcs,
15874                                                formats, num_formats,
15875                                                i9xx_format_modifiers,
15876                                                DRM_PLANE_TYPE_PRIMARY,
15877                                                "primary %c", pipe_name(pipe));
15878         else
15879                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
15880                                                possible_crtcs, plane_funcs,
15881                                                formats, num_formats,
15882                                                i9xx_format_modifiers,
15883                                                DRM_PLANE_TYPE_PRIMARY,
15884                                                "plane %c",
15885                                                plane_name(plane->i9xx_plane));
15886         if (ret)
15887                 goto fail;
15888
15889         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
15890                 supported_rotations =
15891                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180 |
15892                         DRM_MODE_REFLECT_X;
15893         } else if (INTEL_GEN(dev_priv) >= 4) {
15894                 supported_rotations =
15895                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180;
15896         } else {
15897                 supported_rotations = DRM_MODE_ROTATE_0;
15898         }
15899
15900         if (INTEL_GEN(dev_priv) >= 4)
15901                 drm_plane_create_rotation_property(&plane->base,
15902                                                    DRM_MODE_ROTATE_0,
15903                                                    supported_rotations);
15904
15905         zpos = 0;
15906         drm_plane_create_zpos_immutable_property(&plane->base, zpos);
15907
15908         drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs);
15909
15910         return plane;
15911
15912 fail:
15913         intel_plane_free(plane);
15914
15915         return ERR_PTR(ret);
15916 }
15917
15918 static struct intel_plane *
15919 intel_cursor_plane_create(struct drm_i915_private *dev_priv,
15920                           enum pipe pipe)
15921 {
15922         unsigned int possible_crtcs;
15923         struct intel_plane *cursor;
15924         int ret, zpos;
15925
15926         cursor = intel_plane_alloc();
15927         if (IS_ERR(cursor))
15928                 return cursor;
15929
15930         cursor->pipe = pipe;
15931         cursor->i9xx_plane = (enum i9xx_plane_id) pipe;
15932         cursor->id = PLANE_CURSOR;
15933         cursor->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, cursor->id);
15934
15935         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
15936                 cursor->max_stride = i845_cursor_max_stride;
15937                 cursor->update_plane = i845_update_cursor;
15938                 cursor->disable_plane = i845_disable_cursor;
15939                 cursor->get_hw_state = i845_cursor_get_hw_state;
15940                 cursor->check_plane = i845_check_cursor;
15941         } else {
15942                 cursor->max_stride = i9xx_cursor_max_stride;
15943                 cursor->update_plane = i9xx_update_cursor;
15944                 cursor->disable_plane = i9xx_disable_cursor;
15945                 cursor->get_hw_state = i9xx_cursor_get_hw_state;
15946                 cursor->check_plane = i9xx_check_cursor;
15947         }
15948
15949         cursor->cursor.base = ~0;
15950         cursor->cursor.cntl = ~0;
15951
15952         if (IS_I845G(dev_priv) || IS_I865G(dev_priv) || HAS_CUR_FBC(dev_priv))
15953                 cursor->cursor.size = ~0;
15954
15955         possible_crtcs = BIT(pipe);
15956
15957         ret = drm_universal_plane_init(&dev_priv->drm, &cursor->base,
15958                                        possible_crtcs, &intel_cursor_plane_funcs,
15959                                        intel_cursor_formats,
15960                                        ARRAY_SIZE(intel_cursor_formats),
15961                                        cursor_format_modifiers,
15962                                        DRM_PLANE_TYPE_CURSOR,
15963                                        "cursor %c", pipe_name(pipe));
15964         if (ret)
15965                 goto fail;
15966
15967         if (INTEL_GEN(dev_priv) >= 4)
15968                 drm_plane_create_rotation_property(&cursor->base,
15969                                                    DRM_MODE_ROTATE_0,
15970                                                    DRM_MODE_ROTATE_0 |
15971                                                    DRM_MODE_ROTATE_180);
15972
15973         zpos = RUNTIME_INFO(dev_priv)->num_sprites[pipe] + 1;
15974         drm_plane_create_zpos_immutable_property(&cursor->base, zpos);
15975
15976         drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs);
15977
15978         return cursor;
15979
15980 fail:
15981         intel_plane_free(cursor);
15982
15983         return ERR_PTR(ret);
15984 }
15985
15986 #define INTEL_CRTC_FUNCS \
15987         .gamma_set = drm_atomic_helper_legacy_gamma_set, \
15988         .set_config = drm_atomic_helper_set_config, \
15989         .destroy = intel_crtc_destroy, \
15990         .page_flip = drm_atomic_helper_page_flip, \
15991         .atomic_duplicate_state = intel_crtc_duplicate_state, \
15992         .atomic_destroy_state = intel_crtc_destroy_state, \
15993         .set_crc_source = intel_crtc_set_crc_source, \
15994         .verify_crc_source = intel_crtc_verify_crc_source, \
15995         .get_crc_sources = intel_crtc_get_crc_sources
15996
15997 static const struct drm_crtc_funcs bdw_crtc_funcs = {
15998         INTEL_CRTC_FUNCS,
15999
16000         .get_vblank_counter = g4x_get_vblank_counter,
16001         .enable_vblank = bdw_enable_vblank,
16002         .disable_vblank = bdw_disable_vblank,
16003 };
16004
16005 static const struct drm_crtc_funcs ilk_crtc_funcs = {
16006         INTEL_CRTC_FUNCS,
16007
16008         .get_vblank_counter = g4x_get_vblank_counter,
16009         .enable_vblank = ilk_enable_vblank,
16010         .disable_vblank = ilk_disable_vblank,
16011 };
16012
16013 static const struct drm_crtc_funcs g4x_crtc_funcs = {
16014         INTEL_CRTC_FUNCS,
16015
16016         .get_vblank_counter = g4x_get_vblank_counter,
16017         .enable_vblank = i965_enable_vblank,
16018         .disable_vblank = i965_disable_vblank,
16019 };
16020
16021 static const struct drm_crtc_funcs i965_crtc_funcs = {
16022         INTEL_CRTC_FUNCS,
16023
16024         .get_vblank_counter = i915_get_vblank_counter,
16025         .enable_vblank = i965_enable_vblank,
16026         .disable_vblank = i965_disable_vblank,
16027 };
16028
16029 static const struct drm_crtc_funcs i915gm_crtc_funcs = {
16030         INTEL_CRTC_FUNCS,
16031
16032         .get_vblank_counter = i915_get_vblank_counter,
16033         .enable_vblank = i915gm_enable_vblank,
16034         .disable_vblank = i915gm_disable_vblank,
16035 };
16036
16037 static const struct drm_crtc_funcs i915_crtc_funcs = {
16038         INTEL_CRTC_FUNCS,
16039
16040         .get_vblank_counter = i915_get_vblank_counter,
16041         .enable_vblank = i8xx_enable_vblank,
16042         .disable_vblank = i8xx_disable_vblank,
16043 };
16044
16045 static const struct drm_crtc_funcs i8xx_crtc_funcs = {
16046         INTEL_CRTC_FUNCS,
16047
16048         /* no hw vblank counter */
16049         .enable_vblank = i8xx_enable_vblank,
16050         .disable_vblank = i8xx_disable_vblank,
16051 };
16052
16053 static struct intel_crtc *intel_crtc_alloc(void)
16054 {
16055         struct intel_crtc_state *crtc_state;
16056         struct intel_crtc *crtc;
16057
16058         crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
16059         if (!crtc)
16060                 return ERR_PTR(-ENOMEM);
16061
16062         crtc_state = intel_crtc_state_alloc(crtc);
16063         if (!crtc_state) {
16064                 kfree(crtc);
16065                 return ERR_PTR(-ENOMEM);
16066         }
16067
16068         crtc->base.state = &crtc_state->uapi;
16069         crtc->config = crtc_state;
16070
16071         return crtc;
16072 }
16073
16074 static void intel_crtc_free(struct intel_crtc *crtc)
16075 {
16076         intel_crtc_destroy_state(&crtc->base, crtc->base.state);
16077         kfree(crtc);
16078 }
16079
16080 static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
16081 {
16082         struct intel_plane *primary, *cursor;
16083         const struct drm_crtc_funcs *funcs;
16084         struct intel_crtc *crtc;
16085         int sprite, ret;
16086
16087         crtc = intel_crtc_alloc();
16088         if (IS_ERR(crtc))
16089                 return PTR_ERR(crtc);
16090
16091         crtc->pipe = pipe;
16092         crtc->num_scalers = RUNTIME_INFO(dev_priv)->num_scalers[pipe];
16093
16094         primary = intel_primary_plane_create(dev_priv, pipe);
16095         if (IS_ERR(primary)) {
16096                 ret = PTR_ERR(primary);
16097                 goto fail;
16098         }
16099         crtc->plane_ids_mask |= BIT(primary->id);
16100
16101         for_each_sprite(dev_priv, pipe, sprite) {
16102                 struct intel_plane *plane;
16103
16104                 plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
16105                 if (IS_ERR(plane)) {
16106                         ret = PTR_ERR(plane);
16107                         goto fail;
16108                 }
16109                 crtc->plane_ids_mask |= BIT(plane->id);
16110         }
16111
16112         cursor = intel_cursor_plane_create(dev_priv, pipe);
16113         if (IS_ERR(cursor)) {
16114                 ret = PTR_ERR(cursor);
16115                 goto fail;
16116         }
16117         crtc->plane_ids_mask |= BIT(cursor->id);
16118
16119         if (HAS_GMCH(dev_priv)) {
16120                 if (IS_CHERRYVIEW(dev_priv) ||
16121                     IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
16122                         funcs = &g4x_crtc_funcs;
16123                 else if (IS_GEN(dev_priv, 4))
16124                         funcs = &i965_crtc_funcs;
16125                 else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv))
16126                         funcs = &i915gm_crtc_funcs;
16127                 else if (IS_GEN(dev_priv, 3))
16128                         funcs = &i915_crtc_funcs;
16129                 else
16130                         funcs = &i8xx_crtc_funcs;
16131         } else {
16132                 if (INTEL_GEN(dev_priv) >= 8)
16133                         funcs = &bdw_crtc_funcs;
16134                 else
16135                         funcs = &ilk_crtc_funcs;
16136         }
16137
16138         ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base,
16139                                         &primary->base, &cursor->base,
16140                                         funcs, "pipe %c", pipe_name(pipe));
16141         if (ret)
16142                 goto fail;
16143
16144         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->pipe_to_crtc_mapping) ||
16145                dev_priv->pipe_to_crtc_mapping[pipe] != NULL);
16146         dev_priv->pipe_to_crtc_mapping[pipe] = crtc;
16147
16148         if (INTEL_GEN(dev_priv) < 9) {
16149                 enum i9xx_plane_id i9xx_plane = primary->i9xx_plane;
16150
16151                 BUG_ON(i9xx_plane >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
16152                        dev_priv->plane_to_crtc_mapping[i9xx_plane] != NULL);
16153                 dev_priv->plane_to_crtc_mapping[i9xx_plane] = crtc;
16154         }
16155
16156         intel_color_init(crtc);
16157
16158         WARN_ON(drm_crtc_index(&crtc->base) != crtc->pipe);
16159
16160         return 0;
16161
16162 fail:
16163         intel_crtc_free(crtc);
16164
16165         return ret;
16166 }
16167
16168 int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
16169                                       struct drm_file *file)
16170 {
16171         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
16172         struct drm_crtc *drmmode_crtc;
16173         struct intel_crtc *crtc;
16174
16175         drmmode_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
16176         if (!drmmode_crtc)
16177                 return -ENOENT;
16178
16179         crtc = to_intel_crtc(drmmode_crtc);
16180         pipe_from_crtc_id->pipe = crtc->pipe;
16181
16182         return 0;
16183 }
16184
16185 static u32 intel_encoder_possible_clones(struct intel_encoder *encoder)
16186 {
16187         struct drm_device *dev = encoder->base.dev;
16188         struct intel_encoder *source_encoder;
16189         u32 possible_clones = 0;
16190
16191         for_each_intel_encoder(dev, source_encoder) {
16192                 if (encoders_cloneable(encoder, source_encoder))
16193                         possible_clones |= drm_encoder_mask(&source_encoder->base);
16194         }
16195
16196         return possible_clones;
16197 }
16198
16199 static u32 intel_encoder_possible_crtcs(struct intel_encoder *encoder)
16200 {
16201         struct drm_device *dev = encoder->base.dev;
16202         struct intel_crtc *crtc;
16203         u32 possible_crtcs = 0;
16204
16205         for_each_intel_crtc(dev, crtc) {
16206                 if (encoder->pipe_mask & BIT(crtc->pipe))
16207                         possible_crtcs |= drm_crtc_mask(&crtc->base);
16208         }
16209
16210         return possible_crtcs;
16211 }
16212
16213 static bool ilk_has_edp_a(struct drm_i915_private *dev_priv)
16214 {
16215         if (!IS_MOBILE(dev_priv))
16216                 return false;
16217
16218         if ((I915_READ(DP_A) & DP_DETECTED) == 0)
16219                 return false;
16220
16221         if (IS_GEN(dev_priv, 5) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
16222                 return false;
16223
16224         return true;
16225 }
16226
16227 static bool intel_ddi_crt_present(struct drm_i915_private *dev_priv)
16228 {
16229         if (INTEL_GEN(dev_priv) >= 9)
16230                 return false;
16231
16232         if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
16233                 return false;
16234
16235         if (HAS_PCH_LPT_H(dev_priv) &&
16236             I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
16237                 return false;
16238
16239         /* DDI E can't be used if DDI A requires 4 lanes */
16240         if (I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
16241                 return false;
16242
16243         if (!dev_priv->vbt.int_crt_support)
16244                 return false;
16245
16246         return true;
16247 }
16248
16249 void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv)
16250 {
16251         int pps_num;
16252         int pps_idx;
16253
16254         if (HAS_DDI(dev_priv))
16255                 return;
16256         /*
16257          * This w/a is needed at least on CPT/PPT, but to be sure apply it
16258          * everywhere where registers can be write protected.
16259          */
16260         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
16261                 pps_num = 2;
16262         else
16263                 pps_num = 1;
16264
16265         for (pps_idx = 0; pps_idx < pps_num; pps_idx++) {
16266                 u32 val = I915_READ(PP_CONTROL(pps_idx));
16267
16268                 val = (val & ~PANEL_UNLOCK_MASK) | PANEL_UNLOCK_REGS;
16269                 I915_WRITE(PP_CONTROL(pps_idx), val);
16270         }
16271 }
16272
16273 static void intel_pps_init(struct drm_i915_private *dev_priv)
16274 {
16275         if (HAS_PCH_SPLIT(dev_priv) || IS_GEN9_LP(dev_priv))
16276                 dev_priv->pps_mmio_base = PCH_PPS_BASE;
16277         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
16278                 dev_priv->pps_mmio_base = VLV_PPS_BASE;
16279         else
16280                 dev_priv->pps_mmio_base = PPS_BASE;
16281
16282         intel_pps_unlock_regs_wa(dev_priv);
16283 }
16284
16285 static void intel_setup_outputs(struct drm_i915_private *dev_priv)
16286 {
16287         struct intel_encoder *encoder;
16288         bool dpd_is_edp = false;
16289
16290         intel_pps_init(dev_priv);
16291
16292         if (!HAS_DISPLAY(dev_priv) || !INTEL_DISPLAY_ENABLED(dev_priv))
16293                 return;
16294
16295         if (INTEL_GEN(dev_priv) >= 12) {
16296                 intel_ddi_init(dev_priv, PORT_A);
16297                 intel_ddi_init(dev_priv, PORT_B);
16298                 intel_ddi_init(dev_priv, PORT_D);
16299                 intel_ddi_init(dev_priv, PORT_E);
16300                 intel_ddi_init(dev_priv, PORT_F);
16301                 intel_ddi_init(dev_priv, PORT_G);
16302                 intel_ddi_init(dev_priv, PORT_H);
16303                 intel_ddi_init(dev_priv, PORT_I);
16304                 icl_dsi_init(dev_priv);
16305         } else if (IS_ELKHARTLAKE(dev_priv)) {
16306                 intel_ddi_init(dev_priv, PORT_A);
16307                 intel_ddi_init(dev_priv, PORT_B);
16308                 intel_ddi_init(dev_priv, PORT_C);
16309                 intel_ddi_init(dev_priv, PORT_D);
16310                 icl_dsi_init(dev_priv);
16311         } else if (IS_GEN(dev_priv, 11)) {
16312                 intel_ddi_init(dev_priv, PORT_A);
16313                 intel_ddi_init(dev_priv, PORT_B);
16314                 intel_ddi_init(dev_priv, PORT_C);
16315                 intel_ddi_init(dev_priv, PORT_D);
16316                 intel_ddi_init(dev_priv, PORT_E);
16317                 /*
16318                  * On some ICL SKUs port F is not present. No strap bits for
16319                  * this, so rely on VBT.
16320                  * Work around broken VBTs on SKUs known to have no port F.
16321                  */
16322                 if (IS_ICL_WITH_PORT_F(dev_priv) &&
16323                     intel_bios_is_port_present(dev_priv, PORT_F))
16324                         intel_ddi_init(dev_priv, PORT_F);
16325
16326                 icl_dsi_init(dev_priv);
16327         } else if (IS_GEN9_LP(dev_priv)) {
16328                 /*
16329                  * FIXME: Broxton doesn't support port detection via the
16330                  * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to
16331                  * detect the ports.
16332                  */
16333                 intel_ddi_init(dev_priv, PORT_A);
16334                 intel_ddi_init(dev_priv, PORT_B);
16335                 intel_ddi_init(dev_priv, PORT_C);
16336
16337                 vlv_dsi_init(dev_priv);
16338         } else if (HAS_DDI(dev_priv)) {
16339                 int found;
16340
16341                 if (intel_ddi_crt_present(dev_priv))
16342                         intel_crt_init(dev_priv);
16343
16344                 /*
16345                  * Haswell uses DDI functions to detect digital outputs.
16346                  * On SKL pre-D0 the strap isn't connected, so we assume
16347                  * it's there.
16348                  */
16349                 found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
16350                 /* WaIgnoreDDIAStrap: skl */
16351                 if (found || IS_GEN9_BC(dev_priv))
16352                         intel_ddi_init(dev_priv, PORT_A);
16353
16354                 /* DDI B, C, D, and F detection is indicated by the SFUSE_STRAP
16355                  * register */
16356                 found = I915_READ(SFUSE_STRAP);
16357
16358                 if (found & SFUSE_STRAP_DDIB_DETECTED)
16359                         intel_ddi_init(dev_priv, PORT_B);
16360                 if (found & SFUSE_STRAP_DDIC_DETECTED)
16361                         intel_ddi_init(dev_priv, PORT_C);
16362                 if (found & SFUSE_STRAP_DDID_DETECTED)
16363                         intel_ddi_init(dev_priv, PORT_D);
16364                 if (found & SFUSE_STRAP_DDIF_DETECTED)
16365                         intel_ddi_init(dev_priv, PORT_F);
16366                 /*
16367                  * On SKL we don't have a way to detect DDI-E so we rely on VBT.
16368                  */
16369                 if (IS_GEN9_BC(dev_priv) &&
16370                     intel_bios_is_port_present(dev_priv, PORT_E))
16371                         intel_ddi_init(dev_priv, PORT_E);
16372
16373         } else if (HAS_PCH_SPLIT(dev_priv)) {
16374                 int found;
16375
16376                 /*
16377                  * intel_edp_init_connector() depends on this completing first,
16378                  * to prevent the registration of both eDP and LVDS and the
16379                  * incorrect sharing of the PPS.
16380                  */
16381                 intel_lvds_init(dev_priv);
16382                 intel_crt_init(dev_priv);
16383
16384                 dpd_is_edp = intel_dp_is_port_edp(dev_priv, PORT_D);
16385
16386                 if (ilk_has_edp_a(dev_priv))
16387                         intel_dp_init(dev_priv, DP_A, PORT_A);
16388
16389                 if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) {
16390                         /* PCH SDVOB multiplex with HDMIB */
16391                         found = intel_sdvo_init(dev_priv, PCH_SDVOB, PORT_B);
16392                         if (!found)
16393                                 intel_hdmi_init(dev_priv, PCH_HDMIB, PORT_B);
16394                         if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
16395                                 intel_dp_init(dev_priv, PCH_DP_B, PORT_B);
16396                 }
16397
16398                 if (I915_READ(PCH_HDMIC) & SDVO_DETECTED)
16399                         intel_hdmi_init(dev_priv, PCH_HDMIC, PORT_C);
16400
16401                 if (!dpd_is_edp && I915_READ(PCH_HDMID) & SDVO_DETECTED)
16402                         intel_hdmi_init(dev_priv, PCH_HDMID, PORT_D);
16403
16404                 if (I915_READ(PCH_DP_C) & DP_DETECTED)
16405                         intel_dp_init(dev_priv, PCH_DP_C, PORT_C);
16406
16407                 if (I915_READ(PCH_DP_D) & DP_DETECTED)
16408                         intel_dp_init(dev_priv, PCH_DP_D, PORT_D);
16409         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
16410                 bool has_edp, has_port;
16411
16412                 if (IS_VALLEYVIEW(dev_priv) && dev_priv->vbt.int_crt_support)
16413                         intel_crt_init(dev_priv);
16414
16415                 /*
16416                  * The DP_DETECTED bit is the latched state of the DDC
16417                  * SDA pin at boot. However since eDP doesn't require DDC
16418                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
16419                  * eDP ports may have been muxed to an alternate function.
16420                  * Thus we can't rely on the DP_DETECTED bit alone to detect
16421                  * eDP ports. Consult the VBT as well as DP_DETECTED to
16422                  * detect eDP ports.
16423                  *
16424                  * Sadly the straps seem to be missing sometimes even for HDMI
16425                  * ports (eg. on Voyo V3 - CHT x7-Z8700), so check both strap
16426                  * and VBT for the presence of the port. Additionally we can't
16427                  * trust the port type the VBT declares as we've seen at least
16428                  * HDMI ports that the VBT claim are DP or eDP.
16429                  */
16430                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_B);
16431                 has_port = intel_bios_is_port_present(dev_priv, PORT_B);
16432                 if (I915_READ(VLV_DP_B) & DP_DETECTED || has_port)
16433                         has_edp &= intel_dp_init(dev_priv, VLV_DP_B, PORT_B);
16434                 if ((I915_READ(VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp)
16435                         intel_hdmi_init(dev_priv, VLV_HDMIB, PORT_B);
16436
16437                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_C);
16438                 has_port = intel_bios_is_port_present(dev_priv, PORT_C);
16439                 if (I915_READ(VLV_DP_C) & DP_DETECTED || has_port)
16440                         has_edp &= intel_dp_init(dev_priv, VLV_DP_C, PORT_C);
16441                 if ((I915_READ(VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp)
16442                         intel_hdmi_init(dev_priv, VLV_HDMIC, PORT_C);
16443
16444                 if (IS_CHERRYVIEW(dev_priv)) {
16445                         /*
16446                          * eDP not supported on port D,
16447                          * so no need to worry about it
16448                          */
16449                         has_port = intel_bios_is_port_present(dev_priv, PORT_D);
16450                         if (I915_READ(CHV_DP_D) & DP_DETECTED || has_port)
16451                                 intel_dp_init(dev_priv, CHV_DP_D, PORT_D);
16452                         if (I915_READ(CHV_HDMID) & SDVO_DETECTED || has_port)
16453                                 intel_hdmi_init(dev_priv, CHV_HDMID, PORT_D);
16454                 }
16455
16456                 vlv_dsi_init(dev_priv);
16457         } else if (IS_PINEVIEW(dev_priv)) {
16458                 intel_lvds_init(dev_priv);
16459                 intel_crt_init(dev_priv);
16460         } else if (IS_GEN_RANGE(dev_priv, 3, 4)) {
16461                 bool found = false;
16462
16463                 if (IS_MOBILE(dev_priv))
16464                         intel_lvds_init(dev_priv);
16465
16466                 intel_crt_init(dev_priv);
16467
16468                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
16469                         DRM_DEBUG_KMS("probing SDVOB\n");
16470                         found = intel_sdvo_init(dev_priv, GEN3_SDVOB, PORT_B);
16471                         if (!found && IS_G4X(dev_priv)) {
16472                                 DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
16473                                 intel_hdmi_init(dev_priv, GEN4_HDMIB, PORT_B);
16474                         }
16475
16476                         if (!found && IS_G4X(dev_priv))
16477                                 intel_dp_init(dev_priv, DP_B, PORT_B);
16478                 }
16479
16480                 /* Before G4X SDVOC doesn't have its own detect register */
16481
16482                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
16483                         DRM_DEBUG_KMS("probing SDVOC\n");
16484                         found = intel_sdvo_init(dev_priv, GEN3_SDVOC, PORT_C);
16485                 }
16486
16487                 if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) {
16488
16489                         if (IS_G4X(dev_priv)) {
16490                                 DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
16491                                 intel_hdmi_init(dev_priv, GEN4_HDMIC, PORT_C);
16492                         }
16493                         if (IS_G4X(dev_priv))
16494                                 intel_dp_init(dev_priv, DP_C, PORT_C);
16495                 }
16496
16497                 if (IS_G4X(dev_priv) && (I915_READ(DP_D) & DP_DETECTED))
16498                         intel_dp_init(dev_priv, DP_D, PORT_D);
16499
16500                 if (SUPPORTS_TV(dev_priv))
16501                         intel_tv_init(dev_priv);
16502         } else if (IS_GEN(dev_priv, 2)) {
16503                 if (IS_I85X(dev_priv))
16504                         intel_lvds_init(dev_priv);
16505
16506                 intel_crt_init(dev_priv);
16507                 intel_dvo_init(dev_priv);
16508         }
16509
16510         intel_psr_init(dev_priv);
16511
16512         for_each_intel_encoder(&dev_priv->drm, encoder) {
16513                 encoder->base.possible_crtcs =
16514                         intel_encoder_possible_crtcs(encoder);
16515                 encoder->base.possible_clones =
16516                         intel_encoder_possible_clones(encoder);
16517         }
16518
16519         intel_init_pch_refclk(dev_priv);
16520
16521         drm_helper_move_panel_connectors_to_head(&dev_priv->drm);
16522 }
16523
16524 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
16525 {
16526         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
16527
16528         drm_framebuffer_cleanup(fb);
16529         intel_frontbuffer_put(intel_fb->frontbuffer);
16530
16531         kfree(intel_fb);
16532 }
16533
16534 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
16535                                                 struct drm_file *file,
16536                                                 unsigned int *handle)
16537 {
16538         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
16539
16540         if (obj->userptr.mm) {
16541                 DRM_DEBUG("attempting to use a userptr for a framebuffer, denied\n");
16542                 return -EINVAL;
16543         }
16544
16545         return drm_gem_handle_create(file, &obj->base, handle);
16546 }
16547
16548 static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
16549                                         struct drm_file *file,
16550                                         unsigned flags, unsigned color,
16551                                         struct drm_clip_rect *clips,
16552                                         unsigned num_clips)
16553 {
16554         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
16555
16556         i915_gem_object_flush_if_display(obj);
16557         intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB);
16558
16559         return 0;
16560 }
16561
16562 static const struct drm_framebuffer_funcs intel_fb_funcs = {
16563         .destroy = intel_user_framebuffer_destroy,
16564         .create_handle = intel_user_framebuffer_create_handle,
16565         .dirty = intel_user_framebuffer_dirty,
16566 };
16567
16568 static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
16569                                   struct drm_i915_gem_object *obj,
16570                                   struct drm_mode_fb_cmd2 *mode_cmd)
16571 {
16572         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
16573         struct drm_framebuffer *fb = &intel_fb->base;
16574         u32 max_stride;
16575         unsigned int tiling, stride;
16576         int ret = -EINVAL;
16577         int i;
16578
16579         intel_fb->frontbuffer = intel_frontbuffer_get(obj);
16580         if (!intel_fb->frontbuffer)
16581                 return -ENOMEM;
16582
16583         i915_gem_object_lock(obj);
16584         tiling = i915_gem_object_get_tiling(obj);
16585         stride = i915_gem_object_get_stride(obj);
16586         i915_gem_object_unlock(obj);
16587
16588         if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
16589                 /*
16590                  * If there's a fence, enforce that
16591                  * the fb modifier and tiling mode match.
16592                  */
16593                 if (tiling != I915_TILING_NONE &&
16594                     tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
16595                         DRM_DEBUG_KMS("tiling_mode doesn't match fb modifier\n");
16596                         goto err;
16597                 }
16598         } else {
16599                 if (tiling == I915_TILING_X) {
16600                         mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
16601                 } else if (tiling == I915_TILING_Y) {
16602                         DRM_DEBUG_KMS("No Y tiling for legacy addfb\n");
16603                         goto err;
16604                 }
16605         }
16606
16607         if (!drm_any_plane_has_format(&dev_priv->drm,
16608                                       mode_cmd->pixel_format,
16609                                       mode_cmd->modifier[0])) {
16610                 struct drm_format_name_buf format_name;
16611
16612                 DRM_DEBUG_KMS("unsupported pixel format %s / modifier 0x%llx\n",
16613                               drm_get_format_name(mode_cmd->pixel_format,
16614                                                   &format_name),
16615                               mode_cmd->modifier[0]);
16616                 goto err;
16617         }
16618
16619         /*
16620          * gen2/3 display engine uses the fence if present,
16621          * so the tiling mode must match the fb modifier exactly.
16622          */
16623         if (INTEL_GEN(dev_priv) < 4 &&
16624             tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
16625                 DRM_DEBUG_KMS("tiling_mode must match fb modifier exactly on gen2/3\n");
16626                 goto err;
16627         }
16628
16629         max_stride = intel_fb_max_stride(dev_priv, mode_cmd->pixel_format,
16630                                          mode_cmd->modifier[0]);
16631         if (mode_cmd->pitches[0] > max_stride) {
16632                 DRM_DEBUG_KMS("%s pitch (%u) must be at most %d\n",
16633                               mode_cmd->modifier[0] != DRM_FORMAT_MOD_LINEAR ?
16634                               "tiled" : "linear",
16635                               mode_cmd->pitches[0], max_stride);
16636                 goto err;
16637         }
16638
16639         /*
16640          * If there's a fence, enforce that
16641          * the fb pitch and fence stride match.
16642          */
16643         if (tiling != I915_TILING_NONE && mode_cmd->pitches[0] != stride) {
16644                 DRM_DEBUG_KMS("pitch (%d) must match tiling stride (%d)\n",
16645                               mode_cmd->pitches[0], stride);
16646                 goto err;
16647         }
16648
16649         /* FIXME need to adjust LINOFF/TILEOFF accordingly. */
16650         if (mode_cmd->offsets[0] != 0)
16651                 goto err;
16652
16653         drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd);
16654
16655         for (i = 0; i < fb->format->num_planes; i++) {
16656                 u32 stride_alignment;
16657
16658                 if (mode_cmd->handles[i] != mode_cmd->handles[0]) {
16659                         DRM_DEBUG_KMS("bad plane %d handle\n", i);
16660                         goto err;
16661                 }
16662
16663                 stride_alignment = intel_fb_stride_alignment(fb, i);
16664                 if (fb->pitches[i] & (stride_alignment - 1)) {
16665                         DRM_DEBUG_KMS("plane %d pitch (%d) must be at least %u byte aligned\n",
16666                                       i, fb->pitches[i], stride_alignment);
16667                         goto err;
16668                 }
16669
16670                 if (is_gen12_ccs_plane(fb, i)) {
16671                         int ccs_aux_stride = gen12_ccs_aux_stride(fb, i);
16672
16673                         if (fb->pitches[i] != ccs_aux_stride) {
16674                                 DRM_DEBUG_KMS("ccs aux plane %d pitch (%d) must be %d\n",
16675                                               i,
16676                                               fb->pitches[i], ccs_aux_stride);
16677                                 goto err;
16678                         }
16679                 }
16680
16681                 fb->obj[i] = &obj->base;
16682         }
16683
16684         ret = intel_fill_fb_info(dev_priv, fb);
16685         if (ret)
16686                 goto err;
16687
16688         ret = drm_framebuffer_init(&dev_priv->drm, fb, &intel_fb_funcs);
16689         if (ret) {
16690                 DRM_ERROR("framebuffer init failed %d\n", ret);
16691                 goto err;
16692         }
16693
16694         return 0;
16695
16696 err:
16697         intel_frontbuffer_put(intel_fb->frontbuffer);
16698         return ret;
16699 }
16700
16701 static struct drm_framebuffer *
16702 intel_user_framebuffer_create(struct drm_device *dev,
16703                               struct drm_file *filp,
16704                               const struct drm_mode_fb_cmd2 *user_mode_cmd)
16705 {
16706         struct drm_framebuffer *fb;
16707         struct drm_i915_gem_object *obj;
16708         struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
16709
16710         obj = i915_gem_object_lookup(filp, mode_cmd.handles[0]);
16711         if (!obj)
16712                 return ERR_PTR(-ENOENT);
16713
16714         fb = intel_framebuffer_create(obj, &mode_cmd);
16715         i915_gem_object_put(obj);
16716
16717         return fb;
16718 }
16719
16720 static void intel_atomic_state_free(struct drm_atomic_state *state)
16721 {
16722         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
16723
16724         drm_atomic_state_default_release(state);
16725
16726         i915_sw_fence_fini(&intel_state->commit_ready);
16727
16728         kfree(state);
16729 }
16730
16731 static enum drm_mode_status
16732 intel_mode_valid(struct drm_device *dev,
16733                  const struct drm_display_mode *mode)
16734 {
16735         struct drm_i915_private *dev_priv = to_i915(dev);
16736         int hdisplay_max, htotal_max;
16737         int vdisplay_max, vtotal_max;
16738
16739         /*
16740          * Can't reject DBLSCAN here because Xorg ddxen can add piles
16741          * of DBLSCAN modes to the output's mode list when they detect
16742          * the scaling mode property on the connector. And they don't
16743          * ask the kernel to validate those modes in any way until
16744          * modeset time at which point the client gets a protocol error.
16745          * So in order to not upset those clients we silently ignore the
16746          * DBLSCAN flag on such connectors. For other connectors we will
16747          * reject modes with the DBLSCAN flag in encoder->compute_config().
16748          * And we always reject DBLSCAN modes in connector->mode_valid()
16749          * as we never want such modes on the connector's mode list.
16750          */
16751
16752         if (mode->vscan > 1)
16753                 return MODE_NO_VSCAN;
16754
16755         if (mode->flags & DRM_MODE_FLAG_HSKEW)
16756                 return MODE_H_ILLEGAL;
16757
16758         if (mode->flags & (DRM_MODE_FLAG_CSYNC |
16759                            DRM_MODE_FLAG_NCSYNC |
16760                            DRM_MODE_FLAG_PCSYNC))
16761                 return MODE_HSYNC;
16762
16763         if (mode->flags & (DRM_MODE_FLAG_BCAST |
16764                            DRM_MODE_FLAG_PIXMUX |
16765                            DRM_MODE_FLAG_CLKDIV2))
16766                 return MODE_BAD;
16767
16768         /* Transcoder timing limits */
16769         if (INTEL_GEN(dev_priv) >= 11) {
16770                 hdisplay_max = 16384;
16771                 vdisplay_max = 8192;
16772                 htotal_max = 16384;
16773                 vtotal_max = 8192;
16774         } else if (INTEL_GEN(dev_priv) >= 9 ||
16775                    IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
16776                 hdisplay_max = 8192; /* FDI max 4096 handled elsewhere */
16777                 vdisplay_max = 4096;
16778                 htotal_max = 8192;
16779                 vtotal_max = 8192;
16780         } else if (INTEL_GEN(dev_priv) >= 3) {
16781                 hdisplay_max = 4096;
16782                 vdisplay_max = 4096;
16783                 htotal_max = 8192;
16784                 vtotal_max = 8192;
16785         } else {
16786                 hdisplay_max = 2048;
16787                 vdisplay_max = 2048;
16788                 htotal_max = 4096;
16789                 vtotal_max = 4096;
16790         }
16791
16792         if (mode->hdisplay > hdisplay_max ||
16793             mode->hsync_start > htotal_max ||
16794             mode->hsync_end > htotal_max ||
16795             mode->htotal > htotal_max)
16796                 return MODE_H_ILLEGAL;
16797
16798         if (mode->vdisplay > vdisplay_max ||
16799             mode->vsync_start > vtotal_max ||
16800             mode->vsync_end > vtotal_max ||
16801             mode->vtotal > vtotal_max)
16802                 return MODE_V_ILLEGAL;
16803
16804         if (INTEL_GEN(dev_priv) >= 5) {
16805                 if (mode->hdisplay < 64 ||
16806                     mode->htotal - mode->hdisplay < 32)
16807                         return MODE_H_ILLEGAL;
16808
16809                 if (mode->vtotal - mode->vdisplay < 5)
16810                         return MODE_V_ILLEGAL;
16811         } else {
16812                 if (mode->htotal - mode->hdisplay < 32)
16813                         return MODE_H_ILLEGAL;
16814
16815                 if (mode->vtotal - mode->vdisplay < 3)
16816                         return MODE_V_ILLEGAL;
16817         }
16818
16819         return MODE_OK;
16820 }
16821
16822 enum drm_mode_status
16823 intel_mode_valid_max_plane_size(struct drm_i915_private *dev_priv,
16824                                 const struct drm_display_mode *mode)
16825 {
16826         int plane_width_max, plane_height_max;
16827
16828         /*
16829          * intel_mode_valid() should be
16830          * sufficient on older platforms.
16831          */
16832         if (INTEL_GEN(dev_priv) < 9)
16833                 return MODE_OK;
16834
16835         /*
16836          * Most people will probably want a fullscreen
16837          * plane so let's not advertize modes that are
16838          * too big for that.
16839          */
16840         if (INTEL_GEN(dev_priv) >= 11) {
16841                 plane_width_max = 5120;
16842                 plane_height_max = 4320;
16843         } else {
16844                 plane_width_max = 5120;
16845                 plane_height_max = 4096;
16846         }
16847
16848         if (mode->hdisplay > plane_width_max)
16849                 return MODE_H_ILLEGAL;
16850
16851         if (mode->vdisplay > plane_height_max)
16852                 return MODE_V_ILLEGAL;
16853
16854         return MODE_OK;
16855 }
16856
16857 static const struct drm_mode_config_funcs intel_mode_funcs = {
16858         .fb_create = intel_user_framebuffer_create,
16859         .get_format_info = intel_get_format_info,
16860         .output_poll_changed = intel_fbdev_output_poll_changed,
16861         .mode_valid = intel_mode_valid,
16862         .atomic_check = intel_atomic_check,
16863         .atomic_commit = intel_atomic_commit,
16864         .atomic_state_alloc = intel_atomic_state_alloc,
16865         .atomic_state_clear = intel_atomic_state_clear,
16866         .atomic_state_free = intel_atomic_state_free,
16867 };
16868
16869 /**
16870  * intel_init_display_hooks - initialize the display modesetting hooks
16871  * @dev_priv: device private
16872  */
16873 void intel_init_display_hooks(struct drm_i915_private *dev_priv)
16874 {
16875         intel_init_cdclk_hooks(dev_priv);
16876
16877         if (INTEL_GEN(dev_priv) >= 9) {
16878                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
16879                 dev_priv->display.get_initial_plane_config =
16880                         skylake_get_initial_plane_config;
16881                 dev_priv->display.crtc_compute_clock =
16882                         haswell_crtc_compute_clock;
16883                 dev_priv->display.crtc_enable = haswell_crtc_enable;
16884                 dev_priv->display.crtc_disable = haswell_crtc_disable;
16885         } else if (HAS_DDI(dev_priv)) {
16886                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
16887                 dev_priv->display.get_initial_plane_config =
16888                         i9xx_get_initial_plane_config;
16889                 dev_priv->display.crtc_compute_clock =
16890                         haswell_crtc_compute_clock;
16891                 dev_priv->display.crtc_enable = haswell_crtc_enable;
16892                 dev_priv->display.crtc_disable = haswell_crtc_disable;
16893         } else if (HAS_PCH_SPLIT(dev_priv)) {
16894                 dev_priv->display.get_pipe_config = ironlake_get_pipe_config;
16895                 dev_priv->display.get_initial_plane_config =
16896                         i9xx_get_initial_plane_config;
16897                 dev_priv->display.crtc_compute_clock =
16898                         ironlake_crtc_compute_clock;
16899                 dev_priv->display.crtc_enable = ironlake_crtc_enable;
16900                 dev_priv->display.crtc_disable = ironlake_crtc_disable;
16901         } else if (IS_CHERRYVIEW(dev_priv)) {
16902                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
16903                 dev_priv->display.get_initial_plane_config =
16904                         i9xx_get_initial_plane_config;
16905                 dev_priv->display.crtc_compute_clock = chv_crtc_compute_clock;
16906                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
16907                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
16908         } else if (IS_VALLEYVIEW(dev_priv)) {
16909                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
16910                 dev_priv->display.get_initial_plane_config =
16911                         i9xx_get_initial_plane_config;
16912                 dev_priv->display.crtc_compute_clock = vlv_crtc_compute_clock;
16913                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
16914                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
16915         } else if (IS_G4X(dev_priv)) {
16916                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
16917                 dev_priv->display.get_initial_plane_config =
16918                         i9xx_get_initial_plane_config;
16919                 dev_priv->display.crtc_compute_clock = g4x_crtc_compute_clock;
16920                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
16921                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
16922         } else if (IS_PINEVIEW(dev_priv)) {
16923                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
16924                 dev_priv->display.get_initial_plane_config =
16925                         i9xx_get_initial_plane_config;
16926                 dev_priv->display.crtc_compute_clock = pnv_crtc_compute_clock;
16927                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
16928                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
16929         } else if (!IS_GEN(dev_priv, 2)) {
16930                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
16931                 dev_priv->display.get_initial_plane_config =
16932                         i9xx_get_initial_plane_config;
16933                 dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
16934                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
16935                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
16936         } else {
16937                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
16938                 dev_priv->display.get_initial_plane_config =
16939                         i9xx_get_initial_plane_config;
16940                 dev_priv->display.crtc_compute_clock = i8xx_crtc_compute_clock;
16941                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
16942                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
16943         }
16944
16945         if (IS_GEN(dev_priv, 5)) {
16946                 dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
16947         } else if (IS_GEN(dev_priv, 6)) {
16948                 dev_priv->display.fdi_link_train = gen6_fdi_link_train;
16949         } else if (IS_IVYBRIDGE(dev_priv)) {
16950                 /* FIXME: detect B0+ stepping and use auto training */
16951                 dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
16952         }
16953
16954         if (INTEL_GEN(dev_priv) >= 9)
16955                 dev_priv->display.commit_modeset_enables = skl_commit_modeset_enables;
16956         else
16957                 dev_priv->display.commit_modeset_enables = intel_commit_modeset_enables;
16958
16959 }
16960
16961 void intel_modeset_init_hw(struct drm_i915_private *i915)
16962 {
16963         intel_update_cdclk(i915);
16964         intel_dump_cdclk_state(&i915->cdclk.hw, "Current CDCLK");
16965         i915->cdclk.logical = i915->cdclk.actual = i915->cdclk.hw;
16966 }
16967
16968 /*
16969  * Calculate what we think the watermarks should be for the state we've read
16970  * out of the hardware and then immediately program those watermarks so that
16971  * we ensure the hardware settings match our internal state.
16972  *
16973  * We can calculate what we think WM's should be by creating a duplicate of the
16974  * current state (which was constructed during hardware readout) and running it
16975  * through the atomic check code to calculate new watermark values in the
16976  * state object.
16977  */
16978 static void sanitize_watermarks(struct drm_device *dev)
16979 {
16980         struct drm_i915_private *dev_priv = to_i915(dev);
16981         struct drm_atomic_state *state;
16982         struct intel_atomic_state *intel_state;
16983         struct intel_crtc *crtc;
16984         struct intel_crtc_state *crtc_state;
16985         struct drm_modeset_acquire_ctx ctx;
16986         int ret;
16987         int i;
16988
16989         /* Only supported on platforms that use atomic watermark design */
16990         if (!dev_priv->display.optimize_watermarks)
16991                 return;
16992
16993         /*
16994          * We need to hold connection_mutex before calling duplicate_state so
16995          * that the connector loop is protected.
16996          */
16997         drm_modeset_acquire_init(&ctx, 0);
16998 retry:
16999         ret = drm_modeset_lock_all_ctx(dev, &ctx);
17000         if (ret == -EDEADLK) {
17001                 drm_modeset_backoff(&ctx);
17002                 goto retry;
17003         } else if (WARN_ON(ret)) {
17004                 goto fail;
17005         }
17006
17007         state = drm_atomic_helper_duplicate_state(dev, &ctx);
17008         if (WARN_ON(IS_ERR(state)))
17009                 goto fail;
17010
17011         intel_state = to_intel_atomic_state(state);
17012
17013         /*
17014          * Hardware readout is the only time we don't want to calculate
17015          * intermediate watermarks (since we don't trust the current
17016          * watermarks).
17017          */
17018         if (!HAS_GMCH(dev_priv))
17019                 intel_state->skip_intermediate_wm = true;
17020
17021         ret = intel_atomic_check(dev, state);
17022         if (ret) {
17023                 /*
17024                  * If we fail here, it means that the hardware appears to be
17025                  * programmed in a way that shouldn't be possible, given our
17026                  * understanding of watermark requirements.  This might mean a
17027                  * mistake in the hardware readout code or a mistake in the
17028                  * watermark calculations for a given platform.  Raise a WARN
17029                  * so that this is noticeable.
17030                  *
17031                  * If this actually happens, we'll have to just leave the
17032                  * BIOS-programmed watermarks untouched and hope for the best.
17033                  */
17034                 WARN(true, "Could not determine valid watermarks for inherited state\n");
17035                 goto put_state;
17036         }
17037
17038         /* Write calculated watermark values back */
17039         for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
17040                 crtc_state->wm.need_postvbl_update = true;
17041                 dev_priv->display.optimize_watermarks(intel_state, crtc);
17042
17043                 to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
17044         }
17045
17046 put_state:
17047         drm_atomic_state_put(state);
17048 fail:
17049         drm_modeset_drop_locks(&ctx);
17050         drm_modeset_acquire_fini(&ctx);
17051 }
17052
17053 static void intel_update_fdi_pll_freq(struct drm_i915_private *dev_priv)
17054 {
17055         if (IS_GEN(dev_priv, 5)) {
17056                 u32 fdi_pll_clk =
17057                         I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK;
17058
17059                 dev_priv->fdi_pll_freq = (fdi_pll_clk + 2) * 10000;
17060         } else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) {
17061                 dev_priv->fdi_pll_freq = 270000;
17062         } else {
17063                 return;
17064         }
17065
17066         DRM_DEBUG_DRIVER("FDI PLL freq=%d\n", dev_priv->fdi_pll_freq);
17067 }
17068
17069 static int intel_initial_commit(struct drm_device *dev)
17070 {
17071         struct drm_atomic_state *state = NULL;
17072         struct drm_modeset_acquire_ctx ctx;
17073         struct intel_crtc *crtc;
17074         int ret = 0;
17075
17076         state = drm_atomic_state_alloc(dev);
17077         if (!state)
17078                 return -ENOMEM;
17079
17080         drm_modeset_acquire_init(&ctx, 0);
17081
17082 retry:
17083         state->acquire_ctx = &ctx;
17084
17085         for_each_intel_crtc(dev, crtc) {
17086                 struct intel_crtc_state *crtc_state =
17087                         intel_atomic_get_crtc_state(state, crtc);
17088
17089                 if (IS_ERR(crtc_state)) {
17090                         ret = PTR_ERR(crtc_state);
17091                         goto out;
17092                 }
17093
17094                 if (crtc_state->hw.active) {
17095                         ret = drm_atomic_add_affected_planes(state, &crtc->base);
17096                         if (ret)
17097                                 goto out;
17098
17099                         /*
17100                          * FIXME hack to force a LUT update to avoid the
17101                          * plane update forcing the pipe gamma on without
17102                          * having a proper LUT loaded. Remove once we
17103                          * have readout for pipe gamma enable.
17104                          */
17105                         crtc_state->uapi.color_mgmt_changed = true;
17106                 }
17107         }
17108
17109         ret = drm_atomic_commit(state);
17110
17111 out:
17112         if (ret == -EDEADLK) {
17113                 drm_atomic_state_clear(state);
17114                 drm_modeset_backoff(&ctx);
17115                 goto retry;
17116         }
17117
17118         drm_atomic_state_put(state);
17119
17120         drm_modeset_drop_locks(&ctx);
17121         drm_modeset_acquire_fini(&ctx);
17122
17123         return ret;
17124 }
17125
17126 static void intel_mode_config_init(struct drm_i915_private *i915)
17127 {
17128         struct drm_mode_config *mode_config = &i915->drm.mode_config;
17129
17130         drm_mode_config_init(&i915->drm);
17131
17132         mode_config->min_width = 0;
17133         mode_config->min_height = 0;
17134
17135         mode_config->preferred_depth = 24;
17136         mode_config->prefer_shadow = 1;
17137
17138         mode_config->allow_fb_modifiers = true;
17139
17140         mode_config->funcs = &intel_mode_funcs;
17141
17142         /*
17143          * Maximum framebuffer dimensions, chosen to match
17144          * the maximum render engine surface size on gen4+.
17145          */
17146         if (INTEL_GEN(i915) >= 7) {
17147                 mode_config->max_width = 16384;
17148                 mode_config->max_height = 16384;
17149         } else if (INTEL_GEN(i915) >= 4) {
17150                 mode_config->max_width = 8192;
17151                 mode_config->max_height = 8192;
17152         } else if (IS_GEN(i915, 3)) {
17153                 mode_config->max_width = 4096;
17154                 mode_config->max_height = 4096;
17155         } else {
17156                 mode_config->max_width = 2048;
17157                 mode_config->max_height = 2048;
17158         }
17159
17160         if (IS_I845G(i915) || IS_I865G(i915)) {
17161                 mode_config->cursor_width = IS_I845G(i915) ? 64 : 512;
17162                 mode_config->cursor_height = 1023;
17163         } else if (IS_GEN(i915, 2)) {
17164                 mode_config->cursor_width = 64;
17165                 mode_config->cursor_height = 64;
17166         } else {
17167                 mode_config->cursor_width = 256;
17168                 mode_config->cursor_height = 256;
17169         }
17170 }
17171
17172 int intel_modeset_init(struct drm_i915_private *i915)
17173 {
17174         struct drm_device *dev = &i915->drm;
17175         enum pipe pipe;
17176         struct intel_crtc *crtc;
17177         int ret;
17178
17179         i915->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0);
17180         i915->flip_wq = alloc_workqueue("i915_flip", WQ_HIGHPRI |
17181                                         WQ_UNBOUND, WQ_UNBOUND_MAX_ACTIVE);
17182
17183         intel_mode_config_init(i915);
17184
17185         ret = intel_bw_init(i915);
17186         if (ret)
17187                 return ret;
17188
17189         init_llist_head(&i915->atomic_helper.free_list);
17190         INIT_WORK(&i915->atomic_helper.free_work,
17191                   intel_atomic_helper_free_state_worker);
17192
17193         intel_init_quirks(i915);
17194
17195         intel_fbc_init(i915);
17196
17197         intel_init_pm(i915);
17198
17199         intel_panel_sanitize_ssc(i915);
17200
17201         intel_gmbus_setup(i915);
17202
17203         DRM_DEBUG_KMS("%d display pipe%s available.\n",
17204                       INTEL_NUM_PIPES(i915),
17205                       INTEL_NUM_PIPES(i915) > 1 ? "s" : "");
17206
17207         if (HAS_DISPLAY(i915) && INTEL_DISPLAY_ENABLED(i915)) {
17208                 for_each_pipe(i915, pipe) {
17209                         ret = intel_crtc_init(i915, pipe);
17210                         if (ret) {
17211                                 drm_mode_config_cleanup(dev);
17212                                 return ret;
17213                         }
17214                 }
17215         }
17216
17217         intel_shared_dpll_init(dev);
17218         intel_update_fdi_pll_freq(i915);
17219
17220         intel_update_czclk(i915);
17221         intel_modeset_init_hw(i915);
17222
17223         intel_hdcp_component_init(i915);
17224
17225         if (i915->max_cdclk_freq == 0)
17226                 intel_update_max_cdclk(i915);
17227
17228         /* Just disable it once at startup */
17229         intel_vga_disable(i915);
17230         intel_setup_outputs(i915);
17231
17232         drm_modeset_lock_all(dev);
17233         intel_modeset_setup_hw_state(dev, dev->mode_config.acquire_ctx);
17234         drm_modeset_unlock_all(dev);
17235
17236         for_each_intel_crtc(dev, crtc) {
17237                 struct intel_initial_plane_config plane_config = {};
17238
17239                 if (!crtc->active)
17240                         continue;
17241
17242                 /*
17243                  * Note that reserving the BIOS fb up front prevents us
17244                  * from stuffing other stolen allocations like the ring
17245                  * on top.  This prevents some ugliness at boot time, and
17246                  * can even allow for smooth boot transitions if the BIOS
17247                  * fb is large enough for the active pipe configuration.
17248                  */
17249                 i915->display.get_initial_plane_config(crtc, &plane_config);
17250
17251                 /*
17252                  * If the fb is shared between multiple heads, we'll
17253                  * just get the first one.
17254                  */
17255                 intel_find_initial_plane_obj(crtc, &plane_config);
17256         }
17257
17258         /*
17259          * Make sure hardware watermarks really match the state we read out.
17260          * Note that we need to do this after reconstructing the BIOS fb's
17261          * since the watermark calculation done here will use pstate->fb.
17262          */
17263         if (!HAS_GMCH(i915))
17264                 sanitize_watermarks(dev);
17265
17266         /*
17267          * Force all active planes to recompute their states. So that on
17268          * mode_setcrtc after probe, all the intel_plane_state variables
17269          * are already calculated and there is no assert_plane warnings
17270          * during bootup.
17271          */
17272         ret = intel_initial_commit(dev);
17273         if (ret)
17274                 DRM_DEBUG_KMS("Initial commit in probe failed.\n");
17275
17276         return 0;
17277 }
17278
17279 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
17280 {
17281         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
17282         /* 640x480@60Hz, ~25175 kHz */
17283         struct dpll clock = {
17284                 .m1 = 18,
17285                 .m2 = 7,
17286                 .p1 = 13,
17287                 .p2 = 4,
17288                 .n = 2,
17289         };
17290         u32 dpll, fp;
17291         int i;
17292
17293         WARN_ON(i9xx_calc_dpll_params(48000, &clock) != 25154);
17294
17295         DRM_DEBUG_KMS("enabling pipe %c due to force quirk (vco=%d dot=%d)\n",
17296                       pipe_name(pipe), clock.vco, clock.dot);
17297
17298         fp = i9xx_dpll_compute_fp(&clock);
17299         dpll = DPLL_DVO_2X_MODE |
17300                 DPLL_VGA_MODE_DIS |
17301                 ((clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT) |
17302                 PLL_P2_DIVIDE_BY_4 |
17303                 PLL_REF_INPUT_DREFCLK |
17304                 DPLL_VCO_ENABLE;
17305
17306         I915_WRITE(FP0(pipe), fp);
17307         I915_WRITE(FP1(pipe), fp);
17308
17309         I915_WRITE(HTOTAL(pipe), (640 - 1) | ((800 - 1) << 16));
17310         I915_WRITE(HBLANK(pipe), (640 - 1) | ((800 - 1) << 16));
17311         I915_WRITE(HSYNC(pipe), (656 - 1) | ((752 - 1) << 16));
17312         I915_WRITE(VTOTAL(pipe), (480 - 1) | ((525 - 1) << 16));
17313         I915_WRITE(VBLANK(pipe), (480 - 1) | ((525 - 1) << 16));
17314         I915_WRITE(VSYNC(pipe), (490 - 1) | ((492 - 1) << 16));
17315         I915_WRITE(PIPESRC(pipe), ((640 - 1) << 16) | (480 - 1));
17316
17317         /*
17318          * Apparently we need to have VGA mode enabled prior to changing
17319          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
17320          * dividers, even though the register value does change.
17321          */
17322         I915_WRITE(DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS);
17323         I915_WRITE(DPLL(pipe), dpll);
17324
17325         /* Wait for the clocks to stabilize. */
17326         POSTING_READ(DPLL(pipe));
17327         udelay(150);
17328
17329         /* The pixel multiplier can only be updated once the
17330          * DPLL is enabled and the clocks are stable.
17331          *
17332          * So write it again.
17333          */
17334         I915_WRITE(DPLL(pipe), dpll);
17335
17336         /* We do this three times for luck */
17337         for (i = 0; i < 3 ; i++) {
17338                 I915_WRITE(DPLL(pipe), dpll);
17339                 POSTING_READ(DPLL(pipe));
17340                 udelay(150); /* wait for warmup */
17341         }
17342
17343         I915_WRITE(PIPECONF(pipe), PIPECONF_ENABLE | PIPECONF_PROGRESSIVE);
17344         POSTING_READ(PIPECONF(pipe));
17345
17346         intel_wait_for_pipe_scanline_moving(crtc);
17347 }
17348
17349 void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
17350 {
17351         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
17352
17353         DRM_DEBUG_KMS("disabling pipe %c due to force quirk\n",
17354                       pipe_name(pipe));
17355
17356         WARN_ON(I915_READ(DSPCNTR(PLANE_A)) & DISPLAY_PLANE_ENABLE);
17357         WARN_ON(I915_READ(DSPCNTR(PLANE_B)) & DISPLAY_PLANE_ENABLE);
17358         WARN_ON(I915_READ(DSPCNTR(PLANE_C)) & DISPLAY_PLANE_ENABLE);
17359         WARN_ON(I915_READ(CURCNTR(PIPE_A)) & MCURSOR_MODE);
17360         WARN_ON(I915_READ(CURCNTR(PIPE_B)) & MCURSOR_MODE);
17361
17362         I915_WRITE(PIPECONF(pipe), 0);
17363         POSTING_READ(PIPECONF(pipe));
17364
17365         intel_wait_for_pipe_scanline_stopped(crtc);
17366
17367         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
17368         POSTING_READ(DPLL(pipe));
17369 }
17370
17371 static void
17372 intel_sanitize_plane_mapping(struct drm_i915_private *dev_priv)
17373 {
17374         struct intel_crtc *crtc;
17375
17376         if (INTEL_GEN(dev_priv) >= 4)
17377                 return;
17378
17379         for_each_intel_crtc(&dev_priv->drm, crtc) {
17380                 struct intel_plane *plane =
17381                         to_intel_plane(crtc->base.primary);
17382                 struct intel_crtc *plane_crtc;
17383                 enum pipe pipe;
17384
17385                 if (!plane->get_hw_state(plane, &pipe))
17386                         continue;
17387
17388                 if (pipe == crtc->pipe)
17389                         continue;
17390
17391                 DRM_DEBUG_KMS("[PLANE:%d:%s] attached to the wrong pipe, disabling plane\n",
17392                               plane->base.base.id, plane->base.name);
17393
17394                 plane_crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
17395                 intel_plane_disable_noatomic(plane_crtc, plane);
17396         }
17397 }
17398
17399 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
17400 {
17401         struct drm_device *dev = crtc->base.dev;
17402         struct intel_encoder *encoder;
17403
17404         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
17405                 return true;
17406
17407         return false;
17408 }
17409
17410 static struct intel_connector *intel_encoder_find_connector(struct intel_encoder *encoder)
17411 {
17412         struct drm_device *dev = encoder->base.dev;
17413         struct intel_connector *connector;
17414
17415         for_each_connector_on_encoder(dev, &encoder->base, connector)
17416                 return connector;
17417
17418         return NULL;
17419 }
17420
17421 static bool has_pch_trancoder(struct drm_i915_private *dev_priv,
17422                               enum pipe pch_transcoder)
17423 {
17424         return HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
17425                 (HAS_PCH_LPT_H(dev_priv) && pch_transcoder == PIPE_A);
17426 }
17427
17428 static void intel_sanitize_frame_start_delay(const struct intel_crtc_state *crtc_state)
17429 {
17430         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
17431         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
17432         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
17433
17434         if (INTEL_GEN(dev_priv) >= 9 ||
17435             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
17436                 i915_reg_t reg = CHICKEN_TRANS(cpu_transcoder);
17437                 u32 val;
17438
17439                 if (transcoder_is_dsi(cpu_transcoder))
17440                         return;
17441
17442                 val = I915_READ(reg);
17443                 val &= ~HSW_FRAME_START_DELAY_MASK;
17444                 val |= HSW_FRAME_START_DELAY(0);
17445                 I915_WRITE(reg, val);
17446         } else {
17447                 i915_reg_t reg = PIPECONF(cpu_transcoder);
17448                 u32 val;
17449
17450                 val = I915_READ(reg);
17451                 val &= ~PIPECONF_FRAME_START_DELAY_MASK;
17452                 val |= PIPECONF_FRAME_START_DELAY(0);
17453                 I915_WRITE(reg, val);
17454         }
17455
17456         if (!crtc_state->has_pch_encoder)
17457                 return;
17458
17459         if (HAS_PCH_IBX(dev_priv)) {
17460                 i915_reg_t reg = PCH_TRANSCONF(crtc->pipe);
17461                 u32 val;
17462
17463                 val = I915_READ(reg);
17464                 val &= ~TRANS_FRAME_START_DELAY_MASK;
17465                 val |= TRANS_FRAME_START_DELAY(0);
17466                 I915_WRITE(reg, val);
17467         } else {
17468                 enum pipe pch_transcoder = intel_crtc_pch_transcoder(crtc);
17469                 i915_reg_t reg = TRANS_CHICKEN2(pch_transcoder);
17470                 u32 val;
17471
17472                 val = I915_READ(reg);
17473                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
17474                 val |= TRANS_CHICKEN2_FRAME_START_DELAY(0);
17475                 I915_WRITE(reg, val);
17476         }
17477 }
17478
17479 static void intel_sanitize_crtc(struct intel_crtc *crtc,
17480                                 struct drm_modeset_acquire_ctx *ctx)
17481 {
17482         struct drm_device *dev = crtc->base.dev;
17483         struct drm_i915_private *dev_priv = to_i915(dev);
17484         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
17485
17486         if (crtc_state->hw.active) {
17487                 struct intel_plane *plane;
17488
17489                 /* Clear any frame start delays used for debugging left by the BIOS */
17490                 intel_sanitize_frame_start_delay(crtc_state);
17491
17492                 /* Disable everything but the primary plane */
17493                 for_each_intel_plane_on_crtc(dev, crtc, plane) {
17494                         const struct intel_plane_state *plane_state =
17495                                 to_intel_plane_state(plane->base.state);
17496
17497                         if (plane_state->uapi.visible &&
17498                             plane->base.type != DRM_PLANE_TYPE_PRIMARY)
17499                                 intel_plane_disable_noatomic(crtc, plane);
17500                 }
17501
17502                 /*
17503                  * Disable any background color set by the BIOS, but enable the
17504                  * gamma and CSC to match how we program our planes.
17505                  */
17506                 if (INTEL_GEN(dev_priv) >= 9)
17507                         I915_WRITE(SKL_BOTTOM_COLOR(crtc->pipe),
17508                                    SKL_BOTTOM_COLOR_GAMMA_ENABLE |
17509                                    SKL_BOTTOM_COLOR_CSC_ENABLE);
17510         }
17511
17512         /* Adjust the state of the output pipe according to whether we
17513          * have active connectors/encoders. */
17514         if (crtc_state->hw.active && !intel_crtc_has_encoders(crtc))
17515                 intel_crtc_disable_noatomic(crtc, ctx);
17516
17517         if (crtc_state->hw.active || HAS_GMCH(dev_priv)) {
17518                 /*
17519                  * We start out with underrun reporting disabled to avoid races.
17520                  * For correct bookkeeping mark this on active crtcs.
17521                  *
17522                  * Also on gmch platforms we dont have any hardware bits to
17523                  * disable the underrun reporting. Which means we need to start
17524                  * out with underrun reporting disabled also on inactive pipes,
17525                  * since otherwise we'll complain about the garbage we read when
17526                  * e.g. coming up after runtime pm.
17527                  *
17528                  * No protection against concurrent access is required - at
17529                  * worst a fifo underrun happens which also sets this to false.
17530                  */
17531                 crtc->cpu_fifo_underrun_disabled = true;
17532                 /*
17533                  * We track the PCH trancoder underrun reporting state
17534                  * within the crtc. With crtc for pipe A housing the underrun
17535                  * reporting state for PCH transcoder A, crtc for pipe B housing
17536                  * it for PCH transcoder B, etc. LPT-H has only PCH transcoder A,
17537                  * and marking underrun reporting as disabled for the non-existing
17538                  * PCH transcoders B and C would prevent enabling the south
17539                  * error interrupt (see cpt_can_enable_serr_int()).
17540                  */
17541                 if (has_pch_trancoder(dev_priv, crtc->pipe))
17542                         crtc->pch_fifo_underrun_disabled = true;
17543         }
17544 }
17545
17546 static bool has_bogus_dpll_config(const struct intel_crtc_state *crtc_state)
17547 {
17548         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
17549
17550         /*
17551          * Some SNB BIOSen (eg. ASUS K53SV) are known to misprogram
17552          * the hardware when a high res displays plugged in. DPLL P
17553          * divider is zero, and the pipe timings are bonkers. We'll
17554          * try to disable everything in that case.
17555          *
17556          * FIXME would be nice to be able to sanitize this state
17557          * without several WARNs, but for now let's take the easy
17558          * road.
17559          */
17560         return IS_GEN(dev_priv, 6) &&
17561                 crtc_state->hw.active &&
17562                 crtc_state->shared_dpll &&
17563                 crtc_state->port_clock == 0;
17564 }
17565
17566 static void intel_sanitize_encoder(struct intel_encoder *encoder)
17567 {
17568         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
17569         struct intel_connector *connector;
17570         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
17571         struct intel_crtc_state *crtc_state = crtc ?
17572                 to_intel_crtc_state(crtc->base.state) : NULL;
17573
17574         /* We need to check both for a crtc link (meaning that the
17575          * encoder is active and trying to read from a pipe) and the
17576          * pipe itself being active. */
17577         bool has_active_crtc = crtc_state &&
17578                 crtc_state->hw.active;
17579
17580         if (crtc_state && has_bogus_dpll_config(crtc_state)) {
17581                 DRM_DEBUG_KMS("BIOS has misprogrammed the hardware. Disabling pipe %c\n",
17582                               pipe_name(crtc->pipe));
17583                 has_active_crtc = false;
17584         }
17585
17586         connector = intel_encoder_find_connector(encoder);
17587         if (connector && !has_active_crtc) {
17588                 DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n",
17589                               encoder->base.base.id,
17590                               encoder->base.name);
17591
17592                 /* Connector is active, but has no active pipe. This is
17593                  * fallout from our resume register restoring. Disable
17594                  * the encoder manually again. */
17595                 if (crtc_state) {
17596                         struct drm_encoder *best_encoder;
17597
17598                         DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n",
17599                                       encoder->base.base.id,
17600                                       encoder->base.name);
17601
17602                         /* avoid oopsing in case the hooks consult best_encoder */
17603                         best_encoder = connector->base.state->best_encoder;
17604                         connector->base.state->best_encoder = &encoder->base;
17605
17606                         if (encoder->disable)
17607                                 encoder->disable(encoder, crtc_state,
17608                                                  connector->base.state);
17609                         if (encoder->post_disable)
17610                                 encoder->post_disable(encoder, crtc_state,
17611                                                       connector->base.state);
17612
17613                         connector->base.state->best_encoder = best_encoder;
17614                 }
17615                 encoder->base.crtc = NULL;
17616
17617                 /* Inconsistent output/port/pipe state happens presumably due to
17618                  * a bug in one of the get_hw_state functions. Or someplace else
17619                  * in our code, like the register restore mess on resume. Clamp
17620                  * things to off as a safer default. */
17621
17622                 connector->base.dpms = DRM_MODE_DPMS_OFF;
17623                 connector->base.encoder = NULL;
17624         }
17625
17626         /* notify opregion of the sanitized encoder state */
17627         intel_opregion_notify_encoder(encoder, connector && has_active_crtc);
17628
17629         if (INTEL_GEN(dev_priv) >= 11)
17630                 icl_sanitize_encoder_pll_mapping(encoder);
17631 }
17632
17633 /* FIXME read out full plane state for all planes */
17634 static void readout_plane_state(struct drm_i915_private *dev_priv)
17635 {
17636         struct intel_plane *plane;
17637         struct intel_crtc *crtc;
17638
17639         for_each_intel_plane(&dev_priv->drm, plane) {
17640                 struct intel_plane_state *plane_state =
17641                         to_intel_plane_state(plane->base.state);
17642                 struct intel_crtc_state *crtc_state;
17643                 enum pipe pipe = PIPE_A;
17644                 bool visible;
17645
17646                 visible = plane->get_hw_state(plane, &pipe);
17647
17648                 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
17649                 crtc_state = to_intel_crtc_state(crtc->base.state);
17650
17651                 intel_set_plane_visible(crtc_state, plane_state, visible);
17652
17653                 DRM_DEBUG_KMS("[PLANE:%d:%s] hw state readout: %s, pipe %c\n",
17654                               plane->base.base.id, plane->base.name,
17655                               enableddisabled(visible), pipe_name(pipe));
17656         }
17657
17658         for_each_intel_crtc(&dev_priv->drm, crtc) {
17659                 struct intel_crtc_state *crtc_state =
17660                         to_intel_crtc_state(crtc->base.state);
17661
17662                 fixup_active_planes(crtc_state);
17663         }
17664 }
17665
17666 static void intel_modeset_readout_hw_state(struct drm_device *dev)
17667 {
17668         struct drm_i915_private *dev_priv = to_i915(dev);
17669         enum pipe pipe;
17670         struct intel_crtc *crtc;
17671         struct intel_encoder *encoder;
17672         struct intel_connector *connector;
17673         struct drm_connector_list_iter conn_iter;
17674         int i;
17675
17676         dev_priv->active_pipes = 0;
17677
17678         for_each_intel_crtc(dev, crtc) {
17679                 struct intel_crtc_state *crtc_state =
17680                         to_intel_crtc_state(crtc->base.state);
17681
17682                 __drm_atomic_helper_crtc_destroy_state(&crtc_state->uapi);
17683                 intel_crtc_free_hw_state(crtc_state);
17684                 intel_crtc_state_reset(crtc_state, crtc);
17685
17686                 crtc_state->hw.active = crtc_state->hw.enable =
17687                         dev_priv->display.get_pipe_config(crtc, crtc_state);
17688
17689                 crtc->base.enabled = crtc_state->hw.enable;
17690                 crtc->active = crtc_state->hw.active;
17691
17692                 if (crtc_state->hw.active)
17693                         dev_priv->active_pipes |= BIT(crtc->pipe);
17694
17695                 DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n",
17696                               crtc->base.base.id, crtc->base.name,
17697                               enableddisabled(crtc_state->hw.active));
17698         }
17699
17700         readout_plane_state(dev_priv);
17701
17702         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
17703                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
17704
17705                 pll->on = pll->info->funcs->get_hw_state(dev_priv, pll,
17706                                                         &pll->state.hw_state);
17707
17708                 if (IS_ELKHARTLAKE(dev_priv) && pll->on &&
17709                     pll->info->id == DPLL_ID_EHL_DPLL4) {
17710                         pll->wakeref = intel_display_power_get(dev_priv,
17711                                                                POWER_DOMAIN_DPLL_DC_OFF);
17712                 }
17713
17714                 pll->state.crtc_mask = 0;
17715                 for_each_intel_crtc(dev, crtc) {
17716                         struct intel_crtc_state *crtc_state =
17717                                 to_intel_crtc_state(crtc->base.state);
17718
17719                         if (crtc_state->hw.active &&
17720                             crtc_state->shared_dpll == pll)
17721                                 pll->state.crtc_mask |= 1 << crtc->pipe;
17722                 }
17723                 pll->active_mask = pll->state.crtc_mask;
17724
17725                 DRM_DEBUG_KMS("%s hw state readout: crtc_mask 0x%08x, on %i\n",
17726                               pll->info->name, pll->state.crtc_mask, pll->on);
17727         }
17728
17729         for_each_intel_encoder(dev, encoder) {
17730                 pipe = 0;
17731
17732                 if (encoder->get_hw_state(encoder, &pipe)) {
17733                         struct intel_crtc_state *crtc_state;
17734
17735                         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
17736                         crtc_state = to_intel_crtc_state(crtc->base.state);
17737
17738                         encoder->base.crtc = &crtc->base;
17739                         encoder->get_config(encoder, crtc_state);
17740                 } else {
17741                         encoder->base.crtc = NULL;
17742                 }
17743
17744                 DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
17745                               encoder->base.base.id, encoder->base.name,
17746                               enableddisabled(encoder->base.crtc),
17747                               pipe_name(pipe));
17748         }
17749
17750         drm_connector_list_iter_begin(dev, &conn_iter);
17751         for_each_intel_connector_iter(connector, &conn_iter) {
17752                 if (connector->get_hw_state(connector)) {
17753                         struct intel_crtc_state *crtc_state;
17754                         struct intel_crtc *crtc;
17755
17756                         connector->base.dpms = DRM_MODE_DPMS_ON;
17757
17758                         encoder = connector->encoder;
17759                         connector->base.encoder = &encoder->base;
17760
17761                         crtc = to_intel_crtc(encoder->base.crtc);
17762                         crtc_state = crtc ? to_intel_crtc_state(crtc->base.state) : NULL;
17763
17764                         if (crtc_state && crtc_state->hw.active) {
17765                                 /*
17766                                  * This has to be done during hardware readout
17767                                  * because anything calling .crtc_disable may
17768                                  * rely on the connector_mask being accurate.
17769                                  */
17770                                 crtc_state->uapi.connector_mask |=
17771                                         drm_connector_mask(&connector->base);
17772                                 crtc_state->uapi.encoder_mask |=
17773                                         drm_encoder_mask(&encoder->base);
17774                         }
17775                 } else {
17776                         connector->base.dpms = DRM_MODE_DPMS_OFF;
17777                         connector->base.encoder = NULL;
17778                 }
17779                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n",
17780                               connector->base.base.id, connector->base.name,
17781                               enableddisabled(connector->base.encoder));
17782         }
17783         drm_connector_list_iter_end(&conn_iter);
17784
17785         for_each_intel_crtc(dev, crtc) {
17786                 struct intel_bw_state *bw_state =
17787                         to_intel_bw_state(dev_priv->bw_obj.state);
17788                 struct intel_crtc_state *crtc_state =
17789                         to_intel_crtc_state(crtc->base.state);
17790                 struct intel_plane *plane;
17791                 int min_cdclk = 0;
17792
17793                 if (crtc_state->hw.active) {
17794                         struct drm_display_mode *mode = &crtc_state->hw.mode;
17795
17796                         intel_mode_from_pipe_config(&crtc_state->hw.adjusted_mode,
17797                                                     crtc_state);
17798
17799                         *mode = crtc_state->hw.adjusted_mode;
17800                         mode->hdisplay = crtc_state->pipe_src_w;
17801                         mode->vdisplay = crtc_state->pipe_src_h;
17802
17803                         /*
17804                          * The initial mode needs to be set in order to keep
17805                          * the atomic core happy. It wants a valid mode if the
17806                          * crtc's enabled, so we do the above call.
17807                          *
17808                          * But we don't set all the derived state fully, hence
17809                          * set a flag to indicate that a full recalculation is
17810                          * needed on the next commit.
17811                          */
17812                         mode->private_flags = I915_MODE_FLAG_INHERITED;
17813
17814                         intel_crtc_compute_pixel_rate(crtc_state);
17815
17816                         intel_crtc_update_active_timings(crtc_state);
17817
17818                         intel_crtc_copy_hw_to_uapi_state(crtc_state);
17819                 }
17820
17821                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
17822                         const struct intel_plane_state *plane_state =
17823                                 to_intel_plane_state(plane->base.state);
17824
17825                         /*
17826                          * FIXME don't have the fb yet, so can't
17827                          * use intel_plane_data_rate() :(
17828                          */
17829                         if (plane_state->uapi.visible)
17830                                 crtc_state->data_rate[plane->id] =
17831                                         4 * crtc_state->pixel_rate;
17832                         /*
17833                          * FIXME don't have the fb yet, so can't
17834                          * use plane->min_cdclk() :(
17835                          */
17836                         if (plane_state->uapi.visible && plane->min_cdclk) {
17837                                 if (crtc_state->double_wide ||
17838                                     INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
17839                                         crtc_state->min_cdclk[plane->id] =
17840                                                 DIV_ROUND_UP(crtc_state->pixel_rate, 2);
17841                                 else
17842                                         crtc_state->min_cdclk[plane->id] =
17843                                                 crtc_state->pixel_rate;
17844                         }
17845                         DRM_DEBUG_KMS("[PLANE:%d:%s] min_cdclk %d kHz\n",
17846                                       plane->base.base.id, plane->base.name,
17847                                       crtc_state->min_cdclk[plane->id]);
17848                 }
17849
17850                 if (crtc_state->hw.active) {
17851                         min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
17852                         if (WARN_ON(min_cdclk < 0))
17853                                 min_cdclk = 0;
17854                 }
17855
17856                 dev_priv->min_cdclk[crtc->pipe] = min_cdclk;
17857                 dev_priv->min_voltage_level[crtc->pipe] =
17858                         crtc_state->min_voltage_level;
17859
17860                 intel_bw_crtc_update(bw_state, crtc_state);
17861
17862                 intel_pipe_config_sanity_check(dev_priv, crtc_state);
17863         }
17864 }
17865
17866 static void
17867 get_encoder_power_domains(struct drm_i915_private *dev_priv)
17868 {
17869         struct intel_encoder *encoder;
17870
17871         for_each_intel_encoder(&dev_priv->drm, encoder) {
17872                 struct intel_crtc_state *crtc_state;
17873
17874                 if (!encoder->get_power_domains)
17875                         continue;
17876
17877                 /*
17878                  * MST-primary and inactive encoders don't have a crtc state
17879                  * and neither of these require any power domain references.
17880                  */
17881                 if (!encoder->base.crtc)
17882                         continue;
17883
17884                 crtc_state = to_intel_crtc_state(encoder->base.crtc->state);
17885                 encoder->get_power_domains(encoder, crtc_state);
17886         }
17887 }
17888
17889 static void intel_early_display_was(struct drm_i915_private *dev_priv)
17890 {
17891         /* Display WA #1185 WaDisableDARBFClkGating:cnl,glk */
17892         if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
17893                 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
17894                            DARBF_GATING_DIS);
17895
17896         if (IS_HASWELL(dev_priv)) {
17897                 /*
17898                  * WaRsPkgCStateDisplayPMReq:hsw
17899                  * System hang if this isn't done before disabling all planes!
17900                  */
17901                 I915_WRITE(CHICKEN_PAR1_1,
17902                            I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
17903         }
17904 }
17905
17906 static void ibx_sanitize_pch_hdmi_port(struct drm_i915_private *dev_priv,
17907                                        enum port port, i915_reg_t hdmi_reg)
17908 {
17909         u32 val = I915_READ(hdmi_reg);
17910
17911         if (val & SDVO_ENABLE ||
17912             (val & SDVO_PIPE_SEL_MASK) == SDVO_PIPE_SEL(PIPE_A))
17913                 return;
17914
17915         DRM_DEBUG_KMS("Sanitizing transcoder select for HDMI %c\n",
17916                       port_name(port));
17917
17918         val &= ~SDVO_PIPE_SEL_MASK;
17919         val |= SDVO_PIPE_SEL(PIPE_A);
17920
17921         I915_WRITE(hdmi_reg, val);
17922 }
17923
17924 static void ibx_sanitize_pch_dp_port(struct drm_i915_private *dev_priv,
17925                                      enum port port, i915_reg_t dp_reg)
17926 {
17927         u32 val = I915_READ(dp_reg);
17928
17929         if (val & DP_PORT_EN ||
17930             (val & DP_PIPE_SEL_MASK) == DP_PIPE_SEL(PIPE_A))
17931                 return;
17932
17933         DRM_DEBUG_KMS("Sanitizing transcoder select for DP %c\n",
17934                       port_name(port));
17935
17936         val &= ~DP_PIPE_SEL_MASK;
17937         val |= DP_PIPE_SEL(PIPE_A);
17938
17939         I915_WRITE(dp_reg, val);
17940 }
17941
17942 static void ibx_sanitize_pch_ports(struct drm_i915_private *dev_priv)
17943 {
17944         /*
17945          * The BIOS may select transcoder B on some of the PCH
17946          * ports even it doesn't enable the port. This would trip
17947          * assert_pch_dp_disabled() and assert_pch_hdmi_disabled().
17948          * Sanitize the transcoder select bits to prevent that. We
17949          * assume that the BIOS never actually enabled the port,
17950          * because if it did we'd actually have to toggle the port
17951          * on and back off to make the transcoder A select stick
17952          * (see. intel_dp_link_down(), intel_disable_hdmi(),
17953          * intel_disable_sdvo()).
17954          */
17955         ibx_sanitize_pch_dp_port(dev_priv, PORT_B, PCH_DP_B);
17956         ibx_sanitize_pch_dp_port(dev_priv, PORT_C, PCH_DP_C);
17957         ibx_sanitize_pch_dp_port(dev_priv, PORT_D, PCH_DP_D);
17958
17959         /* PCH SDVOB multiplex with HDMIB */
17960         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_B, PCH_HDMIB);
17961         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_C, PCH_HDMIC);
17962         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_D, PCH_HDMID);
17963 }
17964
17965 /* Scan out the current hw modeset state,
17966  * and sanitizes it to the current state
17967  */
17968 static void
17969 intel_modeset_setup_hw_state(struct drm_device *dev,
17970                              struct drm_modeset_acquire_ctx *ctx)
17971 {
17972         struct drm_i915_private *dev_priv = to_i915(dev);
17973         struct intel_encoder *encoder;
17974         struct intel_crtc *crtc;
17975         intel_wakeref_t wakeref;
17976         int i;
17977
17978         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
17979
17980         intel_early_display_was(dev_priv);
17981         intel_modeset_readout_hw_state(dev);
17982
17983         /* HW state is read out, now we need to sanitize this mess. */
17984
17985         /* Sanitize the TypeC port mode upfront, encoders depend on this */
17986         for_each_intel_encoder(dev, encoder) {
17987                 enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
17988
17989                 /* We need to sanitize only the MST primary port. */
17990                 if (encoder->type != INTEL_OUTPUT_DP_MST &&
17991                     intel_phy_is_tc(dev_priv, phy))
17992                         intel_tc_port_sanitize(enc_to_dig_port(&encoder->base));
17993         }
17994
17995         get_encoder_power_domains(dev_priv);
17996
17997         if (HAS_PCH_IBX(dev_priv))
17998                 ibx_sanitize_pch_ports(dev_priv);
17999
18000         /*
18001          * intel_sanitize_plane_mapping() may need to do vblank
18002          * waits, so we need vblank interrupts restored beforehand.
18003          */
18004         for_each_intel_crtc(&dev_priv->drm, crtc) {
18005                 struct intel_crtc_state *crtc_state =
18006                         to_intel_crtc_state(crtc->base.state);
18007
18008                 drm_crtc_vblank_reset(&crtc->base);
18009
18010                 if (crtc_state->hw.active)
18011                         intel_crtc_vblank_on(crtc_state);
18012         }
18013
18014         intel_sanitize_plane_mapping(dev_priv);
18015
18016         for_each_intel_encoder(dev, encoder)
18017                 intel_sanitize_encoder(encoder);
18018
18019         for_each_intel_crtc(&dev_priv->drm, crtc) {
18020                 struct intel_crtc_state *crtc_state =
18021                         to_intel_crtc_state(crtc->base.state);
18022
18023                 intel_sanitize_crtc(crtc, ctx);
18024                 intel_dump_pipe_config(crtc_state, NULL, "[setup_hw_state]");
18025         }
18026
18027         intel_modeset_update_connector_atomic_state(dev);
18028
18029         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
18030                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
18031
18032                 if (!pll->on || pll->active_mask)
18033                         continue;
18034
18035                 DRM_DEBUG_KMS("%s enabled but not in use, disabling\n",
18036                               pll->info->name);
18037
18038                 pll->info->funcs->disable(dev_priv, pll);
18039                 pll->on = false;
18040         }
18041
18042         if (IS_G4X(dev_priv)) {
18043                 g4x_wm_get_hw_state(dev_priv);
18044                 g4x_wm_sanitize(dev_priv);
18045         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
18046                 vlv_wm_get_hw_state(dev_priv);
18047                 vlv_wm_sanitize(dev_priv);
18048         } else if (INTEL_GEN(dev_priv) >= 9) {
18049                 skl_wm_get_hw_state(dev_priv);
18050         } else if (HAS_PCH_SPLIT(dev_priv)) {
18051                 ilk_wm_get_hw_state(dev_priv);
18052         }
18053
18054         for_each_intel_crtc(dev, crtc) {
18055                 struct intel_crtc_state *crtc_state =
18056                         to_intel_crtc_state(crtc->base.state);
18057                 u64 put_domains;
18058
18059                 put_domains = modeset_get_crtc_power_domains(crtc_state);
18060                 if (WARN_ON(put_domains))
18061                         modeset_put_power_domains(dev_priv, put_domains);
18062         }
18063
18064         intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref);
18065 }
18066
18067 void intel_display_resume(struct drm_device *dev)
18068 {
18069         struct drm_i915_private *dev_priv = to_i915(dev);
18070         struct drm_atomic_state *state = dev_priv->modeset_restore_state;
18071         struct drm_modeset_acquire_ctx ctx;
18072         int ret;
18073
18074         dev_priv->modeset_restore_state = NULL;
18075         if (state)
18076                 state->acquire_ctx = &ctx;
18077
18078         drm_modeset_acquire_init(&ctx, 0);
18079
18080         while (1) {
18081                 ret = drm_modeset_lock_all_ctx(dev, &ctx);
18082                 if (ret != -EDEADLK)
18083                         break;
18084
18085                 drm_modeset_backoff(&ctx);
18086         }
18087
18088         if (!ret)
18089                 ret = __intel_display_resume(dev, state, &ctx);
18090
18091         intel_enable_ipc(dev_priv);
18092         drm_modeset_drop_locks(&ctx);
18093         drm_modeset_acquire_fini(&ctx);
18094
18095         if (ret)
18096                 DRM_ERROR("Restoring old state failed with %i\n", ret);
18097         if (state)
18098                 drm_atomic_state_put(state);
18099 }
18100
18101 static void intel_hpd_poll_fini(struct drm_i915_private *i915)
18102 {
18103         struct intel_connector *connector;
18104         struct drm_connector_list_iter conn_iter;
18105
18106         /* Kill all the work that may have been queued by hpd. */
18107         drm_connector_list_iter_begin(&i915->drm, &conn_iter);
18108         for_each_intel_connector_iter(connector, &conn_iter) {
18109                 if (connector->modeset_retry_work.func)
18110                         cancel_work_sync(&connector->modeset_retry_work);
18111                 if (connector->hdcp.shim) {
18112                         cancel_delayed_work_sync(&connector->hdcp.check_work);
18113                         cancel_work_sync(&connector->hdcp.prop_work);
18114                 }
18115         }
18116         drm_connector_list_iter_end(&conn_iter);
18117 }
18118
18119 void intel_modeset_driver_remove(struct drm_i915_private *i915)
18120 {
18121         flush_workqueue(i915->flip_wq);
18122         flush_workqueue(i915->modeset_wq);
18123
18124         flush_work(&i915->atomic_helper.free_work);
18125         WARN_ON(!llist_empty(&i915->atomic_helper.free_list));
18126
18127         /*
18128          * Interrupts and polling as the first thing to avoid creating havoc.
18129          * Too much stuff here (turning of connectors, ...) would
18130          * experience fancy races otherwise.
18131          */
18132         intel_irq_uninstall(i915);
18133
18134         /*
18135          * Due to the hpd irq storm handling the hotplug work can re-arm the
18136          * poll handlers. Hence disable polling after hpd handling is shut down.
18137          */
18138         intel_hpd_poll_fini(i915);
18139
18140         /*
18141          * MST topology needs to be suspended so we don't have any calls to
18142          * fbdev after it's finalized. MST will be destroyed later as part of
18143          * drm_mode_config_cleanup()
18144          */
18145         intel_dp_mst_suspend(i915);
18146
18147         /* poll work can call into fbdev, hence clean that up afterwards */
18148         intel_fbdev_fini(i915);
18149
18150         intel_unregister_dsm_handler();
18151
18152         intel_fbc_global_disable(i915);
18153
18154         /* flush any delayed tasks or pending work */
18155         flush_scheduled_work();
18156
18157         intel_hdcp_component_fini(i915);
18158
18159         drm_mode_config_cleanup(&i915->drm);
18160
18161         intel_overlay_cleanup(i915);
18162
18163         intel_gmbus_teardown(i915);
18164
18165         destroy_workqueue(i915->flip_wq);
18166         destroy_workqueue(i915->modeset_wq);
18167
18168         intel_fbc_cleanup_cfb(i915);
18169 }
18170
18171 #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
18172
18173 struct intel_display_error_state {
18174
18175         u32 power_well_driver;
18176
18177         struct intel_cursor_error_state {
18178                 u32 control;
18179                 u32 position;
18180                 u32 base;
18181                 u32 size;
18182         } cursor[I915_MAX_PIPES];
18183
18184         struct intel_pipe_error_state {
18185                 bool power_domain_on;
18186                 u32 source;
18187                 u32 stat;
18188         } pipe[I915_MAX_PIPES];
18189
18190         struct intel_plane_error_state {
18191                 u32 control;
18192                 u32 stride;
18193                 u32 size;
18194                 u32 pos;
18195                 u32 addr;
18196                 u32 surface;
18197                 u32 tile_offset;
18198         } plane[I915_MAX_PIPES];
18199
18200         struct intel_transcoder_error_state {
18201                 bool available;
18202                 bool power_domain_on;
18203                 enum transcoder cpu_transcoder;
18204
18205                 u32 conf;
18206
18207                 u32 htotal;
18208                 u32 hblank;
18209                 u32 hsync;
18210                 u32 vtotal;
18211                 u32 vblank;
18212                 u32 vsync;
18213         } transcoder[5];
18214 };
18215
18216 struct intel_display_error_state *
18217 intel_display_capture_error_state(struct drm_i915_private *dev_priv)
18218 {
18219         struct intel_display_error_state *error;
18220         int transcoders[] = {
18221                 TRANSCODER_A,
18222                 TRANSCODER_B,
18223                 TRANSCODER_C,
18224                 TRANSCODER_D,
18225                 TRANSCODER_EDP,
18226         };
18227         int i;
18228
18229         BUILD_BUG_ON(ARRAY_SIZE(transcoders) != ARRAY_SIZE(error->transcoder));
18230
18231         if (!HAS_DISPLAY(dev_priv) || !INTEL_DISPLAY_ENABLED(dev_priv))
18232                 return NULL;
18233
18234         error = kzalloc(sizeof(*error), GFP_ATOMIC);
18235         if (error == NULL)
18236                 return NULL;
18237
18238         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
18239                 error->power_well_driver = I915_READ(HSW_PWR_WELL_CTL2);
18240
18241         for_each_pipe(dev_priv, i) {
18242                 error->pipe[i].power_domain_on =
18243                         __intel_display_power_is_enabled(dev_priv,
18244                                                          POWER_DOMAIN_PIPE(i));
18245                 if (!error->pipe[i].power_domain_on)
18246                         continue;
18247
18248                 error->cursor[i].control = I915_READ(CURCNTR(i));
18249                 error->cursor[i].position = I915_READ(CURPOS(i));
18250                 error->cursor[i].base = I915_READ(CURBASE(i));
18251
18252                 error->plane[i].control = I915_READ(DSPCNTR(i));
18253                 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
18254                 if (INTEL_GEN(dev_priv) <= 3) {
18255                         error->plane[i].size = I915_READ(DSPSIZE(i));
18256                         error->plane[i].pos = I915_READ(DSPPOS(i));
18257                 }
18258                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
18259                         error->plane[i].addr = I915_READ(DSPADDR(i));
18260                 if (INTEL_GEN(dev_priv) >= 4) {
18261                         error->plane[i].surface = I915_READ(DSPSURF(i));
18262                         error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
18263                 }
18264
18265                 error->pipe[i].source = I915_READ(PIPESRC(i));
18266
18267                 if (HAS_GMCH(dev_priv))
18268                         error->pipe[i].stat = I915_READ(PIPESTAT(i));
18269         }
18270
18271         for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
18272                 enum transcoder cpu_transcoder = transcoders[i];
18273
18274                 if (!INTEL_INFO(dev_priv)->trans_offsets[cpu_transcoder])
18275                         continue;
18276
18277                 error->transcoder[i].available = true;
18278                 error->transcoder[i].power_domain_on =
18279                         __intel_display_power_is_enabled(dev_priv,
18280                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder));
18281                 if (!error->transcoder[i].power_domain_on)
18282                         continue;
18283
18284                 error->transcoder[i].cpu_transcoder = cpu_transcoder;
18285
18286                 error->transcoder[i].conf = I915_READ(PIPECONF(cpu_transcoder));
18287                 error->transcoder[i].htotal = I915_READ(HTOTAL(cpu_transcoder));
18288                 error->transcoder[i].hblank = I915_READ(HBLANK(cpu_transcoder));
18289                 error->transcoder[i].hsync = I915_READ(HSYNC(cpu_transcoder));
18290                 error->transcoder[i].vtotal = I915_READ(VTOTAL(cpu_transcoder));
18291                 error->transcoder[i].vblank = I915_READ(VBLANK(cpu_transcoder));
18292                 error->transcoder[i].vsync = I915_READ(VSYNC(cpu_transcoder));
18293         }
18294
18295         return error;
18296 }
18297
18298 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
18299
18300 void
18301 intel_display_print_error_state(struct drm_i915_error_state_buf *m,
18302                                 struct intel_display_error_state *error)
18303 {
18304         struct drm_i915_private *dev_priv = m->i915;
18305         int i;
18306
18307         if (!error)
18308                 return;
18309
18310         err_printf(m, "Num Pipes: %d\n", INTEL_NUM_PIPES(dev_priv));
18311         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
18312                 err_printf(m, "PWR_WELL_CTL2: %08x\n",
18313                            error->power_well_driver);
18314         for_each_pipe(dev_priv, i) {
18315                 err_printf(m, "Pipe [%d]:\n", i);
18316                 err_printf(m, "  Power: %s\n",
18317                            onoff(error->pipe[i].power_domain_on));
18318                 err_printf(m, "  SRC: %08x\n", error->pipe[i].source);
18319                 err_printf(m, "  STAT: %08x\n", error->pipe[i].stat);
18320
18321                 err_printf(m, "Plane [%d]:\n", i);
18322                 err_printf(m, "  CNTR: %08x\n", error->plane[i].control);
18323                 err_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
18324                 if (INTEL_GEN(dev_priv) <= 3) {
18325                         err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
18326                         err_printf(m, "  POS: %08x\n", error->plane[i].pos);
18327                 }
18328                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
18329                         err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
18330                 if (INTEL_GEN(dev_priv) >= 4) {
18331                         err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
18332                         err_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
18333                 }
18334
18335                 err_printf(m, "Cursor [%d]:\n", i);
18336                 err_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
18337                 err_printf(m, "  POS: %08x\n", error->cursor[i].position);
18338                 err_printf(m, "  BASE: %08x\n", error->cursor[i].base);
18339         }
18340
18341         for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
18342                 if (!error->transcoder[i].available)
18343                         continue;
18344
18345                 err_printf(m, "CPU transcoder: %s\n",
18346                            transcoder_name(error->transcoder[i].cpu_transcoder));
18347                 err_printf(m, "  Power: %s\n",
18348                            onoff(error->transcoder[i].power_domain_on));
18349                 err_printf(m, "  CONF: %08x\n", error->transcoder[i].conf);
18350                 err_printf(m, "  HTOTAL: %08x\n", error->transcoder[i].htotal);
18351                 err_printf(m, "  HBLANK: %08x\n", error->transcoder[i].hblank);
18352                 err_printf(m, "  HSYNC: %08x\n", error->transcoder[i].hsync);
18353                 err_printf(m, "  VTOTAL: %08x\n", error->transcoder[i].vtotal);
18354                 err_printf(m, "  VBLANK: %08x\n", error->transcoder[i].vblank);
18355                 err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
18356         }
18357 }
18358
18359 #endif