drivers/gpu/drm/i915/display/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/i2c.h>
  28 #include <linux/input.h>
  29 #include <linux/intel-iommu.h>
  30 #include <linux/kernel.h>
  31 #include <linux/module.h>
  32 #include <linux/dma-resv.h>
  33 #include <linux/slab.h>
  34
  35 #include <drm/drm_atomic.h>
  36 #include <drm/drm_atomic_helper.h>
  37 #include <drm/drm_atomic_uapi.h>
  38 #include <drm/drm_dp_helper.h>
  39 #include <drm/drm_edid.h>
  40 #include <drm/drm_fourcc.h>
  41 #include <drm/drm_plane_helper.h>
  42 #include <drm/drm_probe_helper.h>
  43 #include <drm/drm_rect.h>
  44 #include <drm/i915_drm.h>
  45
  46 #include "display/intel_crt.h"
  47 #include "display/intel_ddi.h"
  48 #include "display/intel_dp.h"
  49 #include "display/intel_dsi.h"
  50 #include "display/intel_dvo.h"
  51 #include "display/intel_gmbus.h"
  52 #include "display/intel_hdmi.h"
  53 #include "display/intel_lvds.h"
  54 #include "display/intel_sdvo.h"
  55 #include "display/intel_tv.h"
  56 #include "display/intel_vdsc.h"
  57
  58 #include "gt/intel_rps.h"
  59
  60 #include "i915_drv.h"
  61 #include "i915_trace.h"
  62 #include "intel_acpi.h"
  63 #include "intel_atomic.h"
  64 #include "intel_atomic_plane.h"
  65 #include "intel_bw.h"
  66 #include "intel_cdclk.h"
  67 #include "intel_color.h"
  68 #include "intel_display_types.h"
  69 #include "intel_dp_link_training.h"
  70 #include "intel_fbc.h"
  71 #include "intel_fbdev.h"
  72 #include "intel_fifo_underrun.h"
  73 #include "intel_frontbuffer.h"
  74 #include "intel_hdcp.h"
  75 #include "intel_hotplug.h"
  76 #include "intel_overlay.h"
  77 #include "intel_pipe_crc.h"
  78 #include "intel_pm.h"
  79 #include "intel_psr.h"
  80 #include "intel_quirks.h"
  81 #include "intel_sideband.h"
  82 #include "intel_sprite.h"
  83 #include "intel_tc.h"
  84 #include "intel_vga.h"
  85
  86 /* Primary plane formats for gen <= 3 */
  87 static const u32 i8xx_primary_formats[] = {
  88         DRM_FORMAT_C8,
  89         DRM_FORMAT_XRGB1555,
  90         DRM_FORMAT_RGB565,
  91         DRM_FORMAT_XRGB8888,
  92 };
  93
  94 /* Primary plane formats for ivb (no fp16 due to hw issue) */
  95 static const u32 ivb_primary_formats[] = {
  96         DRM_FORMAT_C8,
  97         DRM_FORMAT_RGB565,
  98         DRM_FORMAT_XRGB8888,
  99         DRM_FORMAT_XBGR8888,
 100         DRM_FORMAT_XRGB2101010,
 101         DRM_FORMAT_XBGR2101010,
 102 };
 103
 104 /* Primary plane formats for gen >= 4, except ivb */
 105 static const u32 i965_primary_formats[] = {
 106         DRM_FORMAT_C8,
 107         DRM_FORMAT_RGB565,
 108         DRM_FORMAT_XRGB8888,
 109         DRM_FORMAT_XBGR8888,
 110         DRM_FORMAT_XRGB2101010,
 111         DRM_FORMAT_XBGR2101010,
 112         DRM_FORMAT_XBGR16161616F,
 113 };
 114
 115 /* Primary plane formats for vlv/chv */
 116 static const u32 vlv_primary_formats[] = {
 117         DRM_FORMAT_C8,
 118         DRM_FORMAT_RGB565,
 119         DRM_FORMAT_XRGB8888,
 120         DRM_FORMAT_XBGR8888,
 121         DRM_FORMAT_ARGB8888,
 122         DRM_FORMAT_ABGR8888,
 123         DRM_FORMAT_XRGB2101010,
 124         DRM_FORMAT_XBGR2101010,
 125         DRM_FORMAT_ARGB2101010,
 126         DRM_FORMAT_ABGR2101010,
 127         DRM_FORMAT_XBGR16161616F,
 128 };
 129
 130 static const u64 i9xx_format_modifiers[] = {
 131         I915_FORMAT_MOD_X_TILED,
 132         DRM_FORMAT_MOD_LINEAR,
 133         DRM_FORMAT_MOD_INVALID
 134 };
 135
 136 /* Cursor formats */
 137 static const u32 intel_cursor_formats[] = {
 138         DRM_FORMAT_ARGB8888,
 139 };
 140
 141 static const u64 cursor_format_modifiers[] = {
 142         DRM_FORMAT_MOD_LINEAR,
 143         DRM_FORMAT_MOD_INVALID
 144 };
 145
 146 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
 147                                 struct intel_crtc_state *pipe_config);
 148 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
 149                                    struct intel_crtc_state *pipe_config);
 150
 151 static int intel_framebuffer_init(struct intel_framebuffer *ifb,
 152                                   struct drm_i915_gem_object *obj,
 153                                   struct drm_mode_fb_cmd2 *mode_cmd);
 154 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state);
 155 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state);
 156 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
 157                                          const struct intel_link_m_n *m_n,
 158                                          const struct intel_link_m_n *m2_n2);
 159 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state);
 160 static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state);
 161 static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state);
 162 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state);
 163 static void vlv_prepare_pll(struct intel_crtc *crtc,
 164                             const struct intel_crtc_state *pipe_config);
 165 static void chv_prepare_pll(struct intel_crtc *crtc,
 166                             const struct intel_crtc_state *pipe_config);
 167 static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state);
 168 static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state);
 169 static void intel_modeset_setup_hw_state(struct drm_device *dev,
 170                                          struct drm_modeset_acquire_ctx *ctx);
 171 static struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc);
 172
 173 struct intel_limit {
 174         struct {
 175                 int min, max;
 176         } dot, vco, n, m, m1, m2, p, p1;
 177
 178         struct {
 179                 int dot_limit;
 180                 int p2_slow, p2_fast;
 181         } p2;
 182 };
 183
 184 /* returns HPLL frequency in kHz */
 185 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv)
 186 {
 187         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 188
 189         /* Obtain SKU information */
 190         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
 191                 CCK_FUSE_HPLL_FREQ_MASK;
 192
 193         return vco_freq[hpll_freq] * 1000;
 194 }
 195
 196 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
 197                       const char *name, u32 reg, int ref_freq)
 198 {
 199         u32 val;
 200         int divider;
 201
 202         val = vlv_cck_read(dev_priv, reg);
 203         divider = val & CCK_FREQUENCY_VALUES;
 204
 205         WARN((val & CCK_FREQUENCY_STATUS) !=
 206              (divider << CCK_FREQUENCY_STATUS_SHIFT),
 207              "%s change in progress\n", name);
 208
 209         return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1);
 210 }
 211
 212 int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 213                            const char *name, u32 reg)
 214 {
 215         int hpll;
 216
 217         vlv_cck_get(dev_priv);
 218
 219         if (dev_priv->hpll_freq == 0)
 220                 dev_priv->hpll_freq = vlv_get_hpll_vco(dev_priv);
 221
 222         hpll = vlv_get_cck_clock(dev_priv, name, reg, dev_priv->hpll_freq);
 223
 224         vlv_cck_put(dev_priv);
 225
 226         return hpll;
 227 }
 228
 229 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 230 {
 231         if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
 232                 return;
 233
 234         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
 235                                                       CCK_CZ_CLOCK_CONTROL);
 236
 237         DRM_DEBUG_DRIVER("CZ clock rate: %d kHz\n", dev_priv->czclk_freq);
 238 }
 239
 240 static inline u32 /* units of 100MHz */
 241 intel_fdi_link_freq(struct drm_i915_private *dev_priv,
 242                     const struct intel_crtc_state *pipe_config)
 243 {
 244         if (HAS_DDI(dev_priv))
 245                 return pipe_config->port_clock; /* SPLL */
 246         else
 247                 return dev_priv->fdi_pll_freq;
 248 }
 249
 250 static const struct intel_limit intel_limits_i8xx_dac = {
 251         .dot = { .min = 25000, .max = 350000 },
 252         .vco = { .min = 908000, .max = 1512000 },
 253         .n = { .min = 2, .max = 16 },
 254         .m = { .min = 96, .max = 140 },
 255         .m1 = { .min = 18, .max = 26 },
 256         .m2 = { .min = 6, .max = 16 },
 257         .p = { .min = 4, .max = 128 },
 258         .p1 = { .min = 2, .max = 33 },
 259         .p2 = { .dot_limit = 165000,
 260                 .p2_slow = 4, .p2_fast = 2 },
 261 };
 262
 263 static const struct intel_limit intel_limits_i8xx_dvo = {
 264         .dot = { .min = 25000, .max = 350000 },
 265         .vco = { .min = 908000, .max = 1512000 },
 266         .n = { .min = 2, .max = 16 },
 267         .m = { .min = 96, .max = 140 },
 268         .m1 = { .min = 18, .max = 26 },
 269         .m2 = { .min = 6, .max = 16 },
 270         .p = { .min = 4, .max = 128 },
 271         .p1 = { .min = 2, .max = 33 },
 272         .p2 = { .dot_limit = 165000,
 273                 .p2_slow = 4, .p2_fast = 4 },
 274 };
 275
 276 static const struct intel_limit intel_limits_i8xx_lvds = {
 277         .dot = { .min = 25000, .max = 350000 },
 278         .vco = { .min = 908000, .max = 1512000 },
 279         .n = { .min = 2, .max = 16 },
 280         .m = { .min = 96, .max = 140 },
 281         .m1 = { .min = 18, .max = 26 },
 282         .m2 = { .min = 6, .max = 16 },
 283         .p = { .min = 4, .max = 128 },
 284         .p1 = { .min = 1, .max = 6 },
 285         .p2 = { .dot_limit = 165000,
 286                 .p2_slow = 14, .p2_fast = 7 },
 287 };
 288
 289 static const struct intel_limit intel_limits_i9xx_sdvo = {
 290         .dot = { .min = 20000, .max = 400000 },
 291         .vco = { .min = 1400000, .max = 2800000 },
 292         .n = { .min = 1, .max = 6 },
 293         .m = { .min = 70, .max = 120 },
 294         .m1 = { .min = 8, .max = 18 },
 295         .m2 = { .min = 3, .max = 7 },
 296         .p = { .min = 5, .max = 80 },
 297         .p1 = { .min = 1, .max = 8 },
 298         .p2 = { .dot_limit = 200000,
 299                 .p2_slow = 10, .p2_fast = 5 },
 300 };
 301
 302 static const struct intel_limit intel_limits_i9xx_lvds = {
 303         .dot = { .min = 20000, .max = 400000 },
 304         .vco = { .min = 1400000, .max = 2800000 },
 305         .n = { .min = 1, .max = 6 },
 306         .m = { .min = 70, .max = 120 },
 307         .m1 = { .min = 8, .max = 18 },
 308         .m2 = { .min = 3, .max = 7 },
 309         .p = { .min = 7, .max = 98 },
 310         .p1 = { .min = 1, .max = 8 },
 311         .p2 = { .dot_limit = 112000,
 312                 .p2_slow = 14, .p2_fast = 7 },
 313 };
 314
 315
 316 static const struct intel_limit intel_limits_g4x_sdvo = {
 317         .dot = { .min = 25000, .max = 270000 },
 318         .vco = { .min = 1750000, .max = 3500000},
 319         .n = { .min = 1, .max = 4 },
 320         .m = { .min = 104, .max = 138 },
 321         .m1 = { .min = 17, .max = 23 },
 322         .m2 = { .min = 5, .max = 11 },
 323         .p = { .min = 10, .max = 30 },
 324         .p1 = { .min = 1, .max = 3},
 325         .p2 = { .dot_limit = 270000,
 326                 .p2_slow = 10,
 327                 .p2_fast = 10
 328         },
 329 };
 330
 331 static const struct intel_limit intel_limits_g4x_hdmi = {
 332         .dot = { .min = 22000, .max = 400000 },
 333         .vco = { .min = 1750000, .max = 3500000},
 334         .n = { .min = 1, .max = 4 },
 335         .m = { .min = 104, .max = 138 },
 336         .m1 = { .min = 16, .max = 23 },
 337         .m2 = { .min = 5, .max = 11 },
 338         .p = { .min = 5, .max = 80 },
 339         .p1 = { .min = 1, .max = 8},
 340         .p2 = { .dot_limit = 165000,
 341                 .p2_slow = 10, .p2_fast = 5 },
 342 };
 343
 344 static const struct intel_limit intel_limits_g4x_single_channel_lvds = {
 345         .dot = { .min = 20000, .max = 115000 },
 346         .vco = { .min = 1750000, .max = 3500000 },
 347         .n = { .min = 1, .max = 3 },
 348         .m = { .min = 104, .max = 138 },
 349         .m1 = { .min = 17, .max = 23 },
 350         .m2 = { .min = 5, .max = 11 },
 351         .p = { .min = 28, .max = 112 },
 352         .p1 = { .min = 2, .max = 8 },
 353         .p2 = { .dot_limit = 0,
 354                 .p2_slow = 14, .p2_fast = 14
 355         },
 356 };
 357
 358 static const struct intel_limit intel_limits_g4x_dual_channel_lvds = {
 359         .dot = { .min = 80000, .max = 224000 },
 360         .vco = { .min = 1750000, .max = 3500000 },
 361         .n = { .min = 1, .max = 3 },
 362         .m = { .min = 104, .max = 138 },
 363         .m1 = { .min = 17, .max = 23 },
 364         .m2 = { .min = 5, .max = 11 },
 365         .p = { .min = 14, .max = 42 },
 366         .p1 = { .min = 2, .max = 6 },
 367         .p2 = { .dot_limit = 0,
 368                 .p2_slow = 7, .p2_fast = 7
 369         },
 370 };
 371
 372 static const struct intel_limit intel_limits_pineview_sdvo = {
 373         .dot = { .min = 20000, .max = 400000},
 374         .vco = { .min = 1700000, .max = 3500000 },
 375         /* Pineview's Ncounter is a ring counter */
 376         .n = { .min = 3, .max = 6 },
 377         .m = { .min = 2, .max = 256 },
 378         /* Pineview only has one combined m divider, which we treat as m2. */
 379         .m1 = { .min = 0, .max = 0 },
 380         .m2 = { .min = 0, .max = 254 },
 381         .p = { .min = 5, .max = 80 },
 382         .p1 = { .min = 1, .max = 8 },
 383         .p2 = { .dot_limit = 200000,
 384                 .p2_slow = 10, .p2_fast = 5 },
 385 };
 386
 387 static const struct intel_limit intel_limits_pineview_lvds = {
 388         .dot = { .min = 20000, .max = 400000 },
 389         .vco = { .min = 1700000, .max = 3500000 },
 390         .n = { .min = 3, .max = 6 },
 391         .m = { .min = 2, .max = 256 },
 392         .m1 = { .min = 0, .max = 0 },
 393         .m2 = { .min = 0, .max = 254 },
 394         .p = { .min = 7, .max = 112 },
 395         .p1 = { .min = 1, .max = 8 },
 396         .p2 = { .dot_limit = 112000,
 397                 .p2_slow = 14, .p2_fast = 14 },
 398 };
 399
 400 /* Ironlake / Sandybridge
 401  *
 402  * We calculate clock using (register_value + 2) for N/M1/M2, so here
 403  * the range value for them is (actual_value - 2).
 404  */
 405 static const struct intel_limit intel_limits_ironlake_dac = {
 406         .dot = { .min = 25000, .max = 350000 },
 407         .vco = { .min = 1760000, .max = 3510000 },
 408         .n = { .min = 1, .max = 5 },
 409         .m = { .min = 79, .max = 127 },
 410         .m1 = { .min = 12, .max = 22 },
 411         .m2 = { .min = 5, .max = 9 },
 412         .p = { .min = 5, .max = 80 },
 413         .p1 = { .min = 1, .max = 8 },
 414         .p2 = { .dot_limit = 225000,
 415                 .p2_slow = 10, .p2_fast = 5 },
 416 };
 417
 418 static const struct intel_limit intel_limits_ironlake_single_lvds = {
 419         .dot = { .min = 25000, .max = 350000 },
 420         .vco = { .min = 1760000, .max = 3510000 },
 421         .n = { .min = 1, .max = 3 },
 422         .m = { .min = 79, .max = 118 },
 423         .m1 = { .min = 12, .max = 22 },
 424         .m2 = { .min = 5, .max = 9 },
 425         .p = { .min = 28, .max = 112 },
 426         .p1 = { .min = 2, .max = 8 },
 427         .p2 = { .dot_limit = 225000,
 428                 .p2_slow = 14, .p2_fast = 14 },
 429 };
 430
 431 static const struct intel_limit intel_limits_ironlake_dual_lvds = {
 432         .dot = { .min = 25000, .max = 350000 },
 433         .vco = { .min = 1760000, .max = 3510000 },
 434         .n = { .min = 1, .max = 3 },
 435         .m = { .min = 79, .max = 127 },
 436         .m1 = { .min = 12, .max = 22 },
 437         .m2 = { .min = 5, .max = 9 },
 438         .p = { .min = 14, .max = 56 },
 439         .p1 = { .min = 2, .max = 8 },
 440         .p2 = { .dot_limit = 225000,
 441                 .p2_slow = 7, .p2_fast = 7 },
 442 };
 443
 444 /* LVDS 100mhz refclk limits. */
 445 static const struct intel_limit intel_limits_ironlake_single_lvds_100m = {
 446         .dot = { .min = 25000, .max = 350000 },
 447         .vco = { .min = 1760000, .max = 3510000 },
 448         .n = { .min = 1, .max = 2 },
 449         .m = { .min = 79, .max = 126 },
 450         .m1 = { .min = 12, .max = 22 },
 451         .m2 = { .min = 5, .max = 9 },
 452         .p = { .min = 28, .max = 112 },
 453         .p1 = { .min = 2, .max = 8 },
 454         .p2 = { .dot_limit = 225000,
 455                 .p2_slow = 14, .p2_fast = 14 },
 456 };
 457
 458 static const struct intel_limit intel_limits_ironlake_dual_lvds_100m = {
 459         .dot = { .min = 25000, .max = 350000 },
 460         .vco = { .min = 1760000, .max = 3510000 },
 461         .n = { .min = 1, .max = 3 },
 462         .m = { .min = 79, .max = 126 },
 463         .m1 = { .min = 12, .max = 22 },
 464         .m2 = { .min = 5, .max = 9 },
 465         .p = { .min = 14, .max = 42 },
 466         .p1 = { .min = 2, .max = 6 },
 467         .p2 = { .dot_limit = 225000,
 468                 .p2_slow = 7, .p2_fast = 7 },
 469 };
 470
 471 static const struct intel_limit intel_limits_vlv = {
 472          /*
 473           * These are the data rate limits (measured in fast clocks)
 474           * since those are the strictest limits we have. The fast
 475           * clock and actual rate limits are more relaxed, so checking
 476           * them would make no difference.
 477           */
 478         .dot = { .min = 25000 * 5, .max = 270000 * 5 },
 479         .vco = { .min = 4000000, .max = 6000000 },
 480         .n = { .min = 1, .max = 7 },
 481         .m1 = { .min = 2, .max = 3 },
 482         .m2 = { .min = 11, .max = 156 },
 483         .p1 = { .min = 2, .max = 3 },
 484         .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */
 485 };
 486
 487 static const struct intel_limit intel_limits_chv = {
 488         /*
 489          * These are the data rate limits (measured in fast clocks)
 490          * since those are the strictest limits we have.  The fast
 491          * clock and actual rate limits are more relaxed, so checking
 492          * them would make no difference.
 493          */
 494         .dot = { .min = 25000 * 5, .max = 540000 * 5},
 495         .vco = { .min = 4800000, .max = 6480000 },
 496         .n = { .min = 1, .max = 1 },
 497         .m1 = { .min = 2, .max = 2 },
 498         .m2 = { .min = 24 << 22, .max = 175 << 22 },
 499         .p1 = { .min = 2, .max = 4 },
 500         .p2 = { .p2_slow = 1, .p2_fast = 14 },
 501 };
 502
 503 static const struct intel_limit intel_limits_bxt = {
 504         /* FIXME: find real dot limits */
 505         .dot = { .min = 0, .max = INT_MAX },
 506         .vco = { .min = 4800000, .max = 6700000 },
 507         .n = { .min = 1, .max = 1 },
 508         .m1 = { .min = 2, .max = 2 },
 509         /* FIXME: find real m2 limits */
 510         .m2 = { .min = 2 << 22, .max = 255 << 22 },
 511         .p1 = { .min = 2, .max = 4 },
 512         .p2 = { .p2_slow = 1, .p2_fast = 20 },
 513 };
 514
 515 /* WA Display #0827: Gen9:all */
 516 static void
 517 skl_wa_827(struct drm_i915_private *dev_priv, enum pipe pipe, bool enable)
 518 {
 519         if (enable)
 520                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 521                            I915_READ(CLKGATE_DIS_PSL(pipe)) |
 522                            DUPS1_GATING_DIS | DUPS2_GATING_DIS);
 523         else
 524                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 525                            I915_READ(CLKGATE_DIS_PSL(pipe)) &
 526                            ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS));
 527 }
 528
 529 /* Wa_2006604312:icl */
 530 static void
 531 icl_wa_scalerclkgating(struct drm_i915_private *dev_priv, enum pipe pipe,
 532                        bool enable)
 533 {
 534         if (enable)
 535                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 536                            I915_READ(CLKGATE_DIS_PSL(pipe)) | DPFR_GATING_DIS);
 537         else
 538                 I915_WRITE(CLKGATE_DIS_PSL(pipe),
 539                            I915_READ(CLKGATE_DIS_PSL(pipe)) & ~DPFR_GATING_DIS);
 540 }
 541
 542 static bool
 543 needs_modeset(const struct intel_crtc_state *state)
 544 {
 545         return drm_atomic_crtc_needs_modeset(&state->uapi);
 546 }
 547
 548 bool
 549 is_trans_port_sync_mode(const struct intel_crtc_state *crtc_state)
 550 {
 551         return (crtc_state->master_transcoder != INVALID_TRANSCODER ||
 552                 crtc_state->sync_mode_slaves_mask);
 553 }
 554
 555 static bool
 556 is_trans_port_sync_master(const struct intel_crtc_state *crtc_state)
 557 {
 558         return (crtc_state->master_transcoder == INVALID_TRANSCODER &&
 559                 crtc_state->sync_mode_slaves_mask);
 560 }
 561
 562 static bool
 563 is_trans_port_sync_slave(const struct intel_crtc_state *crtc_state)
 564 {
 565         return crtc_state->master_transcoder != INVALID_TRANSCODER;
 566 }
 567
 568 /*
 569  * Platform specific helpers to calculate the port PLL loopback- (clock.m),
 570  * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast
 571  * (clock.dot) clock rates. This fast dot clock is fed to the port's IO logic.
 572  * The helpers' return value is the rate of the clock that is fed to the
 573  * display engine's pipe which can be the above fast dot clock rate or a
 574  * divided-down version of it.
 575  */
 576 /* m1 is reserved as 0 in Pineview, n is a ring counter */
 577 static int pnv_calc_dpll_params(int refclk, struct dpll *clock)
 578 {
 579         clock->m = clock->m2 + 2;
 580         clock->p = clock->p1 * clock->p2;
 581         if (WARN_ON(clock->n == 0 || clock->p == 0))
 582                 return 0;
 583         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 584         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 585
 586         return clock->dot;
 587 }
 588
 589 static u32 i9xx_dpll_compute_m(struct dpll *dpll)
 590 {
 591         return 5 * (dpll->m1 + 2) + (dpll->m2 + 2);
 592 }
 593
 594 static int i9xx_calc_dpll_params(int refclk, struct dpll *clock)
 595 {
 596         clock->m = i9xx_dpll_compute_m(clock);
 597         clock->p = clock->p1 * clock->p2;
 598         if (WARN_ON(clock->n + 2 == 0 || clock->p == 0))
 599                 return 0;
 600         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n + 2);
 601         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 602
 603         return clock->dot;
 604 }
 605
 606 static int vlv_calc_dpll_params(int refclk, struct dpll *clock)
 607 {
 608         clock->m = clock->m1 * clock->m2;
 609         clock->p = clock->p1 * clock->p2;
 610         if (WARN_ON(clock->n == 0 || clock->p == 0))
 611                 return 0;
 612         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 613         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 614
 615         return clock->dot / 5;
 616 }
 617
 618 int chv_calc_dpll_params(int refclk, struct dpll *clock)
 619 {
 620         clock->m = clock->m1 * clock->m2;
 621         clock->p = clock->p1 * clock->p2;
 622         if (WARN_ON(clock->n == 0 || clock->p == 0))
 623                 return 0;
 624         clock->vco = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(refclk, clock->m),
 625                                            clock->n << 22);
 626         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 627
 628         return clock->dot / 5;
 629 }
 630
 631 #define INTELPllInvalid(s)   do { /* DRM_DEBUG(s); */ return false; } while (0)
 632
 633 /*
 634  * Returns whether the given set of divisors are valid for a given refclk with
 635  * the given connectors.
 636  */
 637 static bool intel_PLL_is_valid(struct drm_i915_private *dev_priv,
 638                                const struct intel_limit *limit,
 639                                const struct dpll *clock)
 640 {
 641         if (clock->n   < limit->n.min   || limit->n.max   < clock->n)
 642                 INTELPllInvalid("n out of range\n");
 643         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
 644                 INTELPllInvalid("p1 out of range\n");
 645         if (clock->m2  < limit->m2.min  || limit->m2.max  < clock->m2)
 646                 INTELPllInvalid("m2 out of range\n");
 647         if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
 648                 INTELPllInvalid("m1 out of range\n");
 649
 650         if (!IS_PINEVIEW(dev_priv) && !IS_VALLEYVIEW(dev_priv) &&
 651             !IS_CHERRYVIEW(dev_priv) && !IS_GEN9_LP(dev_priv))
 652                 if (clock->m1 <= clock->m2)
 653                         INTELPllInvalid("m1 <= m2\n");
 654
 655         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
 656             !IS_GEN9_LP(dev_priv)) {
 657                 if (clock->p < limit->p.min || limit->p.max < clock->p)
 658                         INTELPllInvalid("p out of range\n");
 659                 if (clock->m < limit->m.min || limit->m.max < clock->m)
 660                         INTELPllInvalid("m out of range\n");
 661         }
 662
 663         if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
 664                 INTELPllInvalid("vco out of range\n");
 665         /* XXX: We may need to be checking "Dot clock" depending on the multiplier,
 666          * connector, etc., rather than just a single range.
 667          */
 668         if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
 669                 INTELPllInvalid("dot out of range\n");
 670
 671         return true;
 672 }
 673
 674 static int
 675 i9xx_select_p2_div(const struct intel_limit *limit,
 676                    const struct intel_crtc_state *crtc_state,
 677                    int target)
 678 {
 679         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 680
 681         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 682                 /*
 683                  * For LVDS just rely on its current settings for dual-channel.
 684                  * We haven't figured out how to reliably set up different
 685                  * single/dual channel state, if we even can.
 686                  */
 687                 if (intel_is_dual_link_lvds(dev_priv))
 688                         return limit->p2.p2_fast;
 689                 else
 690                         return limit->p2.p2_slow;
 691         } else {
 692                 if (target < limit->p2.dot_limit)
 693                         return limit->p2.p2_slow;
 694                 else
 695                         return limit->p2.p2_fast;
 696         }
 697 }
 698
 699 /*
 700  * Returns a set of divisors for the desired target clock with the given
 701  * refclk, or FALSE.  The returned values represent the clock equation:
 702  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 703  *
 704  * Target and reference clocks are specified in kHz.
 705  *
 706  * If match_clock is provided, then best_clock P divider must match the P
 707  * divider from @match_clock used for LVDS downclocking.
 708  */
 709 static bool
 710 i9xx_find_best_dpll(const struct intel_limit *limit,
 711                     struct intel_crtc_state *crtc_state,
 712                     int target, int refclk, struct dpll *match_clock,
 713                     struct dpll *best_clock)
 714 {
 715         struct drm_device *dev = crtc_state->uapi.crtc->dev;
 716         struct dpll clock;
 717         int err = target;
 718
 719         memset(best_clock, 0, sizeof(*best_clock));
 720
 721         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 722
 723         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 724              clock.m1++) {
 725                 for (clock.m2 = limit->m2.min;
 726                      clock.m2 <= limit->m2.max; clock.m2++) {
 727                         if (clock.m2 >= clock.m1)
 728                                 break;
 729                         for (clock.n = limit->n.min;
 730                              clock.n <= limit->n.max; clock.n++) {
 731                                 for (clock.p1 = limit->p1.min;
 732                                         clock.p1 <= limit->p1.max; clock.p1++) {
 733                                         int this_err;
 734
 735                                         i9xx_calc_dpll_params(refclk, &clock);
 736                                         if (!intel_PLL_is_valid(to_i915(dev),
 737                                                                 limit,
 738                                                                 &clock))
 739                                                 continue;
 740                                         if (match_clock &&
 741                                             clock.p != match_clock->p)
 742                                                 continue;
 743
 744                                         this_err = abs(clock.dot - target);
 745                                         if (this_err < err) {
 746                                                 *best_clock = clock;
 747                                                 err = this_err;
 748                                         }
 749                                 }
 750                         }
 751                 }
 752         }
 753
 754         return (err != target);
 755 }
 756
 757 /*
 758  * Returns a set of divisors for the desired target clock with the given
 759  * refclk, or FALSE.  The returned values represent the clock equation:
 760  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 761  *
 762  * Target and reference clocks are specified in kHz.
 763  *
 764  * If match_clock is provided, then best_clock P divider must match the P
 765  * divider from @match_clock used for LVDS downclocking.
 766  */
 767 static bool
 768 pnv_find_best_dpll(const struct intel_limit *limit,
 769                    struct intel_crtc_state *crtc_state,
 770                    int target, int refclk, struct dpll *match_clock,
 771                    struct dpll *best_clock)
 772 {
 773         struct drm_device *dev = crtc_state->uapi.crtc->dev;
 774         struct dpll clock;
 775         int err = target;
 776
 777         memset(best_clock, 0, sizeof(*best_clock));
 778
 779         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 780
 781         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 782              clock.m1++) {
 783                 for (clock.m2 = limit->m2.min;
 784                      clock.m2 <= limit->m2.max; clock.m2++) {
 785                         for (clock.n = limit->n.min;
 786                              clock.n <= limit->n.max; clock.n++) {
 787                                 for (clock.p1 = limit->p1.min;
 788                                         clock.p1 <= limit->p1.max; clock.p1++) {
 789                                         int this_err;
 790
 791                                         pnv_calc_dpll_params(refclk, &clock);
 792                                         if (!intel_PLL_is_valid(to_i915(dev),
 793                                                                 limit,
 794                                                                 &clock))
 795                                                 continue;
 796                                         if (match_clock &&
 797                                             clock.p != match_clock->p)
 798                                                 continue;
 799
 800                                         this_err = abs(clock.dot - target);
 801                                         if (this_err < err) {
 802                                                 *best_clock = clock;
 803                                                 err = this_err;
 804                                         }
 805                                 }
 806                         }
 807                 }
 808         }
 809
 810         return (err != target);
 811 }
 812
 813 /*
 814  * Returns a set of divisors for the desired target clock with the given
 815  * refclk, or FALSE.  The returned values represent the clock equation:
 816  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 817  *
 818  * Target and reference clocks are specified in kHz.
 819  *
 820  * If match_clock is provided, then best_clock P divider must match the P
 821  * divider from @match_clock used for LVDS downclocking.
 822  */
 823 static bool
 824 g4x_find_best_dpll(const struct intel_limit *limit,
 825                    struct intel_crtc_state *crtc_state,
 826                    int target, int refclk, struct dpll *match_clock,
 827                    struct dpll *best_clock)
 828 {
 829         struct drm_device *dev = crtc_state->uapi.crtc->dev;
 830         struct dpll clock;
 831         int max_n;
 832         bool found = false;
 833         /* approximately equals target * 0.00585 */
 834         int err_most = (target >> 8) + (target >> 9);
 835
 836         memset(best_clock, 0, sizeof(*best_clock));
 837
 838         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 839
 840         max_n = limit->n.max;
 841         /* based on hardware requirement, prefer smaller n to precision */
 842         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 843                 /* based on hardware requirement, prefere larger m1,m2 */
 844                 for (clock.m1 = limit->m1.max;
 845                      clock.m1 >= limit->m1.min; clock.m1--) {
 846                         for (clock.m2 = limit->m2.max;
 847                              clock.m2 >= limit->m2.min; clock.m2--) {
 848                                 for (clock.p1 = limit->p1.max;
 849                                      clock.p1 >= limit->p1.min; clock.p1--) {
 850                                         int this_err;
 851
 852                                         i9xx_calc_dpll_params(refclk, &clock);
 853                                         if (!intel_PLL_is_valid(to_i915(dev),
 854                                                                 limit,
 855                                                                 &clock))
 856                                                 continue;
 857
 858                                         this_err = abs(clock.dot - target);
 859                                         if (this_err < err_most) {
 860                                                 *best_clock = clock;
 861                                                 err_most = this_err;
 862                                                 max_n = clock.n;
 863                                                 found = true;
 864                                         }
 865                                 }
 866                         }
 867                 }
 868         }
 869         return found;
 870 }
 871
 872 /*
 873  * Check if the calculated PLL configuration is more optimal compared to the
 874  * best configuration and error found so far. Return the calculated error.
 875  */
 876 static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
 877                                const struct dpll *calculated_clock,
 878                                const struct dpll *best_clock,
 879                                unsigned int best_error_ppm,
 880                                unsigned int *error_ppm)
 881 {
 882         /*
 883          * For CHV ignore the error and consider only the P value.
 884          * Prefer a bigger P value based on HW requirements.
 885          */
 886         if (IS_CHERRYVIEW(to_i915(dev))) {
 887                 *error_ppm = 0;
 888
 889                 return calculated_clock->p > best_clock->p;
 890         }
 891
 892         if (WARN_ON_ONCE(!target_freq))
 893                 return false;
 894
 895         *error_ppm = div_u64(1000000ULL *
 896                                 abs(target_freq - calculated_clock->dot),
 897                              target_freq);
 898         /*
 899          * Prefer a better P value over a better (smaller) error if the error
 900          * is small. Ensure this preference for future configurations too by
 901          * setting the error to 0.
 902          */
 903         if (*error_ppm < 100 && calculated_clock->p > best_clock->p) {
 904                 *error_ppm = 0;
 905
 906                 return true;
 907         }
 908
 909         return *error_ppm + 10 < best_error_ppm;
 910 }
 911
 912 /*
 913  * Returns a set of divisors for the desired target clock with the given
 914  * refclk, or FALSE.  The returned values represent the clock equation:
 915  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 916  */
 917 static bool
 918 vlv_find_best_dpll(const struct intel_limit *limit,
 919                    struct intel_crtc_state *crtc_state,
 920                    int target, int refclk, struct dpll *match_clock,
 921                    struct dpll *best_clock)
 922 {
 923         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 924         struct drm_device *dev = crtc->base.dev;
 925         struct dpll clock;
 926         unsigned int bestppm = 1000000;
 927         /* min update 19.2 MHz */
 928         int max_n = min(limit->n.max, refclk / 19200);
 929         bool found = false;
 930
 931         target *= 5; /* fast clock */
 932
 933         memset(best_clock, 0, sizeof(*best_clock));
 934
 935         /* based on hardware requirement, prefer smaller n to precision */
 936         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 937                 for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 938                         for (clock.p2 = limit->p2.p2_fast; clock.p2 >= limit->p2.p2_slow;
 939                              clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 940                                 clock.p = clock.p1 * clock.p2;
 941                                 /* based on hardware requirement, prefer bigger m1,m2 values */
 942                                 for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) {
 943                                         unsigned int ppm;
 944
 945                                         clock.m2 = DIV_ROUND_CLOSEST(target * clock.p * clock.n,
 946                                                                      refclk * clock.m1);
 947
 948                                         vlv_calc_dpll_params(refclk, &clock);
 949
 950                                         if (!intel_PLL_is_valid(to_i915(dev),
 951                                                                 limit,
 952                                                                 &clock))
 953                                                 continue;
 954
 955                                         if (!vlv_PLL_is_optimal(dev, target,
 956                                                                 &clock,
 957                                                                 best_clock,
 958                                                                 bestppm, &ppm))
 959                                                 continue;
 960
 961                                         *best_clock = clock;
 962                                         bestppm = ppm;
 963                                         found = true;
 964                                 }
 965                         }
 966                 }
 967         }
 968
 969         return found;
 970 }
 971
 972 /*
 973  * Returns a set of divisors for the desired target clock with the given
 974  * refclk, or FALSE.  The returned values represent the clock equation:
 975  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 976  */
 977 static bool
 978 chv_find_best_dpll(const struct intel_limit *limit,
 979                    struct intel_crtc_state *crtc_state,
 980                    int target, int refclk, struct dpll *match_clock,
 981                    struct dpll *best_clock)
 982 {
 983         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 984         struct drm_device *dev = crtc->base.dev;
 985         unsigned int best_error_ppm;
 986         struct dpll clock;
 987         u64 m2;
 988         int found = false;
 989
 990         memset(best_clock, 0, sizeof(*best_clock));
 991         best_error_ppm = 1000000;
 992
 993         /*
 994          * Based on hardware doc, the n always set to 1, and m1 always
 995          * set to 2.  If requires to support 200Mhz refclk, we need to
 996          * revisit this because n may not 1 anymore.
 997          */
 998         clock.n = 1, clock.m1 = 2;
 999         target *= 5;    /* fast clock */
1000
1001         for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
1002                 for (clock.p2 = limit->p2.p2_fast;
1003                                 clock.p2 >= limit->p2.p2_slow;
1004                                 clock.p2 -= clock.p2 > 10 ? 2 : 1) {
1005                         unsigned int error_ppm;
1006
1007                         clock.p = clock.p1 * clock.p2;
1008
1009                         m2 = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(target, clock.p * clock.n) << 22,
1010                                                    refclk * clock.m1);
1011
1012                         if (m2 > INT_MAX/clock.m1)
1013                                 continue;
1014
1015                         clock.m2 = m2;
1016
1017                         chv_calc_dpll_params(refclk, &clock);
1018
1019                         if (!intel_PLL_is_valid(to_i915(dev), limit, &clock))
1020                                 continue;
1021
1022                         if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock,
1023                                                 best_error_ppm, &error_ppm))
1024                                 continue;
1025
1026                         *best_clock = clock;
1027                         best_error_ppm = error_ppm;
1028                         found = true;
1029                 }
1030         }
1031
1032         return found;
1033 }
1034
1035 bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state,
1036                         struct dpll *best_clock)
1037 {
1038         int refclk = 100000;
1039         const struct intel_limit *limit = &intel_limits_bxt;
1040
1041         return chv_find_best_dpll(limit, crtc_state,
1042                                   crtc_state->port_clock, refclk,
1043                                   NULL, best_clock);
1044 }
1045
1046 static bool pipe_scanline_is_moving(struct drm_i915_private *dev_priv,
1047                                     enum pipe pipe)
1048 {
1049         i915_reg_t reg = PIPEDSL(pipe);
1050         u32 line1, line2;
1051         u32 line_mask;
1052
1053         if (IS_GEN(dev_priv, 2))
1054                 line_mask = DSL_LINEMASK_GEN2;
1055         else
1056                 line_mask = DSL_LINEMASK_GEN3;
1057
1058         line1 = I915_READ(reg) & line_mask;
1059         msleep(5);
1060         line2 = I915_READ(reg) & line_mask;
1061
1062         return line1 != line2;
1063 }
1064
1065 static void wait_for_pipe_scanline_moving(struct intel_crtc *crtc, bool state)
1066 {
1067         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1068         enum pipe pipe = crtc->pipe;
1069
1070         /* Wait for the display line to settle/start moving */
1071         if (wait_for(pipe_scanline_is_moving(dev_priv, pipe) == state, 100))
1072                 DRM_ERROR("pipe %c scanline %s wait timed out\n",
1073                           pipe_name(pipe), onoff(state));
1074 }
1075
1076 static void intel_wait_for_pipe_scanline_stopped(struct intel_crtc *crtc)
1077 {
1078         wait_for_pipe_scanline_moving(crtc, false);
1079 }
1080
1081 static void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
1082 {
1083         wait_for_pipe_scanline_moving(crtc, true);
1084 }
1085
1086 static void
1087 intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
1088 {
1089         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
1090         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1091
1092         if (INTEL_GEN(dev_priv) >= 4) {
1093                 enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1094                 i915_reg_t reg = PIPECONF(cpu_transcoder);
1095
1096                 /* Wait for the Pipe State to go off */
1097                 if (intel_de_wait_for_clear(dev_priv, reg,
1098                                             I965_PIPECONF_ACTIVE, 100))
1099                         WARN(1, "pipe_off wait timed out\n");
1100         } else {
1101                 intel_wait_for_pipe_scanline_stopped(crtc);
1102         }
1103 }
1104
1105 /* Only for pre-ILK configs */
1106 void assert_pll(struct drm_i915_private *dev_priv,
1107                 enum pipe pipe, bool state)
1108 {
1109         u32 val;
1110         bool cur_state;
1111
1112         val = I915_READ(DPLL(pipe));
1113         cur_state = !!(val & DPLL_VCO_ENABLE);
1114         I915_STATE_WARN(cur_state != state,
1115              "PLL state assertion failure (expected %s, current %s)\n",
1116                         onoff(state), onoff(cur_state));
1117 }
1118
1119 /* XXX: the dsi pll is shared between MIPI DSI ports */
1120 void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state)
1121 {
1122         u32 val;
1123         bool cur_state;
1124
1125         vlv_cck_get(dev_priv);
1126         val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL);
1127         vlv_cck_put(dev_priv);
1128
1129         cur_state = val & DSI_PLL_VCO_EN;
1130         I915_STATE_WARN(cur_state != state,
1131              "DSI PLL state assertion failure (expected %s, current %s)\n",
1132                         onoff(state), onoff(cur_state));
1133 }
1134
1135 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
1136                           enum pipe pipe, bool state)
1137 {
1138         bool cur_state;
1139
1140         if (HAS_DDI(dev_priv)) {
1141                 /*
1142                  * DDI does not have a specific FDI_TX register.
1143                  *
1144                  * FDI is never fed from EDP transcoder
1145                  * so pipe->transcoder cast is fine here.
1146                  */
1147                 enum transcoder cpu_transcoder = (enum transcoder)pipe;
1148                 u32 val = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
1149                 cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
1150         } else {
1151                 u32 val = I915_READ(FDI_TX_CTL(pipe));
1152                 cur_state = !!(val & FDI_TX_ENABLE);
1153         }
1154         I915_STATE_WARN(cur_state != state,
1155              "FDI TX state assertion failure (expected %s, current %s)\n",
1156                         onoff(state), onoff(cur_state));
1157 }
1158 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
1159 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
1160
1161 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
1162                           enum pipe pipe, bool state)
1163 {
1164         u32 val;
1165         bool cur_state;
1166
1167         val = I915_READ(FDI_RX_CTL(pipe));
1168         cur_state = !!(val & FDI_RX_ENABLE);
1169         I915_STATE_WARN(cur_state != state,
1170              "FDI RX state assertion failure (expected %s, current %s)\n",
1171                         onoff(state), onoff(cur_state));
1172 }
1173 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
1174 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
1175
1176 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
1177                                       enum pipe pipe)
1178 {
1179         u32 val;
1180
1181         /* ILK FDI PLL is always enabled */
1182         if (IS_GEN(dev_priv, 5))
1183                 return;
1184
1185         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
1186         if (HAS_DDI(dev_priv))
1187                 return;
1188
1189         val = I915_READ(FDI_TX_CTL(pipe));
1190         I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
1191 }
1192
1193 void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
1194                        enum pipe pipe, bool state)
1195 {
1196         u32 val;
1197         bool cur_state;
1198
1199         val = I915_READ(FDI_RX_CTL(pipe));
1200         cur_state = !!(val & FDI_RX_PLL_ENABLE);
1201         I915_STATE_WARN(cur_state != state,
1202              "FDI RX PLL assertion failure (expected %s, current %s)\n",
1203                         onoff(state), onoff(cur_state));
1204 }
1205
1206 void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe)
1207 {
1208         i915_reg_t pp_reg;
1209         u32 val;
1210         enum pipe panel_pipe = INVALID_PIPE;
1211         bool locked = true;
1212
1213         if (WARN_ON(HAS_DDI(dev_priv)))
1214                 return;
1215
1216         if (HAS_PCH_SPLIT(dev_priv)) {
1217                 u32 port_sel;
1218
1219                 pp_reg = PP_CONTROL(0);
1220                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1221
1222                 switch (port_sel) {
1223                 case PANEL_PORT_SELECT_LVDS:
1224                         intel_lvds_port_enabled(dev_priv, PCH_LVDS, &panel_pipe);
1225                         break;
1226                 case PANEL_PORT_SELECT_DPA:
1227                         intel_dp_port_enabled(dev_priv, DP_A, PORT_A, &panel_pipe);
1228                         break;
1229                 case PANEL_PORT_SELECT_DPC:
1230                         intel_dp_port_enabled(dev_priv, PCH_DP_C, PORT_C, &panel_pipe);
1231                         break;
1232                 case PANEL_PORT_SELECT_DPD:
1233                         intel_dp_port_enabled(dev_priv, PCH_DP_D, PORT_D, &panel_pipe);
1234                         break;
1235                 default:
1236                         MISSING_CASE(port_sel);
1237                         break;
1238                 }
1239         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1240                 /* presumably write lock depends on pipe, not port select */
1241                 pp_reg = PP_CONTROL(pipe);
1242                 panel_pipe = pipe;
1243         } else {
1244                 u32 port_sel;
1245
1246                 pp_reg = PP_CONTROL(0);
1247                 port_sel = I915_READ(PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK;
1248
1249                 WARN_ON(port_sel != PANEL_PORT_SELECT_LVDS);
1250                 intel_lvds_port_enabled(dev_priv, LVDS, &panel_pipe);
1251         }
1252
1253         val = I915_READ(pp_reg);
1254         if (!(val & PANEL_POWER_ON) ||
1255             ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
1256                 locked = false;
1257
1258         I915_STATE_WARN(panel_pipe == pipe && locked,
1259              "panel assertion failure, pipe %c regs locked\n",
1260              pipe_name(pipe));
1261 }
1262
1263 void assert_pipe(struct drm_i915_private *dev_priv,
1264                  enum transcoder cpu_transcoder, bool state)
1265 {
1266         bool cur_state;
1267         enum intel_display_power_domain power_domain;
1268         intel_wakeref_t wakeref;
1269
1270         /* we keep both pipes enabled on 830 */
1271         if (IS_I830(dev_priv))
1272                 state = true;
1273
1274         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
1275         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
1276         if (wakeref) {
1277                 u32 val = I915_READ(PIPECONF(cpu_transcoder));
1278                 cur_state = !!(val & PIPECONF_ENABLE);
1279
1280                 intel_display_power_put(dev_priv, power_domain, wakeref);
1281         } else {
1282                 cur_state = false;
1283         }
1284
1285         I915_STATE_WARN(cur_state != state,
1286                         "transcoder %s assertion failure (expected %s, current %s)\n",
1287                         transcoder_name(cpu_transcoder),
1288                         onoff(state), onoff(cur_state));
1289 }
1290
1291 static void assert_plane(struct intel_plane *plane, bool state)
1292 {
1293         enum pipe pipe;
1294         bool cur_state;
1295
1296         cur_state = plane->get_hw_state(plane, &pipe);
1297
1298         I915_STATE_WARN(cur_state != state,
1299                         "%s assertion failure (expected %s, current %s)\n",
1300                         plane->base.name, onoff(state), onoff(cur_state));
1301 }
1302
1303 #define assert_plane_enabled(p) assert_plane(p, true)
1304 #define assert_plane_disabled(p) assert_plane(p, false)
1305
1306 static void assert_planes_disabled(struct intel_crtc *crtc)
1307 {
1308         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1309         struct intel_plane *plane;
1310
1311         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
1312                 assert_plane_disabled(plane);
1313 }
1314
1315 static void assert_vblank_disabled(struct drm_crtc *crtc)
1316 {
1317         if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
1318                 drm_crtc_vblank_put(crtc);
1319 }
1320
1321 void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
1322                                     enum pipe pipe)
1323 {
1324         u32 val;
1325         bool enabled;
1326
1327         val = I915_READ(PCH_TRANSCONF(pipe));
1328         enabled = !!(val & TRANS_ENABLE);
1329         I915_STATE_WARN(enabled,
1330              "transcoder assertion failed, should be off on pipe %c but is still active\n",
1331              pipe_name(pipe));
1332 }
1333
1334 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
1335                                    enum pipe pipe, enum port port,
1336                                    i915_reg_t dp_reg)
1337 {
1338         enum pipe port_pipe;
1339         bool state;
1340
1341         state = intel_dp_port_enabled(dev_priv, dp_reg, port, &port_pipe);
1342
1343         I915_STATE_WARN(state && port_pipe == pipe,
1344                         "PCH DP %c enabled on transcoder %c, should be disabled\n",
1345                         port_name(port), pipe_name(pipe));
1346
1347         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1348                         "IBX PCH DP %c still using transcoder B\n",
1349                         port_name(port));
1350 }
1351
1352 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
1353                                      enum pipe pipe, enum port port,
1354                                      i915_reg_t hdmi_reg)
1355 {
1356         enum pipe port_pipe;
1357         bool state;
1358
1359         state = intel_sdvo_port_enabled(dev_priv, hdmi_reg, &port_pipe);
1360
1361         I915_STATE_WARN(state && port_pipe == pipe,
1362                         "PCH HDMI %c enabled on transcoder %c, should be disabled\n",
1363                         port_name(port), pipe_name(pipe));
1364
1365         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && !state && port_pipe == PIPE_B,
1366                         "IBX PCH HDMI %c still using transcoder B\n",
1367                         port_name(port));
1368 }
1369
1370 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
1371                                       enum pipe pipe)
1372 {
1373         enum pipe port_pipe;
1374
1375         assert_pch_dp_disabled(dev_priv, pipe, PORT_B, PCH_DP_B);
1376         assert_pch_dp_disabled(dev_priv, pipe, PORT_C, PCH_DP_C);
1377         assert_pch_dp_disabled(dev_priv, pipe, PORT_D, PCH_DP_D);
1378
1379         I915_STATE_WARN(intel_crt_port_enabled(dev_priv, PCH_ADPA, &port_pipe) &&
1380                         port_pipe == pipe,
1381                         "PCH VGA enabled on transcoder %c, should be disabled\n",
1382                         pipe_name(pipe));
1383
1384         I915_STATE_WARN(intel_lvds_port_enabled(dev_priv, PCH_LVDS, &port_pipe) &&
1385                         port_pipe == pipe,
1386                         "PCH LVDS enabled on transcoder %c, should be disabled\n",
1387                         pipe_name(pipe));
1388
1389         /* PCH SDVOB multiplex with HDMIB */
1390         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_B, PCH_HDMIB);
1391         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_C, PCH_HDMIC);
1392         assert_pch_hdmi_disabled(dev_priv, pipe, PORT_D, PCH_HDMID);
1393 }
1394
1395 static void _vlv_enable_pll(struct intel_crtc *crtc,
1396                             const struct intel_crtc_state *pipe_config)
1397 {
1398         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1399         enum pipe pipe = crtc->pipe;
1400
1401         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1402         POSTING_READ(DPLL(pipe));
1403         udelay(150);
1404
1405         if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1))
1406                 DRM_ERROR("DPLL %d failed to lock\n", pipe);
1407 }
1408
1409 static void vlv_enable_pll(struct intel_crtc *crtc,
1410                            const struct intel_crtc_state *pipe_config)
1411 {
1412         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1413         enum pipe pipe = crtc->pipe;
1414
1415         assert_pipe_disabled(dev_priv, pipe_config->cpu_transcoder);
1416
1417         /* PLL is protected by panel, make sure we can write it */
1418         assert_panel_unlocked(dev_priv, pipe);
1419
1420         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1421                 _vlv_enable_pll(crtc, pipe_config);
1422
1423         I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1424         POSTING_READ(DPLL_MD(pipe));
1425 }
1426
1427
1428 static void _chv_enable_pll(struct intel_crtc *crtc,
1429                             const struct intel_crtc_state *pipe_config)
1430 {
1431         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1432         enum pipe pipe = crtc->pipe;
1433         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1434         u32 tmp;
1435
1436         vlv_dpio_get(dev_priv);
1437
1438         /* Enable back the 10bit clock to display controller */
1439         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1440         tmp |= DPIO_DCLKP_EN;
1441         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp);
1442
1443         vlv_dpio_put(dev_priv);
1444
1445         /*
1446          * Need to wait > 100ns between dclkp clock enable bit and PLL enable.
1447          */
1448         udelay(1);
1449
1450         /* Enable PLL */
1451         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1452
1453         /* Check PLL is locked */
1454         if (intel_de_wait_for_set(dev_priv, DPLL(pipe), DPLL_LOCK_VLV, 1))
1455                 DRM_ERROR("PLL %d failed to lock\n", pipe);
1456 }
1457
1458 static void chv_enable_pll(struct intel_crtc *crtc,
1459                            const struct intel_crtc_state *pipe_config)
1460 {
1461         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1462         enum pipe pipe = crtc->pipe;
1463
1464         assert_pipe_disabled(dev_priv, pipe_config->cpu_transcoder);
1465
1466         /* PLL is protected by panel, make sure we can write it */
1467         assert_panel_unlocked(dev_priv, pipe);
1468
1469         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1470                 _chv_enable_pll(crtc, pipe_config);
1471
1472         if (pipe != PIPE_A) {
1473                 /*
1474                  * WaPixelRepeatModeFixForC0:chv
1475                  *
1476                  * DPLLCMD is AWOL. Use chicken bits to propagate
1477                  * the value from DPLLBMD to either pipe B or C.
1478                  */
1479                 I915_WRITE(CBR4_VLV, CBR_DPLLBMD_PIPE(pipe));
1480                 I915_WRITE(DPLL_MD(PIPE_B), pipe_config->dpll_hw_state.dpll_md);
1481                 I915_WRITE(CBR4_VLV, 0);
1482                 dev_priv->chv_dpll_md[pipe] = pipe_config->dpll_hw_state.dpll_md;
1483
1484                 /*
1485                  * DPLLB VGA mode also seems to cause problems.
1486                  * We should always have it disabled.
1487                  */
1488                 WARN_ON((I915_READ(DPLL(PIPE_B)) & DPLL_VGA_MODE_DIS) == 0);
1489         } else {
1490                 I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1491                 POSTING_READ(DPLL_MD(pipe));
1492         }
1493 }
1494
1495 static bool i9xx_has_pps(struct drm_i915_private *dev_priv)
1496 {
1497         if (IS_I830(dev_priv))
1498                 return false;
1499
1500         return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
1501 }
1502
1503 static void i9xx_enable_pll(struct intel_crtc *crtc,
1504                             const struct intel_crtc_state *crtc_state)
1505 {
1506         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1507         i915_reg_t reg = DPLL(crtc->pipe);
1508         u32 dpll = crtc_state->dpll_hw_state.dpll;
1509         int i;
1510
1511         assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder);
1512
1513         /* PLL is protected by panel, make sure we can write it */
1514         if (i9xx_has_pps(dev_priv))
1515                 assert_panel_unlocked(dev_priv, crtc->pipe);
1516
1517         /*
1518          * Apparently we need to have VGA mode enabled prior to changing
1519          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
1520          * dividers, even though the register value does change.
1521          */
1522         I915_WRITE(reg, dpll & ~DPLL_VGA_MODE_DIS);
1523         I915_WRITE(reg, dpll);
1524
1525         /* Wait for the clocks to stabilize. */
1526         POSTING_READ(reg);
1527         udelay(150);
1528
1529         if (INTEL_GEN(dev_priv) >= 4) {
1530                 I915_WRITE(DPLL_MD(crtc->pipe),
1531                            crtc_state->dpll_hw_state.dpll_md);
1532         } else {
1533                 /* The pixel multiplier can only be updated once the
1534                  * DPLL is enabled and the clocks are stable.
1535                  *
1536                  * So write it again.
1537                  */
1538                 I915_WRITE(reg, dpll);
1539         }
1540
1541         /* We do this three times for luck */
1542         for (i = 0; i < 3; i++) {
1543                 I915_WRITE(reg, dpll);
1544                 POSTING_READ(reg);
1545                 udelay(150); /* wait for warmup */
1546         }
1547 }
1548
1549 static void i9xx_disable_pll(const struct intel_crtc_state *crtc_state)
1550 {
1551         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1552         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1553         enum pipe pipe = crtc->pipe;
1554
1555         /* Don't disable pipe or pipe PLLs if needed */
1556         if (IS_I830(dev_priv))
1557                 return;
1558
1559         /* Make sure the pipe isn't still relying on us */
1560         assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder);
1561
1562         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
1563         POSTING_READ(DPLL(pipe));
1564 }
1565
1566 static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1567 {
1568         u32 val;
1569
1570         /* Make sure the pipe isn't still relying on us */
1571         assert_pipe_disabled(dev_priv, (enum transcoder)pipe);
1572
1573         val = DPLL_INTEGRATED_REF_CLK_VLV |
1574                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1575         if (pipe != PIPE_A)
1576                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1577
1578         I915_WRITE(DPLL(pipe), val);
1579         POSTING_READ(DPLL(pipe));
1580 }
1581
1582 static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1583 {
1584         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1585         u32 val;
1586
1587         /* Make sure the pipe isn't still relying on us */
1588         assert_pipe_disabled(dev_priv, (enum transcoder)pipe);
1589
1590         val = DPLL_SSC_REF_CLK_CHV |
1591                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1592         if (pipe != PIPE_A)
1593                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1594
1595         I915_WRITE(DPLL(pipe), val);
1596         POSTING_READ(DPLL(pipe));
1597
1598         vlv_dpio_get(dev_priv);
1599
1600         /* Disable 10bit clock to display controller */
1601         val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1602         val &= ~DPIO_DCLKP_EN;
1603         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
1604
1605         vlv_dpio_put(dev_priv);
1606 }
1607
1608 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
1609                          struct intel_digital_port *dport,
1610                          unsigned int expected_mask)
1611 {
1612         u32 port_mask;
1613         i915_reg_t dpll_reg;
1614
1615         switch (dport->base.port) {
1616         case PORT_B:
1617                 port_mask = DPLL_PORTB_READY_MASK;
1618                 dpll_reg = DPLL(0);
1619                 break;
1620         case PORT_C:
1621                 port_mask = DPLL_PORTC_READY_MASK;
1622                 dpll_reg = DPLL(0);
1623                 expected_mask <<= 4;
1624                 break;
1625         case PORT_D:
1626                 port_mask = DPLL_PORTD_READY_MASK;
1627                 dpll_reg = DPIO_PHY_STATUS;
1628                 break;
1629         default:
1630                 BUG();
1631         }
1632
1633         if (intel_de_wait_for_register(dev_priv, dpll_reg,
1634                                        port_mask, expected_mask, 1000))
1635                 WARN(1, "timed out waiting for [ENCODER:%d:%s] port ready: got 0x%x, expected 0x%x\n",
1636                      dport->base.base.base.id, dport->base.base.name,
1637                      I915_READ(dpll_reg) & port_mask, expected_mask);
1638 }
1639
1640 static void ironlake_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
1641 {
1642         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1643         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1644         enum pipe pipe = crtc->pipe;
1645         i915_reg_t reg;
1646         u32 val, pipeconf_val;
1647
1648         /* Make sure PCH DPLL is enabled */
1649         assert_shared_dpll_enabled(dev_priv, crtc_state->shared_dpll);
1650
1651         /* FDI must be feeding us bits for PCH ports */
1652         assert_fdi_tx_enabled(dev_priv, pipe);
1653         assert_fdi_rx_enabled(dev_priv, pipe);
1654
1655         if (HAS_PCH_CPT(dev_priv)) {
1656                 reg = TRANS_CHICKEN2(pipe);
1657                 val = I915_READ(reg);
1658                 /*
1659                  * Workaround: Set the timing override bit
1660                  * before enabling the pch transcoder.
1661                  */
1662                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1663                 /* Configure frame start delay to match the CPU */
1664                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
1665                 val |= TRANS_CHICKEN2_FRAME_START_DELAY(0);
1666                 I915_WRITE(reg, val);
1667         }
1668
1669         reg = PCH_TRANSCONF(pipe);
1670         val = I915_READ(reg);
1671         pipeconf_val = I915_READ(PIPECONF(pipe));
1672
1673         if (HAS_PCH_IBX(dev_priv)) {
1674                 /* Configure frame start delay to match the CPU */
1675                 val &= ~TRANS_FRAME_START_DELAY_MASK;
1676                 val |= TRANS_FRAME_START_DELAY(0);
1677
1678                 /*
1679                  * Make the BPC in transcoder be consistent with
1680                  * that in pipeconf reg. For HDMI we must use 8bpc
1681                  * here for both 8bpc and 12bpc.
1682                  */
1683                 val &= ~PIPECONF_BPC_MASK;
1684                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
1685                         val |= PIPECONF_8BPC;
1686                 else
1687                         val |= pipeconf_val & PIPECONF_BPC_MASK;
1688         }
1689
1690         val &= ~TRANS_INTERLACE_MASK;
1691         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK) {
1692                 if (HAS_PCH_IBX(dev_priv) &&
1693                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
1694                         val |= TRANS_LEGACY_INTERLACED_ILK;
1695                 else
1696                         val |= TRANS_INTERLACED;
1697         } else {
1698                 val |= TRANS_PROGRESSIVE;
1699         }
1700
1701         I915_WRITE(reg, val | TRANS_ENABLE);
1702         if (intel_de_wait_for_set(dev_priv, reg, TRANS_STATE_ENABLE, 100))
1703                 DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe));
1704 }
1705
1706 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1707                                       enum transcoder cpu_transcoder)
1708 {
1709         u32 val, pipeconf_val;
1710
1711         /* FDI must be feeding us bits for PCH ports */
1712         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
1713         assert_fdi_rx_enabled(dev_priv, PIPE_A);
1714
1715         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1716         /* Workaround: set timing override bit. */
1717         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1718         /* Configure frame start delay to match the CPU */
1719         val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
1720         val |= TRANS_CHICKEN2_FRAME_START_DELAY(0);
1721         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1722
1723         val = TRANS_ENABLE;
1724         pipeconf_val = I915_READ(PIPECONF(cpu_transcoder));
1725
1726         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
1727             PIPECONF_INTERLACED_ILK)
1728                 val |= TRANS_INTERLACED;
1729         else
1730                 val |= TRANS_PROGRESSIVE;
1731
1732         I915_WRITE(LPT_TRANSCONF, val);
1733         if (intel_de_wait_for_set(dev_priv, LPT_TRANSCONF,
1734                                   TRANS_STATE_ENABLE, 100))
1735                 DRM_ERROR("Failed to enable PCH transcoder\n");
1736 }
1737
1738 static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
1739                                             enum pipe pipe)
1740 {
1741         i915_reg_t reg;
1742         u32 val;
1743
1744         /* FDI relies on the transcoder */
1745         assert_fdi_tx_disabled(dev_priv, pipe);
1746         assert_fdi_rx_disabled(dev_priv, pipe);
1747
1748         /* Ports must be off as well */
1749         assert_pch_ports_disabled(dev_priv, pipe);
1750
1751         reg = PCH_TRANSCONF(pipe);
1752         val = I915_READ(reg);
1753         val &= ~TRANS_ENABLE;
1754         I915_WRITE(reg, val);
1755         /* wait for PCH transcoder off, transcoder state */
1756         if (intel_de_wait_for_clear(dev_priv, reg, TRANS_STATE_ENABLE, 50))
1757                 DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
1758
1759         if (HAS_PCH_CPT(dev_priv)) {
1760                 /* Workaround: Clear the timing override chicken bit again. */
1761                 reg = TRANS_CHICKEN2(pipe);
1762                 val = I915_READ(reg);
1763                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1764                 I915_WRITE(reg, val);
1765         }
1766 }
1767
1768 void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
1769 {
1770         u32 val;
1771
1772         val = I915_READ(LPT_TRANSCONF);
1773         val &= ~TRANS_ENABLE;
1774         I915_WRITE(LPT_TRANSCONF, val);
1775         /* wait for PCH transcoder off, transcoder state */
1776         if (intel_de_wait_for_clear(dev_priv, LPT_TRANSCONF,
1777                                     TRANS_STATE_ENABLE, 50))
1778                 DRM_ERROR("Failed to disable PCH transcoder\n");
1779
1780         /* Workaround: clear timing override bit. */
1781         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1782         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1783         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1784 }
1785
1786 enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
1787 {
1788         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1789
1790         if (HAS_PCH_LPT(dev_priv))
1791                 return PIPE_A;
1792         else
1793                 return crtc->pipe;
1794 }
1795
1796 static u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
1797 {
1798         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1799
1800         /*
1801          * On i965gm the hardware frame counter reads
1802          * zero when the TV encoder is enabled :(
1803          */
1804         if (IS_I965GM(dev_priv) &&
1805             (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
1806                 return 0;
1807
1808         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1809                 return 0xffffffff; /* full 32 bit counter */
1810         else if (INTEL_GEN(dev_priv) >= 3)
1811                 return 0xffffff; /* only 24 bits of frame count */
1812         else
1813                 return 0; /* Gen2 doesn't have a hardware frame counter */
1814 }
1815
1816 static void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
1817 {
1818         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1819
1820         assert_vblank_disabled(&crtc->base);
1821         drm_crtc_set_max_vblank_count(&crtc->base,
1822                                       intel_crtc_max_vblank_count(crtc_state));
1823         drm_crtc_vblank_on(&crtc->base);
1824 }
1825
1826 void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
1827 {
1828         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1829
1830         drm_crtc_vblank_off(&crtc->base);
1831         assert_vblank_disabled(&crtc->base);
1832 }
1833
1834 static void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state)
1835 {
1836         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
1837         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1838         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
1839         enum pipe pipe = crtc->pipe;
1840         i915_reg_t reg;
1841         u32 val;
1842
1843         DRM_DEBUG_KMS("enabling pipe %c\n", pipe_name(pipe));
1844
1845         assert_planes_disabled(crtc);
1846
1847         /*
1848          * A pipe without a PLL won't actually be able to drive bits from
1849          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
1850          * need the check.
1851          */
1852         if (HAS_GMCH(dev_priv)) {
1853                 if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
1854                         assert_dsi_pll_enabled(dev_priv);
1855                 else
1856                         assert_pll_enabled(dev_priv, pipe);
1857         } else {
1858                 if (new_crtc_state->has_pch_encoder) {
1859                         /* if driving the PCH, we need FDI enabled */
1860                         assert_fdi_rx_pll_enabled(dev_priv,
1861                                                   intel_crtc_pch_transcoder(crtc));
1862                         assert_fdi_tx_pll_enabled(dev_priv,
1863                                                   (enum pipe) cpu_transcoder);
1864                 }
1865                 /* FIXME: assert CPU port conditions for SNB+ */
1866         }
1867
1868         trace_intel_pipe_enable(crtc);
1869
1870         reg = PIPECONF(cpu_transcoder);
1871         val = I915_READ(reg);
1872         if (val & PIPECONF_ENABLE) {
1873                 /* we keep both pipes enabled on 830 */
1874                 WARN_ON(!IS_I830(dev_priv));
1875                 return;
1876         }
1877
1878         I915_WRITE(reg, val | PIPECONF_ENABLE);
1879         POSTING_READ(reg);
1880
1881         /*
1882          * Until the pipe starts PIPEDSL reads will return a stale value,
1883          * which causes an apparent vblank timestamp jump when PIPEDSL
1884          * resets to its proper value. That also messes up the frame count
1885          * when it's derived from the timestamps. So let's wait for the
1886          * pipe to start properly before we call drm_crtc_vblank_on()
1887          */
1888         if (intel_crtc_max_vblank_count(new_crtc_state) == 0)
1889                 intel_wait_for_pipe_scanline_moving(crtc);
1890 }
1891
1892 void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state)
1893 {
1894         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
1895         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1896         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
1897         enum pipe pipe = crtc->pipe;
1898         i915_reg_t reg;
1899         u32 val;
1900
1901         DRM_DEBUG_KMS("disabling pipe %c\n", pipe_name(pipe));
1902
1903         /*
1904          * Make sure planes won't keep trying to pump pixels to us,
1905          * or we might hang the display.
1906          */
1907         assert_planes_disabled(crtc);
1908
1909         trace_intel_pipe_disable(crtc);
1910
1911         reg = PIPECONF(cpu_transcoder);
1912         val = I915_READ(reg);
1913         if ((val & PIPECONF_ENABLE) == 0)
1914                 return;
1915
1916         /*
1917          * Double wide has implications for planes
1918          * so best keep it disabled when not needed.
1919          */
1920         if (old_crtc_state->double_wide)
1921                 val &= ~PIPECONF_DOUBLE_WIDE;
1922
1923         /* Don't disable pipe or pipe PLLs if needed */
1924         if (!IS_I830(dev_priv))
1925                 val &= ~PIPECONF_ENABLE;
1926
1927         I915_WRITE(reg, val);
1928         if ((val & PIPECONF_ENABLE) == 0)
1929                 intel_wait_for_pipe_off(old_crtc_state);
1930 }
1931
1932 static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv)
1933 {
1934         return IS_GEN(dev_priv, 2) ? 2048 : 4096;
1935 }
1936
1937 static bool is_ccs_plane(const struct drm_framebuffer *fb, int plane)
1938 {
1939         if (!is_ccs_modifier(fb->modifier))
1940                 return false;
1941
1942         return plane >= fb->format->num_planes / 2;
1943 }
1944
1945 static bool is_gen12_ccs_modifier(u64 modifier)
1946 {
1947         return modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS;
1948 }
1949
1950 static bool is_gen12_ccs_plane(const struct drm_framebuffer *fb, int plane)
1951 {
1952         return is_gen12_ccs_modifier(fb->modifier) && is_ccs_plane(fb, plane);
1953 }
1954
1955 static bool is_aux_plane(const struct drm_framebuffer *fb, int plane)
1956 {
1957         if (is_ccs_modifier(fb->modifier))
1958                 return is_ccs_plane(fb, plane);
1959
1960         return plane == 1;
1961 }
1962
1963 static int main_to_ccs_plane(const struct drm_framebuffer *fb, int main_plane)
1964 {
1965         WARN_ON(!is_ccs_modifier(fb->modifier) ||
1966                 (main_plane && main_plane >= fb->format->num_planes / 2));
1967
1968         return fb->format->num_planes / 2 + main_plane;
1969 }
1970
1971 static int ccs_to_main_plane(const struct drm_framebuffer *fb, int ccs_plane)
1972 {
1973         WARN_ON(!is_ccs_modifier(fb->modifier) ||
1974                 ccs_plane < fb->format->num_planes / 2);
1975
1976         return ccs_plane - fb->format->num_planes / 2;
1977 }
1978
1979 /* Return either the main plane's CCS or - if not a CCS FB - UV plane */
1980 static int
1981 intel_main_to_aux_plane(const struct drm_framebuffer *fb, int main_plane)
1982 {
1983         if (is_ccs_modifier(fb->modifier))
1984                 return main_to_ccs_plane(fb, main_plane);
1985
1986         return 1;
1987 }
1988
1989 bool
1990 intel_format_info_is_yuv_semiplanar(const struct drm_format_info *info,
1991                                     uint64_t modifier)
1992 {
1993         return info->is_yuv &&
1994                info->num_planes == (is_ccs_modifier(modifier) ? 4 : 2);
1995 }
1996
1997 static unsigned int
1998 intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane)
1999 {
2000         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2001         unsigned int cpp = fb->format->cpp[color_plane];
2002
2003         switch (fb->modifier) {
2004         case DRM_FORMAT_MOD_LINEAR:
2005                 return intel_tile_size(dev_priv);
2006         case I915_FORMAT_MOD_X_TILED:
2007                 if (IS_GEN(dev_priv, 2))
2008                         return 128;
2009                 else
2010                         return 512;
2011         case I915_FORMAT_MOD_Y_TILED_CCS:
2012                 if (is_ccs_plane(fb, color_plane))
2013                         return 128;
2014                 /* fall through */
2015         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
2016                 if (is_ccs_plane(fb, color_plane))
2017                         return 64;
2018                 /* fall through */
2019         case I915_FORMAT_MOD_Y_TILED:
2020                 if (IS_GEN(dev_priv, 2) || HAS_128_BYTE_Y_TILING(dev_priv))
2021                         return 128;
2022                 else
2023                         return 512;
2024         case I915_FORMAT_MOD_Yf_TILED_CCS:
2025                 if (is_ccs_plane(fb, color_plane))
2026                         return 128;
2027                 /* fall through */
2028         case I915_FORMAT_MOD_Yf_TILED:
2029                 switch (cpp) {
2030                 case 1:
2031                         return 64;
2032                 case 2:
2033                 case 4:
2034                         return 128;
2035                 case 8:
2036                 case 16:
2037                         return 256;
2038                 default:
2039                         MISSING_CASE(cpp);
2040                         return cpp;
2041                 }
2042                 break;
2043         default:
2044                 MISSING_CASE(fb->modifier);
2045                 return cpp;
2046         }
2047 }
2048
2049 static unsigned int
2050 intel_tile_height(const struct drm_framebuffer *fb, int color_plane)
2051 {
2052         if (is_gen12_ccs_plane(fb, color_plane))
2053                 return 1;
2054
2055         return intel_tile_size(to_i915(fb->dev)) /
2056                 intel_tile_width_bytes(fb, color_plane);
2057 }
2058
2059 /* Return the tile dimensions in pixel units */
2060 static void intel_tile_dims(const struct drm_framebuffer *fb, int color_plane,
2061                             unsigned int *tile_width,
2062                             unsigned int *tile_height)
2063 {
2064         unsigned int tile_width_bytes = intel_tile_width_bytes(fb, color_plane);
2065         unsigned int cpp = fb->format->cpp[color_plane];
2066
2067         *tile_width = tile_width_bytes / cpp;
2068         *tile_height = intel_tile_height(fb, color_plane);
2069 }
2070
2071 unsigned int
2072 intel_fb_align_height(const struct drm_framebuffer *fb,
2073                       int color_plane, unsigned int height)
2074 {
2075         unsigned int tile_height = intel_tile_height(fb, color_plane);
2076
2077         return ALIGN(height, tile_height);
2078 }
2079
2080 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info)
2081 {
2082         unsigned int size = 0;
2083         int i;
2084
2085         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
2086                 size += rot_info->plane[i].width * rot_info->plane[i].height;
2087
2088         return size;
2089 }
2090
2091 unsigned int intel_remapped_info_size(const struct intel_remapped_info *rem_info)
2092 {
2093         unsigned int size = 0;
2094         int i;
2095
2096         for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++)
2097                 size += rem_info->plane[i].width * rem_info->plane[i].height;
2098
2099         return size;
2100 }
2101
2102 static void
2103 intel_fill_fb_ggtt_view(struct i915_ggtt_view *view,
2104                         const struct drm_framebuffer *fb,
2105                         unsigned int rotation)
2106 {
2107         view->type = I915_GGTT_VIEW_NORMAL;
2108         if (drm_rotation_90_or_270(rotation)) {
2109                 view->type = I915_GGTT_VIEW_ROTATED;
2110                 view->rotated = to_intel_framebuffer(fb)->rot_info;
2111         }
2112 }
2113
2114 static unsigned int intel_cursor_alignment(const struct drm_i915_private *dev_priv)
2115 {
2116         if (IS_I830(dev_priv))
2117                 return 16 * 1024;
2118         else if (IS_I85X(dev_priv))
2119                 return 256;
2120         else if (IS_I845G(dev_priv) || IS_I865G(dev_priv))
2121                 return 32;
2122         else
2123                 return 4 * 1024;
2124 }
2125
2126 static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv)
2127 {
2128         if (INTEL_GEN(dev_priv) >= 9)
2129                 return 256 * 1024;
2130         else if (IS_I965G(dev_priv) || IS_I965GM(dev_priv) ||
2131                  IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2132                 return 128 * 1024;
2133         else if (INTEL_GEN(dev_priv) >= 4)
2134                 return 4 * 1024;
2135         else
2136                 return 0;
2137 }
2138
2139 static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb,
2140                                          int color_plane)
2141 {
2142         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2143
2144         /* AUX_DIST needs only 4K alignment */
2145         if (is_aux_plane(fb, color_plane))
2146                 return 4096;
2147
2148         switch (fb->modifier) {
2149         case DRM_FORMAT_MOD_LINEAR:
2150                 return intel_linear_alignment(dev_priv);
2151         case I915_FORMAT_MOD_X_TILED:
2152                 if (INTEL_GEN(dev_priv) >= 9)
2153                         return 256 * 1024;
2154                 return 0;
2155         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
2156                 return 16 * 1024;
2157         case I915_FORMAT_MOD_Y_TILED_CCS:
2158         case I915_FORMAT_MOD_Yf_TILED_CCS:
2159         case I915_FORMAT_MOD_Y_TILED:
2160         case I915_FORMAT_MOD_Yf_TILED:
2161                 return 1 * 1024 * 1024;
2162         default:
2163                 MISSING_CASE(fb->modifier);
2164                 return 0;
2165         }
2166 }
2167
2168 static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
2169 {
2170         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2171         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2172
2173         return INTEL_GEN(dev_priv) < 4 ||
2174                 (plane->has_fbc &&
2175                  plane_state->view.type == I915_GGTT_VIEW_NORMAL);
2176 }
2177
2178 struct i915_vma *
2179 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
2180                            const struct i915_ggtt_view *view,
2181                            bool uses_fence,
2182                            unsigned long *out_flags)
2183 {
2184         struct drm_device *dev = fb->dev;
2185         struct drm_i915_private *dev_priv = to_i915(dev);
2186         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2187         intel_wakeref_t wakeref;
2188         struct i915_vma *vma;
2189         unsigned int pinctl;
2190         u32 alignment;
2191
2192         if (WARN_ON(!i915_gem_object_is_framebuffer(obj)))
2193                 return ERR_PTR(-EINVAL);
2194
2195         alignment = intel_surf_alignment(fb, 0);
2196
2197         /* Note that the w/a also requires 64 PTE of padding following the
2198          * bo. We currently fill all unused PTE with the shadow page and so
2199          * we should always have valid PTE following the scanout preventing
2200          * the VT-d warning.
2201          */
2202         if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024)
2203                 alignment = 256 * 1024;
2204
2205         /*
2206          * Global gtt pte registers are special registers which actually forward
2207          * writes to a chunk of system memory. Which means that there is no risk
2208          * that the register values disappear as soon as we call
2209          * intel_runtime_pm_put(), so it is correct to wrap only the
2210          * pin/unpin/fence and not more.
2211          */
2212         wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
2213
2214         atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
2215
2216         /*
2217          * Valleyview is definitely limited to scanning out the first
2218          * 512MiB. Lets presume this behaviour was inherited from the
2219          * g4x display engine and that all earlier gen are similarly
2220          * limited. Testing suggests that it is a little more
2221          * complicated than this. For example, Cherryview appears quite
2222          * happy to scanout from anywhere within its global aperture.
2223          */
2224         pinctl = 0;
2225         if (HAS_GMCH(dev_priv))
2226                 pinctl |= PIN_MAPPABLE;
2227
2228         vma = i915_gem_object_pin_to_display_plane(obj,
2229                                                    alignment, view, pinctl);
2230         if (IS_ERR(vma))
2231                 goto err;
2232
2233         if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
2234                 int ret;
2235
2236                 /*
2237                  * Install a fence for tiled scan-out. Pre-i965 always needs a
2238                  * fence, whereas 965+ only requires a fence if using
2239                  * framebuffer compression.  For simplicity, we always, when
2240                  * possible, install a fence as the cost is not that onerous.
2241                  *
2242                  * If we fail to fence the tiled scanout, then either the
2243                  * modeset will reject the change (which is highly unlikely as
2244                  * the affected systems, all but one, do not have unmappable
2245                  * space) or we will not be able to enable full powersaving
2246                  * techniques (also likely not to apply due to various limits
2247                  * FBC and the like impose on the size of the buffer, which
2248                  * presumably we violated anyway with this unmappable buffer).
2249                  * Anyway, it is presumably better to stumble onwards with
2250                  * something and try to run the system in a "less than optimal"
2251                  * mode that matches the user configuration.
2252                  */
2253                 ret = i915_vma_pin_fence(vma);
2254                 if (ret != 0 && INTEL_GEN(dev_priv) < 4) {
2255                         i915_gem_object_unpin_from_display_plane(vma);
2256                         vma = ERR_PTR(ret);
2257                         goto err;
2258                 }
2259
2260                 if (ret == 0 && vma->fence)
2261                         *out_flags |= PLANE_HAS_FENCE;
2262         }
2263
2264         i915_vma_get(vma);
2265 err:
2266         atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
2267         intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
2268         return vma;
2269 }
2270
2271 void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags)
2272 {
2273         i915_gem_object_lock(vma->obj);
2274         if (flags & PLANE_HAS_FENCE)
2275                 i915_vma_unpin_fence(vma);
2276         i915_gem_object_unpin_from_display_plane(vma);
2277         i915_gem_object_unlock(vma->obj);
2278
2279         i915_vma_put(vma);
2280 }
2281
2282 static int intel_fb_pitch(const struct drm_framebuffer *fb, int color_plane,
2283                           unsigned int rotation)
2284 {
2285         if (drm_rotation_90_or_270(rotation))
2286                 return to_intel_framebuffer(fb)->rotated[color_plane].pitch;
2287         else
2288                 return fb->pitches[color_plane];
2289 }
2290
2291 /*
2292  * Convert the x/y offsets into a linear offset.
2293  * Only valid with 0/180 degree rotation, which is fine since linear
2294  * offset is only used with linear buffers on pre-hsw and tiled buffers
2295  * with gen2/3, and 90/270 degree rotations isn't supported on any of them.
2296  */
2297 u32 intel_fb_xy_to_linear(int x, int y,
2298                           const struct intel_plane_state *state,
2299                           int color_plane)
2300 {
2301         const struct drm_framebuffer *fb = state->hw.fb;
2302         unsigned int cpp = fb->format->cpp[color_plane];
2303         unsigned int pitch = state->color_plane[color_plane].stride;
2304
2305         return y * pitch + x * cpp;
2306 }
2307
2308 /*
2309  * Add the x/y offsets derived from fb->offsets[] to the user
2310  * specified plane src x/y offsets. The resulting x/y offsets
2311  * specify the start of scanout from the beginning of the gtt mapping.
2312  */
2313 void intel_add_fb_offsets(int *x, int *y,
2314                           const struct intel_plane_state *state,
2315                           int color_plane)
2316
2317 {
2318         *x += state->color_plane[color_plane].x;
2319         *y += state->color_plane[color_plane].y;
2320 }
2321
2322 static u32 intel_adjust_tile_offset(int *x, int *y,
2323                                     unsigned int tile_width,
2324                                     unsigned int tile_height,
2325                                     unsigned int tile_size,
2326                                     unsigned int pitch_tiles,
2327                                     u32 old_offset,
2328                                     u32 new_offset)
2329 {
2330         unsigned int pitch_pixels = pitch_tiles * tile_width;
2331         unsigned int tiles;
2332
2333         WARN_ON(old_offset & (tile_size - 1));
2334         WARN_ON(new_offset & (tile_size - 1));
2335         WARN_ON(new_offset > old_offset);
2336
2337         tiles = (old_offset - new_offset) / tile_size;
2338
2339         *y += tiles / pitch_tiles * tile_height;
2340         *x += tiles % pitch_tiles * tile_width;
2341
2342         /* minimize x in case it got needlessly big */
2343         *y += *x / pitch_pixels * tile_height;
2344         *x %= pitch_pixels;
2345
2346         return new_offset;
2347 }
2348
2349 static bool is_surface_linear(const struct drm_framebuffer *fb, int color_plane)
2350 {
2351         return fb->modifier == DRM_FORMAT_MOD_LINEAR ||
2352                is_gen12_ccs_plane(fb, color_plane);
2353 }
2354
2355 static u32 intel_adjust_aligned_offset(int *x, int *y,
2356                                        const struct drm_framebuffer *fb,
2357                                        int color_plane,
2358                                        unsigned int rotation,
2359                                        unsigned int pitch,
2360                                        u32 old_offset, u32 new_offset)
2361 {
2362         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2363         unsigned int cpp = fb->format->cpp[color_plane];
2364
2365         WARN_ON(new_offset > old_offset);
2366
2367         if (!is_surface_linear(fb, color_plane)) {
2368                 unsigned int tile_size, tile_width, tile_height;
2369                 unsigned int pitch_tiles;
2370
2371                 tile_size = intel_tile_size(dev_priv);
2372                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2373
2374                 if (drm_rotation_90_or_270(rotation)) {
2375                         pitch_tiles = pitch / tile_height;
2376                         swap(tile_width, tile_height);
2377                 } else {
2378                         pitch_tiles = pitch / (tile_width * cpp);
2379                 }
2380
2381                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2382                                          tile_size, pitch_tiles,
2383                                          old_offset, new_offset);
2384         } else {
2385                 old_offset += *y * pitch + *x * cpp;
2386
2387                 *y = (old_offset - new_offset) / pitch;
2388                 *x = ((old_offset - new_offset) - *y * pitch) / cpp;
2389         }
2390
2391         return new_offset;
2392 }
2393
2394 /*
2395  * Adjust the tile offset by moving the difference into
2396  * the x/y offsets.
2397  */
2398 static u32 intel_plane_adjust_aligned_offset(int *x, int *y,
2399                                              const struct intel_plane_state *state,
2400                                              int color_plane,
2401                                              u32 old_offset, u32 new_offset)
2402 {
2403         return intel_adjust_aligned_offset(x, y, state->hw.fb, color_plane,
2404                                            state->hw.rotation,
2405                                            state->color_plane[color_plane].stride,
2406                                            old_offset, new_offset);
2407 }
2408
2409 /*
2410  * Computes the aligned offset to the base tile and adjusts
2411  * x, y. bytes per pixel is assumed to be a power-of-two.
2412  *
2413  * In the 90/270 rotated case, x and y are assumed
2414  * to be already rotated to match the rotated GTT view, and
2415  * pitch is the tile_height aligned framebuffer height.
2416  *
2417  * This function is used when computing the derived information
2418  * under intel_framebuffer, so using any of that information
2419  * here is not allowed. Anything under drm_framebuffer can be
2420  * used. This is why the user has to pass in the pitch since it
2421  * is specified in the rotated orientation.
2422  */
2423 static u32 intel_compute_aligned_offset(struct drm_i915_private *dev_priv,
2424                                         int *x, int *y,
2425                                         const struct drm_framebuffer *fb,
2426                                         int color_plane,
2427                                         unsigned int pitch,
2428                                         unsigned int rotation,
2429                                         u32 alignment)
2430 {
2431         unsigned int cpp = fb->format->cpp[color_plane];
2432         u32 offset, offset_aligned;
2433
2434         if (alignment)
2435                 alignment--;
2436
2437         if (!is_surface_linear(fb, color_plane)) {
2438                 unsigned int tile_size, tile_width, tile_height;
2439                 unsigned int tile_rows, tiles, pitch_tiles;
2440
2441                 tile_size = intel_tile_size(dev_priv);
2442                 intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
2443
2444                 if (drm_rotation_90_or_270(rotation)) {
2445                         pitch_tiles = pitch / tile_height;
2446                         swap(tile_width, tile_height);
2447                 } else {
2448                         pitch_tiles = pitch / (tile_width * cpp);
2449                 }
2450
2451                 tile_rows = *y / tile_height;
2452                 *y %= tile_height;
2453
2454                 tiles = *x / tile_width;
2455                 *x %= tile_width;
2456
2457                 offset = (tile_rows * pitch_tiles + tiles) * tile_size;
2458                 offset_aligned = offset & ~alignment;
2459
2460                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2461                                          tile_size, pitch_tiles,
2462                                          offset, offset_aligned);
2463         } else {
2464                 offset = *y * pitch + *x * cpp;
2465                 offset_aligned = offset & ~alignment;
2466
2467                 *y = (offset & alignment) / pitch;
2468                 *x = ((offset & alignment) - *y * pitch) / cpp;
2469         }
2470
2471         return offset_aligned;
2472 }
2473
2474 static u32 intel_plane_compute_aligned_offset(int *x, int *y,
2475                                               const struct intel_plane_state *state,
2476                                               int color_plane)
2477 {
2478         struct intel_plane *intel_plane = to_intel_plane(state->uapi.plane);
2479         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
2480         const struct drm_framebuffer *fb = state->hw.fb;
2481         unsigned int rotation = state->hw.rotation;
2482         int pitch = state->color_plane[color_plane].stride;
2483         u32 alignment;
2484
2485         if (intel_plane->id == PLANE_CURSOR)
2486                 alignment = intel_cursor_alignment(dev_priv);
2487         else
2488                 alignment = intel_surf_alignment(fb, color_plane);
2489
2490         return intel_compute_aligned_offset(dev_priv, x, y, fb, color_plane,
2491                                             pitch, rotation, alignment);
2492 }
2493
2494 /* Convert the fb->offset[] into x/y offsets */
2495 static int intel_fb_offset_to_xy(int *x, int *y,
2496                                  const struct drm_framebuffer *fb,
2497                                  int color_plane)
2498 {
2499         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2500         unsigned int height;
2501
2502         if (fb->modifier != DRM_FORMAT_MOD_LINEAR &&
2503             fb->offsets[color_plane] % intel_tile_size(dev_priv)) {
2504                 DRM_DEBUG_KMS("Misaligned offset 0x%08x for color plane %d\n",
2505                               fb->offsets[color_plane], color_plane);
2506                 return -EINVAL;
2507         }
2508
2509         height = drm_framebuffer_plane_height(fb->height, fb, color_plane);
2510         height = ALIGN(height, intel_tile_height(fb, color_plane));
2511
2512         /* Catch potential overflows early */
2513         if (add_overflows_t(u32, mul_u32_u32(height, fb->pitches[color_plane]),
2514                             fb->offsets[color_plane])) {
2515                 DRM_DEBUG_KMS("Bad offset 0x%08x or pitch %d for color plane %d\n",
2516                               fb->offsets[color_plane], fb->pitches[color_plane],
2517                               color_plane);
2518                 return -ERANGE;
2519         }
2520
2521         *x = 0;
2522         *y = 0;
2523
2524         intel_adjust_aligned_offset(x, y,
2525                                     fb, color_plane, DRM_MODE_ROTATE_0,
2526                                     fb->pitches[color_plane],
2527                                     fb->offsets[color_plane], 0);
2528
2529         return 0;
2530 }
2531
2532 static unsigned int intel_fb_modifier_to_tiling(u64 fb_modifier)
2533 {
2534         switch (fb_modifier) {
2535         case I915_FORMAT_MOD_X_TILED:
2536                 return I915_TILING_X;
2537         case I915_FORMAT_MOD_Y_TILED:
2538         case I915_FORMAT_MOD_Y_TILED_CCS:
2539         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
2540                 return I915_TILING_Y;
2541         default:
2542                 return I915_TILING_NONE;
2543         }
2544 }
2545
2546 /*
2547  * From the Sky Lake PRM:
2548  * "The Color Control Surface (CCS) contains the compression status of
2549  *  the cache-line pairs. The compression state of the cache-line pair
2550  *  is specified by 2 bits in the CCS. Each CCS cache-line represents
2551  *  an area on the main surface of 16 x16 sets of 128 byte Y-tiled
2552  *  cache-line-pairs. CCS is always Y tiled."
2553  *
2554  * Since cache line pairs refers to horizontally adjacent cache lines,
2555  * each cache line in the CCS corresponds to an area of 32x16 cache
2556  * lines on the main surface. Since each pixel is 4 bytes, this gives
2557  * us a ratio of one byte in the CCS for each 8x16 pixels in the
2558  * main surface.
2559  */
2560 static const struct drm_format_info skl_ccs_formats[] = {
2561         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
2562           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2563         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
2564           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
2565         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
2566           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
2567         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
2568           .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, .has_alpha = true, },
2569 };
2570
2571 /*
2572  * Gen-12 compression uses 4 bits of CCS data for each cache line pair in the
2573  * main surface. And each 64B CCS cache line represents an area of 4x1 Y-tiles
2574  * in the main surface. With 4 byte pixels and each Y-tile having dimensions of
2575  * 32x32 pixels, the ratio turns out to 1B in the CCS for every 2x32 pixels in
2576  * the main surface.
2577  */
2578 static const struct drm_format_info gen12_ccs_formats[] = {
2579         { .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2,
2580           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2581           .hsub = 1, .vsub = 1, },
2582         { .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2,
2583           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2584           .hsub = 1, .vsub = 1, },
2585         { .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2,
2586           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2587           .hsub = 1, .vsub = 1, .has_alpha = true },
2588         { .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2,
2589           .char_per_block = { 4, 1 }, .block_w = { 1, 2 }, .block_h = { 1, 1 },
2590           .hsub = 1, .vsub = 1, .has_alpha = true },
2591 };
2592
2593 static const struct drm_format_info *
2594 lookup_format_info(const struct drm_format_info formats[],
2595                    int num_formats, u32 format)
2596 {
2597         int i;
2598
2599         for (i = 0; i < num_formats; i++) {
2600                 if (formats[i].format == format)
2601                         return &formats[i];
2602         }
2603
2604         return NULL;
2605 }
2606
2607 static const struct drm_format_info *
2608 intel_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
2609 {
2610         switch (cmd->modifier[0]) {
2611         case I915_FORMAT_MOD_Y_TILED_CCS:
2612         case I915_FORMAT_MOD_Yf_TILED_CCS:
2613                 return lookup_format_info(skl_ccs_formats,
2614                                           ARRAY_SIZE(skl_ccs_formats),
2615                                           cmd->pixel_format);
2616         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
2617                 return lookup_format_info(gen12_ccs_formats,
2618                                           ARRAY_SIZE(gen12_ccs_formats),
2619                                           cmd->pixel_format);
2620         default:
2621                 return NULL;
2622         }
2623 }
2624
2625 bool is_ccs_modifier(u64 modifier)
2626 {
2627         return modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS ||
2628                modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
2629                modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
2630 }
2631
2632 static int gen12_ccs_aux_stride(struct drm_framebuffer *fb, int ccs_plane)
2633 {
2634         return DIV_ROUND_UP(fb->pitches[ccs_to_main_plane(fb, ccs_plane)],
2635                             512) * 64;
2636 }
2637
2638 u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv,
2639                               u32 pixel_format, u64 modifier)
2640 {
2641         struct intel_crtc *crtc;
2642         struct intel_plane *plane;
2643
2644         /*
2645          * We assume the primary plane for pipe A has
2646          * the highest stride limits of them all.
2647          */
2648         crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A);
2649         if (!crtc)
2650                 return 0;
2651
2652         plane = to_intel_plane(crtc->base.primary);
2653
2654         return plane->max_stride(plane, pixel_format, modifier,
2655                                  DRM_MODE_ROTATE_0);
2656 }
2657
2658 static
2659 u32 intel_fb_max_stride(struct drm_i915_private *dev_priv,
2660                         u32 pixel_format, u64 modifier)
2661 {
2662         /*
2663          * Arbitrary limit for gen4+ chosen to match the
2664          * render engine max stride.
2665          *
2666          * The new CCS hash mode makes remapping impossible
2667          */
2668         if (!is_ccs_modifier(modifier)) {
2669                 if (INTEL_GEN(dev_priv) >= 7)
2670                         return 256*1024;
2671                 else if (INTEL_GEN(dev_priv) >= 4)
2672                         return 128*1024;
2673         }
2674
2675         return intel_plane_fb_max_stride(dev_priv, pixel_format, modifier);
2676 }
2677
2678 static u32
2679 intel_fb_stride_alignment(const struct drm_framebuffer *fb, int color_plane)
2680 {
2681         struct drm_i915_private *dev_priv = to_i915(fb->dev);
2682         u32 tile_width;
2683
2684         if (is_surface_linear(fb, color_plane)) {
2685                 u32 max_stride = intel_plane_fb_max_stride(dev_priv,
2686                                                            fb->format->format,
2687                                                            fb->modifier);
2688
2689                 /*
2690                  * To make remapping with linear generally feasible
2691                  * we need the stride to be page aligned.
2692                  */
2693                 if (fb->pitches[color_plane] > max_stride &&
2694                     !is_ccs_modifier(fb->modifier))
2695                         return intel_tile_size(dev_priv);
2696                 else
2697                         return 64;
2698         }
2699
2700         tile_width = intel_tile_width_bytes(fb, color_plane);
2701         if (is_ccs_modifier(fb->modifier) && color_plane == 0) {
2702                 /*
2703                  * Display WA #0531: skl,bxt,kbl,glk
2704                  *
2705                  * Render decompression and plane width > 3840
2706                  * combined with horizontal panning requires the
2707                  * plane stride to be a multiple of 4. We'll just
2708                  * require the entire fb to accommodate that to avoid
2709                  * potential runtime errors at plane configuration time.
2710                  */
2711                 if (IS_GEN(dev_priv, 9) && fb->width > 3840)
2712                         tile_width *= 4;
2713                 /*
2714                  * The main surface pitch must be padded to a multiple of four
2715                  * tile widths.
2716                  */
2717                 else if (INTEL_GEN(dev_priv) >= 12)
2718                         tile_width *= 4;
2719         }
2720         return tile_width;
2721 }
2722
2723 bool intel_plane_can_remap(const struct intel_plane_state *plane_state)
2724 {
2725         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2726         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2727         const struct drm_framebuffer *fb = plane_state->hw.fb;
2728         int i;
2729
2730         /* We don't want to deal with remapping with cursors */
2731         if (plane->id == PLANE_CURSOR)
2732                 return false;
2733
2734         /*
2735          * The display engine limits already match/exceed the
2736          * render engine limits, so not much point in remapping.
2737          * Would also need to deal with the fence POT alignment
2738          * and gen2 2KiB GTT tile size.
2739          */
2740         if (INTEL_GEN(dev_priv) < 4)
2741                 return false;
2742
2743         /*
2744          * The new CCS hash mode isn't compatible with remapping as
2745          * the virtual address of the pages affects the compressed data.
2746          */
2747         if (is_ccs_modifier(fb->modifier))
2748                 return false;
2749
2750         /* Linear needs a page aligned stride for remapping */
2751         if (fb->modifier == DRM_FORMAT_MOD_LINEAR) {
2752                 unsigned int alignment = intel_tile_size(dev_priv) - 1;
2753
2754                 for (i = 0; i < fb->format->num_planes; i++) {
2755                         if (fb->pitches[i] & alignment)
2756                                 return false;
2757                 }
2758         }
2759
2760         return true;
2761 }
2762
2763 static bool intel_plane_needs_remap(const struct intel_plane_state *plane_state)
2764 {
2765         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2766         const struct drm_framebuffer *fb = plane_state->hw.fb;
2767         unsigned int rotation = plane_state->hw.rotation;
2768         u32 stride, max_stride;
2769
2770         /*
2771          * No remapping for invisible planes since we don't have
2772          * an actual source viewport to remap.
2773          */
2774         if (!plane_state->uapi.visible)
2775                 return false;
2776
2777         if (!intel_plane_can_remap(plane_state))
2778                 return false;
2779
2780         /*
2781          * FIXME: aux plane limits on gen9+ are
2782          * unclear in Bspec, for now no checking.
2783          */
2784         stride = intel_fb_pitch(fb, 0, rotation);
2785         max_stride = plane->max_stride(plane, fb->format->format,
2786                                        fb->modifier, rotation);
2787
2788         return stride > max_stride;
2789 }
2790
2791 static void
2792 intel_fb_plane_get_subsampling(int *hsub, int *vsub,
2793                                const struct drm_framebuffer *fb,
2794                                int color_plane)
2795 {
2796         int main_plane;
2797
2798         if (color_plane == 0) {
2799                 *hsub = 1;
2800                 *vsub = 1;
2801
2802                 return;
2803         }
2804
2805         /*
2806          * TODO: Deduct the subsampling from the char block for all CCS
2807          * formats and planes.
2808          */
2809         if (!is_gen12_ccs_plane(fb, color_plane)) {
2810                 *hsub = fb->format->hsub;
2811                 *vsub = fb->format->vsub;
2812
2813                 return;
2814         }
2815
2816         main_plane = ccs_to_main_plane(fb, color_plane);
2817         *hsub = drm_format_info_block_width(fb->format, color_plane) /
2818                 drm_format_info_block_width(fb->format, main_plane);
2819
2820         /*
2821          * The min stride check in the core framebuffer_check() function
2822          * assumes that format->hsub applies to every plane except for the
2823          * first plane. That's incorrect for the CCS AUX plane of the first
2824          * plane, but for the above check to pass we must define the block
2825          * width with that subsampling applied to it. Adjust the width here
2826          * accordingly, so we can calculate the actual subsampling factor.
2827          */
2828         if (main_plane == 0)
2829                 *hsub *= fb->format->hsub;
2830
2831         *vsub = 32;
2832 }
2833 static int
2834 intel_fb_check_ccs_xy(struct drm_framebuffer *fb, int ccs_plane, int x, int y)
2835 {
2836         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2837         int main_plane;
2838         int hsub, vsub;
2839         int tile_width, tile_height;
2840         int ccs_x, ccs_y;
2841         int main_x, main_y;
2842
2843         if (!is_ccs_plane(fb, ccs_plane))
2844                 return 0;
2845
2846         intel_tile_dims(fb, ccs_plane, &tile_width, &tile_height);
2847         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, ccs_plane);
2848
2849         tile_width *= hsub;
2850         tile_height *= vsub;
2851
2852         ccs_x = (x * hsub) % tile_width;
2853         ccs_y = (y * vsub) % tile_height;
2854
2855         main_plane = ccs_to_main_plane(fb, ccs_plane);
2856         main_x = intel_fb->normal[main_plane].x % tile_width;
2857         main_y = intel_fb->normal[main_plane].y % tile_height;
2858
2859         /*
2860          * CCS doesn't have its own x/y offset register, so the intra CCS tile
2861          * x/y offsets must match between CCS and the main surface.
2862          */
2863         if (main_x != ccs_x || main_y != ccs_y) {
2864                 DRM_DEBUG_KMS("Bad CCS x/y (main %d,%d ccs %d,%d) full (main %d,%d ccs %d,%d)\n",
2865                               main_x, main_y,
2866                               ccs_x, ccs_y,
2867                               intel_fb->normal[main_plane].x,
2868                               intel_fb->normal[main_plane].y,
2869                               x, y);
2870                 return -EINVAL;
2871         }
2872
2873         return 0;
2874 }
2875
2876 static void
2877 intel_fb_plane_dims(int *w, int *h, struct drm_framebuffer *fb, int color_plane)
2878 {
2879         int hsub, vsub;
2880
2881         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, color_plane);
2882         *w = fb->width / hsub;
2883         *h = fb->height / vsub;
2884 }
2885
2886 /*
2887  * Setup the rotated view for an FB plane and return the size the GTT mapping
2888  * requires for this view.
2889  */
2890 static u32
2891 setup_fb_rotation(int plane, const struct intel_remapped_plane_info *plane_info,
2892                   u32 gtt_offset_rotated, int x, int y,
2893                   unsigned int width, unsigned int height,
2894                   unsigned int tile_size,
2895                   unsigned int tile_width, unsigned int tile_height,
2896                   struct drm_framebuffer *fb)
2897 {
2898         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2899         struct intel_rotation_info *rot_info = &intel_fb->rot_info;
2900         unsigned int pitch_tiles;
2901         struct drm_rect r;
2902
2903         /* Y or Yf modifiers required for 90/270 rotation */
2904         if (fb->modifier != I915_FORMAT_MOD_Y_TILED &&
2905             fb->modifier != I915_FORMAT_MOD_Yf_TILED)
2906                 return 0;
2907
2908         if (WARN_ON(plane >= ARRAY_SIZE(rot_info->plane)))
2909                 return 0;
2910
2911         rot_info->plane[plane] = *plane_info;
2912
2913         intel_fb->rotated[plane].pitch = plane_info->height * tile_height;
2914
2915         /* rotate the x/y offsets to match the GTT view */
2916         drm_rect_init(&r, x, y, width, height);
2917         drm_rect_rotate(&r,
2918                         plane_info->width * tile_width,
2919                         plane_info->height * tile_height,
2920                         DRM_MODE_ROTATE_270);
2921         x = r.x1;
2922         y = r.y1;
2923
2924         /* rotate the tile dimensions to match the GTT view */
2925         pitch_tiles = intel_fb->rotated[plane].pitch / tile_height;
2926         swap(tile_width, tile_height);
2927
2928         /*
2929          * We only keep the x/y offsets, so push all of the
2930          * gtt offset into the x/y offsets.
2931          */
2932         intel_adjust_tile_offset(&x, &y,
2933                                  tile_width, tile_height,
2934                                  tile_size, pitch_tiles,
2935                                  gtt_offset_rotated * tile_size, 0);
2936
2937         /*
2938          * First pixel of the framebuffer from
2939          * the start of the rotated gtt mapping.
2940          */
2941         intel_fb->rotated[plane].x = x;
2942         intel_fb->rotated[plane].y = y;
2943
2944         return plane_info->width * plane_info->height;
2945 }
2946
2947 static int
2948 intel_fill_fb_info(struct drm_i915_private *dev_priv,
2949                    struct drm_framebuffer *fb)
2950 {
2951         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
2952         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2953         u32 gtt_offset_rotated = 0;
2954         unsigned int max_size = 0;
2955         int i, num_planes = fb->format->num_planes;
2956         unsigned int tile_size = intel_tile_size(dev_priv);
2957
2958         for (i = 0; i < num_planes; i++) {
2959                 unsigned int width, height;
2960                 unsigned int cpp, size;
2961                 u32 offset;
2962                 int x, y;
2963                 int ret;
2964
2965                 cpp = fb->format->cpp[i];
2966                 intel_fb_plane_dims(&width, &height, fb, i);
2967
2968                 ret = intel_fb_offset_to_xy(&x, &y, fb, i);
2969                 if (ret) {
2970                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2971                                       i, fb->offsets[i]);
2972                         return ret;
2973                 }
2974
2975                 ret = intel_fb_check_ccs_xy(fb, i, x, y);
2976                 if (ret)
2977                         return ret;
2978
2979                 /*
2980                  * The fence (if used) is aligned to the start of the object
2981                  * so having the framebuffer wrap around across the edge of the
2982                  * fenced region doesn't really work. We have no API to configure
2983                  * the fence start offset within the object (nor could we probably
2984                  * on gen2/3). So it's just easier if we just require that the
2985                  * fb layout agrees with the fence layout. We already check that the
2986                  * fb stride matches the fence stride elsewhere.
2987                  */
2988                 if (i == 0 && i915_gem_object_is_tiled(obj) &&
2989                     (x + width) * cpp > fb->pitches[i]) {
2990                         DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
2991                                       i, fb->offsets[i]);
2992                         return -EINVAL;
2993                 }
2994
2995                 /*
2996                  * First pixel of the framebuffer from
2997                  * the start of the normal gtt mapping.
2998                  */
2999                 intel_fb->normal[i].x = x;
3000                 intel_fb->normal[i].y = y;
3001
3002                 offset = intel_compute_aligned_offset(dev_priv, &x, &y, fb, i,
3003                                                       fb->pitches[i],
3004                                                       DRM_MODE_ROTATE_0,
3005                                                       tile_size);
3006                 offset /= tile_size;
3007
3008                 if (!is_surface_linear(fb, i)) {
3009                         struct intel_remapped_plane_info plane_info;
3010                         unsigned int tile_width, tile_height;
3011
3012                         intel_tile_dims(fb, i, &tile_width, &tile_height);
3013
3014                         plane_info.offset = offset;
3015                         plane_info.stride = DIV_ROUND_UP(fb->pitches[i],
3016                                                          tile_width * cpp);
3017                         plane_info.width = DIV_ROUND_UP(x + width, tile_width);
3018                         plane_info.height = DIV_ROUND_UP(y + height,
3019                                                          tile_height);
3020
3021                         /* how many tiles does this plane need */
3022                         size = plane_info.stride * plane_info.height;
3023                         /*
3024                          * If the plane isn't horizontally tile aligned,
3025                          * we need one more tile.
3026                          */
3027                         if (x != 0)
3028                                 size++;
3029
3030                         gtt_offset_rotated +=
3031                                 setup_fb_rotation(i, &plane_info,
3032                                                   gtt_offset_rotated,
3033                                                   x, y, width, height,
3034                                                   tile_size,
3035                                                   tile_width, tile_height,
3036                                                   fb);
3037                 } else {
3038                         size = DIV_ROUND_UP((y + height) * fb->pitches[i] +
3039                                             x * cpp, tile_size);
3040                 }
3041
3042                 /* how many tiles in total needed in the bo */
3043                 max_size = max(max_size, offset + size);
3044         }
3045
3046         if (mul_u32_u32(max_size, tile_size) > obj->base.size) {
3047                 DRM_DEBUG_KMS("fb too big for bo (need %llu bytes, have %zu bytes)\n",
3048                               mul_u32_u32(max_size, tile_size), obj->base.size);
3049                 return -EINVAL;
3050         }
3051
3052         return 0;
3053 }
3054
3055 static void
3056 intel_plane_remap_gtt(struct intel_plane_state *plane_state)
3057 {
3058         struct drm_i915_private *dev_priv =
3059                 to_i915(plane_state->uapi.plane->dev);
3060         struct drm_framebuffer *fb = plane_state->hw.fb;
3061         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
3062         struct intel_rotation_info *info = &plane_state->view.rotated;
3063         unsigned int rotation = plane_state->hw.rotation;
3064         int i, num_planes = fb->format->num_planes;
3065         unsigned int tile_size = intel_tile_size(dev_priv);
3066         unsigned int src_x, src_y;
3067         unsigned int src_w, src_h;
3068         u32 gtt_offset = 0;
3069
3070         memset(&plane_state->view, 0, sizeof(plane_state->view));
3071         plane_state->view.type = drm_rotation_90_or_270(rotation) ?
3072                 I915_GGTT_VIEW_ROTATED : I915_GGTT_VIEW_REMAPPED;
3073
3074         src_x = plane_state->uapi.src.x1 >> 16;
3075         src_y = plane_state->uapi.src.y1 >> 16;
3076         src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
3077         src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
3078
3079         WARN_ON(is_ccs_modifier(fb->modifier));
3080
3081         /* Make src coordinates relative to the viewport */
3082         drm_rect_translate(&plane_state->uapi.src,
3083                            -(src_x << 16), -(src_y << 16));
3084
3085         /* Rotate src coordinates to match rotated GTT view */
3086         if (drm_rotation_90_or_270(rotation))
3087                 drm_rect_rotate(&plane_state->uapi.src,
3088                                 src_w << 16, src_h << 16,
3089                                 DRM_MODE_ROTATE_270);
3090
3091         for (i = 0; i < num_planes; i++) {
3092                 unsigned int hsub = i ? fb->format->hsub : 1;
3093                 unsigned int vsub = i ? fb->format->vsub : 1;
3094                 unsigned int cpp = fb->format->cpp[i];
3095                 unsigned int tile_width, tile_height;
3096                 unsigned int width, height;
3097                 unsigned int pitch_tiles;
3098                 unsigned int x, y;
3099                 u32 offset;
3100
3101                 intel_tile_dims(fb, i, &tile_width, &tile_height);
3102
3103                 x = src_x / hsub;
3104                 y = src_y / vsub;
3105                 width = src_w / hsub;
3106                 height = src_h / vsub;
3107
3108                 /*
3109                  * First pixel of the src viewport from the
3110                  * start of the normal gtt mapping.
3111                  */
3112                 x += intel_fb->normal[i].x;
3113                 y += intel_fb->normal[i].y;
3114
3115                 offset = intel_compute_aligned_offset(dev_priv, &x, &y,
3116                                                       fb, i, fb->pitches[i],
3117                                                       DRM_MODE_ROTATE_0, tile_size);
3118                 offset /= tile_size;
3119
3120                 WARN_ON(i >= ARRAY_SIZE(info->plane));
3121                 info->plane[i].offset = offset;
3122                 info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i],
3123                                                      tile_width * cpp);
3124                 info->plane[i].width = DIV_ROUND_UP(x + width, tile_width);
3125                 info->plane[i].height = DIV_ROUND_UP(y + height, tile_height);
3126
3127                 if (drm_rotation_90_or_270(rotation)) {
3128                         struct drm_rect r;
3129
3130                         /* rotate the x/y offsets to match the GTT view */
3131                         drm_rect_init(&r, x, y, width, height);
3132                         drm_rect_rotate(&r,
3133                                         info->plane[i].width * tile_width,
3134                                         info->plane[i].height * tile_height,
3135                                         DRM_MODE_ROTATE_270);
3136                         x = r.x1;
3137                         y = r.y1;
3138
3139                         pitch_tiles = info->plane[i].height;
3140                         plane_state->color_plane[i].stride = pitch_tiles * tile_height;
3141
3142                         /* rotate the tile dimensions to match the GTT view */
3143                         swap(tile_width, tile_height);
3144                 } else {
3145                         pitch_tiles = info->plane[i].width;
3146                         plane_state->color_plane[i].stride = pitch_tiles * tile_width * cpp;
3147                 }
3148
3149                 /*
3150                  * We only keep the x/y offsets, so push all of the
3151                  * gtt offset into the x/y offsets.
3152                  */
3153                 intel_adjust_tile_offset(&x, &y,
3154                                          tile_width, tile_height,
3155                                          tile_size, pitch_tiles,
3156                                          gtt_offset * tile_size, 0);
3157
3158                 gtt_offset += info->plane[i].width * info->plane[i].height;
3159
3160                 plane_state->color_plane[i].offset = 0;
3161                 plane_state->color_plane[i].x = x;
3162                 plane_state->color_plane[i].y = y;
3163         }
3164 }
3165
3166 static int
3167 intel_plane_compute_gtt(struct intel_plane_state *plane_state)
3168 {
3169         const struct intel_framebuffer *fb =
3170                 to_intel_framebuffer(plane_state->hw.fb);
3171         unsigned int rotation = plane_state->hw.rotation;
3172         int i, num_planes;
3173
3174         if (!fb)
3175                 return 0;
3176
3177         num_planes = fb->base.format->num_planes;
3178
3179         if (intel_plane_needs_remap(plane_state)) {
3180                 intel_plane_remap_gtt(plane_state);
3181
3182                 /*
3183                  * Sometimes even remapping can't overcome
3184                  * the stride limitations :( Can happen with
3185                  * big plane sizes and suitably misaligned
3186                  * offsets.
3187                  */
3188                 return intel_plane_check_stride(plane_state);
3189         }
3190
3191         intel_fill_fb_ggtt_view(&plane_state->view, &fb->base, rotation);
3192
3193         for (i = 0; i < num_planes; i++) {
3194                 plane_state->color_plane[i].stride = intel_fb_pitch(&fb->base, i, rotation);
3195                 plane_state->color_plane[i].offset = 0;
3196
3197                 if (drm_rotation_90_or_270(rotation)) {
3198                         plane_state->color_plane[i].x = fb->rotated[i].x;
3199                         plane_state->color_plane[i].y = fb->rotated[i].y;
3200                 } else {
3201                         plane_state->color_plane[i].x = fb->normal[i].x;
3202                         plane_state->color_plane[i].y = fb->normal[i].y;
3203                 }
3204         }
3205
3206         /* Rotate src coordinates to match rotated GTT view */
3207         if (drm_rotation_90_or_270(rotation))
3208                 drm_rect_rotate(&plane_state->uapi.src,
3209                                 fb->base.width << 16, fb->base.height << 16,
3210                                 DRM_MODE_ROTATE_270);
3211
3212         return intel_plane_check_stride(plane_state);
3213 }
3214
3215 static int i9xx_format_to_fourcc(int format)
3216 {
3217         switch (format) {
3218         case DISPPLANE_8BPP:
3219                 return DRM_FORMAT_C8;
3220         case DISPPLANE_BGRA555:
3221                 return DRM_FORMAT_ARGB1555;
3222         case DISPPLANE_BGRX555:
3223                 return DRM_FORMAT_XRGB1555;
3224         case DISPPLANE_BGRX565:
3225                 return DRM_FORMAT_RGB565;
3226         default:
3227         case DISPPLANE_BGRX888:
3228                 return DRM_FORMAT_XRGB8888;
3229         case DISPPLANE_RGBX888:
3230                 return DRM_FORMAT_XBGR8888;
3231         case DISPPLANE_BGRA888:
3232                 return DRM_FORMAT_ARGB8888;
3233         case DISPPLANE_RGBA888:
3234                 return DRM_FORMAT_ABGR8888;
3235         case DISPPLANE_BGRX101010:
3236                 return DRM_FORMAT_XRGB2101010;
3237         case DISPPLANE_RGBX101010:
3238                 return DRM_FORMAT_XBGR2101010;
3239         case DISPPLANE_BGRA101010:
3240                 return DRM_FORMAT_ARGB2101010;
3241         case DISPPLANE_RGBA101010:
3242                 return DRM_FORMAT_ABGR2101010;
3243         case DISPPLANE_RGBX161616:
3244                 return DRM_FORMAT_XBGR16161616F;
3245         }
3246 }
3247
3248 int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
3249 {
3250         switch (format) {
3251         case PLANE_CTL_FORMAT_RGB_565:
3252                 return DRM_FORMAT_RGB565;
3253         case PLANE_CTL_FORMAT_NV12:
3254                 return DRM_FORMAT_NV12;
3255         case PLANE_CTL_FORMAT_P010:
3256                 return DRM_FORMAT_P010;
3257         case PLANE_CTL_FORMAT_P012:
3258                 return DRM_FORMAT_P012;
3259         case PLANE_CTL_FORMAT_P016:
3260                 return DRM_FORMAT_P016;
3261         case PLANE_CTL_FORMAT_Y210:
3262                 return DRM_FORMAT_Y210;
3263         case PLANE_CTL_FORMAT_Y212:
3264                 return DRM_FORMAT_Y212;
3265         case PLANE_CTL_FORMAT_Y216:
3266                 return DRM_FORMAT_Y216;
3267         case PLANE_CTL_FORMAT_Y410:
3268                 return DRM_FORMAT_XVYU2101010;
3269         case PLANE_CTL_FORMAT_Y412:
3270                 return DRM_FORMAT_XVYU12_16161616;
3271         case PLANE_CTL_FORMAT_Y416:
3272                 return DRM_FORMAT_XVYU16161616;
3273         default:
3274         case PLANE_CTL_FORMAT_XRGB_8888:
3275                 if (rgb_order) {
3276                         if (alpha)
3277                                 return DRM_FORMAT_ABGR8888;
3278                         else
3279                                 return DRM_FORMAT_XBGR8888;
3280                 } else {
3281                         if (alpha)
3282                                 return DRM_FORMAT_ARGB8888;
3283                         else
3284                                 return DRM_FORMAT_XRGB8888;
3285                 }
3286         case PLANE_CTL_FORMAT_XRGB_2101010:
3287                 if (rgb_order) {
3288                         if (alpha)
3289                                 return DRM_FORMAT_ABGR2101010;
3290                         else
3291                                 return DRM_FORMAT_XBGR2101010;
3292                 } else {
3293                         if (alpha)
3294                                 return DRM_FORMAT_ARGB2101010;
3295                         else
3296                                 return DRM_FORMAT_XRGB2101010;
3297                 }
3298         case PLANE_CTL_FORMAT_XRGB_16161616F:
3299                 if (rgb_order) {
3300                         if (alpha)
3301                                 return DRM_FORMAT_ABGR16161616F;
3302                         else
3303                                 return DRM_FORMAT_XBGR16161616F;
3304                 } else {
3305                         if (alpha)
3306                                 return DRM_FORMAT_ARGB16161616F;
3307                         else
3308                                 return DRM_FORMAT_XRGB16161616F;
3309                 }
3310         }
3311 }
3312
3313 static bool
3314 intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
3315                               struct intel_initial_plane_config *plane_config)
3316 {
3317         struct drm_device *dev = crtc->base.dev;
3318         struct drm_i915_private *dev_priv = to_i915(dev);
3319         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
3320         struct drm_framebuffer *fb = &plane_config->fb->base;
3321         u32 base_aligned = round_down(plane_config->base, PAGE_SIZE);
3322         u32 size_aligned = round_up(plane_config->base + plane_config->size,
3323                                     PAGE_SIZE);
3324         struct drm_i915_gem_object *obj;
3325         bool ret = false;
3326
3327         size_aligned -= base_aligned;
3328
3329         if (plane_config->size == 0)
3330                 return false;
3331
3332         /* If the FB is too big, just don't use it since fbdev is not very
3333          * important and we should probably use that space with FBC or other
3334          * features. */
3335         if (size_aligned * 2 > dev_priv->stolen_usable_size)
3336                 return false;
3337
3338         switch (fb->modifier) {
3339         case DRM_FORMAT_MOD_LINEAR:
3340         case I915_FORMAT_MOD_X_TILED:
3341         case I915_FORMAT_MOD_Y_TILED:
3342                 break;
3343         default:
3344                 DRM_DEBUG_DRIVER("Unsupported modifier for initial FB: 0x%llx\n",
3345                                  fb->modifier);
3346                 return false;
3347         }
3348
3349         obj = i915_gem_object_create_stolen_for_preallocated(dev_priv,
3350                                                              base_aligned,
3351                                                              base_aligned,
3352                                                              size_aligned);
3353         if (IS_ERR(obj))
3354                 return false;
3355
3356         switch (plane_config->tiling) {
3357         case I915_TILING_NONE:
3358                 break;
3359         case I915_TILING_X:
3360         case I915_TILING_Y:
3361                 obj->tiling_and_stride = fb->pitches[0] | plane_config->tiling;
3362                 break;
3363         default:
3364                 MISSING_CASE(plane_config->tiling);
3365                 goto out;
3366         }
3367
3368         mode_cmd.pixel_format = fb->format->format;
3369         mode_cmd.width = fb->width;
3370         mode_cmd.height = fb->height;
3371         mode_cmd.pitches[0] = fb->pitches[0];
3372         mode_cmd.modifier[0] = fb->modifier;
3373         mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
3374
3375         if (intel_framebuffer_init(to_intel_framebuffer(fb), obj, &mode_cmd)) {
3376                 DRM_DEBUG_KMS("intel fb init failed\n");
3377                 goto out;
3378         }
3379
3380
3381         DRM_DEBUG_KMS("initial plane fb obj %p\n", obj);
3382         ret = true;
3383 out:
3384         i915_gem_object_put(obj);
3385         return ret;
3386 }
3387
3388 static void
3389 intel_set_plane_visible(struct intel_crtc_state *crtc_state,
3390                         struct intel_plane_state *plane_state,
3391                         bool visible)
3392 {
3393         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
3394
3395         plane_state->uapi.visible = visible;
3396
3397         if (visible)
3398                 crtc_state->uapi.plane_mask |= drm_plane_mask(&plane->base);
3399         else
3400                 crtc_state->uapi.plane_mask &= ~drm_plane_mask(&plane->base);
3401 }
3402
3403 static void fixup_active_planes(struct intel_crtc_state *crtc_state)
3404 {
3405         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
3406         struct drm_plane *plane;
3407
3408         /*
3409          * Active_planes aliases if multiple "primary" or cursor planes
3410          * have been used on the same (or wrong) pipe. plane_mask uses
3411          * unique ids, hence we can use that to reconstruct active_planes.
3412          */
3413         crtc_state->active_planes = 0;
3414
3415         drm_for_each_plane_mask(plane, &dev_priv->drm,
3416                                 crtc_state->uapi.plane_mask)
3417                 crtc_state->active_planes |= BIT(to_intel_plane(plane)->id);
3418 }
3419
3420 static void intel_plane_disable_noatomic(struct intel_crtc *crtc,
3421                                          struct intel_plane *plane)
3422 {
3423         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3424         struct intel_crtc_state *crtc_state =
3425                 to_intel_crtc_state(crtc->base.state);
3426         struct intel_plane_state *plane_state =
3427                 to_intel_plane_state(plane->base.state);
3428
3429         DRM_DEBUG_KMS("Disabling [PLANE:%d:%s] on [CRTC:%d:%s]\n",
3430                       plane->base.base.id, plane->base.name,
3431                       crtc->base.base.id, crtc->base.name);
3432
3433         intel_set_plane_visible(crtc_state, plane_state, false);
3434         fixup_active_planes(crtc_state);
3435         crtc_state->data_rate[plane->id] = 0;
3436         crtc_state->min_cdclk[plane->id] = 0;
3437
3438         if (plane->id == PLANE_PRIMARY)
3439                 hsw_disable_ips(crtc_state);
3440
3441         /*
3442          * Vblank time updates from the shadow to live plane control register
3443          * are blocked if the memory self-refresh mode is active at that
3444          * moment. So to make sure the plane gets truly disabled, disable
3445          * first the self-refresh mode. The self-refresh enable bit in turn
3446          * will be checked/applied by the HW only at the next frame start
3447          * event which is after the vblank start event, so we need to have a
3448          * wait-for-vblank between disabling the plane and the pipe.
3449          */
3450         if (HAS_GMCH(dev_priv) &&
3451             intel_set_memory_cxsr(dev_priv, false))
3452                 intel_wait_for_vblank(dev_priv, crtc->pipe);
3453
3454         /*
3455          * Gen2 reports pipe underruns whenever all planes are disabled.
3456          * So disable underrun reporting before all the planes get disabled.
3457          */
3458         if (IS_GEN(dev_priv, 2) && !crtc_state->active_planes)
3459                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
3460
3461         intel_disable_plane(plane, crtc_state);
3462 }
3463
3464 static struct intel_frontbuffer *
3465 to_intel_frontbuffer(struct drm_framebuffer *fb)
3466 {
3467         return fb ? to_intel_framebuffer(fb)->frontbuffer : NULL;
3468 }
3469
3470 static void
3471 intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
3472                              struct intel_initial_plane_config *plane_config)
3473 {
3474         struct drm_device *dev = intel_crtc->base.dev;
3475         struct drm_i915_private *dev_priv = to_i915(dev);
3476         struct drm_crtc *c;
3477         struct drm_plane *primary = intel_crtc->base.primary;
3478         struct drm_plane_state *plane_state = primary->state;
3479         struct intel_plane *intel_plane = to_intel_plane(primary);
3480         struct intel_plane_state *intel_state =
3481                 to_intel_plane_state(plane_state);
3482         struct drm_framebuffer *fb;
3483
3484         if (!plane_config->fb)
3485                 return;
3486
3487         if (intel_alloc_initial_plane_obj(intel_crtc, plane_config)) {
3488                 fb = &plane_config->fb->base;
3489                 goto valid_fb;
3490         }
3491
3492         kfree(plane_config->fb);
3493
3494         /*
3495          * Failed to alloc the obj, check to see if we should share
3496          * an fb with another CRTC instead
3497          */
3498         for_each_crtc(dev, c) {
3499                 struct intel_plane_state *state;
3500
3501                 if (c == &intel_crtc->base)
3502                         continue;
3503
3504                 if (!to_intel_crtc(c)->active)
3505                         continue;
3506
3507                 state = to_intel_plane_state(c->primary->state);
3508                 if (!state->vma)
3509                         continue;
3510
3511                 if (intel_plane_ggtt_offset(state) == plane_config->base) {
3512                         fb = state->hw.fb;
3513                         drm_framebuffer_get(fb);
3514                         goto valid_fb;
3515                 }
3516         }
3517
3518         /*
3519          * We've failed to reconstruct the BIOS FB.  Current display state
3520          * indicates that the primary plane is visible, but has a NULL FB,
3521          * which will lead to problems later if we don't fix it up.  The
3522          * simplest solution is to just disable the primary plane now and
3523          * pretend the BIOS never had it enabled.
3524          */
3525         intel_plane_disable_noatomic(intel_crtc, intel_plane);
3526
3527         return;
3528
3529 valid_fb:
3530         intel_state->hw.rotation = plane_config->rotation;
3531         intel_fill_fb_ggtt_view(&intel_state->view, fb,
3532                                 intel_state->hw.rotation);
3533         intel_state->color_plane[0].stride =
3534                 intel_fb_pitch(fb, 0, intel_state->hw.rotation);
3535
3536         intel_state->vma =
3537                 intel_pin_and_fence_fb_obj(fb,
3538                                            &intel_state->view,
3539                                            intel_plane_uses_fence(intel_state),
3540                                            &intel_state->flags);
3541         if (IS_ERR(intel_state->vma)) {
3542                 DRM_ERROR("failed to pin boot fb on pipe %d: %li\n",
3543                           intel_crtc->pipe, PTR_ERR(intel_state->vma));
3544
3545                 intel_state->vma = NULL;
3546                 drm_framebuffer_put(fb);
3547                 return;
3548         }
3549
3550         intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB);
3551
3552         plane_state->src_x = 0;
3553         plane_state->src_y = 0;
3554         plane_state->src_w = fb->width << 16;
3555         plane_state->src_h = fb->height << 16;
3556
3557         plane_state->crtc_x = 0;
3558         plane_state->crtc_y = 0;
3559         plane_state->crtc_w = fb->width;
3560         plane_state->crtc_h = fb->height;
3561
3562         intel_state->uapi.src = drm_plane_state_src(plane_state);
3563         intel_state->uapi.dst = drm_plane_state_dest(plane_state);
3564
3565         if (plane_config->tiling)
3566                 dev_priv->preserve_bios_swizzle = true;
3567
3568         plane_state->fb = fb;
3569         plane_state->crtc = &intel_crtc->base;
3570         intel_plane_copy_uapi_to_hw_state(intel_state, intel_state);
3571
3572         atomic_or(to_intel_plane(primary)->frontbuffer_bit,
3573                   &to_intel_frontbuffer(fb)->bits);
3574 }
3575
3576 static int skl_max_plane_width(const struct drm_framebuffer *fb,
3577                                int color_plane,
3578                                unsigned int rotation)
3579 {
3580         int cpp = fb->format->cpp[color_plane];
3581
3582         switch (fb->modifier) {
3583         case DRM_FORMAT_MOD_LINEAR:
3584         case I915_FORMAT_MOD_X_TILED:
3585                 /*
3586                  * Validated limit is 4k, but has 5k should
3587                  * work apart from the following features:
3588                  * - Ytile (already limited to 4k)
3589                  * - FP16 (already limited to 4k)
3590                  * - render compression (already limited to 4k)
3591                  * - KVMR sprite and cursor (don't care)
3592                  * - horizontal panning (TODO verify this)
3593                  * - pipe and plane scaling (TODO verify this)
3594                  */
3595                 if (cpp == 8)
3596                         return 4096;
3597                 else
3598                         return 5120;
3599         case I915_FORMAT_MOD_Y_TILED_CCS:
3600         case I915_FORMAT_MOD_Yf_TILED_CCS:
3601                 /* FIXME AUX plane? */
3602         case I915_FORMAT_MOD_Y_TILED:
3603         case I915_FORMAT_MOD_Yf_TILED:
3604                 if (cpp == 8)
3605                         return 2048;
3606                 else
3607                         return 4096;
3608         default:
3609                 MISSING_CASE(fb->modifier);
3610                 return 2048;
3611         }
3612 }
3613
3614 static int glk_max_plane_width(const struct drm_framebuffer *fb,
3615                                int color_plane,
3616                                unsigned int rotation)
3617 {
3618         int cpp = fb->format->cpp[color_plane];
3619
3620         switch (fb->modifier) {
3621         case DRM_FORMAT_MOD_LINEAR:
3622         case I915_FORMAT_MOD_X_TILED:
3623                 if (cpp == 8)
3624                         return 4096;
3625                 else
3626                         return 5120;
3627         case I915_FORMAT_MOD_Y_TILED_CCS:
3628         case I915_FORMAT_MOD_Yf_TILED_CCS:
3629                 /* FIXME AUX plane? */
3630         case I915_FORMAT_MOD_Y_TILED:
3631         case I915_FORMAT_MOD_Yf_TILED:
3632                 if (cpp == 8)
3633                         return 2048;
3634                 else
3635                         return 5120;
3636         default:
3637                 MISSING_CASE(fb->modifier);
3638                 return 2048;
3639         }
3640 }
3641
3642 static int icl_max_plane_width(const struct drm_framebuffer *fb,
3643                                int color_plane,
3644                                unsigned int rotation)
3645 {
3646         return 5120;
3647 }
3648
3649 static int skl_max_plane_height(void)
3650 {
3651         return 4096;
3652 }
3653
3654 static int icl_max_plane_height(void)
3655 {
3656         return 4320;
3657 }
3658
3659 static bool skl_check_main_ccs_coordinates(struct intel_plane_state *plane_state,
3660                                            int main_x, int main_y, u32 main_offset)
3661 {
3662         const struct drm_framebuffer *fb = plane_state->hw.fb;
3663         int ccs_plane = main_to_ccs_plane(fb, 0);
3664         int aux_x = plane_state->color_plane[ccs_plane].x;
3665         int aux_y = plane_state->color_plane[ccs_plane].y;
3666         u32 aux_offset = plane_state->color_plane[ccs_plane].offset;
3667         u32 alignment = intel_surf_alignment(fb, ccs_plane);
3668         int hsub;
3669         int vsub;
3670
3671         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, ccs_plane);
3672         while (aux_offset >= main_offset && aux_y <= main_y) {
3673                 int x, y;
3674
3675                 if (aux_x == main_x && aux_y == main_y)
3676                         break;
3677
3678                 if (aux_offset == 0)
3679                         break;
3680
3681                 x = aux_x / hsub;
3682                 y = aux_y / vsub;
3683                 aux_offset = intel_plane_adjust_aligned_offset(&x, &y,
3684                                                                plane_state,
3685                                                                ccs_plane,
3686                                                                aux_offset,
3687                                                                aux_offset -
3688                                                                 alignment);
3689                 aux_x = x * hsub + aux_x % hsub;
3690                 aux_y = y * vsub + aux_y % vsub;
3691         }
3692
3693         if (aux_x != main_x || aux_y != main_y)
3694                 return false;
3695
3696         plane_state->color_plane[ccs_plane].offset = aux_offset;
3697         plane_state->color_plane[ccs_plane].x = aux_x;
3698         plane_state->color_plane[ccs_plane].y = aux_y;
3699
3700         return true;
3701 }
3702
3703 static int skl_check_main_surface(struct intel_plane_state *plane_state)
3704 {
3705         struct drm_i915_private *dev_priv = to_i915(plane_state->uapi.plane->dev);
3706         const struct drm_framebuffer *fb = plane_state->hw.fb;
3707         unsigned int rotation = plane_state->hw.rotation;
3708         int x = plane_state->uapi.src.x1 >> 16;
3709         int y = plane_state->uapi.src.y1 >> 16;
3710         int w = drm_rect_width(&plane_state->uapi.src) >> 16;
3711         int h = drm_rect_height(&plane_state->uapi.src) >> 16;
3712         int max_width;
3713         int max_height;
3714         u32 alignment;
3715         u32 offset;
3716         int aux_plane = intel_main_to_aux_plane(fb, 0);
3717         u32 aux_offset = plane_state->color_plane[aux_plane].offset;
3718
3719         if (INTEL_GEN(dev_priv) >= 11)
3720                 max_width = icl_max_plane_width(fb, 0, rotation);
3721         else if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
3722                 max_width = glk_max_plane_width(fb, 0, rotation);
3723         else
3724                 max_width = skl_max_plane_width(fb, 0, rotation);
3725
3726         if (INTEL_GEN(dev_priv) >= 11)
3727                 max_height = icl_max_plane_height();
3728         else
3729                 max_height = skl_max_plane_height();
3730
3731         if (w > max_width || h > max_height) {
3732                 DRM_DEBUG_KMS("requested Y/RGB source size %dx%d too big (limit %dx%d)\n",
3733                               w, h, max_width, max_height);
3734                 return -EINVAL;
3735         }
3736
3737         intel_add_fb_offsets(&x, &y, plane_state, 0);
3738         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 0);
3739         alignment = intel_surf_alignment(fb, 0);
3740
3741         /*
3742          * AUX surface offset is specified as the distance from the
3743          * main surface offset, and it must be non-negative. Make
3744          * sure that is what we will get.
3745          */
3746         if (offset > aux_offset)
3747                 offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3748                                                            offset, aux_offset & ~(alignment - 1));
3749
3750         /*
3751          * When using an X-tiled surface, the plane blows up
3752          * if the x offset + width exceed the stride.
3753          *
3754          * TODO: linear and Y-tiled seem fine, Yf untested,
3755          */
3756         if (fb->modifier == I915_FORMAT_MOD_X_TILED) {
3757                 int cpp = fb->format->cpp[0];
3758
3759                 while ((x + w) * cpp > plane_state->color_plane[0].stride) {
3760                         if (offset == 0) {
3761                                 DRM_DEBUG_KMS("Unable to find suitable display surface offset due to X-tiling\n");
3762                                 return -EINVAL;
3763                         }
3764
3765                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3766                                                                    offset, offset - alignment);
3767                 }
3768         }
3769
3770         /*
3771          * CCS AUX surface doesn't have its own x/y offsets, we must make sure
3772          * they match with the main surface x/y offsets.
3773          */
3774         if (is_ccs_modifier(fb->modifier)) {
3775                 while (!skl_check_main_ccs_coordinates(plane_state, x, y, offset)) {
3776                         if (offset == 0)
3777                                 break;
3778
3779                         offset = intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
3780                                                                    offset, offset - alignment);
3781                 }
3782
3783                 if (x != plane_state->color_plane[aux_plane].x ||
3784                     y != plane_state->color_plane[aux_plane].y) {
3785                         DRM_DEBUG_KMS("Unable to find suitable display surface offset due to CCS\n");
3786                         return -EINVAL;
3787                 }
3788         }
3789
3790         plane_state->color_plane[0].offset = offset;
3791         plane_state->color_plane[0].x = x;
3792         plane_state->color_plane[0].y = y;
3793
3794         /*
3795          * Put the final coordinates back so that the src
3796          * coordinate checks will see the right values.
3797          */
3798         drm_rect_translate_to(&plane_state->uapi.src,
3799                               x << 16, y << 16);
3800
3801         return 0;
3802 }
3803
3804 static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
3805 {
3806         const struct drm_framebuffer *fb = plane_state->hw.fb;
3807         unsigned int rotation = plane_state->hw.rotation;
3808         int max_width = skl_max_plane_width(fb, 1, rotation);
3809         int max_height = 4096;
3810         int x = plane_state->uapi.src.x1 >> 17;
3811         int y = plane_state->uapi.src.y1 >> 17;
3812         int w = drm_rect_width(&plane_state->uapi.src) >> 17;
3813         int h = drm_rect_height(&plane_state->uapi.src) >> 17;
3814         u32 offset;
3815
3816         intel_add_fb_offsets(&x, &y, plane_state, 1);
3817         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3818
3819         /* FIXME not quite sure how/if these apply to the chroma plane */
3820         if (w > max_width || h > max_height) {
3821                 DRM_DEBUG_KMS("CbCr source size %dx%d too big (limit %dx%d)\n",
3822                               w, h, max_width, max_height);
3823                 return -EINVAL;
3824         }
3825
3826         plane_state->color_plane[1].offset = offset;
3827         plane_state->color_plane[1].x = x;
3828         plane_state->color_plane[1].y = y;
3829
3830         return 0;
3831 }
3832
3833 static int skl_check_ccs_aux_surface(struct intel_plane_state *plane_state)
3834 {
3835         const struct drm_framebuffer *fb = plane_state->hw.fb;
3836         int src_x = plane_state->uapi.src.x1 >> 16;
3837         int src_y = plane_state->uapi.src.y1 >> 16;
3838         int hsub;
3839         int vsub;
3840         int x;
3841         int y;
3842         u32 offset;
3843
3844         intel_fb_plane_get_subsampling(&hsub, &vsub, fb, 1);
3845         x = src_x / hsub;
3846         y = src_y / vsub;
3847         intel_add_fb_offsets(&x, &y, plane_state, 1);
3848         offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 1);
3849
3850         plane_state->color_plane[1].offset = offset;
3851         plane_state->color_plane[1].x = x * hsub + src_x % hsub;
3852         plane_state->color_plane[1].y = y * vsub + src_y % vsub;
3853
3854         return 0;
3855 }
3856
3857 int skl_check_plane_surface(struct intel_plane_state *plane_state)
3858 {
3859         const struct drm_framebuffer *fb = plane_state->hw.fb;
3860         int ret;
3861
3862         ret = intel_plane_compute_gtt(plane_state);
3863         if (ret)
3864                 return ret;
3865
3866         if (!plane_state->uapi.visible)
3867                 return 0;
3868
3869         /*
3870          * Handle the AUX surface first since
3871          * the main surface setup depends on it.
3872          */
3873         if (intel_format_info_is_yuv_semiplanar(fb->format,
3874                                                 fb->modifier)) {
3875                 ret = skl_check_nv12_aux_surface(plane_state);
3876                 if (ret)
3877                         return ret;
3878         } else if (is_ccs_modifier(fb->modifier)) {
3879                 ret = skl_check_ccs_aux_surface(plane_state);
3880                 if (ret)
3881                         return ret;
3882         } else {
3883                 plane_state->color_plane[1].offset = ~0xfff;
3884                 plane_state->color_plane[1].x = 0;
3885                 plane_state->color_plane[1].y = 0;
3886         }
3887
3888         ret = skl_check_main_surface(plane_state);
3889         if (ret)
3890                 return ret;
3891
3892         return 0;
3893 }
3894
3895 static void i9xx_plane_ratio(const struct intel_crtc_state *crtc_state,
3896                              const struct intel_plane_state *plane_state,
3897                              unsigned int *num, unsigned int *den)
3898 {
3899         const struct drm_framebuffer *fb = plane_state->hw.fb;
3900         unsigned int cpp = fb->format->cpp[0];
3901
3902         /*
3903          * g4x bspec says 64bpp pixel rate can't exceed 80%
3904          * of cdclk when the sprite plane is enabled on the
3905          * same pipe. ilk/snb bspec says 64bpp pixel rate is
3906          * never allowed to exceed 80% of cdclk. Let's just go
3907          * with the ilk/snb limit always.
3908          */
3909         if (cpp == 8) {
3910                 *num = 10;
3911                 *den = 8;
3912         } else {
3913                 *num = 1;
3914                 *den = 1;
3915         }
3916 }
3917
3918 static int i9xx_plane_min_cdclk(const struct intel_crtc_state *crtc_state,
3919                                 const struct intel_plane_state *plane_state)
3920 {
3921         unsigned int pixel_rate;
3922         unsigned int num, den;
3923
3924         /*
3925          * Note that crtc_state->pixel_rate accounts for both
3926          * horizontal and vertical panel fitter downscaling factors.
3927          * Pre-HSW bspec tells us to only consider the horizontal
3928          * downscaling factor here. We ignore that and just consider
3929          * both for simplicity.
3930          */
3931         pixel_rate = crtc_state->pixel_rate;
3932
3933         i9xx_plane_ratio(crtc_state, plane_state, &num, &den);
3934
3935         /* two pixels per clock with double wide pipe */
3936         if (crtc_state->double_wide)
3937                 den *= 2;
3938
3939         return DIV_ROUND_UP(pixel_rate * num, den);
3940 }
3941
3942 unsigned int
3943 i9xx_plane_max_stride(struct intel_plane *plane,
3944                       u32 pixel_format, u64 modifier,
3945                       unsigned int rotation)
3946 {
3947         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
3948
3949         if (!HAS_GMCH(dev_priv)) {
3950                 return 32*1024;
3951         } else if (INTEL_GEN(dev_priv) >= 4) {
3952                 if (modifier == I915_FORMAT_MOD_X_TILED)
3953                         return 16*1024;
3954                 else
3955                         return 32*1024;
3956         } else if (INTEL_GEN(dev_priv) >= 3) {
3957                 if (modifier == I915_FORMAT_MOD_X_TILED)
3958                         return 8*1024;
3959                 else
3960                         return 16*1024;
3961         } else {
3962                 if (plane->i9xx_plane == PLANE_C)
3963                         return 4*1024;
3964                 else
3965                         return 8*1024;
3966         }
3967 }
3968
3969 static u32 i9xx_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
3970 {
3971         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3972         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3973         u32 dspcntr = 0;
3974
3975         if (crtc_state->gamma_enable)
3976                 dspcntr |= DISPPLANE_GAMMA_ENABLE;
3977
3978         if (crtc_state->csc_enable)
3979                 dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
3980
3981         if (INTEL_GEN(dev_priv) < 5)
3982                 dspcntr |= DISPPLANE_SEL_PIPE(crtc->pipe);
3983
3984         return dspcntr;
3985 }
3986
3987 static u32 i9xx_plane_ctl(const struct intel_crtc_state *crtc_state,
3988                           const struct intel_plane_state *plane_state)
3989 {
3990         struct drm_i915_private *dev_priv =
3991                 to_i915(plane_state->uapi.plane->dev);
3992         const struct drm_framebuffer *fb = plane_state->hw.fb;
3993         unsigned int rotation = plane_state->hw.rotation;
3994         u32 dspcntr;
3995
3996         dspcntr = DISPLAY_PLANE_ENABLE;
3997
3998         if (IS_G4X(dev_priv) || IS_GEN(dev_priv, 5) ||
3999             IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
4000                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
4001
4002         switch (fb->format->format) {
4003         case DRM_FORMAT_C8:
4004                 dspcntr |= DISPPLANE_8BPP;
4005                 break;
4006         case DRM_FORMAT_XRGB1555:
4007                 dspcntr |= DISPPLANE_BGRX555;
4008                 break;
4009         case DRM_FORMAT_ARGB1555:
4010                 dspcntr |= DISPPLANE_BGRA555;
4011                 break;
4012         case DRM_FORMAT_RGB565:
4013                 dspcntr |= DISPPLANE_BGRX565;
4014                 break;
4015         case DRM_FORMAT_XRGB8888:
4016                 dspcntr |= DISPPLANE_BGRX888;
4017                 break;
4018         case DRM_FORMAT_XBGR8888:
4019                 dspcntr |= DISPPLANE_RGBX888;
4020                 break;
4021         case DRM_FORMAT_ARGB8888:
4022                 dspcntr |= DISPPLANE_BGRA888;
4023                 break;
4024         case DRM_FORMAT_ABGR8888:
4025                 dspcntr |= DISPPLANE_RGBA888;
4026                 break;
4027         case DRM_FORMAT_XRGB2101010:
4028                 dspcntr |= DISPPLANE_BGRX101010;
4029                 break;
4030         case DRM_FORMAT_XBGR2101010:
4031                 dspcntr |= DISPPLANE_RGBX101010;
4032                 break;
4033         case DRM_FORMAT_ARGB2101010:
4034                 dspcntr |= DISPPLANE_BGRA101010;
4035                 break;
4036         case DRM_FORMAT_ABGR2101010:
4037                 dspcntr |= DISPPLANE_RGBA101010;
4038                 break;
4039         case DRM_FORMAT_XBGR16161616F:
4040                 dspcntr |= DISPPLANE_RGBX161616;
4041                 break;
4042         default:
4043                 MISSING_CASE(fb->format->format);
4044                 return 0;
4045         }
4046
4047         if (INTEL_GEN(dev_priv) >= 4 &&
4048             fb->modifier == I915_FORMAT_MOD_X_TILED)
4049                 dspcntr |= DISPPLANE_TILED;
4050
4051         if (rotation & DRM_MODE_ROTATE_180)
4052                 dspcntr |= DISPPLANE_ROTATE_180;
4053
4054         if (rotation & DRM_MODE_REFLECT_X)
4055                 dspcntr |= DISPPLANE_MIRROR;
4056
4057         return dspcntr;
4058 }
4059
4060 int i9xx_check_plane_surface(struct intel_plane_state *plane_state)
4061 {
4062         struct drm_i915_private *dev_priv =
4063                 to_i915(plane_state->uapi.plane->dev);
4064         const struct drm_framebuffer *fb = plane_state->hw.fb;
4065         int src_x, src_y, src_w;
4066         u32 offset;
4067         int ret;
4068
4069         ret = intel_plane_compute_gtt(plane_state);
4070         if (ret)
4071                 return ret;
4072
4073         if (!plane_state->uapi.visible)
4074                 return 0;
4075
4076         src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4077         src_x = plane_state->uapi.src.x1 >> 16;
4078         src_y = plane_state->uapi.src.y1 >> 16;
4079
4080         /* Undocumented hardware limit on i965/g4x/vlv/chv */
4081         if (HAS_GMCH(dev_priv) && fb->format->cpp[0] == 8 && src_w > 2048)
4082                 return -EINVAL;
4083
4084         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
4085
4086         if (INTEL_GEN(dev_priv) >= 4)
4087                 offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
4088                                                             plane_state, 0);
4089         else
4090                 offset = 0;
4091
4092         /*
4093          * Put the final coordinates back so that the src
4094          * coordinate checks will see the right values.
4095          */
4096         drm_rect_translate_to(&plane_state->uapi.src,
4097                               src_x << 16, src_y << 16);
4098
4099         /* HSW/BDW do this automagically in hardware */
4100         if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) {
4101                 unsigned int rotation = plane_state->hw.rotation;
4102                 int src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4103                 int src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
4104
4105                 if (rotation & DRM_MODE_ROTATE_180) {
4106                         src_x += src_w - 1;
4107                         src_y += src_h - 1;
4108                 } else if (rotation & DRM_MODE_REFLECT_X) {
4109                         src_x += src_w - 1;
4110                 }
4111         }
4112
4113         plane_state->color_plane[0].offset = offset;
4114         plane_state->color_plane[0].x = src_x;
4115         plane_state->color_plane[0].y = src_y;
4116
4117         return 0;
4118 }
4119
4120 static bool i9xx_plane_has_windowing(struct intel_plane *plane)
4121 {
4122         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4123         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4124
4125         if (IS_CHERRYVIEW(dev_priv))
4126                 return i9xx_plane == PLANE_B;
4127         else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
4128                 return false;
4129         else if (IS_GEN(dev_priv, 4))
4130                 return i9xx_plane == PLANE_C;
4131         else
4132                 return i9xx_plane == PLANE_B ||
4133                         i9xx_plane == PLANE_C;
4134 }
4135
4136 static int
4137 i9xx_plane_check(struct intel_crtc_state *crtc_state,
4138                  struct intel_plane_state *plane_state)
4139 {
4140         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4141         int ret;
4142
4143         ret = chv_plane_check_rotation(plane_state);
4144         if (ret)
4145                 return ret;
4146
4147         ret = drm_atomic_helper_check_plane_state(&plane_state->uapi,
4148                                                   &crtc_state->uapi,
4149                                                   DRM_PLANE_HELPER_NO_SCALING,
4150                                                   DRM_PLANE_HELPER_NO_SCALING,
4151                                                   i9xx_plane_has_windowing(plane),
4152                                                   true);
4153         if (ret)
4154                 return ret;
4155
4156         ret = i9xx_check_plane_surface(plane_state);
4157         if (ret)
4158                 return ret;
4159
4160         if (!plane_state->uapi.visible)
4161                 return 0;
4162
4163         ret = intel_plane_check_src_coordinates(plane_state);
4164         if (ret)
4165                 return ret;
4166
4167         plane_state->ctl = i9xx_plane_ctl(crtc_state, plane_state);
4168
4169         return 0;
4170 }
4171
4172 static void i9xx_update_plane(struct intel_plane *plane,
4173                               const struct intel_crtc_state *crtc_state,
4174                               const struct intel_plane_state *plane_state)
4175 {
4176         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4177         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4178         u32 linear_offset;
4179         int x = plane_state->color_plane[0].x;
4180         int y = plane_state->color_plane[0].y;
4181         int crtc_x = plane_state->uapi.dst.x1;
4182         int crtc_y = plane_state->uapi.dst.y1;
4183         int crtc_w = drm_rect_width(&plane_state->uapi.dst);
4184         int crtc_h = drm_rect_height(&plane_state->uapi.dst);
4185         unsigned long irqflags;
4186         u32 dspaddr_offset;
4187         u32 dspcntr;
4188
4189         dspcntr = plane_state->ctl | i9xx_plane_ctl_crtc(crtc_state);
4190
4191         linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
4192
4193         if (INTEL_GEN(dev_priv) >= 4)
4194                 dspaddr_offset = plane_state->color_plane[0].offset;
4195         else
4196                 dspaddr_offset = linear_offset;
4197
4198         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
4199
4200         I915_WRITE_FW(DSPSTRIDE(i9xx_plane), plane_state->color_plane[0].stride);
4201
4202         if (INTEL_GEN(dev_priv) < 4) {
4203                 /*
4204                  * PLANE_A doesn't actually have a full window
4205                  * generator but let's assume we still need to
4206                  * program whatever is there.
4207                  */
4208                 I915_WRITE_FW(DSPPOS(i9xx_plane), (crtc_y << 16) | crtc_x);
4209                 I915_WRITE_FW(DSPSIZE(i9xx_plane),
4210                               ((crtc_h - 1) << 16) | (crtc_w - 1));
4211         } else if (IS_CHERRYVIEW(dev_priv) && i9xx_plane == PLANE_B) {
4212                 I915_WRITE_FW(PRIMPOS(i9xx_plane), (crtc_y << 16) | crtc_x);
4213                 I915_WRITE_FW(PRIMSIZE(i9xx_plane),
4214                               ((crtc_h - 1) << 16) | (crtc_w - 1));
4215                 I915_WRITE_FW(PRIMCNSTALPHA(i9xx_plane), 0);
4216         }
4217
4218         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
4219                 I915_WRITE_FW(DSPOFFSET(i9xx_plane), (y << 16) | x);
4220         } else if (INTEL_GEN(dev_priv) >= 4) {
4221                 I915_WRITE_FW(DSPLINOFF(i9xx_plane), linear_offset);
4222                 I915_WRITE_FW(DSPTILEOFF(i9xx_plane), (y << 16) | x);
4223         }
4224
4225         /*
4226          * The control register self-arms if the plane was previously
4227          * disabled. Try to make the plane enable atomic by writing
4228          * the control register just before the surface register.
4229          */
4230         I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr);
4231         if (INTEL_GEN(dev_priv) >= 4)
4232                 I915_WRITE_FW(DSPSURF(i9xx_plane),
4233                               intel_plane_ggtt_offset(plane_state) +
4234                               dspaddr_offset);
4235         else
4236                 I915_WRITE_FW(DSPADDR(i9xx_plane),
4237                               intel_plane_ggtt_offset(plane_state) +
4238                               dspaddr_offset);
4239
4240         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
4241 }
4242
4243 static void i9xx_disable_plane(struct intel_plane *plane,
4244                                const struct intel_crtc_state *crtc_state)
4245 {
4246         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4247         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4248         unsigned long irqflags;
4249         u32 dspcntr;
4250
4251         /*
4252          * DSPCNTR pipe gamma enable on g4x+ and pipe csc
4253          * enable on ilk+ affect the pipe bottom color as
4254          * well, so we must configure them even if the plane
4255          * is disabled.
4256          *
4257          * On pre-g4x there is no way to gamma correct the
4258          * pipe bottom color but we'll keep on doing this
4259          * anyway so that the crtc state readout works correctly.
4260          */
4261         dspcntr = i9xx_plane_ctl_crtc(crtc_state);
4262
4263         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
4264
4265         I915_WRITE_FW(DSPCNTR(i9xx_plane), dspcntr);
4266         if (INTEL_GEN(dev_priv) >= 4)
4267                 I915_WRITE_FW(DSPSURF(i9xx_plane), 0);
4268         else
4269                 I915_WRITE_FW(DSPADDR(i9xx_plane), 0);
4270
4271         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
4272 }
4273
4274 static bool i9xx_plane_get_hw_state(struct intel_plane *plane,
4275                                     enum pipe *pipe)
4276 {
4277         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
4278         enum intel_display_power_domain power_domain;
4279         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
4280         intel_wakeref_t wakeref;
4281         bool ret;
4282         u32 val;
4283
4284         /*
4285          * Not 100% correct for planes that can move between pipes,
4286          * but that's only the case for gen2-4 which don't have any
4287          * display power wells.
4288          */
4289         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
4290         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4291         if (!wakeref)
4292                 return false;
4293
4294         val = I915_READ(DSPCNTR(i9xx_plane));
4295
4296         ret = val & DISPLAY_PLANE_ENABLE;
4297
4298         if (INTEL_GEN(dev_priv) >= 5)
4299                 *pipe = plane->pipe;
4300         else
4301                 *pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
4302                         DISPPLANE_SEL_PIPE_SHIFT;
4303
4304         intel_display_power_put(dev_priv, power_domain, wakeref);
4305
4306         return ret;
4307 }
4308
4309 static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
4310 {
4311         struct drm_device *dev = intel_crtc->base.dev;
4312         struct drm_i915_private *dev_priv = to_i915(dev);
4313
4314         I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, id), 0);
4315         I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, id), 0);
4316         I915_WRITE(SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0);
4317 }
4318
4319 /*
4320  * This function detaches (aka. unbinds) unused scalers in hardware
4321  */
4322 static void skl_detach_scalers(const struct intel_crtc_state *crtc_state)
4323 {
4324         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->uapi.crtc);
4325         const struct intel_crtc_scaler_state *scaler_state =
4326                 &crtc_state->scaler_state;
4327         int i;
4328
4329         /* loop through and disable scalers that aren't in use */
4330         for (i = 0; i < intel_crtc->num_scalers; i++) {
4331                 if (!scaler_state->scalers[i].in_use)
4332                         skl_detach_scaler(intel_crtc, i);
4333         }
4334 }
4335
4336 static unsigned int skl_plane_stride_mult(const struct drm_framebuffer *fb,
4337                                           int color_plane, unsigned int rotation)
4338 {
4339         /*
4340          * The stride is either expressed as a multiple of 64 bytes chunks for
4341          * linear buffers or in number of tiles for tiled buffers.
4342          */
4343         if (is_surface_linear(fb, color_plane))
4344                 return 64;
4345         else if (drm_rotation_90_or_270(rotation))
4346                 return intel_tile_height(fb, color_plane);
4347         else
4348                 return intel_tile_width_bytes(fb, color_plane);
4349 }
4350
4351 u32 skl_plane_stride(const struct intel_plane_state *plane_state,
4352                      int color_plane)
4353 {
4354         const struct drm_framebuffer *fb = plane_state->hw.fb;
4355         unsigned int rotation = plane_state->hw.rotation;
4356         u32 stride = plane_state->color_plane[color_plane].stride;
4357
4358         if (color_plane >= fb->format->num_planes)
4359                 return 0;
4360
4361         return stride / skl_plane_stride_mult(fb, color_plane, rotation);
4362 }
4363
4364 static u32 skl_plane_ctl_format(u32 pixel_format)
4365 {
4366         switch (pixel_format) {
4367         case DRM_FORMAT_C8:
4368                 return PLANE_CTL_FORMAT_INDEXED;
4369         case DRM_FORMAT_RGB565:
4370                 return PLANE_CTL_FORMAT_RGB_565;
4371         case DRM_FORMAT_XBGR8888:
4372         case DRM_FORMAT_ABGR8888:
4373                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX;
4374         case DRM_FORMAT_XRGB8888:
4375         case DRM_FORMAT_ARGB8888:
4376                 return PLANE_CTL_FORMAT_XRGB_8888;
4377         case DRM_FORMAT_XBGR2101010:
4378         case DRM_FORMAT_ABGR2101010:
4379                 return PLANE_CTL_FORMAT_XRGB_2101010 | PLANE_CTL_ORDER_RGBX;
4380         case DRM_FORMAT_XRGB2101010:
4381         case DRM_FORMAT_ARGB2101010:
4382                 return PLANE_CTL_FORMAT_XRGB_2101010;
4383         case DRM_FORMAT_XBGR16161616F:
4384         case DRM_FORMAT_ABGR16161616F:
4385                 return PLANE_CTL_FORMAT_XRGB_16161616F | PLANE_CTL_ORDER_RGBX;
4386         case DRM_FORMAT_XRGB16161616F:
4387         case DRM_FORMAT_ARGB16161616F:
4388                 return PLANE_CTL_FORMAT_XRGB_16161616F;
4389         case DRM_FORMAT_YUYV:
4390                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV;
4391         case DRM_FORMAT_YVYU:
4392                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YVYU;
4393         case DRM_FORMAT_UYVY:
4394                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_UYVY;
4395         case DRM_FORMAT_VYUY:
4396                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY;
4397         case DRM_FORMAT_NV12:
4398                 return PLANE_CTL_FORMAT_NV12;
4399         case DRM_FORMAT_P010:
4400                 return PLANE_CTL_FORMAT_P010;
4401         case DRM_FORMAT_P012:
4402                 return PLANE_CTL_FORMAT_P012;
4403         case DRM_FORMAT_P016:
4404                 return PLANE_CTL_FORMAT_P016;
4405         case DRM_FORMAT_Y210:
4406                 return PLANE_CTL_FORMAT_Y210;
4407         case DRM_FORMAT_Y212:
4408                 return PLANE_CTL_FORMAT_Y212;
4409         case DRM_FORMAT_Y216:
4410                 return PLANE_CTL_FORMAT_Y216;
4411         case DRM_FORMAT_XVYU2101010:
4412                 return PLANE_CTL_FORMAT_Y410;
4413         case DRM_FORMAT_XVYU12_16161616:
4414                 return PLANE_CTL_FORMAT_Y412;
4415         case DRM_FORMAT_XVYU16161616:
4416                 return PLANE_CTL_FORMAT_Y416;
4417         default:
4418                 MISSING_CASE(pixel_format);
4419         }
4420
4421         return 0;
4422 }
4423
4424 static u32 skl_plane_ctl_alpha(const struct intel_plane_state *plane_state)
4425 {
4426         if (!plane_state->hw.fb->format->has_alpha)
4427                 return PLANE_CTL_ALPHA_DISABLE;
4428
4429         switch (plane_state->hw.pixel_blend_mode) {
4430         case DRM_MODE_BLEND_PIXEL_NONE:
4431                 return PLANE_CTL_ALPHA_DISABLE;
4432         case DRM_MODE_BLEND_PREMULTI:
4433                 return PLANE_CTL_ALPHA_SW_PREMULTIPLY;
4434         case DRM_MODE_BLEND_COVERAGE:
4435                 return PLANE_CTL_ALPHA_HW_PREMULTIPLY;
4436         default:
4437                 MISSING_CASE(plane_state->hw.pixel_blend_mode);
4438                 return PLANE_CTL_ALPHA_DISABLE;
4439         }
4440 }
4441
4442 static u32 glk_plane_color_ctl_alpha(const struct intel_plane_state *plane_state)
4443 {
4444         if (!plane_state->hw.fb->format->has_alpha)
4445                 return PLANE_COLOR_ALPHA_DISABLE;
4446
4447         switch (plane_state->hw.pixel_blend_mode) {
4448         case DRM_MODE_BLEND_PIXEL_NONE:
4449                 return PLANE_COLOR_ALPHA_DISABLE;
4450         case DRM_MODE_BLEND_PREMULTI:
4451                 return PLANE_COLOR_ALPHA_SW_PREMULTIPLY;
4452         case DRM_MODE_BLEND_COVERAGE:
4453                 return PLANE_COLOR_ALPHA_HW_PREMULTIPLY;
4454         default:
4455                 MISSING_CASE(plane_state->hw.pixel_blend_mode);
4456                 return PLANE_COLOR_ALPHA_DISABLE;
4457         }
4458 }
4459
4460 static u32 skl_plane_ctl_tiling(u64 fb_modifier)
4461 {
4462         switch (fb_modifier) {
4463         case DRM_FORMAT_MOD_LINEAR:
4464                 break;
4465         case I915_FORMAT_MOD_X_TILED:
4466                 return PLANE_CTL_TILED_X;
4467         case I915_FORMAT_MOD_Y_TILED:
4468                 return PLANE_CTL_TILED_Y;
4469         case I915_FORMAT_MOD_Y_TILED_CCS:
4470                 return PLANE_CTL_TILED_Y | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
4471         case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS:
4472                 return PLANE_CTL_TILED_Y |
4473                        PLANE_CTL_RENDER_DECOMPRESSION_ENABLE |
4474                        PLANE_CTL_CLEAR_COLOR_DISABLE;
4475         case I915_FORMAT_MOD_Yf_TILED:
4476                 return PLANE_CTL_TILED_YF;
4477         case I915_FORMAT_MOD_Yf_TILED_CCS:
4478                 return PLANE_CTL_TILED_YF | PLANE_CTL_RENDER_DECOMPRESSION_ENABLE;
4479         default:
4480                 MISSING_CASE(fb_modifier);
4481         }
4482
4483         return 0;
4484 }
4485
4486 static u32 skl_plane_ctl_rotate(unsigned int rotate)
4487 {
4488         switch (rotate) {
4489         case DRM_MODE_ROTATE_0:
4490                 break;
4491         /*
4492          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
4493          * while i915 HW rotation is clockwise, thats why this swapping.
4494          */
4495         case DRM_MODE_ROTATE_90:
4496                 return PLANE_CTL_ROTATE_270;
4497         case DRM_MODE_ROTATE_180:
4498                 return PLANE_CTL_ROTATE_180;
4499         case DRM_MODE_ROTATE_270:
4500                 return PLANE_CTL_ROTATE_90;
4501         default:
4502                 MISSING_CASE(rotate);
4503         }
4504
4505         return 0;
4506 }
4507
4508 static u32 cnl_plane_ctl_flip(unsigned int reflect)
4509 {
4510         switch (reflect) {
4511         case 0:
4512                 break;
4513         case DRM_MODE_REFLECT_X:
4514                 return PLANE_CTL_FLIP_HORIZONTAL;
4515         case DRM_MODE_REFLECT_Y:
4516         default:
4517                 MISSING_CASE(reflect);
4518         }
4519
4520         return 0;
4521 }
4522
4523 u32 skl_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
4524 {
4525         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4526         u32 plane_ctl = 0;
4527
4528         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
4529                 return plane_ctl;
4530
4531         if (crtc_state->gamma_enable)
4532                 plane_ctl |= PLANE_CTL_PIPE_GAMMA_ENABLE;
4533
4534         if (crtc_state->csc_enable)
4535                 plane_ctl |= PLANE_CTL_PIPE_CSC_ENABLE;
4536
4537         return plane_ctl;
4538 }
4539
4540 u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
4541                   const struct intel_plane_state *plane_state)
4542 {
4543         struct drm_i915_private *dev_priv =
4544                 to_i915(plane_state->uapi.plane->dev);
4545         const struct drm_framebuffer *fb = plane_state->hw.fb;
4546         unsigned int rotation = plane_state->hw.rotation;
4547         const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
4548         u32 plane_ctl;
4549
4550         plane_ctl = PLANE_CTL_ENABLE;
4551
4552         if (INTEL_GEN(dev_priv) < 10 && !IS_GEMINILAKE(dev_priv)) {
4553                 plane_ctl |= skl_plane_ctl_alpha(plane_state);
4554                 plane_ctl |= PLANE_CTL_PLANE_GAMMA_DISABLE;
4555
4556                 if (plane_state->hw.color_encoding == DRM_COLOR_YCBCR_BT709)
4557                         plane_ctl |= PLANE_CTL_YUV_TO_RGB_CSC_FORMAT_BT709;
4558
4559                 if (plane_state->hw.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
4560                         plane_ctl |= PLANE_CTL_YUV_RANGE_CORRECTION_DISABLE;
4561         }
4562
4563         plane_ctl |= skl_plane_ctl_format(fb->format->format);
4564         plane_ctl |= skl_plane_ctl_tiling(fb->modifier);
4565         plane_ctl |= skl_plane_ctl_rotate(rotation & DRM_MODE_ROTATE_MASK);
4566
4567         if (INTEL_GEN(dev_priv) >= 10)
4568                 plane_ctl |= cnl_plane_ctl_flip(rotation &
4569                                                 DRM_MODE_REFLECT_MASK);
4570
4571         if (key->flags & I915_SET_COLORKEY_DESTINATION)
4572                 plane_ctl |= PLANE_CTL_KEY_ENABLE_DESTINATION;
4573         else if (key->flags & I915_SET_COLORKEY_SOURCE)
4574                 plane_ctl |= PLANE_CTL_KEY_ENABLE_SOURCE;
4575
4576         return plane_ctl;
4577 }
4578
4579 u32 glk_plane_color_ctl_crtc(const struct intel_crtc_state *crtc_state)
4580 {
4581         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4582         u32 plane_color_ctl = 0;
4583
4584         if (INTEL_GEN(dev_priv) >= 11)
4585                 return plane_color_ctl;
4586
4587         if (crtc_state->gamma_enable)
4588                 plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE;
4589
4590         if (crtc_state->csc_enable)
4591                 plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE;
4592
4593         return plane_color_ctl;
4594 }
4595
4596 u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
4597                         const struct intel_plane_state *plane_state)
4598 {
4599         struct drm_i915_private *dev_priv =
4600                 to_i915(plane_state->uapi.plane->dev);
4601         const struct drm_framebuffer *fb = plane_state->hw.fb;
4602         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4603         u32 plane_color_ctl = 0;
4604
4605         plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE;
4606         plane_color_ctl |= glk_plane_color_ctl_alpha(plane_state);
4607
4608         if (fb->format->is_yuv && !icl_is_hdr_plane(dev_priv, plane->id)) {
4609                 if (plane_state->hw.color_encoding == DRM_COLOR_YCBCR_BT709)
4610                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709;
4611                 else
4612                         plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV601_TO_RGB709;
4613
4614                 if (plane_state->hw.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
4615                         plane_color_ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE;
4616         } else if (fb->format->is_yuv) {
4617                 plane_color_ctl |= PLANE_COLOR_INPUT_CSC_ENABLE;
4618         }
4619
4620         return plane_color_ctl;
4621 }
4622
4623 static int
4624 __intel_display_resume(struct drm_device *dev,
4625                        struct drm_atomic_state *state,
4626                        struct drm_modeset_acquire_ctx *ctx)
4627 {
4628         struct drm_crtc_state *crtc_state;
4629         struct drm_crtc *crtc;
4630         int i, ret;
4631
4632         intel_modeset_setup_hw_state(dev, ctx);
4633         intel_vga_redisable(to_i915(dev));
4634
4635         if (!state)
4636                 return 0;
4637
4638         /*
4639          * We've duplicated the state, pointers to the old state are invalid.
4640          *
4641          * Don't attempt to use the old state until we commit the duplicated state.
4642          */
4643         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
4644                 /*
4645                  * Force recalculation even if we restore
4646                  * current state. With fast modeset this may not result
4647                  * in a modeset when the state is compatible.
4648                  */
4649                 crtc_state->mode_changed = true;
4650         }
4651
4652         /* ignore any reset values/BIOS leftovers in the WM registers */
4653         if (!HAS_GMCH(to_i915(dev)))
4654                 to_intel_atomic_state(state)->skip_intermediate_wm = true;
4655
4656         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
4657
4658         WARN_ON(ret == -EDEADLK);
4659         return ret;
4660 }
4661
4662 static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv)
4663 {
4664         return (INTEL_INFO(dev_priv)->gpu_reset_clobbers_display &&
4665                 intel_has_gpu_reset(&dev_priv->gt));
4666 }
4667
4668 void intel_prepare_reset(struct drm_i915_private *dev_priv)
4669 {
4670         struct drm_device *dev = &dev_priv->drm;
4671         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
4672         struct drm_atomic_state *state;
4673         int ret;
4674
4675         /* reset doesn't touch the display */
4676         if (!i915_modparams.force_reset_modeset_test &&
4677             !gpu_reset_clobbers_display(dev_priv))
4678                 return;
4679
4680         /* We have a modeset vs reset deadlock, defensively unbreak it. */
4681         set_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags);
4682         smp_mb__after_atomic();
4683         wake_up_bit(&dev_priv->gt.reset.flags, I915_RESET_MODESET);
4684
4685         if (atomic_read(&dev_priv->gpu_error.pending_fb_pin)) {
4686                 DRM_DEBUG_KMS("Modeset potentially stuck, unbreaking through wedging\n");
4687                 intel_gt_set_wedged(&dev_priv->gt);
4688         }
4689
4690         /*
4691          * Need mode_config.mutex so that we don't
4692          * trample ongoing ->detect() and whatnot.
4693          */
4694         mutex_lock(&dev->mode_config.mutex);
4695         drm_modeset_acquire_init(ctx, 0);
4696         while (1) {
4697                 ret = drm_modeset_lock_all_ctx(dev, ctx);
4698                 if (ret != -EDEADLK)
4699                         break;
4700
4701                 drm_modeset_backoff(ctx);
4702         }
4703         /*
4704          * Disabling the crtcs gracefully seems nicer. Also the
4705          * g33 docs say we should at least disable all the planes.
4706          */
4707         state = drm_atomic_helper_duplicate_state(dev, ctx);
4708         if (IS_ERR(state)) {
4709                 ret = PTR_ERR(state);
4710                 DRM_ERROR("Duplicating state failed with %i\n", ret);
4711                 return;
4712         }
4713
4714         ret = drm_atomic_helper_disable_all(dev, ctx);
4715         if (ret) {
4716                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
4717                 drm_atomic_state_put(state);
4718                 return;
4719         }
4720
4721         dev_priv->modeset_restore_state = state;
4722         state->acquire_ctx = ctx;
4723 }
4724
4725 void intel_finish_reset(struct drm_i915_private *dev_priv)
4726 {
4727         struct drm_device *dev = &dev_priv->drm;
4728         struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx;
4729         struct drm_atomic_state *state;
4730         int ret;
4731
4732         /* reset doesn't touch the display */
4733         if (!test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags))
4734                 return;
4735
4736         state = fetch_and_zero(&dev_priv->modeset_restore_state);
4737         if (!state)
4738                 goto unlock;
4739
4740         /* reset doesn't touch the display */
4741         if (!gpu_reset_clobbers_display(dev_priv)) {
4742                 /* for testing only restore the display */
4743                 ret = __intel_display_resume(dev, state, ctx);
4744                 if (ret)
4745                         DRM_ERROR("Restoring old state failed with %i\n", ret);
4746         } else {
4747                 /*
4748                  * The display has been reset as well,
4749                  * so need a full re-initialization.
4750                  */
4751                 intel_pps_unlock_regs_wa(dev_priv);
4752                 intel_modeset_init_hw(dev_priv);
4753                 intel_init_clock_gating(dev_priv);
4754
4755                 spin_lock_irq(&dev_priv->irq_lock);
4756                 if (dev_priv->display.hpd_irq_setup)
4757                         dev_priv->display.hpd_irq_setup(dev_priv);
4758                 spin_unlock_irq(&dev_priv->irq_lock);
4759
4760                 ret = __intel_display_resume(dev, state, ctx);
4761                 if (ret)
4762                         DRM_ERROR("Restoring old state failed with %i\n", ret);
4763
4764                 intel_hpd_init(dev_priv);
4765         }
4766
4767         drm_atomic_state_put(state);
4768 unlock:
4769         drm_modeset_drop_locks(ctx);
4770         drm_modeset_acquire_fini(ctx);
4771         mutex_unlock(&dev->mode_config.mutex);
4772
4773         clear_bit_unlock(I915_RESET_MODESET, &dev_priv->gt.reset.flags);
4774 }
4775
4776 static void icl_set_pipe_chicken(struct intel_crtc *crtc)
4777 {
4778         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4779         enum pipe pipe = crtc->pipe;
4780         u32 tmp;
4781
4782         tmp = I915_READ(PIPE_CHICKEN(pipe));
4783
4784         /*
4785          * Display WA #1153: icl
4786          * enable hardware to bypass the alpha math
4787          * and rounding for per-pixel values 00 and 0xff
4788          */
4789         tmp |= PER_PIXEL_ALPHA_BYPASS_EN;
4790         /*
4791          * Display WA # 1605353570: icl
4792          * Set the pixel rounding bit to 1 for allowing
4793          * passthrough of Frame buffer pixels unmodified
4794          * across pipe
4795          */
4796         tmp |= PIXEL_ROUNDING_TRUNC_FB_PASSTHRU;
4797         I915_WRITE(PIPE_CHICKEN(pipe), tmp);
4798 }
4799
4800 static void icl_enable_trans_port_sync(const struct intel_crtc_state *crtc_state)
4801 {
4802         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4803         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4804         u32 trans_ddi_func_ctl2_val;
4805         u8 master_select;
4806
4807         /*
4808          * Configure the master select and enable Transcoder Port Sync for
4809          * Slave CRTCs transcoder.
4810          */
4811         if (crtc_state->master_transcoder == INVALID_TRANSCODER)
4812                 return;
4813
4814         if (crtc_state->master_transcoder == TRANSCODER_EDP)
4815                 master_select = 0;
4816         else
4817                 master_select = crtc_state->master_transcoder + 1;
4818
4819         /* Set the master select bits for Tranascoder Port Sync */
4820         trans_ddi_func_ctl2_val = (PORT_SYNC_MODE_MASTER_SELECT(master_select) &
4821                                    PORT_SYNC_MODE_MASTER_SELECT_MASK) <<
4822                 PORT_SYNC_MODE_MASTER_SELECT_SHIFT;
4823         /* Enable Transcoder Port Sync */
4824         trans_ddi_func_ctl2_val |= PORT_SYNC_MODE_ENABLE;
4825
4826         I915_WRITE(TRANS_DDI_FUNC_CTL2(crtc_state->cpu_transcoder),
4827                    trans_ddi_func_ctl2_val);
4828 }
4829
4830 static void intel_fdi_normal_train(struct intel_crtc *crtc)
4831 {
4832         struct drm_device *dev = crtc->base.dev;
4833         struct drm_i915_private *dev_priv = to_i915(dev);
4834         enum pipe pipe = crtc->pipe;
4835         i915_reg_t reg;
4836         u32 temp;
4837
4838         /* enable normal train */
4839         reg = FDI_TX_CTL(pipe);
4840         temp = I915_READ(reg);
4841         if (IS_IVYBRIDGE(dev_priv)) {
4842                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
4843                 temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
4844         } else {
4845                 temp &= ~FDI_LINK_TRAIN_NONE;
4846                 temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
4847         }
4848         I915_WRITE(reg, temp);
4849
4850         reg = FDI_RX_CTL(pipe);
4851         temp = I915_READ(reg);
4852         if (HAS_PCH_CPT(dev_priv)) {
4853                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
4854                 temp |= FDI_LINK_TRAIN_NORMAL_CPT;
4855         } else {
4856                 temp &= ~FDI_LINK_TRAIN_NONE;
4857                 temp |= FDI_LINK_TRAIN_NONE;
4858         }
4859         I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
4860
4861         /* wait one idle pattern time */
4862         POSTING_READ(reg);
4863         udelay(1000);
4864
4865         /* IVB wants error correction enabled */
4866         if (IS_IVYBRIDGE(dev_priv))
4867                 I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
4868                            FDI_FE_ERRC_ENABLE);
4869 }
4870
4871 /* The FDI link training functions for ILK/Ibexpeak. */
4872 static void ironlake_fdi_link_train(struct intel_crtc *crtc,
4873                                     const struct intel_crtc_state *crtc_state)
4874 {
4875         struct drm_device *dev = crtc->base.dev;
4876         struct drm_i915_private *dev_priv = to_i915(dev);
4877         enum pipe pipe = crtc->pipe;
4878         i915_reg_t reg;
4879         u32 temp, tries;
4880
4881         /* FDI needs bits from pipe first */
4882         assert_pipe_enabled(dev_priv, crtc_state->cpu_transcoder);
4883
4884         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4885            for train result */
4886         reg = FDI_RX_IMR(pipe);
4887         temp = I915_READ(reg);
4888         temp &= ~FDI_RX_SYMBOL_LOCK;
4889         temp &= ~FDI_RX_BIT_LOCK;
4890         I915_WRITE(reg, temp);
4891         I915_READ(reg);
4892         udelay(150);
4893
4894         /* enable CPU FDI TX and PCH FDI RX */
4895         reg = FDI_TX_CTL(pipe);
4896         temp = I915_READ(reg);
4897         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4898         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4899         temp &= ~FDI_LINK_TRAIN_NONE;
4900         temp |= FDI_LINK_TRAIN_PATTERN_1;
4901         I915_WRITE(reg, temp | FDI_TX_ENABLE);
4902
4903         reg = FDI_RX_CTL(pipe);
4904         temp = I915_READ(reg);
4905         temp &= ~FDI_LINK_TRAIN_NONE;
4906         temp |= FDI_LINK_TRAIN_PATTERN_1;
4907         I915_WRITE(reg, temp | FDI_RX_ENABLE);
4908
4909         POSTING_READ(reg);
4910         udelay(150);
4911
4912         /* Ironlake workaround, enable clock pointer after FDI enable*/
4913         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
4914         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR |
4915                    FDI_RX_PHASE_SYNC_POINTER_EN);
4916
4917         reg = FDI_RX_IIR(pipe);
4918         for (tries = 0; tries < 5; tries++) {
4919                 temp = I915_READ(reg);
4920                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4921
4922                 if ((temp & FDI_RX_BIT_LOCK)) {
4923                         DRM_DEBUG_KMS("FDI train 1 done.\n");
4924                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
4925                         break;
4926                 }
4927         }
4928         if (tries == 5)
4929                 DRM_ERROR("FDI train 1 fail!\n");
4930
4931         /* Train 2 */
4932         reg = FDI_TX_CTL(pipe);
4933         temp = I915_READ(reg);
4934         temp &= ~FDI_LINK_TRAIN_NONE;
4935         temp |= FDI_LINK_TRAIN_PATTERN_2;
4936         I915_WRITE(reg, temp);
4937
4938         reg = FDI_RX_CTL(pipe);
4939         temp = I915_READ(reg);
4940         temp &= ~FDI_LINK_TRAIN_NONE;
4941         temp |= FDI_LINK_TRAIN_PATTERN_2;
4942         I915_WRITE(reg, temp);
4943
4944         POSTING_READ(reg);
4945         udelay(150);
4946
4947         reg = FDI_RX_IIR(pipe);
4948         for (tries = 0; tries < 5; tries++) {
4949                 temp = I915_READ(reg);
4950                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
4951
4952                 if (temp & FDI_RX_SYMBOL_LOCK) {
4953                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
4954                         DRM_DEBUG_KMS("FDI train 2 done.\n");
4955                         break;
4956                 }
4957         }
4958         if (tries == 5)
4959                 DRM_ERROR("FDI train 2 fail!\n");
4960
4961         DRM_DEBUG_KMS("FDI train done\n");
4962
4963 }
4964
4965 static const int snb_b_fdi_train_param[] = {
4966         FDI_LINK_TRAIN_400MV_0DB_SNB_B,
4967         FDI_LINK_TRAIN_400MV_6DB_SNB_B,
4968         FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
4969         FDI_LINK_TRAIN_800MV_0DB_SNB_B,
4970 };
4971
4972 /* The FDI link training functions for SNB/Cougarpoint. */
4973 static void gen6_fdi_link_train(struct intel_crtc *crtc,
4974                                 const struct intel_crtc_state *crtc_state)
4975 {
4976         struct drm_device *dev = crtc->base.dev;
4977         struct drm_i915_private *dev_priv = to_i915(dev);
4978         enum pipe pipe = crtc->pipe;
4979         i915_reg_t reg;
4980         u32 temp, i, retry;
4981
4982         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
4983            for train result */
4984         reg = FDI_RX_IMR(pipe);
4985         temp = I915_READ(reg);
4986         temp &= ~FDI_RX_SYMBOL_LOCK;
4987         temp &= ~FDI_RX_BIT_LOCK;
4988         I915_WRITE(reg, temp);
4989
4990         POSTING_READ(reg);
4991         udelay(150);
4992
4993         /* enable CPU FDI TX and PCH FDI RX */
4994         reg = FDI_TX_CTL(pipe);
4995         temp = I915_READ(reg);
4996         temp &= ~FDI_DP_PORT_WIDTH_MASK;
4997         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
4998         temp &= ~FDI_LINK_TRAIN_NONE;
4999         temp |= FDI_LINK_TRAIN_PATTERN_1;
5000         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5001         /* SNB-B */
5002         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
5003         I915_WRITE(reg, temp | FDI_TX_ENABLE);
5004
5005         I915_WRITE(FDI_RX_MISC(pipe),
5006                    FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
5007
5008         reg = FDI_RX_CTL(pipe);
5009         temp = I915_READ(reg);
5010         if (HAS_PCH_CPT(dev_priv)) {
5011                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5012                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
5013         } else {
5014                 temp &= ~FDI_LINK_TRAIN_NONE;
5015                 temp |= FDI_LINK_TRAIN_PATTERN_1;
5016         }
5017         I915_WRITE(reg, temp | FDI_RX_ENABLE);
5018
5019         POSTING_READ(reg);
5020         udelay(150);
5021
5022         for (i = 0; i < 4; i++) {
5023                 reg = FDI_TX_CTL(pipe);
5024                 temp = I915_READ(reg);
5025                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5026                 temp |= snb_b_fdi_train_param[i];
5027                 I915_WRITE(reg, temp);
5028
5029                 POSTING_READ(reg);
5030                 udelay(500);
5031
5032                 for (retry = 0; retry < 5; retry++) {
5033                         reg = FDI_RX_IIR(pipe);
5034                         temp = I915_READ(reg);
5035                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
5036                         if (temp & FDI_RX_BIT_LOCK) {
5037                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
5038                                 DRM_DEBUG_KMS("FDI train 1 done.\n");
5039                                 break;
5040                         }
5041                         udelay(50);
5042                 }
5043                 if (retry < 5)
5044                         break;
5045         }
5046         if (i == 4)
5047                 DRM_ERROR("FDI train 1 fail!\n");
5048
5049         /* Train 2 */
5050         reg = FDI_TX_CTL(pipe);
5051         temp = I915_READ(reg);
5052         temp &= ~FDI_LINK_TRAIN_NONE;
5053         temp |= FDI_LINK_TRAIN_PATTERN_2;
5054         if (IS_GEN(dev_priv, 6)) {
5055                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5056                 /* SNB-B */
5057                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
5058         }
5059         I915_WRITE(reg, temp);
5060
5061         reg = FDI_RX_CTL(pipe);
5062         temp = I915_READ(reg);
5063         if (HAS_PCH_CPT(dev_priv)) {
5064                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5065                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
5066         } else {
5067                 temp &= ~FDI_LINK_TRAIN_NONE;
5068                 temp |= FDI_LINK_TRAIN_PATTERN_2;
5069         }
5070         I915_WRITE(reg, temp);
5071
5072         POSTING_READ(reg);
5073         udelay(150);
5074
5075         for (i = 0; i < 4; i++) {
5076                 reg = FDI_TX_CTL(pipe);
5077                 temp = I915_READ(reg);
5078                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5079                 temp |= snb_b_fdi_train_param[i];
5080                 I915_WRITE(reg, temp);
5081
5082                 POSTING_READ(reg);
5083                 udelay(500);
5084
5085                 for (retry = 0; retry < 5; retry++) {
5086                         reg = FDI_RX_IIR(pipe);
5087                         temp = I915_READ(reg);
5088                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
5089                         if (temp & FDI_RX_SYMBOL_LOCK) {
5090                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
5091                                 DRM_DEBUG_KMS("FDI train 2 done.\n");
5092                                 break;
5093                         }
5094                         udelay(50);
5095                 }
5096                 if (retry < 5)
5097                         break;
5098         }
5099         if (i == 4)
5100                 DRM_ERROR("FDI train 2 fail!\n");
5101
5102         DRM_DEBUG_KMS("FDI train done.\n");
5103 }
5104
5105 /* Manual link training for Ivy Bridge A0 parts */
5106 static void ivb_manual_fdi_link_train(struct intel_crtc *crtc,
5107                                       const struct intel_crtc_state *crtc_state)
5108 {
5109         struct drm_device *dev = crtc->base.dev;
5110         struct drm_i915_private *dev_priv = to_i915(dev);
5111         enum pipe pipe = crtc->pipe;
5112         i915_reg_t reg;
5113         u32 temp, i, j;
5114
5115         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
5116            for train result */
5117         reg = FDI_RX_IMR(pipe);
5118         temp = I915_READ(reg);
5119         temp &= ~FDI_RX_SYMBOL_LOCK;
5120         temp &= ~FDI_RX_BIT_LOCK;
5121         I915_WRITE(reg, temp);
5122
5123         POSTING_READ(reg);
5124         udelay(150);
5125
5126         DRM_DEBUG_KMS("FDI_RX_IIR before link train 0x%x\n",
5127                       I915_READ(FDI_RX_IIR(pipe)));
5128
5129         /* Try each vswing and preemphasis setting twice before moving on */
5130         for (j = 0; j < ARRAY_SIZE(snb_b_fdi_train_param) * 2; j++) {
5131                 /* disable first in case we need to retry */
5132                 reg = FDI_TX_CTL(pipe);
5133                 temp = I915_READ(reg);
5134                 temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
5135                 temp &= ~FDI_TX_ENABLE;
5136                 I915_WRITE(reg, temp);
5137
5138                 reg = FDI_RX_CTL(pipe);
5139                 temp = I915_READ(reg);
5140                 temp &= ~FDI_LINK_TRAIN_AUTO;
5141                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5142                 temp &= ~FDI_RX_ENABLE;
5143                 I915_WRITE(reg, temp);
5144
5145                 /* enable CPU FDI TX and PCH FDI RX */
5146                 reg = FDI_TX_CTL(pipe);
5147                 temp = I915_READ(reg);
5148                 temp &= ~FDI_DP_PORT_WIDTH_MASK;
5149                 temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
5150                 temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
5151                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
5152                 temp |= snb_b_fdi_train_param[j/2];
5153                 temp |= FDI_COMPOSITE_SYNC;
5154                 I915_WRITE(reg, temp | FDI_TX_ENABLE);
5155
5156                 I915_WRITE(FDI_RX_MISC(pipe),
5157                            FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
5158
5159                 reg = FDI_RX_CTL(pipe);
5160                 temp = I915_READ(reg);
5161                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
5162                 temp |= FDI_COMPOSITE_SYNC;
5163                 I915_WRITE(reg, temp | FDI_RX_ENABLE);
5164
5165                 POSTING_READ(reg);
5166                 udelay(1); /* should be 0.5us */
5167
5168                 for (i = 0; i < 4; i++) {
5169                         reg = FDI_RX_IIR(pipe);
5170                         temp = I915_READ(reg);
5171                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
5172
5173                         if (temp & FDI_RX_BIT_LOCK ||
5174                             (I915_READ(reg) & FDI_RX_BIT_LOCK)) {
5175                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
5176                                 DRM_DEBUG_KMS("FDI train 1 done, level %i.\n",
5177                                               i);
5178                                 break;
5179                         }
5180                         udelay(1); /* should be 0.5us */
5181                 }
5182                 if (i == 4) {
5183                         DRM_DEBUG_KMS("FDI train 1 fail on vswing %d\n", j / 2);
5184                         continue;
5185                 }
5186
5187                 /* Train 2 */
5188                 reg = FDI_TX_CTL(pipe);
5189                 temp = I915_READ(reg);
5190                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
5191                 temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
5192                 I915_WRITE(reg, temp);
5193
5194                 reg = FDI_RX_CTL(pipe);
5195                 temp = I915_READ(reg);
5196                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5197                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
5198                 I915_WRITE(reg, temp);
5199
5200                 POSTING_READ(reg);
5201                 udelay(2); /* should be 1.5us */
5202
5203                 for (i = 0; i < 4; i++) {
5204                         reg = FDI_RX_IIR(pipe);
5205                         temp = I915_READ(reg);
5206                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
5207
5208                         if (temp & FDI_RX_SYMBOL_LOCK ||
5209                             (I915_READ(reg) & FDI_RX_SYMBOL_LOCK)) {
5210                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
5211                                 DRM_DEBUG_KMS("FDI train 2 done, level %i.\n",
5212                                               i);
5213                                 goto train_done;
5214                         }
5215                         udelay(2); /* should be 1.5us */
5216                 }
5217                 if (i == 4)
5218                         DRM_DEBUG_KMS("FDI train 2 fail on vswing %d\n", j / 2);
5219         }
5220
5221 train_done:
5222         DRM_DEBUG_KMS("FDI train done.\n");
5223 }
5224
5225 static void ironlake_fdi_pll_enable(const struct intel_crtc_state *crtc_state)
5226 {
5227         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->uapi.crtc);
5228         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
5229         enum pipe pipe = intel_crtc->pipe;
5230         i915_reg_t reg;
5231         u32 temp;
5232
5233         /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
5234         reg = FDI_RX_CTL(pipe);
5235         temp = I915_READ(reg);
5236         temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16));
5237         temp |= FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
5238         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
5239         I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
5240
5241         POSTING_READ(reg);
5242         udelay(200);
5243
5244         /* Switch from Rawclk to PCDclk */
5245         temp = I915_READ(reg);
5246         I915_WRITE(reg, temp | FDI_PCDCLK);
5247
5248         POSTING_READ(reg);
5249         udelay(200);
5250
5251         /* Enable CPU FDI TX PLL, always on for Ironlake */
5252         reg = FDI_TX_CTL(pipe);
5253         temp = I915_READ(reg);
5254         if ((temp & FDI_TX_PLL_ENABLE) == 0) {
5255                 I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
5256
5257                 POSTING_READ(reg);
5258                 udelay(100);
5259         }
5260 }
5261
5262 static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc)
5263 {
5264         struct drm_device *dev = intel_crtc->base.dev;
5265         struct drm_i915_private *dev_priv = to_i915(dev);
5266         enum pipe pipe = intel_crtc->pipe;
5267         i915_reg_t reg;
5268         u32 temp;
5269
5270         /* Switch from PCDclk to Rawclk */
5271         reg = FDI_RX_CTL(pipe);
5272         temp = I915_READ(reg);
5273         I915_WRITE(reg, temp & ~FDI_PCDCLK);
5274
5275         /* Disable CPU FDI TX PLL */
5276         reg = FDI_TX_CTL(pipe);
5277         temp = I915_READ(reg);
5278         I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
5279
5280         POSTING_READ(reg);
5281         udelay(100);
5282
5283         reg = FDI_RX_CTL(pipe);
5284         temp = I915_READ(reg);
5285         I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
5286
5287         /* Wait for the clocks to turn off. */
5288         POSTING_READ(reg);
5289         udelay(100);
5290 }
5291
5292 static void ironlake_fdi_disable(struct intel_crtc *crtc)
5293 {
5294         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5295         enum pipe pipe = crtc->pipe;
5296         i915_reg_t reg;
5297         u32 temp;
5298
5299         /* disable CPU FDI tx and PCH FDI rx */
5300         reg = FDI_TX_CTL(pipe);
5301         temp = I915_READ(reg);
5302         I915_WRITE(reg, temp & ~FDI_TX_ENABLE);
5303         POSTING_READ(reg);
5304
5305         reg = FDI_RX_CTL(pipe);
5306         temp = I915_READ(reg);
5307         temp &= ~(0x7 << 16);
5308         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
5309         I915_WRITE(reg, temp & ~FDI_RX_ENABLE);
5310
5311         POSTING_READ(reg);
5312         udelay(100);
5313
5314         /* Ironlake workaround, disable clock pointer after downing FDI */
5315         if (HAS_PCH_IBX(dev_priv))
5316                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
5317
5318         /* still set train pattern 1 */
5319         reg = FDI_TX_CTL(pipe);
5320         temp = I915_READ(reg);
5321         temp &= ~FDI_LINK_TRAIN_NONE;
5322         temp |= FDI_LINK_TRAIN_PATTERN_1;
5323         I915_WRITE(reg, temp);
5324
5325         reg = FDI_RX_CTL(pipe);
5326         temp = I915_READ(reg);
5327         if (HAS_PCH_CPT(dev_priv)) {
5328                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
5329                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
5330         } else {
5331                 temp &= ~FDI_LINK_TRAIN_NONE;
5332                 temp |= FDI_LINK_TRAIN_PATTERN_1;
5333         }
5334         /* BPC in FDI rx is consistent with that in PIPECONF */
5335         temp &= ~(0x07 << 16);
5336         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
5337         I915_WRITE(reg, temp);
5338
5339         POSTING_READ(reg);
5340         udelay(100);
5341 }
5342
5343 bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv)
5344 {
5345         struct drm_crtc *crtc;
5346         bool cleanup_done;
5347
5348         drm_for_each_crtc(crtc, &dev_priv->drm) {
5349                 struct drm_crtc_commit *commit;
5350                 spin_lock(&crtc->commit_lock);
5351                 commit = list_first_entry_or_null(&crtc->commit_list,
5352                                                   struct drm_crtc_commit, commit_entry);
5353                 cleanup_done = commit ?
5354                         try_wait_for_completion(&commit->cleanup_done) : true;
5355                 spin_unlock(&crtc->commit_lock);
5356
5357                 if (cleanup_done)
5358                         continue;
5359
5360                 drm_crtc_wait_one_vblank(crtc);
5361
5362                 return true;
5363         }
5364
5365         return false;
5366 }
5367
5368 void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
5369 {
5370         u32 temp;
5371
5372         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
5373
5374         mutex_lock(&dev_priv->sb_lock);
5375
5376         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5377         temp |= SBI_SSCCTL_DISABLE;
5378         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
5379
5380         mutex_unlock(&dev_priv->sb_lock);
5381 }
5382
5383 /* Program iCLKIP clock to the desired frequency */
5384 static void lpt_program_iclkip(const struct intel_crtc_state *crtc_state)
5385 {
5386         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5387         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5388         int clock = crtc_state->hw.adjusted_mode.crtc_clock;
5389         u32 divsel, phaseinc, auxdiv, phasedir = 0;
5390         u32 temp;
5391
5392         lpt_disable_iclkip(dev_priv);
5393
5394         /* The iCLK virtual clock root frequency is in MHz,
5395          * but the adjusted_mode->crtc_clock in in KHz. To get the
5396          * divisors, it is necessary to divide one by another, so we
5397          * convert the virtual clock precision to KHz here for higher
5398          * precision.
5399          */
5400         for (auxdiv = 0; auxdiv < 2; auxdiv++) {
5401                 u32 iclk_virtual_root_freq = 172800 * 1000;
5402                 u32 iclk_pi_range = 64;
5403                 u32 desired_divisor;
5404
5405                 desired_divisor = DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
5406                                                     clock << auxdiv);
5407                 divsel = (desired_divisor / iclk_pi_range) - 2;
5408                 phaseinc = desired_divisor % iclk_pi_range;
5409
5410                 /*
5411                  * Near 20MHz is a corner case which is
5412                  * out of range for the 7-bit divisor
5413                  */
5414                 if (divsel <= 0x7f)
5415                         break;
5416         }
5417
5418         /* This should not happen with any sane values */
5419         WARN_ON(SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
5420                 ~SBI_SSCDIVINTPHASE_DIVSEL_MASK);
5421         WARN_ON(SBI_SSCDIVINTPHASE_DIR(phasedir) &
5422                 ~SBI_SSCDIVINTPHASE_INCVAL_MASK);
5423
5424         DRM_DEBUG_KMS("iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
5425                         clock,
5426                         auxdiv,
5427                         divsel,
5428                         phasedir,
5429                         phaseinc);
5430
5431         mutex_lock(&dev_priv->sb_lock);
5432
5433         /* Program SSCDIVINTPHASE6 */
5434         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
5435         temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
5436         temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
5437         temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
5438         temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
5439         temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
5440         temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
5441         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK);
5442
5443         /* Program SSCAUXDIV */
5444         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
5445         temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
5446         temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
5447         intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK);
5448
5449         /* Enable modulator and associated divider */
5450         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5451         temp &= ~SBI_SSCCTL_DISABLE;
5452         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
5453
5454         mutex_unlock(&dev_priv->sb_lock);
5455
5456         /* Wait for initialization time */
5457         udelay(24);
5458
5459         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE);
5460 }
5461
5462 int lpt_get_iclkip(struct drm_i915_private *dev_priv)
5463 {
5464         u32 divsel, phaseinc, auxdiv;
5465         u32 iclk_virtual_root_freq = 172800 * 1000;
5466         u32 iclk_pi_range = 64;
5467         u32 desired_divisor;
5468         u32 temp;
5469
5470         if ((I915_READ(PIXCLK_GATE) & PIXCLK_GATE_UNGATE) == 0)
5471                 return 0;
5472
5473         mutex_lock(&dev_priv->sb_lock);
5474
5475         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
5476         if (temp & SBI_SSCCTL_DISABLE) {
5477                 mutex_unlock(&dev_priv->sb_lock);
5478                 return 0;
5479         }
5480
5481         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
5482         divsel = (temp & SBI_SSCDIVINTPHASE_DIVSEL_MASK) >>
5483                 SBI_SSCDIVINTPHASE_DIVSEL_SHIFT;
5484         phaseinc = (temp & SBI_SSCDIVINTPHASE_INCVAL_MASK) >>
5485                 SBI_SSCDIVINTPHASE_INCVAL_SHIFT;
5486
5487         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
5488         auxdiv = (temp & SBI_SSCAUXDIV_FINALDIV2SEL_MASK) >>
5489                 SBI_SSCAUXDIV_FINALDIV2SEL_SHIFT;
5490
5491         mutex_unlock(&dev_priv->sb_lock);
5492
5493         desired_divisor = (divsel + 2) * iclk_pi_range + phaseinc;
5494
5495         return DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
5496                                  desired_divisor << auxdiv);
5497 }
5498
5499 static void ironlake_pch_transcoder_set_timings(const struct intel_crtc_state *crtc_state,
5500                                                 enum pipe pch_transcoder)
5501 {
5502         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5503         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5504         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
5505
5506         I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder),
5507                    I915_READ(HTOTAL(cpu_transcoder)));
5508         I915_WRITE(PCH_TRANS_HBLANK(pch_transcoder),
5509                    I915_READ(HBLANK(cpu_transcoder)));
5510         I915_WRITE(PCH_TRANS_HSYNC(pch_transcoder),
5511                    I915_READ(HSYNC(cpu_transcoder)));
5512
5513         I915_WRITE(PCH_TRANS_VTOTAL(pch_transcoder),
5514                    I915_READ(VTOTAL(cpu_transcoder)));
5515         I915_WRITE(PCH_TRANS_VBLANK(pch_transcoder),
5516                    I915_READ(VBLANK(cpu_transcoder)));
5517         I915_WRITE(PCH_TRANS_VSYNC(pch_transcoder),
5518                    I915_READ(VSYNC(cpu_transcoder)));
5519         I915_WRITE(PCH_TRANS_VSYNCSHIFT(pch_transcoder),
5520                    I915_READ(VSYNCSHIFT(cpu_transcoder)));
5521 }
5522
5523 static void cpt_set_fdi_bc_bifurcation(struct drm_i915_private *dev_priv, bool enable)
5524 {
5525         u32 temp;
5526
5527         temp = I915_READ(SOUTH_CHICKEN1);
5528         if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable)
5529                 return;
5530
5531         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_B)) & FDI_RX_ENABLE);
5532         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_C)) & FDI_RX_ENABLE);
5533
5534         temp &= ~FDI_BC_BIFURCATION_SELECT;
5535         if (enable)
5536                 temp |= FDI_BC_BIFURCATION_SELECT;
5537
5538         DRM_DEBUG_KMS("%sabling fdi C rx\n", enable ? "en" : "dis");
5539         I915_WRITE(SOUTH_CHICKEN1, temp);
5540         POSTING_READ(SOUTH_CHICKEN1);
5541 }
5542
5543 static void ivybridge_update_fdi_bc_bifurcation(const struct intel_crtc_state *crtc_state)
5544 {
5545         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5546         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5547
5548         switch (crtc->pipe) {
5549         case PIPE_A:
5550                 break;
5551         case PIPE_B:
5552                 if (crtc_state->fdi_lanes > 2)
5553                         cpt_set_fdi_bc_bifurcation(dev_priv, false);
5554                 else
5555                         cpt_set_fdi_bc_bifurcation(dev_priv, true);
5556
5557                 break;
5558         case PIPE_C:
5559                 cpt_set_fdi_bc_bifurcation(dev_priv, true);
5560
5561                 break;
5562         default:
5563                 BUG();
5564         }
5565 }
5566
5567 /*
5568  * Finds the encoder associated with the given CRTC. This can only be
5569  * used when we know that the CRTC isn't feeding multiple encoders!
5570  */
5571 static struct intel_encoder *
5572 intel_get_crtc_new_encoder(const struct intel_atomic_state *state,
5573                            const struct intel_crtc_state *crtc_state)
5574 {
5575         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5576         const struct drm_connector_state *connector_state;
5577         const struct drm_connector *connector;
5578         struct intel_encoder *encoder = NULL;
5579         int num_encoders = 0;
5580         int i;
5581
5582         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
5583                 if (connector_state->crtc != &crtc->base)
5584                         continue;
5585
5586                 encoder = to_intel_encoder(connector_state->best_encoder);
5587                 num_encoders++;
5588         }
5589
5590         WARN(num_encoders != 1, "%d encoders for pipe %c\n",
5591              num_encoders, pipe_name(crtc->pipe));
5592
5593         return encoder;
5594 }
5595
5596 /*
5597  * Enable PCH resources required for PCH ports:
5598  *   - PCH PLLs
5599  *   - FDI training & RX/TX
5600  *   - update transcoder timings
5601  *   - DP transcoding bits
5602  *   - transcoder
5603  */
5604 static void ironlake_pch_enable(const struct intel_atomic_state *state,
5605                                 const struct intel_crtc_state *crtc_state)
5606 {
5607         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5608         struct drm_device *dev = crtc->base.dev;
5609         struct drm_i915_private *dev_priv = to_i915(dev);
5610         enum pipe pipe = crtc->pipe;
5611         u32 temp;
5612
5613         assert_pch_transcoder_disabled(dev_priv, pipe);
5614
5615         if (IS_IVYBRIDGE(dev_priv))
5616                 ivybridge_update_fdi_bc_bifurcation(crtc_state);
5617
5618         /* Write the TU size bits before fdi link training, so that error
5619          * detection works. */
5620         I915_WRITE(FDI_RX_TUSIZE1(pipe),
5621                    I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
5622
5623         /* For PCH output, training FDI link */
5624         dev_priv->display.fdi_link_train(crtc, crtc_state);
5625
5626         /* We need to program the right clock selection before writing the pixel
5627          * mutliplier into the DPLL. */
5628         if (HAS_PCH_CPT(dev_priv)) {
5629                 u32 sel;
5630
5631                 temp = I915_READ(PCH_DPLL_SEL);
5632                 temp |= TRANS_DPLL_ENABLE(pipe);
5633                 sel = TRANS_DPLLB_SEL(pipe);
5634                 if (crtc_state->shared_dpll ==
5635                     intel_get_shared_dpll_by_id(dev_priv, DPLL_ID_PCH_PLL_B))
5636                         temp |= sel;
5637                 else
5638                         temp &= ~sel;
5639                 I915_WRITE(PCH_DPLL_SEL, temp);
5640         }
5641
5642         /* XXX: pch pll's can be enabled any time before we enable the PCH
5643          * transcoder, and we actually should do this to not upset any PCH
5644          * transcoder that already use the clock when we share it.
5645          *
5646          * Note that enable_shared_dpll tries to do the right thing, but
5647          * get_shared_dpll unconditionally resets the pll - we need that to have
5648          * the right LVDS enable sequence. */
5649         intel_enable_shared_dpll(crtc_state);
5650
5651         /* set transcoder timing, panel must allow it */
5652         assert_panel_unlocked(dev_priv, pipe);
5653         ironlake_pch_transcoder_set_timings(crtc_state, pipe);
5654
5655         intel_fdi_normal_train(crtc);
5656
5657         /* For PCH DP, enable TRANS_DP_CTL */
5658         if (HAS_PCH_CPT(dev_priv) &&
5659             intel_crtc_has_dp_encoder(crtc_state)) {
5660                 const struct drm_display_mode *adjusted_mode =
5661                         &crtc_state->hw.adjusted_mode;
5662                 u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
5663                 i915_reg_t reg = TRANS_DP_CTL(pipe);
5664                 enum port port;
5665
5666                 temp = I915_READ(reg);
5667                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
5668                           TRANS_DP_SYNC_MASK |
5669                           TRANS_DP_BPC_MASK);
5670                 temp |= TRANS_DP_OUTPUT_ENABLE;
5671                 temp |= bpc << 9; /* same format but at 11:9 */
5672
5673                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
5674                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
5675                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
5676                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
5677
5678                 port = intel_get_crtc_new_encoder(state, crtc_state)->port;
5679                 WARN_ON(port < PORT_B || port > PORT_D);
5680                 temp |= TRANS_DP_PORT_SEL(port);
5681
5682                 I915_WRITE(reg, temp);
5683         }
5684
5685         ironlake_enable_pch_transcoder(crtc_state);
5686 }
5687
5688 static void lpt_pch_enable(const struct intel_atomic_state *state,
5689                            const struct intel_crtc_state *crtc_state)
5690 {
5691         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5692         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5693         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
5694
5695         assert_pch_transcoder_disabled(dev_priv, PIPE_A);
5696
5697         lpt_program_iclkip(crtc_state);
5698
5699         /* Set transcoder timing. */
5700         ironlake_pch_transcoder_set_timings(crtc_state, PIPE_A);
5701
5702         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
5703 }
5704
5705 static void cpt_verify_modeset(struct drm_i915_private *dev_priv,
5706                                enum pipe pipe)
5707 {
5708         i915_reg_t dslreg = PIPEDSL(pipe);
5709         u32 temp;
5710
5711         temp = I915_READ(dslreg);
5712         udelay(500);
5713         if (wait_for(I915_READ(dslreg) != temp, 5)) {
5714                 if (wait_for(I915_READ(dslreg) != temp, 5))
5715                         DRM_ERROR("mode set failed: pipe %c stuck\n", pipe_name(pipe));
5716         }
5717 }
5718
5719 /*
5720  * The hardware phase 0.0 refers to the center of the pixel.
5721  * We want to start from the top/left edge which is phase
5722  * -0.5. That matches how the hardware calculates the scaling
5723  * factors (from top-left of the first pixel to bottom-right
5724  * of the last pixel, as opposed to the pixel centers).
5725  *
5726  * For 4:2:0 subsampled chroma planes we obviously have to
5727  * adjust that so that the chroma sample position lands in
5728  * the right spot.
5729  *
5730  * Note that for packed YCbCr 4:2:2 formats there is no way to
5731  * control chroma siting. The hardware simply replicates the
5732  * chroma samples for both of the luma samples, and thus we don't
5733  * actually get the expected MPEG2 chroma siting convention :(
5734  * The same behaviour is observed on pre-SKL platforms as well.
5735  *
5736  * Theory behind the formula (note that we ignore sub-pixel
5737  * source coordinates):
5738  * s = source sample position
5739  * d = destination sample position
5740  *
5741  * Downscaling 4:1:
5742  * -0.5
5743  * | 0.0
5744  * | |     1.5 (initial phase)
5745  * | |     |
5746  * v v     v
5747  * | s | s | s | s |
5748  * |       d       |
5749  *
5750  * Upscaling 1:4:
5751  * -0.5
5752  * | -0.375 (initial phase)
5753  * | |     0.0
5754  * | |     |
5755  * v v     v
5756  * |       s       |
5757  * | d | d | d | d |
5758  */
5759 u16 skl_scaler_calc_phase(int sub, int scale, bool chroma_cosited)
5760 {
5761         int phase = -0x8000;
5762         u16 trip = 0;
5763
5764         if (chroma_cosited)
5765                 phase += (sub - 1) * 0x8000 / sub;
5766
5767         phase += scale / (2 * sub);
5768
5769         /*
5770          * Hardware initial phase limited to [-0.5:1.5].
5771          * Since the max hardware scale factor is 3.0, we
5772          * should never actually excdeed 1.0 here.
5773          */
5774         WARN_ON(phase < -0x8000 || phase > 0x18000);
5775
5776         if (phase < 0)
5777                 phase = 0x10000 + phase;
5778         else
5779                 trip = PS_PHASE_TRIP;
5780
5781         return ((phase >> 2) & PS_PHASE_MASK) | trip;
5782 }
5783
5784 #define SKL_MIN_SRC_W 8
5785 #define SKL_MAX_SRC_W 4096
5786 #define SKL_MIN_SRC_H 8
5787 #define SKL_MAX_SRC_H 4096
5788 #define SKL_MIN_DST_W 8
5789 #define SKL_MAX_DST_W 4096
5790 #define SKL_MIN_DST_H 8
5791 #define SKL_MAX_DST_H 4096
5792 #define ICL_MAX_SRC_W 5120
5793 #define ICL_MAX_SRC_H 4096
5794 #define ICL_MAX_DST_W 5120
5795 #define ICL_MAX_DST_H 4096
5796 #define SKL_MIN_YUV_420_SRC_W 16
5797 #define SKL_MIN_YUV_420_SRC_H 16
5798
5799 static int
5800 skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
5801                   unsigned int scaler_user, int *scaler_id,
5802                   int src_w, int src_h, int dst_w, int dst_h,
5803                   const struct drm_format_info *format,
5804                   u64 modifier, bool need_scaler)
5805 {
5806         struct intel_crtc_scaler_state *scaler_state =
5807                 &crtc_state->scaler_state;
5808         struct intel_crtc *intel_crtc =
5809                 to_intel_crtc(crtc_state->uapi.crtc);
5810         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
5811         const struct drm_display_mode *adjusted_mode =
5812                 &crtc_state->hw.adjusted_mode;
5813
5814         /*
5815          * Src coordinates are already rotated by 270 degrees for
5816          * the 90/270 degree plane rotation cases (to match the
5817          * GTT mapping), hence no need to account for rotation here.
5818          */
5819         if (src_w != dst_w || src_h != dst_h)
5820                 need_scaler = true;
5821
5822         /*
5823          * Scaling/fitting not supported in IF-ID mode in GEN9+
5824          * TODO: Interlace fetch mode doesn't support YUV420 planar formats.
5825          * Once NV12 is enabled, handle it here while allocating scaler
5826          * for NV12.
5827          */
5828         if (INTEL_GEN(dev_priv) >= 9 && crtc_state->hw.enable &&
5829             need_scaler && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
5830                 DRM_DEBUG_KMS("Pipe/Plane scaling not supported with IF-ID mode\n");
5831                 return -EINVAL;
5832         }
5833
5834         /*
5835          * if plane is being disabled or scaler is no more required or force detach
5836          *  - free scaler binded to this plane/crtc
5837          *  - in order to do this, update crtc->scaler_usage
5838          *
5839          * Here scaler state in crtc_state is set free so that
5840          * scaler can be assigned to other user. Actual register
5841          * update to free the scaler is done in plane/panel-fit programming.
5842          * For this purpose crtc/plane_state->scaler_id isn't reset here.
5843          */
5844         if (force_detach || !need_scaler) {
5845                 if (*scaler_id >= 0) {
5846                         scaler_state->scaler_users &= ~(1 << scaler_user);
5847                         scaler_state->scalers[*scaler_id].in_use = 0;
5848
5849                         DRM_DEBUG_KMS("scaler_user index %u.%u: "
5850                                 "Staged freeing scaler id %d scaler_users = 0x%x\n",
5851                                 intel_crtc->pipe, scaler_user, *scaler_id,
5852                                 scaler_state->scaler_users);
5853                         *scaler_id = -1;
5854                 }
5855                 return 0;
5856         }
5857
5858         if (format && intel_format_info_is_yuv_semiplanar(format, modifier) &&
5859             (src_h < SKL_MIN_YUV_420_SRC_H || src_w < SKL_MIN_YUV_420_SRC_W)) {
5860                 DRM_DEBUG_KMS("Planar YUV: src dimensions not met\n");
5861                 return -EINVAL;
5862         }
5863
5864         /* range checks */
5865         if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H ||
5866             dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
5867             (INTEL_GEN(dev_priv) >= 11 &&
5868              (src_w > ICL_MAX_SRC_W || src_h > ICL_MAX_SRC_H ||
5869               dst_w > ICL_MAX_DST_W || dst_h > ICL_MAX_DST_H)) ||
5870             (INTEL_GEN(dev_priv) < 11 &&
5871              (src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
5872               dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H))) {
5873                 DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u "
5874                         "size is out of scaler range\n",
5875                         intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h);
5876                 return -EINVAL;
5877         }
5878
5879         /* mark this plane as a scaler user in crtc_state */
5880         scaler_state->scaler_users |= (1 << scaler_user);
5881         DRM_DEBUG_KMS("scaler_user index %u.%u: "
5882                 "staged scaling request for %ux%u->%ux%u scaler_users = 0x%x\n",
5883                 intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h,
5884                 scaler_state->scaler_users);
5885
5886         return 0;
5887 }
5888
5889 /**
5890  * skl_update_scaler_crtc - Stages update to scaler state for a given crtc.
5891  *
5892  * @state: crtc's scaler state
5893  *
5894  * Return
5895  *     0 - scaler_usage updated successfully
5896  *    error - requested scaling cannot be supported or other error condition
5897  */
5898 int skl_update_scaler_crtc(struct intel_crtc_state *state)
5899 {
5900         const struct drm_display_mode *adjusted_mode = &state->hw.adjusted_mode;
5901         bool need_scaler = false;
5902
5903         if (state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
5904                 need_scaler = true;
5905
5906         return skl_update_scaler(state, !state->hw.active, SKL_CRTC_INDEX,
5907                                  &state->scaler_state.scaler_id,
5908                                  state->pipe_src_w, state->pipe_src_h,
5909                                  adjusted_mode->crtc_hdisplay,
5910                                  adjusted_mode->crtc_vdisplay, NULL, 0,
5911                                  need_scaler);
5912 }
5913
5914 /**
5915  * skl_update_scaler_plane - Stages update to scaler state for a given plane.
5916  * @crtc_state: crtc's scaler state
5917  * @plane_state: atomic plane state to update
5918  *
5919  * Return
5920  *     0 - scaler_usage updated successfully
5921  *    error - requested scaling cannot be supported or other error condition
5922  */
5923 static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
5924                                    struct intel_plane_state *plane_state)
5925 {
5926         struct intel_plane *intel_plane =
5927                 to_intel_plane(plane_state->uapi.plane);
5928         struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
5929         struct drm_framebuffer *fb = plane_state->hw.fb;
5930         int ret;
5931         bool force_detach = !fb || !plane_state->uapi.visible;
5932         bool need_scaler = false;
5933
5934         /* Pre-gen11 and SDR planes always need a scaler for planar formats. */
5935         if (!icl_is_hdr_plane(dev_priv, intel_plane->id) &&
5936             fb && intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
5937                 need_scaler = true;
5938
5939         ret = skl_update_scaler(crtc_state, force_detach,
5940                                 drm_plane_index(&intel_plane->base),
5941                                 &plane_state->scaler_id,
5942                                 drm_rect_width(&plane_state->uapi.src) >> 16,
5943                                 drm_rect_height(&plane_state->uapi.src) >> 16,
5944                                 drm_rect_width(&plane_state->uapi.dst),
5945                                 drm_rect_height(&plane_state->uapi.dst),
5946                                 fb ? fb->format : NULL,
5947                                 fb ? fb->modifier : 0,
5948                                 need_scaler);
5949
5950         if (ret || plane_state->scaler_id < 0)
5951                 return ret;
5952
5953         /* check colorkey */
5954         if (plane_state->ckey.flags) {
5955                 DRM_DEBUG_KMS("[PLANE:%d:%s] scaling with color key not allowed",
5956                               intel_plane->base.base.id,
5957                               intel_plane->base.name);
5958                 return -EINVAL;
5959         }
5960
5961         /* Check src format */
5962         switch (fb->format->format) {
5963         case DRM_FORMAT_RGB565:
5964         case DRM_FORMAT_XBGR8888:
5965         case DRM_FORMAT_XRGB8888:
5966         case DRM_FORMAT_ABGR8888:
5967         case DRM_FORMAT_ARGB8888:
5968         case DRM_FORMAT_XRGB2101010:
5969         case DRM_FORMAT_XBGR2101010:
5970         case DRM_FORMAT_ARGB2101010:
5971         case DRM_FORMAT_ABGR2101010:
5972         case DRM_FORMAT_YUYV:
5973         case DRM_FORMAT_YVYU:
5974         case DRM_FORMAT_UYVY:
5975         case DRM_FORMAT_VYUY:
5976         case DRM_FORMAT_NV12:
5977         case DRM_FORMAT_P010:
5978         case DRM_FORMAT_P012:
5979         case DRM_FORMAT_P016:
5980         case DRM_FORMAT_Y210:
5981         case DRM_FORMAT_Y212:
5982         case DRM_FORMAT_Y216:
5983         case DRM_FORMAT_XVYU2101010:
5984         case DRM_FORMAT_XVYU12_16161616:
5985         case DRM_FORMAT_XVYU16161616:
5986                 break;
5987         case DRM_FORMAT_XBGR16161616F:
5988         case DRM_FORMAT_ABGR16161616F:
5989         case DRM_FORMAT_XRGB16161616F:
5990         case DRM_FORMAT_ARGB16161616F:
5991                 if (INTEL_GEN(dev_priv) >= 11)
5992                         break;
5993                 /* fall through */
5994         default:
5995                 DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n",
5996                               intel_plane->base.base.id, intel_plane->base.name,
5997                               fb->base.id, fb->format->format);
5998                 return -EINVAL;
5999         }
6000
6001         return 0;
6002 }
6003
6004 void skylake_scaler_disable(const struct intel_crtc_state *old_crtc_state)
6005 {
6006         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
6007         int i;
6008
6009         for (i = 0; i < crtc->num_scalers; i++)
6010                 skl_detach_scaler(crtc, i);
6011 }
6012
6013 static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state)
6014 {
6015         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6016         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6017         enum pipe pipe = crtc->pipe;
6018         const struct intel_crtc_scaler_state *scaler_state =
6019                 &crtc_state->scaler_state;
6020
6021         if (crtc_state->pch_pfit.enabled) {
6022                 u16 uv_rgb_hphase, uv_rgb_vphase;
6023                 int pfit_w, pfit_h, hscale, vscale;
6024                 int id;
6025
6026                 if (WARN_ON(crtc_state->scaler_state.scaler_id < 0))
6027                         return;
6028
6029                 pfit_w = (crtc_state->pch_pfit.size >> 16) & 0xFFFF;
6030                 pfit_h = crtc_state->pch_pfit.size & 0xFFFF;
6031
6032                 hscale = (crtc_state->pipe_src_w << 16) / pfit_w;
6033                 vscale = (crtc_state->pipe_src_h << 16) / pfit_h;
6034
6035                 uv_rgb_hphase = skl_scaler_calc_phase(1, hscale, false);
6036                 uv_rgb_vphase = skl_scaler_calc_phase(1, vscale, false);
6037
6038                 id = scaler_state->scaler_id;
6039                 I915_WRITE(SKL_PS_CTRL(pipe, id), PS_SCALER_EN |
6040                         PS_FILTER_MEDIUM | scaler_state->scalers[id].mode);
6041                 I915_WRITE_FW(SKL_PS_VPHASE(pipe, id),
6042                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_vphase));
6043                 I915_WRITE_FW(SKL_PS_HPHASE(pipe, id),
6044                               PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_hphase));
6045                 I915_WRITE(SKL_PS_WIN_POS(pipe, id), crtc_state->pch_pfit.pos);
6046                 I915_WRITE(SKL_PS_WIN_SZ(pipe, id), crtc_state->pch_pfit.size);
6047         }
6048 }
6049
6050 static void ironlake_pfit_enable(const struct intel_crtc_state *crtc_state)
6051 {
6052         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6053         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6054         enum pipe pipe = crtc->pipe;
6055
6056         if (crtc_state->pch_pfit.enabled) {
6057                 /* Force use of hard-coded filter coefficients
6058                  * as some pre-programmed values are broken,
6059                  * e.g. x201.
6060                  */
6061                 if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv))
6062                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
6063                                                  PF_PIPE_SEL_IVB(pipe));
6064                 else
6065                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
6066                 I915_WRITE(PF_WIN_POS(pipe), crtc_state->pch_pfit.pos);
6067                 I915_WRITE(PF_WIN_SZ(pipe), crtc_state->pch_pfit.size);
6068         }
6069 }
6070
6071 void hsw_enable_ips(const struct intel_crtc_state *crtc_state)
6072 {
6073         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6074         struct drm_device *dev = crtc->base.dev;
6075         struct drm_i915_private *dev_priv = to_i915(dev);
6076
6077         if (!crtc_state->ips_enabled)
6078                 return;
6079
6080         /*
6081          * We can only enable IPS after we enable a plane and wait for a vblank
6082          * This function is called from post_plane_update, which is run after
6083          * a vblank wait.
6084          */
6085         WARN_ON(!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)));
6086
6087         if (IS_BROADWELL(dev_priv)) {
6088                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL,
6089                                                 IPS_ENABLE | IPS_PCODE_CONTROL));
6090                 /* Quoting Art Runyan: "its not safe to expect any particular
6091                  * value in IPS_CTL bit 31 after enabling IPS through the
6092                  * mailbox." Moreover, the mailbox may return a bogus state,
6093                  * so we need to just enable it and continue on.
6094                  */
6095         } else {
6096                 I915_WRITE(IPS_CTL, IPS_ENABLE);
6097                 /* The bit only becomes 1 in the next vblank, so this wait here
6098                  * is essentially intel_wait_for_vblank. If we don't have this
6099                  * and don't wait for vblanks until the end of crtc_enable, then
6100                  * the HW state readout code will complain that the expected
6101                  * IPS_CTL value is not the one we read. */
6102                 if (intel_de_wait_for_set(dev_priv, IPS_CTL, IPS_ENABLE, 50))
6103                         DRM_ERROR("Timed out waiting for IPS enable\n");
6104         }
6105 }
6106
6107 void hsw_disable_ips(const struct intel_crtc_state *crtc_state)
6108 {
6109         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6110         struct drm_device *dev = crtc->base.dev;
6111         struct drm_i915_private *dev_priv = to_i915(dev);
6112
6113         if (!crtc_state->ips_enabled)
6114                 return;
6115
6116         if (IS_BROADWELL(dev_priv)) {
6117                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
6118                 /*
6119                  * Wait for PCODE to finish disabling IPS. The BSpec specified
6120                  * 42ms timeout value leads to occasional timeouts so use 100ms
6121                  * instead.
6122                  */
6123                 if (intel_de_wait_for_clear(dev_priv, IPS_CTL, IPS_ENABLE, 100))
6124                         DRM_ERROR("Timed out waiting for IPS disable\n");
6125         } else {
6126                 I915_WRITE(IPS_CTL, 0);
6127                 POSTING_READ(IPS_CTL);
6128         }
6129
6130         /* We need to wait for a vblank before we can disable the plane. */
6131         intel_wait_for_vblank(dev_priv, crtc->pipe);
6132 }
6133
6134 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
6135 {
6136         if (intel_crtc->overlay)
6137                 (void) intel_overlay_switch_off(intel_crtc->overlay);
6138
6139         /* Let userspace switch the overlay on again. In most cases userspace
6140          * has to recompute where to put it anyway.
6141          */
6142 }
6143
6144 static bool hsw_pre_update_disable_ips(const struct intel_crtc_state *old_crtc_state,
6145                                        const struct intel_crtc_state *new_crtc_state)
6146 {
6147         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6148         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6149
6150         if (!old_crtc_state->ips_enabled)
6151                 return false;
6152
6153         if (needs_modeset(new_crtc_state))
6154                 return true;
6155
6156         /*
6157          * Workaround : Do not read or write the pipe palette/gamma data while
6158          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
6159          *
6160          * Disable IPS before we program the LUT.
6161          */
6162         if (IS_HASWELL(dev_priv) &&
6163             (new_crtc_state->uapi.color_mgmt_changed ||
6164              new_crtc_state->update_pipe) &&
6165             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
6166                 return true;
6167
6168         return !new_crtc_state->ips_enabled;
6169 }
6170
6171 static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_state,
6172                                        const struct intel_crtc_state *new_crtc_state)
6173 {
6174         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6175         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6176
6177         if (!new_crtc_state->ips_enabled)
6178                 return false;
6179
6180         if (needs_modeset(new_crtc_state))
6181                 return true;
6182
6183         /*
6184          * Workaround : Do not read or write the pipe palette/gamma data while
6185          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
6186          *
6187          * Re-enable IPS after the LUT has been programmed.
6188          */
6189         if (IS_HASWELL(dev_priv) &&
6190             (new_crtc_state->uapi.color_mgmt_changed ||
6191              new_crtc_state->update_pipe) &&
6192             new_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)
6193                 return true;
6194
6195         /*
6196          * We can't read out IPS on broadwell, assume the worst and
6197          * forcibly enable IPS on the first fastset.
6198          */
6199         if (new_crtc_state->update_pipe &&
6200             old_crtc_state->hw.adjusted_mode.private_flags & I915_MODE_FLAG_INHERITED)
6201                 return true;
6202
6203         return !old_crtc_state->ips_enabled;
6204 }
6205
6206 static bool needs_nv12_wa(const struct intel_crtc_state *crtc_state)
6207 {
6208         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
6209
6210         if (!crtc_state->nv12_planes)
6211                 return false;
6212
6213         /* WA Display #0827: Gen9:all */
6214         if (IS_GEN(dev_priv, 9) && !IS_GEMINILAKE(dev_priv))
6215                 return true;
6216
6217         return false;
6218 }
6219
6220 static bool needs_scalerclk_wa(const struct intel_crtc_state *crtc_state)
6221 {
6222         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
6223
6224         /* Wa_2006604312:icl */
6225         if (crtc_state->scaler_state.scaler_users > 0 && IS_ICELAKE(dev_priv))
6226                 return true;
6227
6228         return false;
6229 }
6230
6231 static bool planes_enabling(const struct intel_crtc_state *old_crtc_state,
6232                             const struct intel_crtc_state *new_crtc_state)
6233 {
6234         return (!old_crtc_state->active_planes || needs_modeset(new_crtc_state)) &&
6235                 new_crtc_state->active_planes;
6236 }
6237
6238 static bool planes_disabling(const struct intel_crtc_state *old_crtc_state,
6239                              const struct intel_crtc_state *new_crtc_state)
6240 {
6241         return old_crtc_state->active_planes &&
6242                 (!new_crtc_state->active_planes || needs_modeset(new_crtc_state));
6243 }
6244
6245 static void intel_post_plane_update(struct intel_atomic_state *state,
6246                                     struct intel_crtc *crtc)
6247 {
6248         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6249         struct intel_plane *primary = to_intel_plane(crtc->base.primary);
6250         const struct intel_crtc_state *old_crtc_state =
6251                 intel_atomic_get_old_crtc_state(state, crtc);
6252         const struct intel_crtc_state *new_crtc_state =
6253                 intel_atomic_get_new_crtc_state(state, crtc);
6254         const struct intel_plane_state *new_primary_state =
6255                 intel_atomic_get_new_plane_state(state, primary);
6256         enum pipe pipe = crtc->pipe;
6257
6258         intel_frontbuffer_flip(dev_priv, new_crtc_state->fb_bits);
6259
6260         if (new_crtc_state->update_wm_post && new_crtc_state->hw.active)
6261                 intel_update_watermarks(crtc);
6262
6263         if (hsw_post_update_enable_ips(old_crtc_state, new_crtc_state))
6264                 hsw_enable_ips(new_crtc_state);
6265
6266         if (new_primary_state)
6267                 intel_fbc_post_update(crtc);
6268
6269         if (needs_nv12_wa(old_crtc_state) &&
6270             !needs_nv12_wa(new_crtc_state))
6271                 skl_wa_827(dev_priv, pipe, false);
6272
6273         if (needs_scalerclk_wa(old_crtc_state) &&
6274             !needs_scalerclk_wa(new_crtc_state))
6275                 icl_wa_scalerclkgating(dev_priv, pipe, false);
6276 }
6277
6278 static void intel_pre_plane_update(struct intel_atomic_state *state,
6279                                    struct intel_crtc *crtc)
6280 {
6281         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6282         struct intel_plane *primary = to_intel_plane(crtc->base.primary);
6283         const struct intel_crtc_state *old_crtc_state =
6284                 intel_atomic_get_old_crtc_state(state, crtc);
6285         const struct intel_crtc_state *new_crtc_state =
6286                 intel_atomic_get_new_crtc_state(state, crtc);
6287         const struct intel_plane_state *new_primary_state =
6288                 intel_atomic_get_new_plane_state(state, primary);
6289         enum pipe pipe = crtc->pipe;
6290
6291         if (hsw_pre_update_disable_ips(old_crtc_state, new_crtc_state))
6292                 hsw_disable_ips(old_crtc_state);
6293
6294         if (new_primary_state &&
6295             intel_fbc_pre_update(crtc, new_crtc_state, new_primary_state))
6296                 intel_wait_for_vblank(dev_priv, pipe);
6297
6298         /* Display WA 827 */
6299         if (!needs_nv12_wa(old_crtc_state) &&
6300             needs_nv12_wa(new_crtc_state))
6301                 skl_wa_827(dev_priv, pipe, true);
6302
6303         /* Wa_2006604312:icl */
6304         if (!needs_scalerclk_wa(old_crtc_state) &&
6305             needs_scalerclk_wa(new_crtc_state))
6306                 icl_wa_scalerclkgating(dev_priv, pipe, true);
6307
6308         /*
6309          * Vblank time updates from the shadow to live plane control register
6310          * are blocked if the memory self-refresh mode is active at that
6311          * moment. So to make sure the plane gets truly disabled, disable
6312          * first the self-refresh mode. The self-refresh enable bit in turn
6313          * will be checked/applied by the HW only at the next frame start
6314          * event which is after the vblank start event, so we need to have a
6315          * wait-for-vblank between disabling the plane and the pipe.
6316          */
6317         if (HAS_GMCH(dev_priv) && old_crtc_state->hw.active &&
6318             new_crtc_state->disable_cxsr && intel_set_memory_cxsr(dev_priv, false))
6319                 intel_wait_for_vblank(dev_priv, pipe);
6320
6321         /*
6322          * IVB workaround: must disable low power watermarks for at least
6323          * one frame before enabling scaling.  LP watermarks can be re-enabled
6324          * when scaling is disabled.
6325          *
6326          * WaCxSRDisabledForSpriteScaling:ivb
6327          */
6328         if (old_crtc_state->hw.active &&
6329             new_crtc_state->disable_lp_wm && ilk_disable_lp_wm(dev_priv))
6330                 intel_wait_for_vblank(dev_priv, pipe);
6331
6332         /*
6333          * If we're doing a modeset we don't need to do any
6334          * pre-vblank watermark programming here.
6335          */
6336         if (!needs_modeset(new_crtc_state)) {
6337                 /*
6338                  * For platforms that support atomic watermarks, program the
6339                  * 'intermediate' watermarks immediately.  On pre-gen9 platforms, these
6340                  * will be the intermediate values that are safe for both pre- and
6341                  * post- vblank; when vblank happens, the 'active' values will be set
6342                  * to the final 'target' values and we'll do this again to get the
6343                  * optimal watermarks.  For gen9+ platforms, the values we program here
6344                  * will be the final target values which will get automatically latched
6345                  * at vblank time; no further programming will be necessary.
6346                  *
6347                  * If a platform hasn't been transitioned to atomic watermarks yet,
6348                  * we'll continue to update watermarks the old way, if flags tell
6349                  * us to.
6350                  */
6351                 if (dev_priv->display.initial_watermarks)
6352                         dev_priv->display.initial_watermarks(state, crtc);
6353                 else if (new_crtc_state->update_wm_pre)
6354                         intel_update_watermarks(crtc);
6355         }
6356
6357         /*
6358          * Gen2 reports pipe underruns whenever all planes are disabled.
6359          * So disable underrun reporting before all the planes get disabled.
6360          *
6361          * We do this after .initial_watermarks() so that we have a
6362          * chance of catching underruns with the intermediate watermarks
6363          * vs. the old plane configuration.
6364          */
6365         if (IS_GEN(dev_priv, 2) && planes_disabling(old_crtc_state, new_crtc_state))
6366                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6367 }
6368
6369 static void intel_crtc_disable_planes(struct intel_atomic_state *state,
6370                                       struct intel_crtc *crtc)
6371 {
6372         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6373         const struct intel_crtc_state *new_crtc_state =
6374                 intel_atomic_get_new_crtc_state(state, crtc);
6375         unsigned int update_mask = new_crtc_state->update_planes;
6376         const struct intel_plane_state *old_plane_state;
6377         struct intel_plane *plane;
6378         unsigned fb_bits = 0;
6379         int i;
6380
6381         intel_crtc_dpms_overlay_disable(crtc);
6382
6383         for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
6384                 if (crtc->pipe != plane->pipe ||
6385                     !(update_mask & BIT(plane->id)))
6386                         continue;
6387
6388                 intel_disable_plane(plane, new_crtc_state);
6389
6390                 if (old_plane_state->uapi.visible)
6391                         fb_bits |= plane->frontbuffer_bit;
6392         }
6393
6394         intel_frontbuffer_flip(dev_priv, fb_bits);
6395 }
6396
6397 /*
6398  * intel_connector_primary_encoder - get the primary encoder for a connector
6399  * @connector: connector for which to return the encoder
6400  *
6401  * Returns the primary encoder for a connector. There is a 1:1 mapping from
6402  * all connectors to their encoder, except for DP-MST connectors which have
6403  * both a virtual and a primary encoder. These DP-MST primary encoders can be
6404  * pointed to by as many DP-MST connectors as there are pipes.
6405  */
6406 static struct intel_encoder *
6407 intel_connector_primary_encoder(struct intel_connector *connector)
6408 {
6409         struct intel_encoder *encoder;
6410
6411         if (connector->mst_port)
6412                 return &dp_to_dig_port(connector->mst_port)->base;
6413
6414         encoder = intel_attached_encoder(&connector->base);
6415         WARN_ON(!encoder);
6416
6417         return encoder;
6418 }
6419
6420 static void intel_encoders_update_prepare(struct intel_atomic_state *state)
6421 {
6422         struct drm_connector_state *new_conn_state;
6423         struct drm_connector *connector;
6424         int i;
6425
6426         for_each_new_connector_in_state(&state->base, connector, new_conn_state,
6427                                         i) {
6428                 struct intel_connector *intel_connector;
6429                 struct intel_encoder *encoder;
6430                 struct intel_crtc *crtc;
6431
6432                 if (!intel_connector_needs_modeset(state, connector))
6433                         continue;
6434
6435                 intel_connector = to_intel_connector(connector);
6436                 encoder = intel_connector_primary_encoder(intel_connector);
6437                 if (!encoder->update_prepare)
6438                         continue;
6439
6440                 crtc = new_conn_state->crtc ?
6441                         to_intel_crtc(new_conn_state->crtc) : NULL;
6442                 encoder->update_prepare(state, encoder, crtc);
6443         }
6444 }
6445
6446 static void intel_encoders_update_complete(struct intel_atomic_state *state)
6447 {
6448         struct drm_connector_state *new_conn_state;
6449         struct drm_connector *connector;
6450         int i;
6451
6452         for_each_new_connector_in_state(&state->base, connector, new_conn_state,
6453                                         i) {
6454                 struct intel_connector *intel_connector;
6455                 struct intel_encoder *encoder;
6456                 struct intel_crtc *crtc;
6457
6458                 if (!intel_connector_needs_modeset(state, connector))
6459                         continue;
6460
6461                 intel_connector = to_intel_connector(connector);
6462                 encoder = intel_connector_primary_encoder(intel_connector);
6463                 if (!encoder->update_complete)
6464                         continue;
6465
6466                 crtc = new_conn_state->crtc ?
6467                         to_intel_crtc(new_conn_state->crtc) : NULL;
6468                 encoder->update_complete(state, encoder, crtc);
6469         }
6470 }
6471
6472 static void intel_encoders_pre_pll_enable(struct intel_atomic_state *state,
6473                                           struct intel_crtc *crtc)
6474 {
6475         const struct intel_crtc_state *crtc_state =
6476                 intel_atomic_get_new_crtc_state(state, crtc);
6477         const struct drm_connector_state *conn_state;
6478         struct drm_connector *conn;
6479         int i;
6480
6481         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6482                 struct intel_encoder *encoder =
6483                         to_intel_encoder(conn_state->best_encoder);
6484
6485                 if (conn_state->crtc != &crtc->base)
6486                         continue;
6487
6488                 if (encoder->pre_pll_enable)
6489                         encoder->pre_pll_enable(encoder, crtc_state, conn_state);
6490         }
6491 }
6492
6493 static void intel_encoders_pre_enable(struct intel_atomic_state *state,
6494                                       struct intel_crtc *crtc)
6495 {
6496         const struct intel_crtc_state *crtc_state =
6497                 intel_atomic_get_new_crtc_state(state, crtc);
6498         const struct drm_connector_state *conn_state;
6499         struct drm_connector *conn;
6500         int i;
6501
6502         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6503                 struct intel_encoder *encoder =
6504                         to_intel_encoder(conn_state->best_encoder);
6505
6506                 if (conn_state->crtc != &crtc->base)
6507                         continue;
6508
6509                 if (encoder->pre_enable)
6510                         encoder->pre_enable(encoder, crtc_state, conn_state);
6511         }
6512 }
6513
6514 static void intel_encoders_enable(struct intel_atomic_state *state,
6515                                   struct intel_crtc *crtc)
6516 {
6517         const struct intel_crtc_state *crtc_state =
6518                 intel_atomic_get_new_crtc_state(state, crtc);
6519         const struct drm_connector_state *conn_state;
6520         struct drm_connector *conn;
6521         int i;
6522
6523         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6524                 struct intel_encoder *encoder =
6525                         to_intel_encoder(conn_state->best_encoder);
6526
6527                 if (conn_state->crtc != &crtc->base)
6528                         continue;
6529
6530                 if (encoder->enable)
6531                         encoder->enable(encoder, crtc_state, conn_state);
6532                 intel_opregion_notify_encoder(encoder, true);
6533         }
6534 }
6535
6536 static void intel_encoders_disable(struct intel_atomic_state *state,
6537                                    struct intel_crtc *crtc)
6538 {
6539         const struct intel_crtc_state *old_crtc_state =
6540                 intel_atomic_get_old_crtc_state(state, crtc);
6541         const struct drm_connector_state *old_conn_state;
6542         struct drm_connector *conn;
6543         int i;
6544
6545         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6546                 struct intel_encoder *encoder =
6547                         to_intel_encoder(old_conn_state->best_encoder);
6548
6549                 if (old_conn_state->crtc != &crtc->base)
6550                         continue;
6551
6552                 intel_opregion_notify_encoder(encoder, false);
6553                 if (encoder->disable)
6554                         encoder->disable(encoder, old_crtc_state, old_conn_state);
6555         }
6556 }
6557
6558 static void intel_encoders_post_disable(struct intel_atomic_state *state,
6559                                         struct intel_crtc *crtc)
6560 {
6561         const struct intel_crtc_state *old_crtc_state =
6562                 intel_atomic_get_old_crtc_state(state, crtc);
6563         const struct drm_connector_state *old_conn_state;
6564         struct drm_connector *conn;
6565         int i;
6566
6567         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6568                 struct intel_encoder *encoder =
6569                         to_intel_encoder(old_conn_state->best_encoder);
6570
6571                 if (old_conn_state->crtc != &crtc->base)
6572                         continue;
6573
6574                 if (encoder->post_disable)
6575                         encoder->post_disable(encoder, old_crtc_state, old_conn_state);
6576         }
6577 }
6578
6579 static void intel_encoders_post_pll_disable(struct intel_atomic_state *state,
6580                                             struct intel_crtc *crtc)
6581 {
6582         const struct intel_crtc_state *old_crtc_state =
6583                 intel_atomic_get_old_crtc_state(state, crtc);
6584         const struct drm_connector_state *old_conn_state;
6585         struct drm_connector *conn;
6586         int i;
6587
6588         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
6589                 struct intel_encoder *encoder =
6590                         to_intel_encoder(old_conn_state->best_encoder);
6591
6592                 if (old_conn_state->crtc != &crtc->base)
6593                         continue;
6594
6595                 if (encoder->post_pll_disable)
6596                         encoder->post_pll_disable(encoder, old_crtc_state, old_conn_state);
6597         }
6598 }
6599
6600 static void intel_encoders_update_pipe(struct intel_atomic_state *state,
6601                                        struct intel_crtc *crtc)
6602 {
6603         const struct intel_crtc_state *crtc_state =
6604                 intel_atomic_get_new_crtc_state(state, crtc);
6605         const struct drm_connector_state *conn_state;
6606         struct drm_connector *conn;
6607         int i;
6608
6609         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
6610                 struct intel_encoder *encoder =
6611                         to_intel_encoder(conn_state->best_encoder);
6612
6613                 if (conn_state->crtc != &crtc->base)
6614                         continue;
6615
6616                 if (encoder->update_pipe)
6617                         encoder->update_pipe(encoder, crtc_state, conn_state);
6618         }
6619 }
6620
6621 static void intel_disable_primary_plane(const struct intel_crtc_state *crtc_state)
6622 {
6623         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6624         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
6625
6626         plane->disable_plane(plane, crtc_state);
6627 }
6628
6629 static void ironlake_crtc_enable(struct intel_atomic_state *state,
6630                                  struct intel_crtc *crtc)
6631 {
6632         const struct intel_crtc_state *new_crtc_state =
6633                 intel_atomic_get_new_crtc_state(state, crtc);
6634         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6635         enum pipe pipe = crtc->pipe;
6636
6637         if (WARN_ON(crtc->active))
6638                 return;
6639
6640         /*
6641          * Sometimes spurious CPU pipe underruns happen during FDI
6642          * training, at least with VGA+HDMI cloning. Suppress them.
6643          *
6644          * On ILK we get an occasional spurious CPU pipe underruns
6645          * between eDP port A enable and vdd enable. Also PCH port
6646          * enable seems to result in the occasional CPU pipe underrun.
6647          *
6648          * Spurious PCH underruns also occur during PCH enabling.
6649          */
6650         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6651         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
6652
6653         if (new_crtc_state->has_pch_encoder)
6654                 intel_prepare_shared_dpll(new_crtc_state);
6655
6656         if (intel_crtc_has_dp_encoder(new_crtc_state))
6657                 intel_dp_set_m_n(new_crtc_state, M1_N1);
6658
6659         intel_set_pipe_timings(new_crtc_state);
6660         intel_set_pipe_src_size(new_crtc_state);
6661
6662         if (new_crtc_state->has_pch_encoder)
6663                 intel_cpu_transcoder_set_m_n(new_crtc_state,
6664                                              &new_crtc_state->fdi_m_n, NULL);
6665
6666         ironlake_set_pipeconf(new_crtc_state);
6667
6668         crtc->active = true;
6669
6670         intel_encoders_pre_enable(state, crtc);
6671
6672         if (new_crtc_state->has_pch_encoder) {
6673                 /* Note: FDI PLL enabling _must_ be done before we enable the
6674                  * cpu pipes, hence this is separate from all the other fdi/pch
6675                  * enabling. */
6676                 ironlake_fdi_pll_enable(new_crtc_state);
6677         } else {
6678                 assert_fdi_tx_disabled(dev_priv, pipe);
6679                 assert_fdi_rx_disabled(dev_priv, pipe);
6680         }
6681
6682         ironlake_pfit_enable(new_crtc_state);
6683
6684         /*
6685          * On ILK+ LUT must be loaded before the pipe is running but with
6686          * clocks enabled
6687          */
6688         intel_color_load_luts(new_crtc_state);
6689         intel_color_commit(new_crtc_state);
6690         /* update DSPCNTR to configure gamma for pipe bottom color */
6691         intel_disable_primary_plane(new_crtc_state);
6692
6693         if (dev_priv->display.initial_watermarks)
6694                 dev_priv->display.initial_watermarks(state, crtc);
6695         intel_enable_pipe(new_crtc_state);
6696
6697         if (new_crtc_state->has_pch_encoder)
6698                 ironlake_pch_enable(state, new_crtc_state);
6699
6700         intel_crtc_vblank_on(new_crtc_state);
6701
6702         intel_encoders_enable(state, crtc);
6703
6704         if (HAS_PCH_CPT(dev_priv))
6705                 cpt_verify_modeset(dev_priv, pipe);
6706
6707         /*
6708          * Must wait for vblank to avoid spurious PCH FIFO underruns.
6709          * And a second vblank wait is needed at least on ILK with
6710          * some interlaced HDMI modes. Let's do the double wait always
6711          * in case there are more corner cases we don't know about.
6712          */
6713         if (new_crtc_state->has_pch_encoder) {
6714                 intel_wait_for_vblank(dev_priv, pipe);
6715                 intel_wait_for_vblank(dev_priv, pipe);
6716         }
6717         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6718         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
6719 }
6720
6721 /* IPS only exists on ULT machines and is tied to pipe A. */
6722 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
6723 {
6724         return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A;
6725 }
6726
6727 static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv,
6728                                             enum pipe pipe, bool apply)
6729 {
6730         u32 val = I915_READ(CLKGATE_DIS_PSL(pipe));
6731         u32 mask = DPF_GATING_DIS | DPF_RAM_GATING_DIS | DPFR_GATING_DIS;
6732
6733         if (apply)
6734                 val |= mask;
6735         else
6736                 val &= ~mask;
6737
6738         I915_WRITE(CLKGATE_DIS_PSL(pipe), val);
6739 }
6740
6741 static void icl_pipe_mbus_enable(struct intel_crtc *crtc)
6742 {
6743         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6744         enum pipe pipe = crtc->pipe;
6745         u32 val;
6746
6747         val = MBUS_DBOX_A_CREDIT(2);
6748
6749         if (INTEL_GEN(dev_priv) >= 12) {
6750                 val |= MBUS_DBOX_BW_CREDIT(2);
6751                 val |= MBUS_DBOX_B_CREDIT(12);
6752         } else {
6753                 val |= MBUS_DBOX_BW_CREDIT(1);
6754                 val |= MBUS_DBOX_B_CREDIT(8);
6755         }
6756
6757         I915_WRITE(PIPE_MBUS_DBOX_CTL(pipe), val);
6758 }
6759
6760 static void hsw_set_frame_start_delay(const struct intel_crtc_state *crtc_state)
6761 {
6762         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6763         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6764         i915_reg_t reg = CHICKEN_TRANS(crtc_state->cpu_transcoder);
6765         u32 val;
6766
6767         val = I915_READ(reg);
6768         val &= ~HSW_FRAME_START_DELAY_MASK;
6769         val |= HSW_FRAME_START_DELAY(0);
6770         I915_WRITE(reg, val);
6771 }
6772
6773 static void haswell_crtc_enable(struct intel_atomic_state *state,
6774                                 struct intel_crtc *crtc)
6775 {
6776         const struct intel_crtc_state *new_crtc_state =
6777                 intel_atomic_get_new_crtc_state(state, crtc);
6778         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6779         enum pipe pipe = crtc->pipe, hsw_workaround_pipe;
6780         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
6781         bool psl_clkgate_wa;
6782
6783         if (WARN_ON(crtc->active))
6784                 return;
6785
6786         intel_encoders_pre_pll_enable(state, crtc);
6787
6788         if (new_crtc_state->shared_dpll)
6789                 intel_enable_shared_dpll(new_crtc_state);
6790
6791         intel_encoders_pre_enable(state, crtc);
6792
6793         if (intel_crtc_has_dp_encoder(new_crtc_state))
6794                 intel_dp_set_m_n(new_crtc_state, M1_N1);
6795
6796         if (!transcoder_is_dsi(cpu_transcoder))
6797                 intel_set_pipe_timings(new_crtc_state);
6798
6799         if (INTEL_GEN(dev_priv) >= 11)
6800                 icl_enable_trans_port_sync(new_crtc_state);
6801
6802         intel_set_pipe_src_size(new_crtc_state);
6803
6804         if (cpu_transcoder != TRANSCODER_EDP &&
6805             !transcoder_is_dsi(cpu_transcoder))
6806                 I915_WRITE(PIPE_MULT(cpu_transcoder),
6807                            new_crtc_state->pixel_multiplier - 1);
6808
6809         if (new_crtc_state->has_pch_encoder)
6810                 intel_cpu_transcoder_set_m_n(new_crtc_state,
6811                                              &new_crtc_state->fdi_m_n, NULL);
6812
6813         if (!transcoder_is_dsi(cpu_transcoder)) {
6814                 hsw_set_frame_start_delay(new_crtc_state);
6815                 haswell_set_pipeconf(new_crtc_state);
6816         }
6817
6818         if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
6819                 bdw_set_pipemisc(new_crtc_state);
6820
6821         crtc->active = true;
6822
6823         /* Display WA #1180: WaDisableScalarClockGating: glk, cnl */
6824         psl_clkgate_wa = (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) &&
6825                 new_crtc_state->pch_pfit.enabled;
6826         if (psl_clkgate_wa)
6827                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, true);
6828
6829         if (INTEL_GEN(dev_priv) >= 9)
6830                 skylake_pfit_enable(new_crtc_state);
6831         else
6832                 ironlake_pfit_enable(new_crtc_state);
6833
6834         /*
6835          * On ILK+ LUT must be loaded before the pipe is running but with
6836          * clocks enabled
6837          */
6838         intel_color_load_luts(new_crtc_state);
6839         intel_color_commit(new_crtc_state);
6840         /* update DSPCNTR to configure gamma/csc for pipe bottom color */
6841         if (INTEL_GEN(dev_priv) < 9)
6842                 intel_disable_primary_plane(new_crtc_state);
6843
6844         if (INTEL_GEN(dev_priv) >= 11)
6845                 icl_set_pipe_chicken(crtc);
6846
6847         if (!transcoder_is_dsi(cpu_transcoder))
6848                 intel_ddi_enable_transcoder_func(new_crtc_state);
6849
6850         if (dev_priv->display.initial_watermarks)
6851                 dev_priv->display.initial_watermarks(state, crtc);
6852
6853         if (INTEL_GEN(dev_priv) >= 11)
6854                 icl_pipe_mbus_enable(crtc);
6855
6856         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
6857         if (!transcoder_is_dsi(cpu_transcoder))
6858                 intel_enable_pipe(new_crtc_state);
6859
6860         if (new_crtc_state->has_pch_encoder)
6861                 lpt_pch_enable(state, new_crtc_state);
6862
6863         intel_crtc_vblank_on(new_crtc_state);
6864
6865         intel_encoders_enable(state, crtc);
6866
6867         if (psl_clkgate_wa) {
6868                 intel_wait_for_vblank(dev_priv, pipe);
6869                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, false);
6870         }
6871
6872         /* If we change the relative order between pipe/planes enabling, we need
6873          * to change the workaround. */
6874         hsw_workaround_pipe = new_crtc_state->hsw_workaround_pipe;
6875         if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
6876                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
6877                 intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
6878         }
6879 }
6880
6881 void ironlake_pfit_disable(const struct intel_crtc_state *old_crtc_state)
6882 {
6883         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
6884         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6885         enum pipe pipe = crtc->pipe;
6886
6887         /* To avoid upsetting the power well on haswell only disable the pfit if
6888          * it's in use. The hw state code will make sure we get this right. */
6889         if (old_crtc_state->pch_pfit.enabled) {
6890                 I915_WRITE(PF_CTL(pipe), 0);
6891                 I915_WRITE(PF_WIN_POS(pipe), 0);
6892                 I915_WRITE(PF_WIN_SZ(pipe), 0);
6893         }
6894 }
6895
6896 static void ironlake_crtc_disable(struct intel_atomic_state *state,
6897                                   struct intel_crtc *crtc)
6898 {
6899         const struct intel_crtc_state *old_crtc_state =
6900                 intel_atomic_get_old_crtc_state(state, crtc);
6901         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6902         enum pipe pipe = crtc->pipe;
6903
6904         /*
6905          * Sometimes spurious CPU pipe underruns happen when the
6906          * pipe is already disabled, but FDI RX/TX is still enabled.
6907          * Happens at least with VGA+HDMI cloning. Suppress them.
6908          */
6909         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6910         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
6911
6912         intel_encoders_disable(state, crtc);
6913
6914         intel_crtc_vblank_off(old_crtc_state);
6915
6916         intel_disable_pipe(old_crtc_state);
6917
6918         ironlake_pfit_disable(old_crtc_state);
6919
6920         if (old_crtc_state->has_pch_encoder)
6921                 ironlake_fdi_disable(crtc);
6922
6923         intel_encoders_post_disable(state, crtc);
6924
6925         if (old_crtc_state->has_pch_encoder) {
6926                 ironlake_disable_pch_transcoder(dev_priv, pipe);
6927
6928                 if (HAS_PCH_CPT(dev_priv)) {
6929                         i915_reg_t reg;
6930                         u32 temp;
6931
6932                         /* disable TRANS_DP_CTL */
6933                         reg = TRANS_DP_CTL(pipe);
6934                         temp = I915_READ(reg);
6935                         temp &= ~(TRANS_DP_OUTPUT_ENABLE |
6936                                   TRANS_DP_PORT_SEL_MASK);
6937                         temp |= TRANS_DP_PORT_SEL_NONE;
6938                         I915_WRITE(reg, temp);
6939
6940                         /* disable DPLL_SEL */
6941                         temp = I915_READ(PCH_DPLL_SEL);
6942                         temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
6943                         I915_WRITE(PCH_DPLL_SEL, temp);
6944                 }
6945
6946                 ironlake_fdi_pll_disable(crtc);
6947         }
6948
6949         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6950         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
6951 }
6952
6953 static void haswell_crtc_disable(struct intel_atomic_state *state,
6954                                  struct intel_crtc *crtc)
6955 {
6956         /*
6957          * FIXME collapse everything to one hook.
6958          * Need care with mst->ddi interactions.
6959          */
6960         intel_encoders_disable(state, crtc);
6961         intel_encoders_post_disable(state, crtc);
6962 }
6963
6964 static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state)
6965 {
6966         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
6967         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6968
6969         if (!crtc_state->gmch_pfit.control)
6970                 return;
6971
6972         /*
6973          * The panel fitter should only be adjusted whilst the pipe is disabled,
6974          * according to register description and PRM.
6975          */
6976         WARN_ON(I915_READ(PFIT_CONTROL) & PFIT_ENABLE);
6977         assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder);
6978
6979         I915_WRITE(PFIT_PGM_RATIOS, crtc_state->gmch_pfit.pgm_ratios);
6980         I915_WRITE(PFIT_CONTROL, crtc_state->gmch_pfit.control);
6981
6982         /* Border color in case we don't scale up to the full screen. Black by
6983          * default, change to something else for debugging. */
6984         I915_WRITE(BCLRPAT(crtc->pipe), 0);
6985 }
6986
6987 bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy)
6988 {
6989         if (phy == PHY_NONE)
6990                 return false;
6991
6992         if (IS_ELKHARTLAKE(dev_priv))
6993                 return phy <= PHY_C;
6994
6995         if (INTEL_GEN(dev_priv) >= 11)
6996                 return phy <= PHY_B;
6997
6998         return false;
6999 }
7000
7001 bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy)
7002 {
7003         if (INTEL_GEN(dev_priv) >= 12)
7004                 return phy >= PHY_D && phy <= PHY_I;
7005
7006         if (INTEL_GEN(dev_priv) >= 11 && !IS_ELKHARTLAKE(dev_priv))
7007                 return phy >= PHY_C && phy <= PHY_F;
7008
7009         return false;
7010 }
7011
7012 enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port)
7013 {
7014         if (IS_ELKHARTLAKE(i915) && port == PORT_D)
7015                 return PHY_A;
7016
7017         return (enum phy)port;
7018 }
7019
7020 enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port)
7021 {
7022         if (!intel_phy_is_tc(dev_priv, intel_port_to_phy(dev_priv, port)))
7023                 return PORT_TC_NONE;
7024
7025         if (INTEL_GEN(dev_priv) >= 12)
7026                 return port - PORT_D;
7027
7028         return port - PORT_C;
7029 }
7030
7031 enum intel_display_power_domain intel_port_to_power_domain(enum port port)
7032 {
7033         switch (port) {
7034         case PORT_A:
7035                 return POWER_DOMAIN_PORT_DDI_A_LANES;
7036         case PORT_B:
7037                 return POWER_DOMAIN_PORT_DDI_B_LANES;
7038         case PORT_C:
7039                 return POWER_DOMAIN_PORT_DDI_C_LANES;
7040         case PORT_D:
7041                 return POWER_DOMAIN_PORT_DDI_D_LANES;
7042         case PORT_E:
7043                 return POWER_DOMAIN_PORT_DDI_E_LANES;
7044         case PORT_F:
7045                 return POWER_DOMAIN_PORT_DDI_F_LANES;
7046         case PORT_G:
7047                 return POWER_DOMAIN_PORT_DDI_G_LANES;
7048         default:
7049                 MISSING_CASE(port);
7050                 return POWER_DOMAIN_PORT_OTHER;
7051         }
7052 }
7053
7054 enum intel_display_power_domain
7055 intel_aux_power_domain(struct intel_digital_port *dig_port)
7056 {
7057         struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
7058         enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port);
7059
7060         if (intel_phy_is_tc(dev_priv, phy) &&
7061             dig_port->tc_mode == TC_PORT_TBT_ALT) {
7062                 switch (dig_port->aux_ch) {
7063                 case AUX_CH_C:
7064                         return POWER_DOMAIN_AUX_C_TBT;
7065                 case AUX_CH_D:
7066                         return POWER_DOMAIN_AUX_D_TBT;
7067                 case AUX_CH_E:
7068                         return POWER_DOMAIN_AUX_E_TBT;
7069                 case AUX_CH_F:
7070                         return POWER_DOMAIN_AUX_F_TBT;
7071                 case AUX_CH_G:
7072                         return POWER_DOMAIN_AUX_G_TBT;
7073                 default:
7074                         MISSING_CASE(dig_port->aux_ch);
7075                         return POWER_DOMAIN_AUX_C_TBT;
7076                 }
7077         }
7078
7079         switch (dig_port->aux_ch) {
7080         case AUX_CH_A:
7081                 return POWER_DOMAIN_AUX_A;
7082         case AUX_CH_B:
7083                 return POWER_DOMAIN_AUX_B;
7084         case AUX_CH_C:
7085                 return POWER_DOMAIN_AUX_C;
7086         case AUX_CH_D:
7087                 return POWER_DOMAIN_AUX_D;
7088         case AUX_CH_E:
7089                 return POWER_DOMAIN_AUX_E;
7090         case AUX_CH_F:
7091                 return POWER_DOMAIN_AUX_F;
7092         case AUX_CH_G:
7093                 return POWER_DOMAIN_AUX_G;
7094         default:
7095                 MISSING_CASE(dig_port->aux_ch);
7096                 return POWER_DOMAIN_AUX_A;
7097         }
7098 }
7099
7100 static u64 get_crtc_power_domains(struct intel_crtc_state *crtc_state)
7101 {
7102         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7103         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7104         struct drm_encoder *encoder;
7105         enum pipe pipe = crtc->pipe;
7106         u64 mask;
7107         enum transcoder transcoder = crtc_state->cpu_transcoder;
7108
7109         if (!crtc_state->hw.active)
7110                 return 0;
7111
7112         mask = BIT_ULL(POWER_DOMAIN_PIPE(pipe));
7113         mask |= BIT_ULL(POWER_DOMAIN_TRANSCODER(transcoder));
7114         if (crtc_state->pch_pfit.enabled ||
7115             crtc_state->pch_pfit.force_thru)
7116                 mask |= BIT_ULL(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
7117
7118         drm_for_each_encoder_mask(encoder, &dev_priv->drm,
7119                                   crtc_state->uapi.encoder_mask) {
7120                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
7121
7122                 mask |= BIT_ULL(intel_encoder->power_domain);
7123         }
7124
7125         if (HAS_DDI(dev_priv) && crtc_state->has_audio)
7126                 mask |= BIT_ULL(POWER_DOMAIN_AUDIO);
7127
7128         if (crtc_state->shared_dpll)
7129                 mask |= BIT_ULL(POWER_DOMAIN_DISPLAY_CORE);
7130
7131         return mask;
7132 }
7133
7134 static u64
7135 modeset_get_crtc_power_domains(struct intel_crtc_state *crtc_state)
7136 {
7137         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7138         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7139         enum intel_display_power_domain domain;
7140         u64 domains, new_domains, old_domains;
7141
7142         old_domains = crtc->enabled_power_domains;
7143         crtc->enabled_power_domains = new_domains =
7144                 get_crtc_power_domains(crtc_state);
7145
7146         domains = new_domains & ~old_domains;
7147
7148         for_each_power_domain(domain, domains)
7149                 intel_display_power_get(dev_priv, domain);
7150
7151         return old_domains & ~new_domains;
7152 }
7153
7154 static void modeset_put_power_domains(struct drm_i915_private *dev_priv,
7155                                       u64 domains)
7156 {
7157         enum intel_display_power_domain domain;
7158
7159         for_each_power_domain(domain, domains)
7160                 intel_display_power_put_unchecked(dev_priv, domain);
7161 }
7162
7163 static void valleyview_crtc_enable(struct intel_atomic_state *state,
7164                                    struct intel_crtc *crtc)
7165 {
7166         const struct intel_crtc_state *new_crtc_state =
7167                 intel_atomic_get_new_crtc_state(state, crtc);
7168         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7169         enum pipe pipe = crtc->pipe;
7170
7171         if (WARN_ON(crtc->active))
7172                 return;
7173
7174         if (intel_crtc_has_dp_encoder(new_crtc_state))
7175                 intel_dp_set_m_n(new_crtc_state, M1_N1);
7176
7177         intel_set_pipe_timings(new_crtc_state);
7178         intel_set_pipe_src_size(new_crtc_state);
7179
7180         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
7181                 I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY);
7182                 I915_WRITE(CHV_CANVAS(pipe), 0);
7183         }
7184
7185         i9xx_set_pipeconf(new_crtc_state);
7186
7187         crtc->active = true;
7188
7189         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
7190
7191         intel_encoders_pre_pll_enable(state, crtc);
7192
7193         if (IS_CHERRYVIEW(dev_priv)) {
7194                 chv_prepare_pll(crtc, new_crtc_state);
7195                 chv_enable_pll(crtc, new_crtc_state);
7196         } else {
7197                 vlv_prepare_pll(crtc, new_crtc_state);
7198                 vlv_enable_pll(crtc, new_crtc_state);
7199         }
7200
7201         intel_encoders_pre_enable(state, crtc);
7202
7203         i9xx_pfit_enable(new_crtc_state);
7204
7205         intel_color_load_luts(new_crtc_state);
7206         intel_color_commit(new_crtc_state);
7207         /* update DSPCNTR to configure gamma for pipe bottom color */
7208         intel_disable_primary_plane(new_crtc_state);
7209
7210         dev_priv->display.initial_watermarks(state, crtc);
7211         intel_enable_pipe(new_crtc_state);
7212
7213         intel_crtc_vblank_on(new_crtc_state);
7214
7215         intel_encoders_enable(state, crtc);
7216 }
7217
7218 static void i9xx_set_pll_dividers(const struct intel_crtc_state *crtc_state)
7219 {
7220         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7221         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7222
7223         I915_WRITE(FP0(crtc->pipe), crtc_state->dpll_hw_state.fp0);
7224         I915_WRITE(FP1(crtc->pipe), crtc_state->dpll_hw_state.fp1);
7225 }
7226
7227 static void i9xx_crtc_enable(struct intel_atomic_state *state,
7228                              struct intel_crtc *crtc)
7229 {
7230         const struct intel_crtc_state *new_crtc_state =
7231                 intel_atomic_get_new_crtc_state(state, crtc);
7232         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7233         enum pipe pipe = crtc->pipe;
7234
7235         if (WARN_ON(crtc->active))
7236                 return;
7237
7238         i9xx_set_pll_dividers(new_crtc_state);
7239
7240         if (intel_crtc_has_dp_encoder(new_crtc_state))
7241                 intel_dp_set_m_n(new_crtc_state, M1_N1);
7242
7243         intel_set_pipe_timings(new_crtc_state);
7244         intel_set_pipe_src_size(new_crtc_state);
7245
7246         i9xx_set_pipeconf(new_crtc_state);
7247
7248         crtc->active = true;
7249
7250         if (!IS_GEN(dev_priv, 2))
7251                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
7252
7253         intel_encoders_pre_enable(state, crtc);
7254
7255         i9xx_enable_pll(crtc, new_crtc_state);
7256
7257         i9xx_pfit_enable(new_crtc_state);
7258
7259         intel_color_load_luts(new_crtc_state);
7260         intel_color_commit(new_crtc_state);
7261         /* update DSPCNTR to configure gamma for pipe bottom color */
7262         intel_disable_primary_plane(new_crtc_state);
7263
7264         if (dev_priv->display.initial_watermarks)
7265                 dev_priv->display.initial_watermarks(state, crtc);
7266         else
7267                 intel_update_watermarks(crtc);
7268         intel_enable_pipe(new_crtc_state);
7269
7270         intel_crtc_vblank_on(new_crtc_state);
7271
7272         intel_encoders_enable(state, crtc);
7273 }
7274
7275 static void i9xx_pfit_disable(const struct intel_crtc_state *old_crtc_state)
7276 {
7277         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
7278         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7279
7280         if (!old_crtc_state->gmch_pfit.control)
7281                 return;
7282
7283         assert_pipe_disabled(dev_priv, old_crtc_state->cpu_transcoder);
7284
7285         DRM_DEBUG_KMS("disabling pfit, current: 0x%08x\n",
7286                       I915_READ(PFIT_CONTROL));
7287         I915_WRITE(PFIT_CONTROL, 0);
7288 }
7289
7290 static void i9xx_crtc_disable(struct intel_atomic_state *state,
7291                               struct intel_crtc *crtc)
7292 {
7293         struct intel_crtc_state *old_crtc_state =
7294                 intel_atomic_get_old_crtc_state(state, crtc);
7295         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7296         enum pipe pipe = crtc->pipe;
7297
7298         /*
7299          * On gen2 planes are double buffered but the pipe isn't, so we must
7300          * wait for planes to fully turn off before disabling the pipe.
7301          */
7302         if (IS_GEN(dev_priv, 2))
7303                 intel_wait_for_vblank(dev_priv, pipe);
7304
7305         intel_encoders_disable(state, crtc);
7306
7307         intel_crtc_vblank_off(old_crtc_state);
7308
7309         intel_disable_pipe(old_crtc_state);
7310
7311         i9xx_pfit_disable(old_crtc_state);
7312
7313         intel_encoders_post_disable(state, crtc);
7314
7315         if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DSI)) {
7316                 if (IS_CHERRYVIEW(dev_priv))
7317                         chv_disable_pll(dev_priv, pipe);
7318                 else if (IS_VALLEYVIEW(dev_priv))
7319                         vlv_disable_pll(dev_priv, pipe);
7320                 else
7321                         i9xx_disable_pll(old_crtc_state);
7322         }
7323
7324         intel_encoders_post_pll_disable(state, crtc);
7325
7326         if (!IS_GEN(dev_priv, 2))
7327                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
7328
7329         if (!dev_priv->display.initial_watermarks)
7330                 intel_update_watermarks(crtc);
7331
7332         /* clock the pipe down to 640x480@60 to potentially save power */
7333         if (IS_I830(dev_priv))
7334                 i830_enable_pipe(dev_priv, pipe);
7335 }
7336
7337 static void intel_crtc_disable_noatomic(struct intel_crtc *crtc,
7338                                         struct drm_modeset_acquire_ctx *ctx)
7339 {
7340         struct intel_encoder *encoder;
7341         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7342         struct intel_bw_state *bw_state =
7343                 to_intel_bw_state(dev_priv->bw_obj.state);
7344         struct intel_crtc_state *crtc_state =
7345                 to_intel_crtc_state(crtc->base.state);
7346         enum intel_display_power_domain domain;
7347         struct intel_plane *plane;
7348         struct drm_atomic_state *state;
7349         struct intel_crtc_state *temp_crtc_state;
7350         enum pipe pipe = crtc->pipe;
7351         u64 domains;
7352         int ret;
7353
7354         if (!crtc_state->hw.active)
7355                 return;
7356
7357         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
7358                 const struct intel_plane_state *plane_state =
7359                         to_intel_plane_state(plane->base.state);
7360
7361                 if (plane_state->uapi.visible)
7362                         intel_plane_disable_noatomic(crtc, plane);
7363         }
7364
7365         state = drm_atomic_state_alloc(&dev_priv->drm);
7366         if (!state) {
7367                 DRM_DEBUG_KMS("failed to disable [CRTC:%d:%s], out of memory",
7368                               crtc->base.base.id, crtc->base.name);
7369                 return;
7370         }
7371
7372         state->acquire_ctx = ctx;
7373
7374         /* Everything's already locked, -EDEADLK can't happen. */
7375         temp_crtc_state = intel_atomic_get_crtc_state(state, crtc);
7376         ret = drm_atomic_add_affected_connectors(state, &crtc->base);
7377
7378         WARN_ON(IS_ERR(temp_crtc_state) || ret);
7379
7380         dev_priv->display.crtc_disable(to_intel_atomic_state(state), crtc);
7381
7382         drm_atomic_state_put(state);
7383
7384         DRM_DEBUG_KMS("[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
7385                       crtc->base.base.id, crtc->base.name);
7386
7387         crtc->active = false;
7388         crtc->base.enabled = false;
7389
7390         WARN_ON(drm_atomic_set_mode_for_crtc(&crtc_state->uapi, NULL) < 0);
7391         crtc_state->uapi.active = false;
7392         crtc_state->uapi.connector_mask = 0;
7393         crtc_state->uapi.encoder_mask = 0;
7394         intel_crtc_free_hw_state(crtc_state);
7395         memset(&crtc_state->hw, 0, sizeof(crtc_state->hw));
7396
7397         for_each_encoder_on_crtc(&dev_priv->drm, &crtc->base, encoder)
7398                 encoder->base.crtc = NULL;
7399
7400         intel_fbc_disable(crtc);
7401         intel_update_watermarks(crtc);
7402         intel_disable_shared_dpll(crtc_state);
7403
7404         domains = crtc->enabled_power_domains;
7405         for_each_power_domain(domain, domains)
7406                 intel_display_power_put_unchecked(dev_priv, domain);
7407         crtc->enabled_power_domains = 0;
7408
7409         dev_priv->active_pipes &= ~BIT(pipe);
7410         dev_priv->min_cdclk[pipe] = 0;
7411         dev_priv->min_voltage_level[pipe] = 0;
7412
7413         bw_state->data_rate[pipe] = 0;
7414         bw_state->num_active_planes[pipe] = 0;
7415 }
7416
7417 /*
7418  * turn all crtc's off, but do not adjust state
7419  * This has to be paired with a call to intel_modeset_setup_hw_state.
7420  */
7421 int intel_display_suspend(struct drm_device *dev)
7422 {
7423         struct drm_i915_private *dev_priv = to_i915(dev);
7424         struct drm_atomic_state *state;
7425         int ret;
7426
7427         state = drm_atomic_helper_suspend(dev);
7428         ret = PTR_ERR_OR_ZERO(state);
7429         if (ret)
7430                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
7431         else
7432                 dev_priv->modeset_restore_state = state;
7433         return ret;
7434 }
7435
7436 void intel_encoder_destroy(struct drm_encoder *encoder)
7437 {
7438         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
7439
7440         drm_encoder_cleanup(encoder);
7441         kfree(intel_encoder);
7442 }
7443
7444 /* Cross check the actual hw state with our own modeset state tracking (and it's
7445  * internal consistency). */
7446 static void intel_connector_verify_state(struct intel_crtc_state *crtc_state,
7447                                          struct drm_connector_state *conn_state)
7448 {
7449         struct intel_connector *connector = to_intel_connector(conn_state->connector);
7450
7451         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
7452                       connector->base.base.id,
7453                       connector->base.name);
7454
7455         if (connector->get_hw_state(connector)) {
7456                 struct intel_encoder *encoder = connector->encoder;
7457
7458                 I915_STATE_WARN(!crtc_state,
7459                          "connector enabled without attached crtc\n");
7460
7461                 if (!crtc_state)
7462                         return;
7463
7464                 I915_STATE_WARN(!crtc_state->hw.active,
7465                                 "connector is active, but attached crtc isn't\n");
7466
7467                 if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST)
7468                         return;
7469
7470                 I915_STATE_WARN(conn_state->best_encoder != &encoder->base,
7471                         "atomic encoder doesn't match attached encoder\n");
7472
7473                 I915_STATE_WARN(conn_state->crtc != encoder->base.crtc,
7474                         "attached encoder crtc differs from connector crtc\n");
7475         } else {
7476                 I915_STATE_WARN(crtc_state && crtc_state->hw.active,
7477                                 "attached crtc is active, but connector isn't\n");
7478                 I915_STATE_WARN(!crtc_state && conn_state->best_encoder,
7479                         "best encoder set without crtc!\n");
7480         }
7481 }
7482
7483 static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state)
7484 {
7485         if (crtc_state->hw.enable && crtc_state->has_pch_encoder)
7486                 return crtc_state->fdi_lanes;
7487
7488         return 0;
7489 }
7490
7491 static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
7492                                      struct intel_crtc_state *pipe_config)
7493 {
7494         struct drm_i915_private *dev_priv = to_i915(dev);
7495         struct drm_atomic_state *state = pipe_config->uapi.state;
7496         struct intel_crtc *other_crtc;
7497         struct intel_crtc_state *other_crtc_state;
7498
7499         DRM_DEBUG_KMS("checking fdi config on pipe %c, lanes %i\n",
7500                       pipe_name(pipe), pipe_config->fdi_lanes);
7501         if (pipe_config->fdi_lanes > 4) {
7502                 DRM_DEBUG_KMS("invalid fdi lane config on pipe %c: %i lanes\n",
7503                               pipe_name(pipe), pipe_config->fdi_lanes);
7504                 return -EINVAL;
7505         }
7506
7507         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
7508                 if (pipe_config->fdi_lanes > 2) {
7509                         DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n",
7510                                       pipe_config->fdi_lanes);
7511                         return -EINVAL;
7512                 } else {
7513                         return 0;
7514                 }
7515         }
7516
7517         if (INTEL_NUM_PIPES(dev_priv) == 2)
7518                 return 0;
7519
7520         /* Ivybridge 3 pipe is really complicated */
7521         switch (pipe) {
7522         case PIPE_A:
7523                 return 0;
7524         case PIPE_B:
7525                 if (pipe_config->fdi_lanes <= 2)
7526                         return 0;
7527
7528                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_C);
7529                 other_crtc_state =
7530                         intel_atomic_get_crtc_state(state, other_crtc);
7531                 if (IS_ERR(other_crtc_state))
7532                         return PTR_ERR(other_crtc_state);
7533
7534                 if (pipe_required_fdi_lanes(other_crtc_state) > 0) {
7535                         DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %c: %i lanes\n",
7536                                       pipe_name(pipe), pipe_config->fdi_lanes);
7537                         return -EINVAL;
7538                 }
7539                 return 0;
7540         case PIPE_C:
7541                 if (pipe_config->fdi_lanes > 2) {
7542                         DRM_DEBUG_KMS("only 2 lanes on pipe %c: required %i lanes\n",
7543                                       pipe_name(pipe), pipe_config->fdi_lanes);
7544                         return -EINVAL;
7545                 }
7546
7547                 other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_B);
7548                 other_crtc_state =
7549                         intel_atomic_get_crtc_state(state, other_crtc);
7550                 if (IS_ERR(other_crtc_state))
7551                         return PTR_ERR(other_crtc_state);
7552
7553                 if (pipe_required_fdi_lanes(other_crtc_state) > 2) {
7554                         DRM_DEBUG_KMS("fdi link B uses too many lanes to enable link C\n");
7555                         return -EINVAL;
7556                 }
7557                 return 0;
7558         default:
7559                 BUG();
7560         }
7561 }
7562
7563 #define RETRY 1
7564 static int ironlake_fdi_compute_config(struct intel_crtc *intel_crtc,
7565                                        struct intel_crtc_state *pipe_config)
7566 {
7567         struct drm_device *dev = intel_crtc->base.dev;
7568         const struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
7569         int lane, link_bw, fdi_dotclock, ret;
7570         bool needs_recompute = false;
7571
7572 retry:
7573         /* FDI is a binary signal running at ~2.7GHz, encoding
7574          * each output octet as 10 bits. The actual frequency
7575          * is stored as a divider into a 100MHz clock, and the
7576          * mode pixel clock is stored in units of 1KHz.
7577          * Hence the bw of each lane in terms of the mode signal
7578          * is:
7579          */
7580         link_bw = intel_fdi_link_freq(to_i915(dev), pipe_config);
7581
7582         fdi_dotclock = adjusted_mode->crtc_clock;
7583
7584         lane = ironlake_get_lanes_required(fdi_dotclock, link_bw,
7585                                            pipe_config->pipe_bpp);
7586
7587         pipe_config->fdi_lanes = lane;
7588
7589         intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock,
7590                                link_bw, &pipe_config->fdi_m_n, false, false);
7591
7592         ret = ironlake_check_fdi_lanes(dev, intel_crtc->pipe, pipe_config);
7593         if (ret == -EDEADLK)
7594                 return ret;
7595
7596         if (ret == -EINVAL && pipe_config->pipe_bpp > 6*3) {
7597                 pipe_config->pipe_bpp -= 2*3;
7598                 DRM_DEBUG_KMS("fdi link bw constraint, reducing pipe bpp to %i\n",
7599                               pipe_config->pipe_bpp);
7600                 needs_recompute = true;
7601                 pipe_config->bw_constrained = true;
7602
7603                 goto retry;
7604         }
7605
7606         if (needs_recompute)
7607                 return RETRY;
7608
7609         return ret;
7610 }
7611
7612 bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state)
7613 {
7614         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7615         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7616
7617         /* IPS only exists on ULT machines and is tied to pipe A. */
7618         if (!hsw_crtc_supports_ips(crtc))
7619                 return false;
7620
7621         if (!i915_modparams.enable_ips)
7622                 return false;
7623
7624         if (crtc_state->pipe_bpp > 24)
7625                 return false;
7626
7627         /*
7628          * We compare against max which means we must take
7629          * the increased cdclk requirement into account when
7630          * calculating the new cdclk.
7631          *
7632          * Should measure whether using a lower cdclk w/o IPS
7633          */
7634         if (IS_BROADWELL(dev_priv) &&
7635             crtc_state->pixel_rate > dev_priv->max_cdclk_freq * 95 / 100)
7636                 return false;
7637
7638         return true;
7639 }
7640
7641 static bool hsw_compute_ips_config(struct intel_crtc_state *crtc_state)
7642 {
7643         struct drm_i915_private *dev_priv =
7644                 to_i915(crtc_state->uapi.crtc->dev);
7645         struct intel_atomic_state *intel_state =
7646                 to_intel_atomic_state(crtc_state->uapi.state);
7647
7648         if (!hsw_crtc_state_ips_capable(crtc_state))
7649                 return false;
7650
7651         /*
7652          * When IPS gets enabled, the pipe CRC changes. Since IPS gets
7653          * enabled and disabled dynamically based on package C states,
7654          * user space can't make reliable use of the CRCs, so let's just
7655          * completely disable it.
7656          */
7657         if (crtc_state->crc_enabled)
7658                 return false;
7659
7660         /* IPS should be fine as long as at least one plane is enabled. */
7661         if (!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)))
7662                 return false;
7663
7664         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
7665         if (IS_BROADWELL(dev_priv) &&
7666             crtc_state->pixel_rate > intel_state->cdclk.logical.cdclk * 95 / 100)
7667                 return false;
7668
7669         return true;
7670 }
7671
7672 static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc)
7673 {
7674         const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7675
7676         /* GDG double wide on either pipe, otherwise pipe A only */
7677         return INTEL_GEN(dev_priv) < 4 &&
7678                 (crtc->pipe == PIPE_A || IS_I915G(dev_priv));
7679 }
7680
7681 static u32 ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config)
7682 {
7683         u32 pixel_rate;
7684
7685         pixel_rate = pipe_config->hw.adjusted_mode.crtc_clock;
7686
7687         /*
7688          * We only use IF-ID interlacing. If we ever use
7689          * PF-ID we'll need to adjust the pixel_rate here.
7690          */
7691
7692         if (pipe_config->pch_pfit.enabled) {
7693                 u64 pipe_w, pipe_h, pfit_w, pfit_h;
7694                 u32 pfit_size = pipe_config->pch_pfit.size;
7695
7696                 pipe_w = pipe_config->pipe_src_w;
7697                 pipe_h = pipe_config->pipe_src_h;
7698
7699                 pfit_w = (pfit_size >> 16) & 0xFFFF;
7700                 pfit_h = pfit_size & 0xFFFF;
7701                 if (pipe_w < pfit_w)
7702                         pipe_w = pfit_w;
7703                 if (pipe_h < pfit_h)
7704                         pipe_h = pfit_h;
7705
7706                 if (WARN_ON(!pfit_w || !pfit_h))
7707                         return pixel_rate;
7708
7709                 pixel_rate = div_u64(mul_u32_u32(pixel_rate, pipe_w * pipe_h),
7710                                      pfit_w * pfit_h);
7711         }
7712
7713         return pixel_rate;
7714 }
7715
7716 static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state)
7717 {
7718         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
7719
7720         if (HAS_GMCH(dev_priv))
7721                 /* FIXME calculate proper pipe pixel rate for GMCH pfit */
7722                 crtc_state->pixel_rate =
7723                         crtc_state->hw.adjusted_mode.crtc_clock;
7724         else
7725                 crtc_state->pixel_rate =
7726                         ilk_pipe_pixel_rate(crtc_state);
7727 }
7728
7729 static int intel_crtc_compute_config(struct intel_crtc *crtc,
7730                                      struct intel_crtc_state *pipe_config)
7731 {
7732         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7733         const struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
7734         int clock_limit = dev_priv->max_dotclk_freq;
7735
7736         if (INTEL_GEN(dev_priv) < 4) {
7737                 clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
7738
7739                 /*
7740                  * Enable double wide mode when the dot clock
7741                  * is > 90% of the (display) core speed.
7742                  */
7743                 if (intel_crtc_supports_double_wide(crtc) &&
7744                     adjusted_mode->crtc_clock > clock_limit) {
7745                         clock_limit = dev_priv->max_dotclk_freq;
7746                         pipe_config->double_wide = true;
7747                 }
7748         }
7749
7750         if (adjusted_mode->crtc_clock > clock_limit) {
7751                 DRM_DEBUG_KMS("requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
7752                               adjusted_mode->crtc_clock, clock_limit,
7753                               yesno(pipe_config->double_wide));
7754                 return -EINVAL;
7755         }
7756
7757         if ((pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
7758              pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) &&
7759              pipe_config->hw.ctm) {
7760                 /*
7761                  * There is only one pipe CSC unit per pipe, and we need that
7762                  * for output conversion from RGB->YCBCR. So if CTM is already
7763                  * applied we can't support YCBCR420 output.
7764                  */
7765                 DRM_DEBUG_KMS("YCBCR420 and CTM together are not possible\n");
7766                 return -EINVAL;
7767         }
7768
7769         /*
7770          * Pipe horizontal size must be even in:
7771          * - DVO ganged mode
7772          * - LVDS dual channel mode
7773          * - Double wide pipe
7774          */
7775         if (pipe_config->pipe_src_w & 1) {
7776                 if (pipe_config->double_wide) {
7777                         DRM_DEBUG_KMS("Odd pipe source width not supported with double wide pipe\n");
7778                         return -EINVAL;
7779                 }
7780
7781                 if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_LVDS) &&
7782                     intel_is_dual_link_lvds(dev_priv)) {
7783                         DRM_DEBUG_KMS("Odd pipe source width not supported with dual link LVDS\n");
7784                         return -EINVAL;
7785                 }
7786         }
7787
7788         /* Cantiga+ cannot handle modes with a hsync front porch of 0.
7789          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
7790          */
7791         if ((INTEL_GEN(dev_priv) > 4 || IS_G4X(dev_priv)) &&
7792                 adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay)
7793                 return -EINVAL;
7794
7795         intel_crtc_compute_pixel_rate(pipe_config);
7796
7797         if (pipe_config->has_pch_encoder)
7798                 return ironlake_fdi_compute_config(crtc, pipe_config);
7799
7800         return 0;
7801 }
7802
7803 static void
7804 intel_reduce_m_n_ratio(u32 *num, u32 *den)
7805 {
7806         while (*num > DATA_LINK_M_N_MASK ||
7807                *den > DATA_LINK_M_N_MASK) {
7808                 *num >>= 1;
7809                 *den >>= 1;
7810         }
7811 }
7812
7813 static void compute_m_n(unsigned int m, unsigned int n,
7814                         u32 *ret_m, u32 *ret_n,
7815                         bool constant_n)
7816 {
7817         /*
7818          * Several DP dongles in particular seem to be fussy about
7819          * too large link M/N values. Give N value as 0x8000 that
7820          * should be acceptable by specific devices. 0x8000 is the
7821          * specified fixed N value for asynchronous clock mode,
7822          * which the devices expect also in synchronous clock mode.
7823          */
7824         if (constant_n)
7825                 *ret_n = 0x8000;
7826         else
7827                 *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
7828
7829         *ret_m = div_u64(mul_u32_u32(m, *ret_n), n);
7830         intel_reduce_m_n_ratio(ret_m, ret_n);
7831 }
7832
7833 void
7834 intel_link_compute_m_n(u16 bits_per_pixel, int nlanes,
7835                        int pixel_clock, int link_clock,
7836                        struct intel_link_m_n *m_n,
7837                        bool constant_n, bool fec_enable)
7838 {
7839         u32 data_clock = bits_per_pixel * pixel_clock;
7840
7841         if (fec_enable)
7842                 data_clock = intel_dp_mode_to_fec_clock(data_clock);
7843
7844         m_n->tu = 64;
7845         compute_m_n(data_clock,
7846                     link_clock * nlanes * 8,
7847                     &m_n->gmch_m, &m_n->gmch_n,
7848                     constant_n);
7849
7850         compute_m_n(pixel_clock, link_clock,
7851                     &m_n->link_m, &m_n->link_n,
7852                     constant_n);
7853 }
7854
7855 static void intel_panel_sanitize_ssc(struct drm_i915_private *dev_priv)
7856 {
7857         /*
7858          * There may be no VBT; and if the BIOS enabled SSC we can
7859          * just keep using it to avoid unnecessary flicker.  Whereas if the
7860          * BIOS isn't using it, don't assume it will work even if the VBT
7861          * indicates as much.
7862          */
7863         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
7864                 bool bios_lvds_use_ssc = I915_READ(PCH_DREF_CONTROL) &
7865                         DREF_SSC1_ENABLE;
7866
7867                 if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) {
7868                         DRM_DEBUG_KMS("SSC %s by BIOS, overriding VBT which says %s\n",
7869                                       enableddisabled(bios_lvds_use_ssc),
7870                                       enableddisabled(dev_priv->vbt.lvds_use_ssc));
7871                         dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc;
7872                 }
7873         }
7874 }
7875
7876 static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
7877 {
7878         if (i915_modparams.panel_use_ssc >= 0)
7879                 return i915_modparams.panel_use_ssc != 0;
7880         return dev_priv->vbt.lvds_use_ssc
7881                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
7882 }
7883
7884 static u32 pnv_dpll_compute_fp(struct dpll *dpll)
7885 {
7886         return (1 << dpll->n) << 16 | dpll->m2;
7887 }
7888
7889 static u32 i9xx_dpll_compute_fp(struct dpll *dpll)
7890 {
7891         return dpll->n << 16 | dpll->m1 << 8 | dpll->m2;
7892 }
7893
7894 static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
7895                                      struct intel_crtc_state *crtc_state,
7896                                      struct dpll *reduced_clock)
7897 {
7898         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7899         u32 fp, fp2 = 0;
7900
7901         if (IS_PINEVIEW(dev_priv)) {
7902                 fp = pnv_dpll_compute_fp(&crtc_state->dpll);
7903                 if (reduced_clock)
7904                         fp2 = pnv_dpll_compute_fp(reduced_clock);
7905         } else {
7906                 fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
7907                 if (reduced_clock)
7908                         fp2 = i9xx_dpll_compute_fp(reduced_clock);
7909         }
7910
7911         crtc_state->dpll_hw_state.fp0 = fp;
7912
7913         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7914             reduced_clock) {
7915                 crtc_state->dpll_hw_state.fp1 = fp2;
7916         } else {
7917                 crtc_state->dpll_hw_state.fp1 = fp;
7918         }
7919 }
7920
7921 static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe
7922                 pipe)
7923 {
7924         u32 reg_val;
7925
7926         /*
7927          * PLLB opamp always calibrates to max value of 0x3f, force enable it
7928          * and set it to a reasonable value instead.
7929          */
7930         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7931         reg_val &= 0xffffff00;
7932         reg_val |= 0x00000030;
7933         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7934
7935         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7936         reg_val &= 0x00ffffff;
7937         reg_val |= 0x8c000000;
7938         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7939
7940         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7941         reg_val &= 0xffffff00;
7942         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7943
7944         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7945         reg_val &= 0x00ffffff;
7946         reg_val |= 0xb0000000;
7947         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7948 }
7949
7950 static void intel_pch_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
7951                                          const struct intel_link_m_n *m_n)
7952 {
7953         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7954         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7955         enum pipe pipe = crtc->pipe;
7956
7957         I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
7958         I915_WRITE(PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
7959         I915_WRITE(PCH_TRANS_LINK_M1(pipe), m_n->link_m);
7960         I915_WRITE(PCH_TRANS_LINK_N1(pipe), m_n->link_n);
7961 }
7962
7963 static bool transcoder_has_m2_n2(struct drm_i915_private *dev_priv,
7964                                  enum transcoder transcoder)
7965 {
7966         if (IS_HASWELL(dev_priv))
7967                 return transcoder == TRANSCODER_EDP;
7968
7969         /*
7970          * Strictly speaking some registers are available before
7971          * gen7, but we only support DRRS on gen7+
7972          */
7973         return IS_GEN(dev_priv, 7) || IS_CHERRYVIEW(dev_priv);
7974 }
7975
7976 static void intel_cpu_transcoder_set_m_n(const struct intel_crtc_state *crtc_state,
7977                                          const struct intel_link_m_n *m_n,
7978                                          const struct intel_link_m_n *m2_n2)
7979 {
7980         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
7981         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7982         enum pipe pipe = crtc->pipe;
7983         enum transcoder transcoder = crtc_state->cpu_transcoder;
7984
7985         if (INTEL_GEN(dev_priv) >= 5) {
7986                 I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m);
7987                 I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
7988                 I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
7989                 I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n);
7990                 /*
7991                  *  M2_N2 registers are set only if DRRS is supported
7992                  * (to make sure the registers are not unnecessarily accessed).
7993                  */
7994                 if (m2_n2 && crtc_state->has_drrs &&
7995                     transcoder_has_m2_n2(dev_priv, transcoder)) {
7996                         I915_WRITE(PIPE_DATA_M2(transcoder),
7997                                         TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
7998                         I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n);
7999                         I915_WRITE(PIPE_LINK_M2(transcoder), m2_n2->link_m);
8000                         I915_WRITE(PIPE_LINK_N2(transcoder), m2_n2->link_n);
8001                 }
8002         } else {
8003                 I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
8004                 I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
8005                 I915_WRITE(PIPE_LINK_M_G4X(pipe), m_n->link_m);
8006                 I915_WRITE(PIPE_LINK_N_G4X(pipe), m_n->link_n);
8007         }
8008 }
8009
8010 void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state, enum link_m_n_set m_n)
8011 {
8012         const struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL;
8013
8014         if (m_n == M1_N1) {
8015                 dp_m_n = &crtc_state->dp_m_n;
8016                 dp_m2_n2 = &crtc_state->dp_m2_n2;
8017         } else if (m_n == M2_N2) {
8018
8019                 /*
8020                  * M2_N2 registers are not supported. Hence m2_n2 divider value
8021                  * needs to be programmed into M1_N1.
8022                  */
8023                 dp_m_n = &crtc_state->dp_m2_n2;
8024         } else {
8025                 DRM_ERROR("Unsupported divider value\n");
8026                 return;
8027         }
8028
8029         if (crtc_state->has_pch_encoder)
8030                 intel_pch_transcoder_set_m_n(crtc_state, &crtc_state->dp_m_n);
8031         else
8032                 intel_cpu_transcoder_set_m_n(crtc_state, dp_m_n, dp_m2_n2);
8033 }
8034
8035 static void vlv_compute_dpll(struct intel_crtc *crtc,
8036                              struct intel_crtc_state *pipe_config)
8037 {
8038         pipe_config->dpll_hw_state.dpll = DPLL_INTEGRATED_REF_CLK_VLV |
8039                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
8040         if (crtc->pipe != PIPE_A)
8041                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
8042
8043         /* DPLL not used with DSI, but still need the rest set up */
8044         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
8045                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE |
8046                         DPLL_EXT_BUFFER_ENABLE_VLV;
8047
8048         pipe_config->dpll_hw_state.dpll_md =
8049                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
8050 }
8051
8052 static void chv_compute_dpll(struct intel_crtc *crtc,
8053                              struct intel_crtc_state *pipe_config)
8054 {
8055         pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV |
8056                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
8057         if (crtc->pipe != PIPE_A)
8058                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
8059
8060         /* DPLL not used with DSI, but still need the rest set up */
8061         if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
8062                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE;
8063
8064         pipe_config->dpll_hw_state.dpll_md =
8065                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
8066 }
8067
8068 static void vlv_prepare_pll(struct intel_crtc *crtc,
8069                             const struct intel_crtc_state *pipe_config)
8070 {
8071         struct drm_device *dev = crtc->base.dev;
8072         struct drm_i915_private *dev_priv = to_i915(dev);
8073         enum pipe pipe = crtc->pipe;
8074         u32 mdiv;
8075         u32 bestn, bestm1, bestm2, bestp1, bestp2;
8076         u32 coreclk, reg_val;
8077
8078         /* Enable Refclk */
8079         I915_WRITE(DPLL(pipe),
8080                    pipe_config->dpll_hw_state.dpll &
8081                    ~(DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV));
8082
8083         /* No need to actually set up the DPLL with DSI */
8084         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8085                 return;
8086
8087         vlv_dpio_get(dev_priv);
8088
8089         bestn = pipe_config->dpll.n;
8090         bestm1 = pipe_config->dpll.m1;
8091         bestm2 = pipe_config->dpll.m2;
8092         bestp1 = pipe_config->dpll.p1;
8093         bestp2 = pipe_config->dpll.p2;
8094
8095         /* See eDP HDMI DPIO driver vbios notes doc */
8096
8097         /* PLL B needs special handling */
8098         if (pipe == PIPE_B)
8099                 vlv_pllb_recal_opamp(dev_priv, pipe);
8100
8101         /* Set up Tx target for periodic Rcomp update */
8102         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9_BCAST, 0x0100000f);
8103
8104         /* Disable target IRef on PLL */
8105         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW8(pipe));
8106         reg_val &= 0x00ffffff;
8107         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW8(pipe), reg_val);
8108
8109         /* Disable fast lock */
8110         vlv_dpio_write(dev_priv, pipe, VLV_CMN_DW0, 0x610);
8111
8112         /* Set idtafcrecal before PLL is enabled */
8113         mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
8114         mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT));
8115         mdiv |= ((bestn << DPIO_N_SHIFT));
8116         mdiv |= (1 << DPIO_K_SHIFT);
8117
8118         /*
8119          * Post divider depends on pixel clock rate, DAC vs digital (and LVDS,
8120          * but we don't support that).
8121          * Note: don't use the DAC post divider as it seems unstable.
8122          */
8123         mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
8124         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
8125
8126         mdiv |= DPIO_ENABLE_CALIBRATION;
8127         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
8128
8129         /* Set HBR and RBR LPF coefficients */
8130         if (pipe_config->port_clock == 162000 ||
8131             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_ANALOG) ||
8132             intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI))
8133                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
8134                                  0x009f0003);
8135         else
8136                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
8137                                  0x00d0000f);
8138
8139         if (intel_crtc_has_dp_encoder(pipe_config)) {
8140                 /* Use SSC source */
8141                 if (pipe == PIPE_A)
8142                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
8143                                          0x0df40000);
8144                 else
8145                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
8146                                          0x0df70000);
8147         } else { /* HDMI or VGA */
8148                 /* Use bend source */
8149                 if (pipe == PIPE_A)
8150                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
8151                                          0x0df70000);
8152                 else
8153                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
8154                                          0x0df40000);
8155         }
8156
8157         coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe));
8158         coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
8159         if (intel_crtc_has_dp_encoder(pipe_config))
8160                 coreclk |= 0x01000000;
8161         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk);
8162
8163         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000);
8164
8165         vlv_dpio_put(dev_priv);
8166 }
8167
8168 static void chv_prepare_pll(struct intel_crtc *crtc,
8169                             const struct intel_crtc_state *pipe_config)
8170 {
8171         struct drm_device *dev = crtc->base.dev;
8172         struct drm_i915_private *dev_priv = to_i915(dev);
8173         enum pipe pipe = crtc->pipe;
8174         enum dpio_channel port = vlv_pipe_to_channel(pipe);
8175         u32 loopfilter, tribuf_calcntr;
8176         u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
8177         u32 dpio_val;
8178         int vco;
8179
8180         /* Enable Refclk and SSC */
8181         I915_WRITE(DPLL(pipe),
8182                    pipe_config->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE);
8183
8184         /* No need to actually set up the DPLL with DSI */
8185         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8186                 return;
8187
8188         bestn = pipe_config->dpll.n;
8189         bestm2_frac = pipe_config->dpll.m2 & 0x3fffff;
8190         bestm1 = pipe_config->dpll.m1;
8191         bestm2 = pipe_config->dpll.m2 >> 22;
8192         bestp1 = pipe_config->dpll.p1;
8193         bestp2 = pipe_config->dpll.p2;
8194         vco = pipe_config->dpll.vco;
8195         dpio_val = 0;
8196         loopfilter = 0;
8197
8198         vlv_dpio_get(dev_priv);
8199
8200         /* p1 and p2 divider */
8201         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port),
8202                         5 << DPIO_CHV_S1_DIV_SHIFT |
8203                         bestp1 << DPIO_CHV_P1_DIV_SHIFT |
8204                         bestp2 << DPIO_CHV_P2_DIV_SHIFT |
8205                         1 << DPIO_CHV_K_DIV_SHIFT);
8206
8207         /* Feedback post-divider - m2 */
8208         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2);
8209
8210         /* Feedback refclk divider - n and m1 */
8211         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port),
8212                         DPIO_CHV_M1_DIV_BY_2 |
8213                         1 << DPIO_CHV_N_DIV_SHIFT);
8214
8215         /* M2 fraction division */
8216         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
8217
8218         /* M2 fraction division enable */
8219         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
8220         dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
8221         dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT);
8222         if (bestm2_frac)
8223                 dpio_val |= DPIO_CHV_FRAC_DIV_EN;
8224         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), dpio_val);
8225
8226         /* Program digital lock detect threshold */
8227         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW9(port));
8228         dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
8229                                         DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
8230         dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT);
8231         if (!bestm2_frac)
8232                 dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
8233         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW9(port), dpio_val);
8234
8235         /* Loop filter */
8236         if (vco == 5400000) {
8237                 loopfilter |= (0x3 << DPIO_CHV_PROP_COEFF_SHIFT);
8238                 loopfilter |= (0x8 << DPIO_CHV_INT_COEFF_SHIFT);
8239                 loopfilter |= (0x1 << DPIO_CHV_GAIN_CTRL_SHIFT);
8240                 tribuf_calcntr = 0x9;
8241         } else if (vco <= 6200000) {
8242                 loopfilter |= (0x5 << DPIO_CHV_PROP_COEFF_SHIFT);
8243                 loopfilter |= (0xB << DPIO_CHV_INT_COEFF_SHIFT);
8244                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
8245                 tribuf_calcntr = 0x9;
8246         } else if (vco <= 6480000) {
8247                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
8248                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
8249                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
8250                 tribuf_calcntr = 0x8;
8251         } else {
8252                 /* Not supported. Apply the same limits as in the max case */
8253                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
8254                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
8255                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
8256                 tribuf_calcntr = 0;
8257         }
8258         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter);
8259
8260         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW8(port));
8261         dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
8262         dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT);
8263         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW8(port), dpio_val);
8264
8265         /* AFC Recal */
8266         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port),
8267                         vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) |
8268                         DPIO_AFC_RECAL);
8269
8270         vlv_dpio_put(dev_priv);
8271 }
8272
8273 /**
8274  * vlv_force_pll_on - forcibly enable just the PLL
8275  * @dev_priv: i915 private structure
8276  * @pipe: pipe PLL to enable
8277  * @dpll: PLL configuration
8278  *
8279  * Enable the PLL for @pipe using the supplied @dpll config. To be used
8280  * in cases where we need the PLL enabled even when @pipe is not going to
8281  * be enabled.
8282  */
8283 int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe,
8284                      const struct dpll *dpll)
8285 {
8286         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
8287         struct intel_crtc_state *pipe_config;
8288
8289         pipe_config = intel_crtc_state_alloc(crtc);
8290         if (!pipe_config)
8291                 return -ENOMEM;
8292
8293         pipe_config->cpu_transcoder = (enum transcoder)pipe;
8294         pipe_config->pixel_multiplier = 1;
8295         pipe_config->dpll = *dpll;
8296
8297         if (IS_CHERRYVIEW(dev_priv)) {
8298                 chv_compute_dpll(crtc, pipe_config);
8299                 chv_prepare_pll(crtc, pipe_config);
8300                 chv_enable_pll(crtc, pipe_config);
8301         } else {
8302                 vlv_compute_dpll(crtc, pipe_config);
8303                 vlv_prepare_pll(crtc, pipe_config);
8304                 vlv_enable_pll(crtc, pipe_config);
8305         }
8306
8307         kfree(pipe_config);
8308
8309         return 0;
8310 }
8311
8312 /**
8313  * vlv_force_pll_off - forcibly disable just the PLL
8314  * @dev_priv: i915 private structure
8315  * @pipe: pipe PLL to disable
8316  *
8317  * Disable the PLL for @pipe. To be used in cases where we need
8318  * the PLL enabled even when @pipe is not going to be enabled.
8319  */
8320 void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe)
8321 {
8322         if (IS_CHERRYVIEW(dev_priv))
8323                 chv_disable_pll(dev_priv, pipe);
8324         else
8325                 vlv_disable_pll(dev_priv, pipe);
8326 }
8327
8328 static void i9xx_compute_dpll(struct intel_crtc *crtc,
8329                               struct intel_crtc_state *crtc_state,
8330                               struct dpll *reduced_clock)
8331 {
8332         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8333         u32 dpll;
8334         struct dpll *clock = &crtc_state->dpll;
8335
8336         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
8337
8338         dpll = DPLL_VGA_MODE_DIS;
8339
8340         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
8341                 dpll |= DPLLB_MODE_LVDS;
8342         else
8343                 dpll |= DPLLB_MODE_DAC_SERIAL;
8344
8345         if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
8346             IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
8347                 dpll |= (crtc_state->pixel_multiplier - 1)
8348                         << SDVO_MULTIPLIER_SHIFT_HIRES;
8349         }
8350
8351         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
8352             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
8353                 dpll |= DPLL_SDVO_HIGH_SPEED;
8354
8355         if (intel_crtc_has_dp_encoder(crtc_state))
8356                 dpll |= DPLL_SDVO_HIGH_SPEED;
8357
8358         /* compute bitmask from p1 value */
8359         if (IS_PINEVIEW(dev_priv))
8360                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
8361         else {
8362                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8363                 if (IS_G4X(dev_priv) && reduced_clock)
8364                         dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
8365         }
8366         switch (clock->p2) {
8367         case 5:
8368                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
8369                 break;
8370         case 7:
8371                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
8372                 break;
8373         case 10:
8374                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
8375                 break;
8376         case 14:
8377                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
8378                 break;
8379         }
8380         if (INTEL_GEN(dev_priv) >= 4)
8381                 dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
8382
8383         if (crtc_state->sdvo_tv_clock)
8384                 dpll |= PLL_REF_INPUT_TVCLKINBC;
8385         else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8386                  intel_panel_use_ssc(dev_priv))
8387                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8388         else
8389                 dpll |= PLL_REF_INPUT_DREFCLK;
8390
8391         dpll |= DPLL_VCO_ENABLE;
8392         crtc_state->dpll_hw_state.dpll = dpll;
8393
8394         if (INTEL_GEN(dev_priv) >= 4) {
8395                 u32 dpll_md = (crtc_state->pixel_multiplier - 1)
8396                         << DPLL_MD_UDI_MULTIPLIER_SHIFT;
8397                 crtc_state->dpll_hw_state.dpll_md = dpll_md;
8398         }
8399 }
8400
8401 static void i8xx_compute_dpll(struct intel_crtc *crtc,
8402                               struct intel_crtc_state *crtc_state,
8403                               struct dpll *reduced_clock)
8404 {
8405         struct drm_device *dev = crtc->base.dev;
8406         struct drm_i915_private *dev_priv = to_i915(dev);
8407         u32 dpll;
8408         struct dpll *clock = &crtc_state->dpll;
8409
8410         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
8411
8412         dpll = DPLL_VGA_MODE_DIS;
8413
8414         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8415                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8416         } else {
8417                 if (clock->p1 == 2)
8418                         dpll |= PLL_P1_DIVIDE_BY_TWO;
8419                 else
8420                         dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8421                 if (clock->p2 == 4)
8422                         dpll |= PLL_P2_DIVIDE_BY_4;
8423         }
8424
8425         /*
8426          * Bspec:
8427          * "[Almador Errata}: For the correct operation of the muxed DVO pins
8428          *  (GDEVSELB/I2Cdata, GIRDBY/I2CClk) and (GFRAMEB/DVI_Data,
8429          *  GTRDYB/DVI_Clk): Bit 31 (DPLL VCO Enable) and Bit 30 (2X Clock
8430          *  Enable) must be set to “1” in both the DPLL A Control Register
8431          *  (06014h-06017h) and DPLL B Control Register (06018h-0601Bh)."
8432          *
8433          * For simplicity We simply keep both bits always enabled in
8434          * both DPLLS. The spec says we should disable the DVO 2X clock
8435          * when not needed, but this seems to work fine in practice.
8436          */
8437         if (IS_I830(dev_priv) ||
8438             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO))
8439                 dpll |= DPLL_DVO_2X_MODE;
8440
8441         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
8442             intel_panel_use_ssc(dev_priv))
8443                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8444         else
8445                 dpll |= PLL_REF_INPUT_DREFCLK;
8446
8447         dpll |= DPLL_VCO_ENABLE;
8448         crtc_state->dpll_hw_state.dpll = dpll;
8449 }
8450
8451 static void intel_set_pipe_timings(const struct intel_crtc_state *crtc_state)
8452 {
8453         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8454         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8455         enum pipe pipe = crtc->pipe;
8456         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
8457         const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
8458         u32 crtc_vtotal, crtc_vblank_end;
8459         int vsyncshift = 0;
8460
8461         /* We need to be careful not to changed the adjusted mode, for otherwise
8462          * the hw state checker will get angry at the mismatch. */
8463         crtc_vtotal = adjusted_mode->crtc_vtotal;
8464         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
8465
8466         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
8467                 /* the chip adds 2 halflines automatically */
8468                 crtc_vtotal -= 1;
8469                 crtc_vblank_end -= 1;
8470
8471                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
8472                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
8473                 else
8474                         vsyncshift = adjusted_mode->crtc_hsync_start -
8475                                 adjusted_mode->crtc_htotal / 2;
8476                 if (vsyncshift < 0)
8477                         vsyncshift += adjusted_mode->crtc_htotal;
8478         }
8479
8480         if (INTEL_GEN(dev_priv) > 3)
8481                 I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift);
8482
8483         I915_WRITE(HTOTAL(cpu_transcoder),
8484                    (adjusted_mode->crtc_hdisplay - 1) |
8485                    ((adjusted_mode->crtc_htotal - 1) << 16));
8486         I915_WRITE(HBLANK(cpu_transcoder),
8487                    (adjusted_mode->crtc_hblank_start - 1) |
8488                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
8489         I915_WRITE(HSYNC(cpu_transcoder),
8490                    (adjusted_mode->crtc_hsync_start - 1) |
8491                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
8492
8493         I915_WRITE(VTOTAL(cpu_transcoder),
8494                    (adjusted_mode->crtc_vdisplay - 1) |
8495                    ((crtc_vtotal - 1) << 16));
8496         I915_WRITE(VBLANK(cpu_transcoder),
8497                    (adjusted_mode->crtc_vblank_start - 1) |
8498                    ((crtc_vblank_end - 1) << 16));
8499         I915_WRITE(VSYNC(cpu_transcoder),
8500                    (adjusted_mode->crtc_vsync_start - 1) |
8501                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
8502
8503         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
8504          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
8505          * documented on the DDI_FUNC_CTL register description, EDP Input Select
8506          * bits. */
8507         if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP &&
8508             (pipe == PIPE_B || pipe == PIPE_C))
8509                 I915_WRITE(VTOTAL(pipe), I915_READ(VTOTAL(cpu_transcoder)));
8510
8511 }
8512
8513 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state)
8514 {
8515         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8516         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8517         enum pipe pipe = crtc->pipe;
8518
8519         /* pipesrc controls the size that is scaled from, which should
8520          * always be the user's requested size.
8521          */
8522         I915_WRITE(PIPESRC(pipe),
8523                    ((crtc_state->pipe_src_w - 1) << 16) |
8524                    (crtc_state->pipe_src_h - 1));
8525 }
8526
8527 static bool intel_pipe_is_interlaced(const struct intel_crtc_state *crtc_state)
8528 {
8529         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
8530         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
8531
8532         if (IS_GEN(dev_priv, 2))
8533                 return false;
8534
8535         if (INTEL_GEN(dev_priv) >= 9 ||
8536             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
8537                 return I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK_HSW;
8538         else
8539                 return I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK;
8540 }
8541
8542 static void intel_get_pipe_timings(struct intel_crtc *crtc,
8543                                    struct intel_crtc_state *pipe_config)
8544 {
8545         struct drm_device *dev = crtc->base.dev;
8546         struct drm_i915_private *dev_priv = to_i915(dev);
8547         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
8548         u32 tmp;
8549
8550         tmp = I915_READ(HTOTAL(cpu_transcoder));
8551         pipe_config->hw.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
8552         pipe_config->hw.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
8553
8554         if (!transcoder_is_dsi(cpu_transcoder)) {
8555                 tmp = I915_READ(HBLANK(cpu_transcoder));
8556                 pipe_config->hw.adjusted_mode.crtc_hblank_start =
8557                                                         (tmp & 0xffff) + 1;
8558                 pipe_config->hw.adjusted_mode.crtc_hblank_end =
8559                                                 ((tmp >> 16) & 0xffff) + 1;
8560         }
8561         tmp = I915_READ(HSYNC(cpu_transcoder));
8562         pipe_config->hw.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
8563         pipe_config->hw.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
8564
8565         tmp = I915_READ(VTOTAL(cpu_transcoder));
8566         pipe_config->hw.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
8567         pipe_config->hw.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
8568
8569         if (!transcoder_is_dsi(cpu_transcoder)) {
8570                 tmp = I915_READ(VBLANK(cpu_transcoder));
8571                 pipe_config->hw.adjusted_mode.crtc_vblank_start =
8572                                                         (tmp & 0xffff) + 1;
8573                 pipe_config->hw.adjusted_mode.crtc_vblank_end =
8574                                                 ((tmp >> 16) & 0xffff) + 1;
8575         }
8576         tmp = I915_READ(VSYNC(cpu_transcoder));
8577         pipe_config->hw.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
8578         pipe_config->hw.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
8579
8580         if (intel_pipe_is_interlaced(pipe_config)) {
8581                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
8582                 pipe_config->hw.adjusted_mode.crtc_vtotal += 1;
8583                 pipe_config->hw.adjusted_mode.crtc_vblank_end += 1;
8584         }
8585 }
8586
8587 static void intel_get_pipe_src_size(struct intel_crtc *crtc,
8588                                     struct intel_crtc_state *pipe_config)
8589 {
8590         struct drm_device *dev = crtc->base.dev;
8591         struct drm_i915_private *dev_priv = to_i915(dev);
8592         u32 tmp;
8593
8594         tmp = I915_READ(PIPESRC(crtc->pipe));
8595         pipe_config->pipe_src_h = (tmp & 0xffff) + 1;
8596         pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1;
8597
8598         pipe_config->hw.mode.vdisplay = pipe_config->pipe_src_h;
8599         pipe_config->hw.mode.hdisplay = pipe_config->pipe_src_w;
8600 }
8601
8602 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
8603                                  struct intel_crtc_state *pipe_config)
8604 {
8605         mode->hdisplay = pipe_config->hw.adjusted_mode.crtc_hdisplay;
8606         mode->htotal = pipe_config->hw.adjusted_mode.crtc_htotal;
8607         mode->hsync_start = pipe_config->hw.adjusted_mode.crtc_hsync_start;
8608         mode->hsync_end = pipe_config->hw.adjusted_mode.crtc_hsync_end;
8609
8610         mode->vdisplay = pipe_config->hw.adjusted_mode.crtc_vdisplay;
8611         mode->vtotal = pipe_config->hw.adjusted_mode.crtc_vtotal;
8612         mode->vsync_start = pipe_config->hw.adjusted_mode.crtc_vsync_start;
8613         mode->vsync_end = pipe_config->hw.adjusted_mode.crtc_vsync_end;
8614
8615         mode->flags = pipe_config->hw.adjusted_mode.flags;
8616         mode->type = DRM_MODE_TYPE_DRIVER;
8617
8618         mode->clock = pipe_config->hw.adjusted_mode.crtc_clock;
8619
8620         mode->hsync = drm_mode_hsync(mode);
8621         mode->vrefresh = drm_mode_vrefresh(mode);
8622         drm_mode_set_name(mode);
8623 }
8624
8625 static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
8626 {
8627         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
8628         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8629         u32 pipeconf;
8630
8631         pipeconf = 0;
8632
8633         /* we keep both pipes enabled on 830 */
8634         if (IS_I830(dev_priv))
8635                 pipeconf |= I915_READ(PIPECONF(crtc->pipe)) & PIPECONF_ENABLE;
8636
8637         if (crtc_state->double_wide)
8638                 pipeconf |= PIPECONF_DOUBLE_WIDE;
8639
8640         /* only g4x and later have fancy bpc/dither controls */
8641         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
8642             IS_CHERRYVIEW(dev_priv)) {
8643                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
8644                 if (crtc_state->dither && crtc_state->pipe_bpp != 30)
8645                         pipeconf |= PIPECONF_DITHER_EN |
8646                                     PIPECONF_DITHER_TYPE_SP;
8647
8648                 switch (crtc_state->pipe_bpp) {
8649                 case 18:
8650                         pipeconf |= PIPECONF_6BPC;
8651                         break;
8652                 case 24:
8653                         pipeconf |= PIPECONF_8BPC;
8654                         break;
8655                 case 30:
8656                         pipeconf |= PIPECONF_10BPC;
8657                         break;
8658                 default:
8659                         /* Case prevented by intel_choose_pipe_bpp_dither. */
8660                         BUG();
8661                 }
8662         }
8663
8664         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
8665                 if (INTEL_GEN(dev_priv) < 4 ||
8666                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
8667                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
8668                 else
8669                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
8670         } else {
8671                 pipeconf |= PIPECONF_PROGRESSIVE;
8672         }
8673
8674         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
8675              crtc_state->limited_color_range)
8676                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
8677
8678         pipeconf |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
8679
8680         pipeconf |= PIPECONF_FRAME_START_DELAY(0);
8681
8682         I915_WRITE(PIPECONF(crtc->pipe), pipeconf);
8683         POSTING_READ(PIPECONF(crtc->pipe));
8684 }
8685
8686 static int i8xx_crtc_compute_clock(struct intel_crtc *crtc,
8687                                    struct intel_crtc_state *crtc_state)
8688 {
8689         struct drm_device *dev = crtc->base.dev;
8690         struct drm_i915_private *dev_priv = to_i915(dev);
8691         const struct intel_limit *limit;
8692         int refclk = 48000;
8693
8694         memset(&crtc_state->dpll_hw_state, 0,
8695                sizeof(crtc_state->dpll_hw_state));
8696
8697         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8698                 if (intel_panel_use_ssc(dev_priv)) {
8699                         refclk = dev_priv->vbt.lvds_ssc_freq;
8700                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8701                 }
8702
8703                 limit = &intel_limits_i8xx_lvds;
8704         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO)) {
8705                 limit = &intel_limits_i8xx_dvo;
8706         } else {
8707                 limit = &intel_limits_i8xx_dac;
8708         }
8709
8710         if (!crtc_state->clock_set &&
8711             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8712                                  refclk, NULL, &crtc_state->dpll)) {
8713                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8714                 return -EINVAL;
8715         }
8716
8717         i8xx_compute_dpll(crtc, crtc_state, NULL);
8718
8719         return 0;
8720 }
8721
8722 static int g4x_crtc_compute_clock(struct intel_crtc *crtc,
8723                                   struct intel_crtc_state *crtc_state)
8724 {
8725         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8726         const struct intel_limit *limit;
8727         int refclk = 96000;
8728
8729         memset(&crtc_state->dpll_hw_state, 0,
8730                sizeof(crtc_state->dpll_hw_state));
8731
8732         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8733                 if (intel_panel_use_ssc(dev_priv)) {
8734                         refclk = dev_priv->vbt.lvds_ssc_freq;
8735                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8736                 }
8737
8738                 if (intel_is_dual_link_lvds(dev_priv))
8739                         limit = &intel_limits_g4x_dual_channel_lvds;
8740                 else
8741                         limit = &intel_limits_g4x_single_channel_lvds;
8742         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) ||
8743                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
8744                 limit = &intel_limits_g4x_hdmi;
8745         } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) {
8746                 limit = &intel_limits_g4x_sdvo;
8747         } else {
8748                 /* The option is for other outputs */
8749                 limit = &intel_limits_i9xx_sdvo;
8750         }
8751
8752         if (!crtc_state->clock_set &&
8753             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8754                                 refclk, NULL, &crtc_state->dpll)) {
8755                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8756                 return -EINVAL;
8757         }
8758
8759         i9xx_compute_dpll(crtc, crtc_state, NULL);
8760
8761         return 0;
8762 }
8763
8764 static int pnv_crtc_compute_clock(struct intel_crtc *crtc,
8765                                   struct intel_crtc_state *crtc_state)
8766 {
8767         struct drm_device *dev = crtc->base.dev;
8768         struct drm_i915_private *dev_priv = to_i915(dev);
8769         const struct intel_limit *limit;
8770         int refclk = 96000;
8771
8772         memset(&crtc_state->dpll_hw_state, 0,
8773                sizeof(crtc_state->dpll_hw_state));
8774
8775         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8776                 if (intel_panel_use_ssc(dev_priv)) {
8777                         refclk = dev_priv->vbt.lvds_ssc_freq;
8778                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8779                 }
8780
8781                 limit = &intel_limits_pineview_lvds;
8782         } else {
8783                 limit = &intel_limits_pineview_sdvo;
8784         }
8785
8786         if (!crtc_state->clock_set &&
8787             !pnv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8788                                 refclk, NULL, &crtc_state->dpll)) {
8789                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8790                 return -EINVAL;
8791         }
8792
8793         i9xx_compute_dpll(crtc, crtc_state, NULL);
8794
8795         return 0;
8796 }
8797
8798 static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
8799                                    struct intel_crtc_state *crtc_state)
8800 {
8801         struct drm_device *dev = crtc->base.dev;
8802         struct drm_i915_private *dev_priv = to_i915(dev);
8803         const struct intel_limit *limit;
8804         int refclk = 96000;
8805
8806         memset(&crtc_state->dpll_hw_state, 0,
8807                sizeof(crtc_state->dpll_hw_state));
8808
8809         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8810                 if (intel_panel_use_ssc(dev_priv)) {
8811                         refclk = dev_priv->vbt.lvds_ssc_freq;
8812                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8813                 }
8814
8815                 limit = &intel_limits_i9xx_lvds;
8816         } else {
8817                 limit = &intel_limits_i9xx_sdvo;
8818         }
8819
8820         if (!crtc_state->clock_set &&
8821             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8822                                  refclk, NULL, &crtc_state->dpll)) {
8823                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8824                 return -EINVAL;
8825         }
8826
8827         i9xx_compute_dpll(crtc, crtc_state, NULL);
8828
8829         return 0;
8830 }
8831
8832 static int chv_crtc_compute_clock(struct intel_crtc *crtc,
8833                                   struct intel_crtc_state *crtc_state)
8834 {
8835         int refclk = 100000;
8836         const struct intel_limit *limit = &intel_limits_chv;
8837
8838         memset(&crtc_state->dpll_hw_state, 0,
8839                sizeof(crtc_state->dpll_hw_state));
8840
8841         if (!crtc_state->clock_set &&
8842             !chv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8843                                 refclk, NULL, &crtc_state->dpll)) {
8844                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8845                 return -EINVAL;
8846         }
8847
8848         chv_compute_dpll(crtc, crtc_state);
8849
8850         return 0;
8851 }
8852
8853 static int vlv_crtc_compute_clock(struct intel_crtc *crtc,
8854                                   struct intel_crtc_state *crtc_state)
8855 {
8856         int refclk = 100000;
8857         const struct intel_limit *limit = &intel_limits_vlv;
8858
8859         memset(&crtc_state->dpll_hw_state, 0,
8860                sizeof(crtc_state->dpll_hw_state));
8861
8862         if (!crtc_state->clock_set &&
8863             !vlv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8864                                 refclk, NULL, &crtc_state->dpll)) {
8865                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8866                 return -EINVAL;
8867         }
8868
8869         vlv_compute_dpll(crtc, crtc_state);
8870
8871         return 0;
8872 }
8873
8874 static bool i9xx_has_pfit(struct drm_i915_private *dev_priv)
8875 {
8876         if (IS_I830(dev_priv))
8877                 return false;
8878
8879         return INTEL_GEN(dev_priv) >= 4 ||
8880                 IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
8881 }
8882
8883 static void i9xx_get_pfit_config(struct intel_crtc *crtc,
8884                                  struct intel_crtc_state *pipe_config)
8885 {
8886         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8887         u32 tmp;
8888
8889         if (!i9xx_has_pfit(dev_priv))
8890                 return;
8891
8892         tmp = I915_READ(PFIT_CONTROL);
8893         if (!(tmp & PFIT_ENABLE))
8894                 return;
8895
8896         /* Check whether the pfit is attached to our pipe. */
8897         if (INTEL_GEN(dev_priv) < 4) {
8898                 if (crtc->pipe != PIPE_B)
8899                         return;
8900         } else {
8901                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
8902                         return;
8903         }
8904
8905         pipe_config->gmch_pfit.control = tmp;
8906         pipe_config->gmch_pfit.pgm_ratios = I915_READ(PFIT_PGM_RATIOS);
8907 }
8908
8909 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
8910                                struct intel_crtc_state *pipe_config)
8911 {
8912         struct drm_device *dev = crtc->base.dev;
8913         struct drm_i915_private *dev_priv = to_i915(dev);
8914         enum pipe pipe = crtc->pipe;
8915         struct dpll clock;
8916         u32 mdiv;
8917         int refclk = 100000;
8918
8919         /* In case of DSI, DPLL will not be used */
8920         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8921                 return;
8922
8923         vlv_dpio_get(dev_priv);
8924         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
8925         vlv_dpio_put(dev_priv);
8926
8927         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
8928         clock.m2 = mdiv & DPIO_M2DIV_MASK;
8929         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
8930         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
8931         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
8932
8933         pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
8934 }
8935
8936 static void
8937 i9xx_get_initial_plane_config(struct intel_crtc *crtc,
8938                               struct intel_initial_plane_config *plane_config)
8939 {
8940         struct drm_device *dev = crtc->base.dev;
8941         struct drm_i915_private *dev_priv = to_i915(dev);
8942         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
8943         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
8944         enum pipe pipe;
8945         u32 val, base, offset;
8946         int fourcc, pixel_format;
8947         unsigned int aligned_height;
8948         struct drm_framebuffer *fb;
8949         struct intel_framebuffer *intel_fb;
8950
8951         if (!plane->get_hw_state(plane, &pipe))
8952                 return;
8953
8954         WARN_ON(pipe != crtc->pipe);
8955
8956         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
8957         if (!intel_fb) {
8958                 DRM_DEBUG_KMS("failed to alloc fb\n");
8959                 return;
8960         }
8961
8962         fb = &intel_fb->base;
8963
8964         fb->dev = dev;
8965
8966         val = I915_READ(DSPCNTR(i9xx_plane));
8967
8968         if (INTEL_GEN(dev_priv) >= 4) {
8969                 if (val & DISPPLANE_TILED) {
8970                         plane_config->tiling = I915_TILING_X;
8971                         fb->modifier = I915_FORMAT_MOD_X_TILED;
8972                 }
8973
8974                 if (val & DISPPLANE_ROTATE_180)
8975                         plane_config->rotation = DRM_MODE_ROTATE_180;
8976         }
8977
8978         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B &&
8979             val & DISPPLANE_MIRROR)
8980                 plane_config->rotation |= DRM_MODE_REFLECT_X;
8981
8982         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
8983         fourcc = i9xx_format_to_fourcc(pixel_format);
8984         fb->format = drm_format_info(fourcc);
8985
8986         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
8987                 offset = I915_READ(DSPOFFSET(i9xx_plane));
8988                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
8989         } else if (INTEL_GEN(dev_priv) >= 4) {
8990                 if (plane_config->tiling)
8991                         offset = I915_READ(DSPTILEOFF(i9xx_plane));
8992                 else
8993                         offset = I915_READ(DSPLINOFF(i9xx_plane));
8994                 base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
8995         } else {
8996                 base = I915_READ(DSPADDR(i9xx_plane));
8997         }
8998         plane_config->base = base;
8999
9000         val = I915_READ(PIPESRC(pipe));
9001         fb->width = ((val >> 16) & 0xfff) + 1;
9002         fb->height = ((val >> 0) & 0xfff) + 1;
9003
9004         val = I915_READ(DSPSTRIDE(i9xx_plane));
9005         fb->pitches[0] = val & 0xffffffc0;
9006
9007         aligned_height = intel_fb_align_height(fb, 0, fb->height);
9008
9009         plane_config->size = fb->pitches[0] * aligned_height;
9010
9011         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
9012                       crtc->base.name, plane->base.name, fb->width, fb->height,
9013                       fb->format->cpp[0] * 8, base, fb->pitches[0],
9014                       plane_config->size);
9015
9016         plane_config->fb = intel_fb;
9017 }
9018
9019 static void chv_crtc_clock_get(struct intel_crtc *crtc,
9020                                struct intel_crtc_state *pipe_config)
9021 {
9022         struct drm_device *dev = crtc->base.dev;
9023         struct drm_i915_private *dev_priv = to_i915(dev);
9024         enum pipe pipe = crtc->pipe;
9025         enum dpio_channel port = vlv_pipe_to_channel(pipe);
9026         struct dpll clock;
9027         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
9028         int refclk = 100000;
9029
9030         /* In case of DSI, DPLL will not be used */
9031         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
9032                 return;
9033
9034         vlv_dpio_get(dev_priv);
9035         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
9036         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
9037         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
9038         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
9039         pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
9040         vlv_dpio_put(dev_priv);
9041
9042         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
9043         clock.m2 = (pll_dw0 & 0xff) << 22;
9044         if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
9045                 clock.m2 |= pll_dw2 & 0x3fffff;
9046         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
9047         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
9048         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
9049
9050         pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
9051 }
9052
9053 static enum intel_output_format
9054 bdw_get_pipemisc_output_format(struct intel_crtc *crtc)
9055 {
9056         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9057         u32 tmp;
9058
9059         tmp = I915_READ(PIPEMISC(crtc->pipe));
9060
9061         if (tmp & PIPEMISC_YUV420_ENABLE) {
9062                 /* We support 4:2:0 in full blend mode only */
9063                 WARN_ON((tmp & PIPEMISC_YUV420_MODE_FULL_BLEND) == 0);
9064
9065                 return INTEL_OUTPUT_FORMAT_YCBCR420;
9066         } else if (tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV) {
9067                 return INTEL_OUTPUT_FORMAT_YCBCR444;
9068         } else {
9069                 return INTEL_OUTPUT_FORMAT_RGB;
9070         }
9071 }
9072
9073 static void i9xx_get_pipe_color_config(struct intel_crtc_state *crtc_state)
9074 {
9075         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9076         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
9077         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9078         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
9079         u32 tmp;
9080
9081         tmp = I915_READ(DSPCNTR(i9xx_plane));
9082
9083         if (tmp & DISPPLANE_GAMMA_ENABLE)
9084                 crtc_state->gamma_enable = true;
9085
9086         if (!HAS_GMCH(dev_priv) &&
9087             tmp & DISPPLANE_PIPE_CSC_ENABLE)
9088                 crtc_state->csc_enable = true;
9089 }
9090
9091 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
9092                                  struct intel_crtc_state *pipe_config)
9093 {
9094         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9095         enum intel_display_power_domain power_domain;
9096         intel_wakeref_t wakeref;
9097         u32 tmp;
9098         bool ret;
9099
9100         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
9101         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
9102         if (!wakeref)
9103                 return false;
9104
9105         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
9106         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9107         pipe_config->shared_dpll = NULL;
9108         pipe_config->master_transcoder = INVALID_TRANSCODER;
9109
9110         ret = false;
9111
9112         tmp = I915_READ(PIPECONF(crtc->pipe));
9113         if (!(tmp & PIPECONF_ENABLE))
9114                 goto out;
9115
9116         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
9117             IS_CHERRYVIEW(dev_priv)) {
9118                 switch (tmp & PIPECONF_BPC_MASK) {
9119                 case PIPECONF_6BPC:
9120                         pipe_config->pipe_bpp = 18;
9121                         break;
9122                 case PIPECONF_8BPC:
9123                         pipe_config->pipe_bpp = 24;
9124                         break;
9125                 case PIPECONF_10BPC:
9126                         pipe_config->pipe_bpp = 30;
9127                         break;
9128                 default:
9129                         break;
9130                 }
9131         }
9132
9133         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
9134             (tmp & PIPECONF_COLOR_RANGE_SELECT))
9135                 pipe_config->limited_color_range = true;
9136
9137         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_I9XX) >>
9138                 PIPECONF_GAMMA_MODE_SHIFT;
9139
9140         if (IS_CHERRYVIEW(dev_priv))
9141                 pipe_config->cgm_mode = I915_READ(CGM_PIPE_MODE(crtc->pipe));
9142
9143         i9xx_get_pipe_color_config(pipe_config);
9144         intel_color_get_config(pipe_config);
9145
9146         if (INTEL_GEN(dev_priv) < 4)
9147                 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
9148
9149         intel_get_pipe_timings(crtc, pipe_config);
9150         intel_get_pipe_src_size(crtc, pipe_config);
9151
9152         i9xx_get_pfit_config(crtc, pipe_config);
9153
9154         if (INTEL_GEN(dev_priv) >= 4) {
9155                 /* No way to read it out on pipes B and C */
9156                 if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
9157                         tmp = dev_priv->chv_dpll_md[crtc->pipe];
9158                 else
9159                         tmp = I915_READ(DPLL_MD(crtc->pipe));
9160                 pipe_config->pixel_multiplier =
9161                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
9162                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
9163                 pipe_config->dpll_hw_state.dpll_md = tmp;
9164         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
9165                    IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
9166                 tmp = I915_READ(DPLL(crtc->pipe));
9167                 pipe_config->pixel_multiplier =
9168                         ((tmp & SDVO_MULTIPLIER_MASK)
9169                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
9170         } else {
9171                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
9172                  * port and will be fixed up in the encoder->get_config
9173                  * function. */
9174                 pipe_config->pixel_multiplier = 1;
9175         }
9176         pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
9177         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
9178                 pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
9179                 pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe));
9180         } else {
9181                 /* Mask out read-only status bits. */
9182                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
9183                                                      DPLL_PORTC_READY_MASK |
9184                                                      DPLL_PORTB_READY_MASK);
9185         }
9186
9187         if (IS_CHERRYVIEW(dev_priv))
9188                 chv_crtc_clock_get(crtc, pipe_config);
9189         else if (IS_VALLEYVIEW(dev_priv))
9190                 vlv_crtc_clock_get(crtc, pipe_config);
9191         else
9192                 i9xx_crtc_clock_get(crtc, pipe_config);
9193
9194         /*
9195          * Normally the dotclock is filled in by the encoder .get_config()
9196          * but in case the pipe is enabled w/o any ports we need a sane
9197          * default.
9198          */
9199         pipe_config->hw.adjusted_mode.crtc_clock =
9200                 pipe_config->port_clock / pipe_config->pixel_multiplier;
9201
9202         ret = true;
9203
9204 out:
9205         intel_display_power_put(dev_priv, power_domain, wakeref);
9206
9207         return ret;
9208 }
9209
9210 static void ironlake_init_pch_refclk(struct drm_i915_private *dev_priv)
9211 {
9212         struct intel_encoder *encoder;
9213         int i;
9214         u32 val, final;
9215         bool has_lvds = false;
9216         bool has_cpu_edp = false;
9217         bool has_panel = false;
9218         bool has_ck505 = false;
9219         bool can_ssc = false;
9220         bool using_ssc_source = false;
9221
9222         /* We need to take the global config into account */
9223         for_each_intel_encoder(&dev_priv->drm, encoder) {
9224                 switch (encoder->type) {
9225                 case INTEL_OUTPUT_LVDS:
9226                         has_panel = true;
9227                         has_lvds = true;
9228                         break;
9229                 case INTEL_OUTPUT_EDP:
9230                         has_panel = true;
9231                         if (encoder->port == PORT_A)
9232                                 has_cpu_edp = true;
9233                         break;
9234                 default:
9235                         break;
9236                 }
9237         }
9238
9239         if (HAS_PCH_IBX(dev_priv)) {
9240                 has_ck505 = dev_priv->vbt.display_clock_mode;
9241                 can_ssc = has_ck505;
9242         } else {
9243                 has_ck505 = false;
9244                 can_ssc = true;
9245         }
9246
9247         /* Check if any DPLLs are using the SSC source */
9248         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
9249                 u32 temp = I915_READ(PCH_DPLL(i));
9250
9251                 if (!(temp & DPLL_VCO_ENABLE))
9252                         continue;
9253
9254                 if ((temp & PLL_REF_INPUT_MASK) ==
9255                     PLLB_REF_INPUT_SPREADSPECTRUMIN) {
9256                         using_ssc_source = true;
9257                         break;
9258                 }
9259         }
9260
9261         DRM_DEBUG_KMS("has_panel %d has_lvds %d has_ck505 %d using_ssc_source %d\n",
9262                       has_panel, has_lvds, has_ck505, using_ssc_source);
9263
9264         /* Ironlake: try to setup display ref clock before DPLL
9265          * enabling. This is only under driver's control after
9266          * PCH B stepping, previous chipset stepping should be
9267          * ignoring this setting.
9268          */
9269         val = I915_READ(PCH_DREF_CONTROL);
9270
9271         /* As we must carefully and slowly disable/enable each source in turn,
9272          * compute the final state we want first and check if we need to
9273          * make any changes at all.
9274          */
9275         final = val;
9276         final &= ~DREF_NONSPREAD_SOURCE_MASK;
9277         if (has_ck505)
9278                 final |= DREF_NONSPREAD_CK505_ENABLE;
9279         else
9280                 final |= DREF_NONSPREAD_SOURCE_ENABLE;
9281
9282         final &= ~DREF_SSC_SOURCE_MASK;
9283         final &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9284         final &= ~DREF_SSC1_ENABLE;
9285
9286         if (has_panel) {
9287                 final |= DREF_SSC_SOURCE_ENABLE;
9288
9289                 if (intel_panel_use_ssc(dev_priv) && can_ssc)
9290                         final |= DREF_SSC1_ENABLE;
9291
9292                 if (has_cpu_edp) {
9293                         if (intel_panel_use_ssc(dev_priv) && can_ssc)
9294                                 final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
9295                         else
9296                                 final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
9297                 } else
9298                         final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9299         } else if (using_ssc_source) {
9300                 final |= DREF_SSC_SOURCE_ENABLE;
9301                 final |= DREF_SSC1_ENABLE;
9302         }
9303
9304         if (final == val)
9305                 return;
9306
9307         /* Always enable nonspread source */
9308         val &= ~DREF_NONSPREAD_SOURCE_MASK;
9309
9310         if (has_ck505)
9311                 val |= DREF_NONSPREAD_CK505_ENABLE;
9312         else
9313                 val |= DREF_NONSPREAD_SOURCE_ENABLE;
9314
9315         if (has_panel) {
9316                 val &= ~DREF_SSC_SOURCE_MASK;
9317                 val |= DREF_SSC_SOURCE_ENABLE;
9318
9319                 /* SSC must be turned on before enabling the CPU output  */
9320                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
9321                         DRM_DEBUG_KMS("Using SSC on panel\n");
9322                         val |= DREF_SSC1_ENABLE;
9323                 } else
9324                         val &= ~DREF_SSC1_ENABLE;
9325
9326                 /* Get SSC going before enabling the outputs */
9327                 I915_WRITE(PCH_DREF_CONTROL, val);
9328                 POSTING_READ(PCH_DREF_CONTROL);
9329                 udelay(200);
9330
9331                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9332
9333                 /* Enable CPU source on CPU attached eDP */
9334                 if (has_cpu_edp) {
9335                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
9336                                 DRM_DEBUG_KMS("Using SSC on eDP\n");
9337                                 val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
9338                         } else
9339                                 val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
9340                 } else
9341                         val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9342
9343                 I915_WRITE(PCH_DREF_CONTROL, val);
9344                 POSTING_READ(PCH_DREF_CONTROL);
9345                 udelay(200);
9346         } else {
9347                 DRM_DEBUG_KMS("Disabling CPU source output\n");
9348
9349                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
9350
9351                 /* Turn off CPU output */
9352                 val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
9353
9354                 I915_WRITE(PCH_DREF_CONTROL, val);
9355                 POSTING_READ(PCH_DREF_CONTROL);
9356                 udelay(200);
9357
9358                 if (!using_ssc_source) {
9359                         DRM_DEBUG_KMS("Disabling SSC source\n");
9360
9361                         /* Turn off the SSC source */
9362                         val &= ~DREF_SSC_SOURCE_MASK;
9363                         val |= DREF_SSC_SOURCE_DISABLE;
9364
9365                         /* Turn off SSC1 */
9366                         val &= ~DREF_SSC1_ENABLE;
9367
9368                         I915_WRITE(PCH_DREF_CONTROL, val);
9369                         POSTING_READ(PCH_DREF_CONTROL);
9370                         udelay(200);
9371                 }
9372         }
9373
9374         BUG_ON(val != final);
9375 }
9376
9377 static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv)
9378 {
9379         u32 tmp;
9380
9381         tmp = I915_READ(SOUTH_CHICKEN2);
9382         tmp |= FDI_MPHY_IOSFSB_RESET_CTL;
9383         I915_WRITE(SOUTH_CHICKEN2, tmp);
9384
9385         if (wait_for_us(I915_READ(SOUTH_CHICKEN2) &
9386                         FDI_MPHY_IOSFSB_RESET_STATUS, 100))
9387                 DRM_ERROR("FDI mPHY reset assert timeout\n");
9388
9389         tmp = I915_READ(SOUTH_CHICKEN2);
9390         tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL;
9391         I915_WRITE(SOUTH_CHICKEN2, tmp);
9392
9393         if (wait_for_us((I915_READ(SOUTH_CHICKEN2) &
9394                          FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100))
9395                 DRM_ERROR("FDI mPHY reset de-assert timeout\n");
9396 }
9397
9398 /* WaMPhyProgramming:hsw */
9399 static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv)
9400 {
9401         u32 tmp;
9402
9403         tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY);
9404         tmp &= ~(0xFF << 24);
9405         tmp |= (0x12 << 24);
9406         intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY);
9407
9408         tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY);
9409         tmp |= (1 << 11);
9410         intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY);
9411
9412         tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY);
9413         tmp |= (1 << 11);
9414         intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY);
9415
9416         tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY);
9417         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
9418         intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY);
9419
9420         tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY);
9421         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
9422         intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY);
9423
9424         tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY);
9425         tmp &= ~(7 << 13);
9426         tmp |= (5 << 13);
9427         intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY);
9428
9429         tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY);
9430         tmp &= ~(7 << 13);
9431         tmp |= (5 << 13);
9432         intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY);
9433
9434         tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY);
9435         tmp &= ~0xFF;
9436         tmp |= 0x1C;
9437         intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY);
9438
9439         tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY);
9440         tmp &= ~0xFF;
9441         tmp |= 0x1C;
9442         intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY);
9443
9444         tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY);
9445         tmp &= ~(0xFF << 16);
9446         tmp |= (0x1C << 16);
9447         intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY);
9448
9449         tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY);
9450         tmp &= ~(0xFF << 16);
9451         tmp |= (0x1C << 16);
9452         intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY);
9453
9454         tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY);
9455         tmp |= (1 << 27);
9456         intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY);
9457
9458         tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY);
9459         tmp |= (1 << 27);
9460         intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY);
9461
9462         tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY);
9463         tmp &= ~(0xF << 28);
9464         tmp |= (4 << 28);
9465         intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY);
9466
9467         tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY);
9468         tmp &= ~(0xF << 28);
9469         tmp |= (4 << 28);
9470         intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY);
9471 }
9472
9473 /* Implements 3 different sequences from BSpec chapter "Display iCLK
9474  * Programming" based on the parameters passed:
9475  * - Sequence to enable CLKOUT_DP
9476  * - Sequence to enable CLKOUT_DP without spread
9477  * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O
9478  */
9479 static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv,
9480                                  bool with_spread, bool with_fdi)
9481 {
9482         u32 reg, tmp;
9483
9484         if (WARN(with_fdi && !with_spread, "FDI requires downspread\n"))
9485                 with_spread = true;
9486         if (WARN(HAS_PCH_LPT_LP(dev_priv) &&
9487             with_fdi, "LP PCH doesn't have FDI\n"))
9488                 with_fdi = false;
9489
9490         mutex_lock(&dev_priv->sb_lock);
9491
9492         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9493         tmp &= ~SBI_SSCCTL_DISABLE;
9494         tmp |= SBI_SSCCTL_PATHALT;
9495         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9496
9497         udelay(24);
9498
9499         if (with_spread) {
9500                 tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9501                 tmp &= ~SBI_SSCCTL_PATHALT;
9502                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9503
9504                 if (with_fdi) {
9505                         lpt_reset_fdi_mphy(dev_priv);
9506                         lpt_program_fdi_mphy(dev_priv);
9507                 }
9508         }
9509
9510         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
9511         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
9512         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
9513         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
9514
9515         mutex_unlock(&dev_priv->sb_lock);
9516 }
9517
9518 /* Sequence to disable CLKOUT_DP */
9519 void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv)
9520 {
9521         u32 reg, tmp;
9522
9523         mutex_lock(&dev_priv->sb_lock);
9524
9525         reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
9526         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
9527         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
9528         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
9529
9530         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
9531         if (!(tmp & SBI_SSCCTL_DISABLE)) {
9532                 if (!(tmp & SBI_SSCCTL_PATHALT)) {
9533                         tmp |= SBI_SSCCTL_PATHALT;
9534                         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9535                         udelay(32);
9536                 }
9537                 tmp |= SBI_SSCCTL_DISABLE;
9538                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
9539         }
9540
9541         mutex_unlock(&dev_priv->sb_lock);
9542 }
9543
9544 #define BEND_IDX(steps) ((50 + (steps)) / 5)
9545
9546 static const u16 sscdivintphase[] = {
9547         [BEND_IDX( 50)] = 0x3B23,
9548         [BEND_IDX( 45)] = 0x3B23,
9549         [BEND_IDX( 40)] = 0x3C23,
9550         [BEND_IDX( 35)] = 0x3C23,
9551         [BEND_IDX( 30)] = 0x3D23,
9552         [BEND_IDX( 25)] = 0x3D23,
9553         [BEND_IDX( 20)] = 0x3E23,
9554         [BEND_IDX( 15)] = 0x3E23,
9555         [BEND_IDX( 10)] = 0x3F23,
9556         [BEND_IDX(  5)] = 0x3F23,
9557         [BEND_IDX(  0)] = 0x0025,
9558         [BEND_IDX( -5)] = 0x0025,
9559         [BEND_IDX(-10)] = 0x0125,
9560         [BEND_IDX(-15)] = 0x0125,
9561         [BEND_IDX(-20)] = 0x0225,
9562         [BEND_IDX(-25)] = 0x0225,
9563         [BEND_IDX(-30)] = 0x0325,
9564         [BEND_IDX(-35)] = 0x0325,
9565         [BEND_IDX(-40)] = 0x0425,
9566         [BEND_IDX(-45)] = 0x0425,
9567         [BEND_IDX(-50)] = 0x0525,
9568 };
9569
9570 /*
9571  * Bend CLKOUT_DP
9572  * steps -50 to 50 inclusive, in steps of 5
9573  * < 0 slow down the clock, > 0 speed up the clock, 0 == no bend (135MHz)
9574  * change in clock period = -(steps / 10) * 5.787 ps
9575  */
9576 static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps)
9577 {
9578         u32 tmp;
9579         int idx = BEND_IDX(steps);
9580
9581         if (WARN_ON(steps % 5 != 0))
9582                 return;
9583
9584         if (WARN_ON(idx >= ARRAY_SIZE(sscdivintphase)))
9585                 return;
9586
9587         mutex_lock(&dev_priv->sb_lock);
9588
9589         if (steps % 10 != 0)
9590                 tmp = 0xAAAAAAAB;
9591         else
9592                 tmp = 0x00000000;
9593         intel_sbi_write(dev_priv, SBI_SSCDITHPHASE, tmp, SBI_ICLK);
9594
9595         tmp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE, SBI_ICLK);
9596         tmp &= 0xffff0000;
9597         tmp |= sscdivintphase[idx];
9598         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE, tmp, SBI_ICLK);
9599
9600         mutex_unlock(&dev_priv->sb_lock);
9601 }
9602
9603 #undef BEND_IDX
9604
9605 static bool spll_uses_pch_ssc(struct drm_i915_private *dev_priv)
9606 {
9607         u32 fuse_strap = I915_READ(FUSE_STRAP);
9608         u32 ctl = I915_READ(SPLL_CTL);
9609
9610         if ((ctl & SPLL_PLL_ENABLE) == 0)
9611                 return false;
9612
9613         if ((ctl & SPLL_REF_MASK) == SPLL_REF_MUXED_SSC &&
9614             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
9615                 return true;
9616
9617         if (IS_BROADWELL(dev_priv) &&
9618             (ctl & SPLL_REF_MASK) == SPLL_REF_PCH_SSC_BDW)
9619                 return true;
9620
9621         return false;
9622 }
9623
9624 static bool wrpll_uses_pch_ssc(struct drm_i915_private *dev_priv,
9625                                enum intel_dpll_id id)
9626 {
9627         u32 fuse_strap = I915_READ(FUSE_STRAP);
9628         u32 ctl = I915_READ(WRPLL_CTL(id));
9629
9630         if ((ctl & WRPLL_PLL_ENABLE) == 0)
9631                 return false;
9632
9633         if ((ctl & WRPLL_REF_MASK) == WRPLL_REF_PCH_SSC)
9634                 return true;
9635
9636         if ((IS_BROADWELL(dev_priv) || IS_HSW_ULT(dev_priv)) &&
9637             (ctl & WRPLL_REF_MASK) == WRPLL_REF_MUXED_SSC_BDW &&
9638             (fuse_strap & HSW_CPU_SSC_ENABLE) == 0)
9639                 return true;
9640
9641         return false;
9642 }
9643
9644 static void lpt_init_pch_refclk(struct drm_i915_private *dev_priv)
9645 {
9646         struct intel_encoder *encoder;
9647         bool has_fdi = false;
9648
9649         for_each_intel_encoder(&dev_priv->drm, encoder) {
9650                 switch (encoder->type) {
9651                 case INTEL_OUTPUT_ANALOG:
9652                         has_fdi = true;
9653                         break;
9654                 default:
9655                         break;
9656                 }
9657         }
9658
9659         /*
9660          * The BIOS may have decided to use the PCH SSC
9661          * reference so we must not disable it until the
9662          * relevant PLLs have stopped relying on it. We'll
9663          * just leave the PCH SSC reference enabled in case
9664          * any active PLL is using it. It will get disabled
9665          * after runtime suspend if we don't have FDI.
9666          *
9667          * TODO: Move the whole reference clock handling
9668          * to the modeset sequence proper so that we can
9669          * actually enable/disable/reconfigure these things
9670          * safely. To do that we need to introduce a real
9671          * clock hierarchy. That would also allow us to do
9672          * clock bending finally.
9673          */
9674         dev_priv->pch_ssc_use = 0;
9675
9676         if (spll_uses_pch_ssc(dev_priv)) {
9677                 DRM_DEBUG_KMS("SPLL using PCH SSC\n");
9678                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_SPLL);
9679         }
9680
9681         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL1)) {
9682                 DRM_DEBUG_KMS("WRPLL1 using PCH SSC\n");
9683                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_WRPLL1);
9684         }
9685
9686         if (wrpll_uses_pch_ssc(dev_priv, DPLL_ID_WRPLL2)) {
9687                 DRM_DEBUG_KMS("WRPLL2 using PCH SSC\n");
9688                 dev_priv->pch_ssc_use |= BIT(DPLL_ID_WRPLL2);
9689         }
9690
9691         if (dev_priv->pch_ssc_use)
9692                 return;
9693
9694         if (has_fdi) {
9695                 lpt_bend_clkout_dp(dev_priv, 0);
9696                 lpt_enable_clkout_dp(dev_priv, true, true);
9697         } else {
9698                 lpt_disable_clkout_dp(dev_priv);
9699         }
9700 }
9701
9702 /*
9703  * Initialize reference clocks when the driver loads
9704  */
9705 void intel_init_pch_refclk(struct drm_i915_private *dev_priv)
9706 {
9707         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
9708                 ironlake_init_pch_refclk(dev_priv);
9709         else if (HAS_PCH_LPT(dev_priv))
9710                 lpt_init_pch_refclk(dev_priv);
9711 }
9712
9713 static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state)
9714 {
9715         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9716         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9717         enum pipe pipe = crtc->pipe;
9718         u32 val;
9719
9720         val = 0;
9721
9722         switch (crtc_state->pipe_bpp) {
9723         case 18:
9724                 val |= PIPECONF_6BPC;
9725                 break;
9726         case 24:
9727                 val |= PIPECONF_8BPC;
9728                 break;
9729         case 30:
9730                 val |= PIPECONF_10BPC;
9731                 break;
9732         case 36:
9733                 val |= PIPECONF_12BPC;
9734                 break;
9735         default:
9736                 /* Case prevented by intel_choose_pipe_bpp_dither. */
9737                 BUG();
9738         }
9739
9740         if (crtc_state->dither)
9741                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
9742
9743         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
9744                 val |= PIPECONF_INTERLACED_ILK;
9745         else
9746                 val |= PIPECONF_PROGRESSIVE;
9747
9748         /*
9749          * This would end up with an odd purple hue over
9750          * the entire display. Make sure we don't do it.
9751          */
9752         WARN_ON(crtc_state->limited_color_range &&
9753                 crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB);
9754
9755         if (crtc_state->limited_color_range)
9756                 val |= PIPECONF_COLOR_RANGE_SELECT;
9757
9758         if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
9759                 val |= PIPECONF_OUTPUT_COLORSPACE_YUV709;
9760
9761         val |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
9762
9763         val |= PIPECONF_FRAME_START_DELAY(0);
9764
9765         I915_WRITE(PIPECONF(pipe), val);
9766         POSTING_READ(PIPECONF(pipe));
9767 }
9768
9769 static void haswell_set_pipeconf(const struct intel_crtc_state *crtc_state)
9770 {
9771         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9772         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9773         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
9774         u32 val = 0;
9775
9776         if (IS_HASWELL(dev_priv) && crtc_state->dither)
9777                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
9778
9779         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
9780                 val |= PIPECONF_INTERLACED_ILK;
9781         else
9782                 val |= PIPECONF_PROGRESSIVE;
9783
9784         if (IS_HASWELL(dev_priv) &&
9785             crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
9786                 val |= PIPECONF_OUTPUT_COLORSPACE_YUV_HSW;
9787
9788         I915_WRITE(PIPECONF(cpu_transcoder), val);
9789         POSTING_READ(PIPECONF(cpu_transcoder));
9790 }
9791
9792 static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state)
9793 {
9794         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
9795         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9796         u32 val = 0;
9797
9798         switch (crtc_state->pipe_bpp) {
9799         case 18:
9800                 val |= PIPEMISC_DITHER_6_BPC;
9801                 break;
9802         case 24:
9803                 val |= PIPEMISC_DITHER_8_BPC;
9804                 break;
9805         case 30:
9806                 val |= PIPEMISC_DITHER_10_BPC;
9807                 break;
9808         case 36:
9809                 val |= PIPEMISC_DITHER_12_BPC;
9810                 break;
9811         default:
9812                 MISSING_CASE(crtc_state->pipe_bpp);
9813                 break;
9814         }
9815
9816         if (crtc_state->dither)
9817                 val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
9818
9819         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
9820             crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444)
9821                 val |= PIPEMISC_OUTPUT_COLORSPACE_YUV;
9822
9823         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
9824                 val |= PIPEMISC_YUV420_ENABLE |
9825                         PIPEMISC_YUV420_MODE_FULL_BLEND;
9826
9827         if (INTEL_GEN(dev_priv) >= 11 &&
9828             (crtc_state->active_planes & ~(icl_hdr_plane_mask() |
9829                                            BIT(PLANE_CURSOR))) == 0)
9830                 val |= PIPEMISC_HDR_MODE_PRECISION;
9831
9832         I915_WRITE(PIPEMISC(crtc->pipe), val);
9833 }
9834
9835 int bdw_get_pipemisc_bpp(struct intel_crtc *crtc)
9836 {
9837         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9838         u32 tmp;
9839
9840         tmp = I915_READ(PIPEMISC(crtc->pipe));
9841
9842         switch (tmp & PIPEMISC_DITHER_BPC_MASK) {
9843         case PIPEMISC_DITHER_6_BPC:
9844                 return 18;
9845         case PIPEMISC_DITHER_8_BPC:
9846                 return 24;
9847         case PIPEMISC_DITHER_10_BPC:
9848                 return 30;
9849         case PIPEMISC_DITHER_12_BPC:
9850                 return 36;
9851         default:
9852                 MISSING_CASE(tmp);
9853                 return 0;
9854         }
9855 }
9856
9857 int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp)
9858 {
9859         /*
9860          * Account for spread spectrum to avoid
9861          * oversubscribing the link. Max center spread
9862          * is 2.5%; use 5% for safety's sake.
9863          */
9864         u32 bps = target_clock * bpp * 21 / 20;
9865         return DIV_ROUND_UP(bps, link_bw * 8);
9866 }
9867
9868 static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor)
9869 {
9870         return i9xx_dpll_compute_m(dpll) < factor * dpll->n;
9871 }
9872
9873 static void ironlake_compute_dpll(struct intel_crtc *crtc,
9874                                   struct intel_crtc_state *crtc_state,
9875                                   struct dpll *reduced_clock)
9876 {
9877         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9878         u32 dpll, fp, fp2;
9879         int factor;
9880
9881         /* Enable autotuning of the PLL clock (if permissible) */
9882         factor = 21;
9883         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
9884                 if ((intel_panel_use_ssc(dev_priv) &&
9885                      dev_priv->vbt.lvds_ssc_freq == 100000) ||
9886                     (HAS_PCH_IBX(dev_priv) &&
9887                      intel_is_dual_link_lvds(dev_priv)))
9888                         factor = 25;
9889         } else if (crtc_state->sdvo_tv_clock) {
9890                 factor = 20;
9891         }
9892
9893         fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
9894
9895         if (ironlake_needs_fb_cb_tune(&crtc_state->dpll, factor))
9896                 fp |= FP_CB_TUNE;
9897
9898         if (reduced_clock) {
9899                 fp2 = i9xx_dpll_compute_fp(reduced_clock);
9900
9901                 if (reduced_clock->m < factor * reduced_clock->n)
9902                         fp2 |= FP_CB_TUNE;
9903         } else {
9904                 fp2 = fp;
9905         }
9906
9907         dpll = 0;
9908
9909         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
9910                 dpll |= DPLLB_MODE_LVDS;
9911         else
9912                 dpll |= DPLLB_MODE_DAC_SERIAL;
9913
9914         dpll |= (crtc_state->pixel_multiplier - 1)
9915                 << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
9916
9917         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
9918             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
9919                 dpll |= DPLL_SDVO_HIGH_SPEED;
9920
9921         if (intel_crtc_has_dp_encoder(crtc_state))
9922                 dpll |= DPLL_SDVO_HIGH_SPEED;
9923
9924         /*
9925          * The high speed IO clock is only really required for
9926          * SDVO/HDMI/DP, but we also enable it for CRT to make it
9927          * possible to share the DPLL between CRT and HDMI. Enabling
9928          * the clock needlessly does no real harm, except use up a
9929          * bit of power potentially.
9930          *
9931          * We'll limit this to IVB with 3 pipes, since it has only two
9932          * DPLLs and so DPLL sharing is the only way to get three pipes
9933          * driving PCH ports at the same time. On SNB we could do this,
9934          * and potentially avoid enabling the second DPLL, but it's not
9935          * clear if it''s a win or loss power wise. No point in doing
9936          * this on ILK at all since it has a fixed DPLL<->pipe mapping.
9937          */
9938         if (INTEL_NUM_PIPES(dev_priv) == 3 &&
9939             intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG))
9940                 dpll |= DPLL_SDVO_HIGH_SPEED;
9941
9942         /* compute bitmask from p1 value */
9943         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
9944         /* also FPA1 */
9945         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
9946
9947         switch (crtc_state->dpll.p2) {
9948         case 5:
9949                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
9950                 break;
9951         case 7:
9952                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
9953                 break;
9954         case 10:
9955                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
9956                 break;
9957         case 14:
9958                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
9959                 break;
9960         }
9961
9962         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
9963             intel_panel_use_ssc(dev_priv))
9964                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
9965         else
9966                 dpll |= PLL_REF_INPUT_DREFCLK;
9967
9968         dpll |= DPLL_VCO_ENABLE;
9969
9970         crtc_state->dpll_hw_state.dpll = dpll;
9971         crtc_state->dpll_hw_state.fp0 = fp;
9972         crtc_state->dpll_hw_state.fp1 = fp2;
9973 }
9974
9975 static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
9976                                        struct intel_crtc_state *crtc_state)
9977 {
9978         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9979         struct intel_atomic_state *state =
9980                 to_intel_atomic_state(crtc_state->uapi.state);
9981         const struct intel_limit *limit;
9982         int refclk = 120000;
9983
9984         memset(&crtc_state->dpll_hw_state, 0,
9985                sizeof(crtc_state->dpll_hw_state));
9986
9987         /* CPU eDP is the only output that doesn't need a PCH PLL of its own. */
9988         if (!crtc_state->has_pch_encoder)
9989                 return 0;
9990
9991         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
9992                 if (intel_panel_use_ssc(dev_priv)) {
9993                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n",
9994                                       dev_priv->vbt.lvds_ssc_freq);
9995                         refclk = dev_priv->vbt.lvds_ssc_freq;
9996                 }
9997
9998                 if (intel_is_dual_link_lvds(dev_priv)) {
9999                         if (refclk == 100000)
10000                                 limit = &intel_limits_ironlake_dual_lvds_100m;
10001                         else
10002                                 limit = &intel_limits_ironlake_dual_lvds;
10003                 } else {
10004                         if (refclk == 100000)
10005                                 limit = &intel_limits_ironlake_single_lvds_100m;
10006                         else
10007                                 limit = &intel_limits_ironlake_single_lvds;
10008                 }
10009         } else {
10010                 limit = &intel_limits_ironlake_dac;
10011         }
10012
10013         if (!crtc_state->clock_set &&
10014             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
10015                                 refclk, NULL, &crtc_state->dpll)) {
10016                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
10017                 return -EINVAL;
10018         }
10019
10020         ironlake_compute_dpll(crtc, crtc_state, NULL);
10021
10022         if (!intel_reserve_shared_dplls(state, crtc, NULL)) {
10023                 DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
10024                               pipe_name(crtc->pipe));
10025                 return -EINVAL;
10026         }
10027
10028         return 0;
10029 }
10030
10031 static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
10032                                          struct intel_link_m_n *m_n)
10033 {
10034         struct drm_device *dev = crtc->base.dev;
10035         struct drm_i915_private *dev_priv = to_i915(dev);
10036         enum pipe pipe = crtc->pipe;
10037
10038         m_n->link_m = I915_READ(PCH_TRANS_LINK_M1(pipe));
10039         m_n->link_n = I915_READ(PCH_TRANS_LINK_N1(pipe));
10040         m_n->gmch_m = I915_READ(PCH_TRANS_DATA_M1(pipe))
10041                 & ~TU_SIZE_MASK;
10042         m_n->gmch_n = I915_READ(PCH_TRANS_DATA_N1(pipe));
10043         m_n->tu = ((I915_READ(PCH_TRANS_DATA_M1(pipe))
10044                     & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
10045 }
10046
10047 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
10048                                          enum transcoder transcoder,
10049                                          struct intel_link_m_n *m_n,
10050                                          struct intel_link_m_n *m2_n2)
10051 {
10052         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10053         enum pipe pipe = crtc->pipe;
10054
10055         if (INTEL_GEN(dev_priv) >= 5) {
10056                 m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder));
10057                 m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder));
10058                 m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder))
10059                         & ~TU_SIZE_MASK;
10060                 m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder));
10061                 m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder))
10062                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
10063
10064                 if (m2_n2 && transcoder_has_m2_n2(dev_priv, transcoder)) {
10065                         m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder));
10066                         m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder));
10067                         m2_n2->gmch_m = I915_READ(PIPE_DATA_M2(transcoder))
10068                                         & ~TU_SIZE_MASK;
10069                         m2_n2->gmch_n = I915_READ(PIPE_DATA_N2(transcoder));
10070                         m2_n2->tu = ((I915_READ(PIPE_DATA_M2(transcoder))
10071                                         & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
10072                 }
10073         } else {
10074                 m_n->link_m = I915_READ(PIPE_LINK_M_G4X(pipe));
10075                 m_n->link_n = I915_READ(PIPE_LINK_N_G4X(pipe));
10076                 m_n->gmch_m = I915_READ(PIPE_DATA_M_G4X(pipe))
10077                         & ~TU_SIZE_MASK;
10078                 m_n->gmch_n = I915_READ(PIPE_DATA_N_G4X(pipe));
10079                 m_n->tu = ((I915_READ(PIPE_DATA_M_G4X(pipe))
10080                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
10081         }
10082 }
10083
10084 void intel_dp_get_m_n(struct intel_crtc *crtc,
10085                       struct intel_crtc_state *pipe_config)
10086 {
10087         if (pipe_config->has_pch_encoder)
10088                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
10089         else
10090                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
10091                                              &pipe_config->dp_m_n,
10092                                              &pipe_config->dp_m2_n2);
10093 }
10094
10095 static void ironlake_get_fdi_m_n_config(struct intel_crtc *crtc,
10096                                         struct intel_crtc_state *pipe_config)
10097 {
10098         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
10099                                      &pipe_config->fdi_m_n, NULL);
10100 }
10101
10102 static void skylake_get_pfit_config(struct intel_crtc *crtc,
10103                                     struct intel_crtc_state *pipe_config)
10104 {
10105         struct drm_device *dev = crtc->base.dev;
10106         struct drm_i915_private *dev_priv = to_i915(dev);
10107         struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state;
10108         u32 ps_ctrl = 0;
10109         int id = -1;
10110         int i;
10111
10112         /* find scaler attached to this pipe */
10113         for (i = 0; i < crtc->num_scalers; i++) {
10114                 ps_ctrl = I915_READ(SKL_PS_CTRL(crtc->pipe, i));
10115                 if (ps_ctrl & PS_SCALER_EN && !(ps_ctrl & PS_PLANE_SEL_MASK)) {
10116                         id = i;
10117                         pipe_config->pch_pfit.enabled = true;
10118                         pipe_config->pch_pfit.pos = I915_READ(SKL_PS_WIN_POS(crtc->pipe, i));
10119                         pipe_config->pch_pfit.size = I915_READ(SKL_PS_WIN_SZ(crtc->pipe, i));
10120                         scaler_state->scalers[i].in_use = true;
10121                         break;
10122                 }
10123         }
10124
10125         scaler_state->scaler_id = id;
10126         if (id >= 0) {
10127                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
10128         } else {
10129                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
10130         }
10131 }
10132
10133 static void
10134 skylake_get_initial_plane_config(struct intel_crtc *crtc,
10135                                  struct intel_initial_plane_config *plane_config)
10136 {
10137         struct drm_device *dev = crtc->base.dev;
10138         struct drm_i915_private *dev_priv = to_i915(dev);
10139         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
10140         enum plane_id plane_id = plane->id;
10141         enum pipe pipe;
10142         u32 val, base, offset, stride_mult, tiling, alpha;
10143         int fourcc, pixel_format;
10144         unsigned int aligned_height;
10145         struct drm_framebuffer *fb;
10146         struct intel_framebuffer *intel_fb;
10147
10148         if (!plane->get_hw_state(plane, &pipe))
10149                 return;
10150
10151         WARN_ON(pipe != crtc->pipe);
10152
10153         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
10154         if (!intel_fb) {
10155                 DRM_DEBUG_KMS("failed to alloc fb\n");
10156                 return;
10157         }
10158
10159         fb = &intel_fb->base;
10160
10161         fb->dev = dev;
10162
10163         val = I915_READ(PLANE_CTL(pipe, plane_id));
10164
10165         if (INTEL_GEN(dev_priv) >= 11)
10166                 pixel_format = val & ICL_PLANE_CTL_FORMAT_MASK;
10167         else
10168                 pixel_format = val & PLANE_CTL_FORMAT_MASK;
10169
10170         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
10171                 alpha = I915_READ(PLANE_COLOR_CTL(pipe, plane_id));
10172                 alpha &= PLANE_COLOR_ALPHA_MASK;
10173         } else {
10174                 alpha = val & PLANE_CTL_ALPHA_MASK;
10175         }
10176
10177         fourcc = skl_format_to_fourcc(pixel_format,
10178                                       val & PLANE_CTL_ORDER_RGBX, alpha);
10179         fb->format = drm_format_info(fourcc);
10180
10181         tiling = val & PLANE_CTL_TILED_MASK;
10182         switch (tiling) {
10183         case PLANE_CTL_TILED_LINEAR:
10184                 fb->modifier = DRM_FORMAT_MOD_LINEAR;
10185                 break;
10186         case PLANE_CTL_TILED_X:
10187                 plane_config->tiling = I915_TILING_X;
10188                 fb->modifier = I915_FORMAT_MOD_X_TILED;
10189                 break;
10190         case PLANE_CTL_TILED_Y:
10191                 plane_config->tiling = I915_TILING_Y;
10192                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
10193                         fb->modifier = INTEL_GEN(dev_priv) >= 12 ?
10194                                 I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS :
10195                                 I915_FORMAT_MOD_Y_TILED_CCS;
10196                 else
10197                         fb->modifier = I915_FORMAT_MOD_Y_TILED;
10198                 break;
10199         case PLANE_CTL_TILED_YF:
10200                 if (val & PLANE_CTL_RENDER_DECOMPRESSION_ENABLE)
10201                         fb->modifier = I915_FORMAT_MOD_Yf_TILED_CCS;
10202                 else
10203                         fb->modifier = I915_FORMAT_MOD_Yf_TILED;
10204                 break;
10205         default:
10206                 MISSING_CASE(tiling);
10207                 goto error;
10208         }
10209
10210         /*
10211          * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
10212          * while i915 HW rotation is clockwise, thats why this swapping.
10213          */
10214         switch (val & PLANE_CTL_ROTATE_MASK) {
10215         case PLANE_CTL_ROTATE_0:
10216                 plane_config->rotation = DRM_MODE_ROTATE_0;
10217                 break;
10218         case PLANE_CTL_ROTATE_90:
10219                 plane_config->rotation = DRM_MODE_ROTATE_270;
10220                 break;
10221         case PLANE_CTL_ROTATE_180:
10222                 plane_config->rotation = DRM_MODE_ROTATE_180;
10223                 break;
10224         case PLANE_CTL_ROTATE_270:
10225                 plane_config->rotation = DRM_MODE_ROTATE_90;
10226                 break;
10227         }
10228
10229         if (INTEL_GEN(dev_priv) >= 10 &&
10230             val & PLANE_CTL_FLIP_HORIZONTAL)
10231                 plane_config->rotation |= DRM_MODE_REFLECT_X;
10232
10233         base = I915_READ(PLANE_SURF(pipe, plane_id)) & 0xfffff000;
10234         plane_config->base = base;
10235
10236         offset = I915_READ(PLANE_OFFSET(pipe, plane_id));
10237
10238         val = I915_READ(PLANE_SIZE(pipe, plane_id));
10239         fb->height = ((val >> 16) & 0xffff) + 1;
10240         fb->width = ((val >> 0) & 0xffff) + 1;
10241
10242         val = I915_READ(PLANE_STRIDE(pipe, plane_id));
10243         stride_mult = skl_plane_stride_mult(fb, 0, DRM_MODE_ROTATE_0);
10244         fb->pitches[0] = (val & 0x3ff) * stride_mult;
10245
10246         aligned_height = intel_fb_align_height(fb, 0, fb->height);
10247
10248         plane_config->size = fb->pitches[0] * aligned_height;
10249
10250         DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
10251                       crtc->base.name, plane->base.name, fb->width, fb->height,
10252                       fb->format->cpp[0] * 8, base, fb->pitches[0],
10253                       plane_config->size);
10254
10255         plane_config->fb = intel_fb;
10256         return;
10257
10258 error:
10259         kfree(intel_fb);
10260 }
10261
10262 static void ironlake_get_pfit_config(struct intel_crtc *crtc,
10263                                      struct intel_crtc_state *pipe_config)
10264 {
10265         struct drm_device *dev = crtc->base.dev;
10266         struct drm_i915_private *dev_priv = to_i915(dev);
10267         u32 tmp;
10268
10269         tmp = I915_READ(PF_CTL(crtc->pipe));
10270
10271         if (tmp & PF_ENABLE) {
10272                 pipe_config->pch_pfit.enabled = true;
10273                 pipe_config->pch_pfit.pos = I915_READ(PF_WIN_POS(crtc->pipe));
10274                 pipe_config->pch_pfit.size = I915_READ(PF_WIN_SZ(crtc->pipe));
10275
10276                 /* We currently do not free assignements of panel fitters on
10277                  * ivb/hsw (since we don't use the higher upscaling modes which
10278                  * differentiates them) so just WARN about this case for now. */
10279                 if (IS_GEN(dev_priv, 7)) {
10280                         WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) !=
10281                                 PF_PIPE_SEL_IVB(crtc->pipe));
10282                 }
10283         }
10284 }
10285
10286 static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
10287                                      struct intel_crtc_state *pipe_config)
10288 {
10289         struct drm_device *dev = crtc->base.dev;
10290         struct drm_i915_private *dev_priv = to_i915(dev);
10291         enum intel_display_power_domain power_domain;
10292         intel_wakeref_t wakeref;
10293         u32 tmp;
10294         bool ret;
10295
10296         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
10297         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
10298         if (!wakeref)
10299                 return false;
10300
10301         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
10302         pipe_config->shared_dpll = NULL;
10303         pipe_config->master_transcoder = INVALID_TRANSCODER;
10304
10305         ret = false;
10306         tmp = I915_READ(PIPECONF(crtc->pipe));
10307         if (!(tmp & PIPECONF_ENABLE))
10308                 goto out;
10309
10310         switch (tmp & PIPECONF_BPC_MASK) {
10311         case PIPECONF_6BPC:
10312                 pipe_config->pipe_bpp = 18;
10313                 break;
10314         case PIPECONF_8BPC:
10315                 pipe_config->pipe_bpp = 24;
10316                 break;
10317         case PIPECONF_10BPC:
10318                 pipe_config->pipe_bpp = 30;
10319                 break;
10320         case PIPECONF_12BPC:
10321                 pipe_config->pipe_bpp = 36;
10322                 break;
10323         default:
10324                 break;
10325         }
10326
10327         if (tmp & PIPECONF_COLOR_RANGE_SELECT)
10328                 pipe_config->limited_color_range = true;
10329
10330         switch (tmp & PIPECONF_OUTPUT_COLORSPACE_MASK) {
10331         case PIPECONF_OUTPUT_COLORSPACE_YUV601:
10332         case PIPECONF_OUTPUT_COLORSPACE_YUV709:
10333                 pipe_config->output_format = INTEL_OUTPUT_FORMAT_YCBCR444;
10334                 break;
10335         default:
10336                 pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
10337                 break;
10338         }
10339
10340         pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_ILK) >>
10341                 PIPECONF_GAMMA_MODE_SHIFT;
10342
10343         pipe_config->csc_mode = I915_READ(PIPE_CSC_MODE(crtc->pipe));
10344
10345         i9xx_get_pipe_color_config(pipe_config);
10346         intel_color_get_config(pipe_config);
10347
10348         if (I915_READ(PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
10349                 struct intel_shared_dpll *pll;
10350                 enum intel_dpll_id pll_id;
10351
10352                 pipe_config->has_pch_encoder = true;
10353
10354                 tmp = I915_READ(FDI_RX_CTL(crtc->pipe));
10355                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
10356                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
10357
10358                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
10359
10360                 if (HAS_PCH_IBX(dev_priv)) {
10361                         /*
10362                          * The pipe->pch transcoder and pch transcoder->pll
10363                          * mapping is fixed.
10364                          */
10365                         pll_id = (enum intel_dpll_id) crtc->pipe;
10366                 } else {
10367                         tmp = I915_READ(PCH_DPLL_SEL);
10368                         if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
10369                                 pll_id = DPLL_ID_PCH_PLL_B;
10370                         else
10371                                 pll_id= DPLL_ID_PCH_PLL_A;
10372                 }
10373
10374                 pipe_config->shared_dpll =
10375                         intel_get_shared_dpll_by_id(dev_priv, pll_id);
10376                 pll = pipe_config->shared_dpll;
10377
10378                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
10379                                                 &pipe_config->dpll_hw_state));
10380
10381                 tmp = pipe_config->dpll_hw_state.dpll;
10382                 pipe_config->pixel_multiplier =
10383                         ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
10384                          >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
10385
10386                 ironlake_pch_clock_get(crtc, pipe_config);
10387         } else {
10388                 pipe_config->pixel_multiplier = 1;
10389         }
10390
10391         intel_get_pipe_timings(crtc, pipe_config);
10392         intel_get_pipe_src_size(crtc, pipe_config);
10393
10394         ironlake_get_pfit_config(crtc, pipe_config);
10395
10396         ret = true;
10397
10398 out:
10399         intel_display_power_put(dev_priv, power_domain, wakeref);
10400
10401         return ret;
10402 }
10403 static int haswell_crtc_compute_clock(struct intel_crtc *crtc,
10404                                       struct intel_crtc_state *crtc_state)
10405 {
10406         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10407         struct intel_atomic_state *state =
10408                 to_intel_atomic_state(crtc_state->uapi.state);
10409
10410         if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) ||
10411             INTEL_GEN(dev_priv) >= 11) {
10412                 struct intel_encoder *encoder =
10413                         intel_get_crtc_new_encoder(state, crtc_state);
10414
10415                 if (!intel_reserve_shared_dplls(state, crtc, encoder)) {
10416                         DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
10417                                       pipe_name(crtc->pipe));
10418                         return -EINVAL;
10419                 }
10420         }
10421
10422         return 0;
10423 }
10424
10425 static void cannonlake_get_ddi_pll(struct drm_i915_private *dev_priv,
10426                                    enum port port,
10427                                    struct intel_crtc_state *pipe_config)
10428 {
10429         enum intel_dpll_id id;
10430         u32 temp;
10431
10432         temp = I915_READ(DPCLKA_CFGCR0) & DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
10433         id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port);
10434
10435         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL2))
10436                 return;
10437
10438         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10439 }
10440
10441 static void icelake_get_ddi_pll(struct drm_i915_private *dev_priv,
10442                                 enum port port,
10443                                 struct intel_crtc_state *pipe_config)
10444 {
10445         enum phy phy = intel_port_to_phy(dev_priv, port);
10446         enum icl_port_dpll_id port_dpll_id;
10447         enum intel_dpll_id id;
10448         u32 temp;
10449
10450         if (intel_phy_is_combo(dev_priv, phy)) {
10451                 temp = I915_READ(ICL_DPCLKA_CFGCR0) &
10452                         ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy);
10453                 id = temp >> ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy);
10454                 port_dpll_id = ICL_PORT_DPLL_DEFAULT;
10455         } else if (intel_phy_is_tc(dev_priv, phy)) {
10456                 u32 clk_sel = I915_READ(DDI_CLK_SEL(port)) & DDI_CLK_SEL_MASK;
10457
10458                 if (clk_sel == DDI_CLK_SEL_MG) {
10459                         id = icl_tc_port_to_pll_id(intel_port_to_tc(dev_priv,
10460                                                                     port));
10461                         port_dpll_id = ICL_PORT_DPLL_MG_PHY;
10462                 } else {
10463                         WARN_ON(clk_sel < DDI_CLK_SEL_TBT_162);
10464                         id = DPLL_ID_ICL_TBTPLL;
10465                         port_dpll_id = ICL_PORT_DPLL_DEFAULT;
10466                 }
10467         } else {
10468                 WARN(1, "Invalid port %x\n", port);
10469                 return;
10470         }
10471
10472         pipe_config->icl_port_dplls[port_dpll_id].pll =
10473                 intel_get_shared_dpll_by_id(dev_priv, id);
10474
10475         icl_set_active_port_dpll(pipe_config, port_dpll_id);
10476 }
10477
10478 static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv,
10479                                 enum port port,
10480                                 struct intel_crtc_state *pipe_config)
10481 {
10482         enum intel_dpll_id id;
10483
10484         switch (port) {
10485         case PORT_A:
10486                 id = DPLL_ID_SKL_DPLL0;
10487                 break;
10488         case PORT_B:
10489                 id = DPLL_ID_SKL_DPLL1;
10490                 break;
10491         case PORT_C:
10492                 id = DPLL_ID_SKL_DPLL2;
10493                 break;
10494         default:
10495                 DRM_ERROR("Incorrect port type\n");
10496                 return;
10497         }
10498
10499         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10500 }
10501
10502 static void skylake_get_ddi_pll(struct drm_i915_private *dev_priv,
10503                                 enum port port,
10504                                 struct intel_crtc_state *pipe_config)
10505 {
10506         enum intel_dpll_id id;
10507         u32 temp;
10508
10509         temp = I915_READ(DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port);
10510         id = temp >> (port * 3 + 1);
10511
10512         if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL3))
10513                 return;
10514
10515         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10516 }
10517
10518 static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
10519                                 enum port port,
10520                                 struct intel_crtc_state *pipe_config)
10521 {
10522         enum intel_dpll_id id;
10523         u32 ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
10524
10525         switch (ddi_pll_sel) {
10526         case PORT_CLK_SEL_WRPLL1:
10527                 id = DPLL_ID_WRPLL1;
10528                 break;
10529         case PORT_CLK_SEL_WRPLL2:
10530                 id = DPLL_ID_WRPLL2;
10531                 break;
10532         case PORT_CLK_SEL_SPLL:
10533                 id = DPLL_ID_SPLL;
10534                 break;
10535         case PORT_CLK_SEL_LCPLL_810:
10536                 id = DPLL_ID_LCPLL_810;
10537                 break;
10538         case PORT_CLK_SEL_LCPLL_1350:
10539                 id = DPLL_ID_LCPLL_1350;
10540                 break;
10541         case PORT_CLK_SEL_LCPLL_2700:
10542                 id = DPLL_ID_LCPLL_2700;
10543                 break;
10544         default:
10545                 MISSING_CASE(ddi_pll_sel);
10546                 /* fall through */
10547         case PORT_CLK_SEL_NONE:
10548                 return;
10549         }
10550
10551         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10552 }
10553
10554 static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
10555                                      struct intel_crtc_state *pipe_config,
10556                                      u64 *power_domain_mask,
10557                                      intel_wakeref_t *wakerefs)
10558 {
10559         struct drm_device *dev = crtc->base.dev;
10560         struct drm_i915_private *dev_priv = to_i915(dev);
10561         enum intel_display_power_domain power_domain;
10562         unsigned long panel_transcoder_mask = 0;
10563         unsigned long enabled_panel_transcoders = 0;
10564         enum transcoder panel_transcoder;
10565         intel_wakeref_t wf;
10566         u32 tmp;
10567
10568         if (INTEL_GEN(dev_priv) >= 11)
10569                 panel_transcoder_mask |=
10570                         BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1);
10571
10572         if (HAS_TRANSCODER_EDP(dev_priv))
10573                 panel_transcoder_mask |= BIT(TRANSCODER_EDP);
10574
10575         /*
10576          * The pipe->transcoder mapping is fixed with the exception of the eDP
10577          * and DSI transcoders handled below.
10578          */
10579         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
10580
10581         /*
10582          * XXX: Do intel_display_power_get_if_enabled before reading this (for
10583          * consistency and less surprising code; it's in always on power).
10584          */
10585         for_each_set_bit(panel_transcoder,
10586                          &panel_transcoder_mask,
10587                          ARRAY_SIZE(INTEL_INFO(dev_priv)->trans_offsets)) {
10588                 bool force_thru = false;
10589                 enum pipe trans_pipe;
10590
10591                 tmp = I915_READ(TRANS_DDI_FUNC_CTL(panel_transcoder));
10592                 if (!(tmp & TRANS_DDI_FUNC_ENABLE))
10593                         continue;
10594
10595                 /*
10596                  * Log all enabled ones, only use the first one.
10597                  *
10598                  * FIXME: This won't work for two separate DSI displays.
10599                  */
10600                 enabled_panel_transcoders |= BIT(panel_transcoder);
10601                 if (enabled_panel_transcoders != BIT(panel_transcoder))
10602                         continue;
10603
10604                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
10605                 default:
10606                         WARN(1, "unknown pipe linked to transcoder %s\n",
10607                              transcoder_name(panel_transcoder));
10608                         /* fall through */
10609                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
10610                         force_thru = true;
10611                         /* fall through */
10612                 case TRANS_DDI_EDP_INPUT_A_ON:
10613                         trans_pipe = PIPE_A;
10614                         break;
10615                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
10616                         trans_pipe = PIPE_B;
10617                         break;
10618                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
10619                         trans_pipe = PIPE_C;
10620                         break;
10621                 case TRANS_DDI_EDP_INPUT_D_ONOFF:
10622                         trans_pipe = PIPE_D;
10623                         break;
10624                 }
10625
10626                 if (trans_pipe == crtc->pipe) {
10627                         pipe_config->cpu_transcoder = panel_transcoder;
10628                         pipe_config->pch_pfit.force_thru = force_thru;
10629                 }
10630         }
10631
10632         /*
10633          * Valid combos: none, eDP, DSI0, DSI1, DSI0+DSI1
10634          */
10635         WARN_ON((enabled_panel_transcoders & BIT(TRANSCODER_EDP)) &&
10636                 enabled_panel_transcoders != BIT(TRANSCODER_EDP));
10637
10638         power_domain = POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder);
10639         WARN_ON(*power_domain_mask & BIT_ULL(power_domain));
10640
10641         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10642         if (!wf)
10643                 return false;
10644
10645         wakerefs[power_domain] = wf;
10646         *power_domain_mask |= BIT_ULL(power_domain);
10647
10648         tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
10649
10650         return tmp & PIPECONF_ENABLE;
10651 }
10652
10653 static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
10654                                          struct intel_crtc_state *pipe_config,
10655                                          u64 *power_domain_mask,
10656                                          intel_wakeref_t *wakerefs)
10657 {
10658         struct drm_device *dev = crtc->base.dev;
10659         struct drm_i915_private *dev_priv = to_i915(dev);
10660         enum intel_display_power_domain power_domain;
10661         enum transcoder cpu_transcoder;
10662         intel_wakeref_t wf;
10663         enum port port;
10664         u32 tmp;
10665
10666         for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) {
10667                 if (port == PORT_A)
10668                         cpu_transcoder = TRANSCODER_DSI_A;
10669                 else
10670                         cpu_transcoder = TRANSCODER_DSI_C;
10671
10672                 power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
10673                 WARN_ON(*power_domain_mask & BIT_ULL(power_domain));
10674
10675                 wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10676                 if (!wf)
10677                         continue;
10678
10679                 wakerefs[power_domain] = wf;
10680                 *power_domain_mask |= BIT_ULL(power_domain);
10681
10682                 /*
10683                  * The PLL needs to be enabled with a valid divider
10684                  * configuration, otherwise accessing DSI registers will hang
10685                  * the machine. See BSpec North Display Engine
10686                  * registers/MIPI[BXT]. We can break out here early, since we
10687                  * need the same DSI PLL to be enabled for both DSI ports.
10688                  */
10689                 if (!bxt_dsi_pll_is_enabled(dev_priv))
10690                         break;
10691
10692                 /* XXX: this works for video mode only */
10693                 tmp = I915_READ(BXT_MIPI_PORT_CTRL(port));
10694                 if (!(tmp & DPI_ENABLE))
10695                         continue;
10696
10697                 tmp = I915_READ(MIPI_CTRL(port));
10698                 if ((tmp & BXT_PIPE_SELECT_MASK) != BXT_PIPE_SELECT(crtc->pipe))
10699                         continue;
10700
10701                 pipe_config->cpu_transcoder = cpu_transcoder;
10702                 break;
10703         }
10704
10705         return transcoder_is_dsi(pipe_config->cpu_transcoder);
10706 }
10707
10708 static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
10709                                        struct intel_crtc_state *pipe_config)
10710 {
10711         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10712         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
10713         struct intel_shared_dpll *pll;
10714         enum port port;
10715         u32 tmp;
10716
10717         if (transcoder_is_dsi(cpu_transcoder)) {
10718                 port = (cpu_transcoder == TRANSCODER_DSI_A) ?
10719                                                 PORT_A : PORT_B;
10720         } else {
10721                 tmp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
10722                 if (INTEL_GEN(dev_priv) >= 12)
10723                         port = TGL_TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp);
10724                 else
10725                         port = TRANS_DDI_FUNC_CTL_VAL_TO_PORT(tmp);
10726         }
10727
10728         if (INTEL_GEN(dev_priv) >= 11)
10729                 icelake_get_ddi_pll(dev_priv, port, pipe_config);
10730         else if (IS_CANNONLAKE(dev_priv))
10731                 cannonlake_get_ddi_pll(dev_priv, port, pipe_config);
10732         else if (IS_GEN9_BC(dev_priv))
10733                 skylake_get_ddi_pll(dev_priv, port, pipe_config);
10734         else if (IS_GEN9_LP(dev_priv))
10735                 bxt_get_ddi_pll(dev_priv, port, pipe_config);
10736         else
10737                 haswell_get_ddi_pll(dev_priv, port, pipe_config);
10738
10739         pll = pipe_config->shared_dpll;
10740         if (pll) {
10741                 WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll,
10742                                                 &pipe_config->dpll_hw_state));
10743         }
10744
10745         /*
10746          * Haswell has only FDI/PCH transcoder A. It is which is connected to
10747          * DDI E. So just check whether this pipe is wired to DDI E and whether
10748          * the PCH transcoder is on.
10749          */
10750         if (INTEL_GEN(dev_priv) < 9 &&
10751             (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) {
10752                 pipe_config->has_pch_encoder = true;
10753
10754                 tmp = I915_READ(FDI_RX_CTL(PIPE_A));
10755                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
10756                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
10757
10758                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
10759         }
10760 }
10761
10762 static enum transcoder transcoder_master_readout(struct drm_i915_private *dev_priv,
10763                                                  enum transcoder cpu_transcoder)
10764 {
10765         u32 trans_port_sync, master_select;
10766
10767         trans_port_sync = I915_READ(TRANS_DDI_FUNC_CTL2(cpu_transcoder));
10768
10769         if ((trans_port_sync & PORT_SYNC_MODE_ENABLE) == 0)
10770                 return INVALID_TRANSCODER;
10771
10772         master_select = trans_port_sync &
10773                         PORT_SYNC_MODE_MASTER_SELECT_MASK;
10774         if (master_select == 0)
10775                 return TRANSCODER_EDP;
10776         else
10777                 return master_select - 1;
10778 }
10779
10780 static void icelake_get_trans_port_sync_config(struct intel_crtc_state *crtc_state)
10781 {
10782         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
10783         u32 transcoders;
10784         enum transcoder cpu_transcoder;
10785
10786         crtc_state->master_transcoder = transcoder_master_readout(dev_priv,
10787                                                                   crtc_state->cpu_transcoder);
10788
10789         transcoders = BIT(TRANSCODER_A) |
10790                 BIT(TRANSCODER_B) |
10791                 BIT(TRANSCODER_C) |
10792                 BIT(TRANSCODER_D);
10793         for_each_cpu_transcoder_masked(dev_priv, cpu_transcoder, transcoders) {
10794                 enum intel_display_power_domain power_domain;
10795                 intel_wakeref_t trans_wakeref;
10796
10797                 power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
10798                 trans_wakeref = intel_display_power_get_if_enabled(dev_priv,
10799                                                                    power_domain);
10800
10801                 if (!trans_wakeref)
10802                         continue;
10803
10804                 if (transcoder_master_readout(dev_priv, cpu_transcoder) ==
10805                     crtc_state->cpu_transcoder)
10806                         crtc_state->sync_mode_slaves_mask |= BIT(cpu_transcoder);
10807
10808                 intel_display_power_put(dev_priv, power_domain, trans_wakeref);
10809         }
10810
10811         WARN_ON(crtc_state->master_transcoder != INVALID_TRANSCODER &&
10812                 crtc_state->sync_mode_slaves_mask);
10813 }
10814
10815 static bool haswell_get_pipe_config(struct intel_crtc *crtc,
10816                                     struct intel_crtc_state *pipe_config)
10817 {
10818         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10819         intel_wakeref_t wakerefs[POWER_DOMAIN_NUM], wf;
10820         enum intel_display_power_domain power_domain;
10821         u64 power_domain_mask;
10822         bool active;
10823
10824         pipe_config->master_transcoder = INVALID_TRANSCODER;
10825
10826         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
10827         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10828         if (!wf)
10829                 return false;
10830
10831         wakerefs[power_domain] = wf;
10832         power_domain_mask = BIT_ULL(power_domain);
10833
10834         pipe_config->shared_dpll = NULL;
10835
10836         active = hsw_get_transcoder_state(crtc, pipe_config,
10837                                           &power_domain_mask, wakerefs);
10838
10839         if (IS_GEN9_LP(dev_priv) &&
10840             bxt_get_dsi_transcoder_state(crtc, pipe_config,
10841                                          &power_domain_mask, wakerefs)) {
10842                 WARN_ON(active);
10843                 active = true;
10844         }
10845
10846         if (!active)
10847                 goto out;
10848
10849         if (!transcoder_is_dsi(pipe_config->cpu_transcoder) ||
10850             INTEL_GEN(dev_priv) >= 11) {
10851                 haswell_get_ddi_port_state(crtc, pipe_config);
10852                 intel_get_pipe_timings(crtc, pipe_config);
10853         }
10854
10855         intel_get_pipe_src_size(crtc, pipe_config);
10856
10857         if (IS_HASWELL(dev_priv)) {
10858                 u32 tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
10859
10860                 if (tmp & PIPECONF_OUTPUT_COLORSPACE_YUV_HSW)
10861                         pipe_config->output_format = INTEL_OUTPUT_FORMAT_YCBCR444;
10862                 else
10863                         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
10864         } else {
10865                 pipe_config->output_format =
10866                         bdw_get_pipemisc_output_format(crtc);
10867
10868                 /*
10869                  * Currently there is no interface defined to
10870                  * check user preference between RGB/YCBCR444
10871                  * or YCBCR420. So the only possible case for
10872                  * YCBCR444 usage is driving YCBCR420 output
10873                  * with LSPCON, when pipe is configured for
10874                  * YCBCR444 output and LSPCON takes care of
10875                  * downsampling it.
10876                  */
10877                 pipe_config->lspcon_downsampling =
10878                         pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR444;
10879         }
10880
10881         pipe_config->gamma_mode = I915_READ(GAMMA_MODE(crtc->pipe));
10882
10883         pipe_config->csc_mode = I915_READ(PIPE_CSC_MODE(crtc->pipe));
10884
10885         if (INTEL_GEN(dev_priv) >= 9) {
10886                 u32 tmp = I915_READ(SKL_BOTTOM_COLOR(crtc->pipe));
10887
10888                 if (tmp & SKL_BOTTOM_COLOR_GAMMA_ENABLE)
10889                         pipe_config->gamma_enable = true;
10890
10891                 if (tmp & SKL_BOTTOM_COLOR_CSC_ENABLE)
10892                         pipe_config->csc_enable = true;
10893         } else {
10894                 i9xx_get_pipe_color_config(pipe_config);
10895         }
10896
10897         intel_color_get_config(pipe_config);
10898
10899         power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
10900         WARN_ON(power_domain_mask & BIT_ULL(power_domain));
10901
10902         wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
10903         if (wf) {
10904                 wakerefs[power_domain] = wf;
10905                 power_domain_mask |= BIT_ULL(power_domain);
10906
10907                 if (INTEL_GEN(dev_priv) >= 9)
10908                         skylake_get_pfit_config(crtc, pipe_config);
10909                 else
10910                         ironlake_get_pfit_config(crtc, pipe_config);
10911         }
10912
10913         if (hsw_crtc_supports_ips(crtc)) {
10914                 if (IS_HASWELL(dev_priv))
10915                         pipe_config->ips_enabled = I915_READ(IPS_CTL) & IPS_ENABLE;
10916                 else {
10917                         /*
10918                          * We cannot readout IPS state on broadwell, set to
10919                          * true so we can set it to a defined state on first
10920                          * commit.
10921                          */
10922                         pipe_config->ips_enabled = true;
10923                 }
10924         }
10925
10926         if (pipe_config->cpu_transcoder != TRANSCODER_EDP &&
10927             !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
10928                 pipe_config->pixel_multiplier =
10929                         I915_READ(PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
10930         } else {
10931                 pipe_config->pixel_multiplier = 1;
10932         }
10933
10934         if (INTEL_GEN(dev_priv) >= 11 &&
10935             !transcoder_is_dsi(pipe_config->cpu_transcoder))
10936                 icelake_get_trans_port_sync_config(pipe_config);
10937
10938 out:
10939         for_each_power_domain(power_domain, power_domain_mask)
10940                 intel_display_power_put(dev_priv,
10941                                         power_domain, wakerefs[power_domain]);
10942
10943         return active;
10944 }
10945
10946 static u32 intel_cursor_base(const struct intel_plane_state *plane_state)
10947 {
10948         struct drm_i915_private *dev_priv =
10949                 to_i915(plane_state->uapi.plane->dev);
10950         const struct drm_framebuffer *fb = plane_state->hw.fb;
10951         const struct drm_i915_gem_object *obj = intel_fb_obj(fb);
10952         u32 base;
10953
10954         if (INTEL_INFO(dev_priv)->display.cursor_needs_physical)
10955                 base = obj->phys_handle->busaddr;
10956         else
10957                 base = intel_plane_ggtt_offset(plane_state);
10958
10959         return base + plane_state->color_plane[0].offset;
10960 }
10961
10962 static u32 intel_cursor_position(const struct intel_plane_state *plane_state)
10963 {
10964         int x = plane_state->uapi.dst.x1;
10965         int y = plane_state->uapi.dst.y1;
10966         u32 pos = 0;
10967
10968         if (x < 0) {
10969                 pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
10970                 x = -x;
10971         }
10972         pos |= x << CURSOR_X_SHIFT;
10973
10974         if (y < 0) {
10975                 pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
10976                 y = -y;
10977         }
10978         pos |= y << CURSOR_Y_SHIFT;
10979
10980         return pos;
10981 }
10982
10983 static bool intel_cursor_size_ok(const struct intel_plane_state *plane_state)
10984 {
10985         const struct drm_mode_config *config =
10986                 &plane_state->uapi.plane->dev->mode_config;
10987         int width = drm_rect_width(&plane_state->uapi.dst);
10988         int height = drm_rect_height(&plane_state->uapi.dst);
10989
10990         return width > 0 && width <= config->cursor_width &&
10991                 height > 0 && height <= config->cursor_height;
10992 }
10993
10994 static int intel_cursor_check_surface(struct intel_plane_state *plane_state)
10995 {
10996         struct drm_i915_private *dev_priv =
10997                 to_i915(plane_state->uapi.plane->dev);
10998         unsigned int rotation = plane_state->hw.rotation;
10999         int src_x, src_y;
11000         u32 offset;
11001         int ret;
11002
11003         ret = intel_plane_compute_gtt(plane_state);
11004         if (ret)
11005                 return ret;
11006
11007         if (!plane_state->uapi.visible)
11008                 return 0;
11009
11010         src_x = plane_state->uapi.src.x1 >> 16;
11011         src_y = plane_state->uapi.src.y1 >> 16;
11012
11013         intel_add_fb_offsets(&src_x, &src_y, plane_state, 0);
11014         offset = intel_plane_compute_aligned_offset(&src_x, &src_y,
11015                                                     plane_state, 0);
11016
11017         if (src_x != 0 || src_y != 0) {
11018                 DRM_DEBUG_KMS("Arbitrary cursor panning not supported\n");
11019                 return -EINVAL;
11020         }
11021
11022         /*
11023          * Put the final coordinates back so that the src
11024          * coordinate checks will see the right values.
11025          */
11026         drm_rect_translate_to(&plane_state->uapi.src,
11027                               src_x << 16, src_y << 16);
11028
11029         /* ILK+ do this automagically in hardware */
11030         if (HAS_GMCH(dev_priv) && rotation & DRM_MODE_ROTATE_180) {
11031                 const struct drm_framebuffer *fb = plane_state->hw.fb;
11032                 int src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
11033                 int src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
11034
11035                 offset += (src_h * src_w - 1) * fb->format->cpp[0];
11036         }
11037
11038         plane_state->color_plane[0].offset = offset;
11039         plane_state->color_plane[0].x = src_x;
11040         plane_state->color_plane[0].y = src_y;
11041
11042         return 0;
11043 }
11044
11045 static int intel_check_cursor(struct intel_crtc_state *crtc_state,
11046                               struct intel_plane_state *plane_state)
11047 {
11048         const struct drm_framebuffer *fb = plane_state->hw.fb;
11049         int ret;
11050
11051         if (fb && fb->modifier != DRM_FORMAT_MOD_LINEAR) {
11052                 DRM_DEBUG_KMS("cursor cannot be tiled\n");
11053                 return -EINVAL;
11054         }
11055
11056         ret = drm_atomic_helper_check_plane_state(&plane_state->uapi,
11057                                                   &crtc_state->uapi,
11058                                                   DRM_PLANE_HELPER_NO_SCALING,
11059                                                   DRM_PLANE_HELPER_NO_SCALING,
11060                                                   true, true);
11061         if (ret)
11062                 return ret;
11063
11064         /* Use the unclipped src/dst rectangles, which we program to hw */
11065         plane_state->uapi.src = drm_plane_state_src(&plane_state->uapi);
11066         plane_state->uapi.dst = drm_plane_state_dest(&plane_state->uapi);
11067
11068         ret = intel_cursor_check_surface(plane_state);
11069         if (ret)
11070                 return ret;
11071
11072         if (!plane_state->uapi.visible)
11073                 return 0;
11074
11075         ret = intel_plane_check_src_coordinates(plane_state);
11076         if (ret)
11077                 return ret;
11078
11079         return 0;
11080 }
11081
11082 static unsigned int
11083 i845_cursor_max_stride(struct intel_plane *plane,
11084                        u32 pixel_format, u64 modifier,
11085                        unsigned int rotation)
11086 {
11087         return 2048;
11088 }
11089
11090 static u32 i845_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state)
11091 {
11092         u32 cntl = 0;
11093
11094         if (crtc_state->gamma_enable)
11095                 cntl |= CURSOR_GAMMA_ENABLE;
11096
11097         return cntl;
11098 }
11099
11100 static u32 i845_cursor_ctl(const struct intel_crtc_state *crtc_state,
11101                            const struct intel_plane_state *plane_state)
11102 {
11103         return CURSOR_ENABLE |
11104                 CURSOR_FORMAT_ARGB |
11105                 CURSOR_STRIDE(plane_state->color_plane[0].stride);
11106 }
11107
11108 static bool i845_cursor_size_ok(const struct intel_plane_state *plane_state)
11109 {
11110         int width = drm_rect_width(&plane_state->uapi.dst);
11111
11112         /*
11113          * 845g/865g are only limited by the width of their cursors,
11114          * the height is arbitrary up to the precision of the register.
11115          */
11116         return intel_cursor_size_ok(plane_state) && IS_ALIGNED(width, 64);
11117 }
11118
11119 static int i845_check_cursor(struct intel_crtc_state *crtc_state,
11120                              struct intel_plane_state *plane_state)
11121 {
11122         const struct drm_framebuffer *fb = plane_state->hw.fb;
11123         int ret;
11124
11125         ret = intel_check_cursor(crtc_state, plane_state);
11126         if (ret)
11127                 return ret;
11128
11129         /* if we want to turn off the cursor ignore width and height */
11130         if (!fb)
11131                 return 0;
11132
11133         /* Check for which cursor types we support */
11134         if (!i845_cursor_size_ok(plane_state)) {
11135                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
11136                           drm_rect_width(&plane_state->uapi.dst),
11137                           drm_rect_height(&plane_state->uapi.dst));
11138                 return -EINVAL;
11139         }
11140
11141         WARN_ON(plane_state->uapi.visible &&
11142                 plane_state->color_plane[0].stride != fb->pitches[0]);
11143
11144         switch (fb->pitches[0]) {
11145         case 256:
11146         case 512:
11147         case 1024:
11148         case 2048:
11149                 break;
11150         default:
11151                 DRM_DEBUG_KMS("Invalid cursor stride (%u)\n",
11152                               fb->pitches[0]);
11153                 return -EINVAL;
11154         }
11155
11156         plane_state->ctl = i845_cursor_ctl(crtc_state, plane_state);
11157
11158         return 0;
11159 }
11160
11161 static void i845_update_cursor(struct intel_plane *plane,
11162                                const struct intel_crtc_state *crtc_state,
11163                                const struct intel_plane_state *plane_state)
11164 {
11165         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11166         u32 cntl = 0, base = 0, pos = 0, size = 0;
11167         unsigned long irqflags;
11168
11169         if (plane_state && plane_state->uapi.visible) {
11170                 unsigned int width = drm_rect_width(&plane_state->uapi.dst);
11171                 unsigned int height = drm_rect_height(&plane_state->uapi.dst);
11172
11173                 cntl = plane_state->ctl |
11174                         i845_cursor_ctl_crtc(crtc_state);
11175
11176                 size = (height << 12) | width;
11177
11178                 base = intel_cursor_base(plane_state);
11179                 pos = intel_cursor_position(plane_state);
11180         }
11181
11182         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
11183
11184         /* On these chipsets we can only modify the base/size/stride
11185          * whilst the cursor is disabled.
11186          */
11187         if (plane->cursor.base != base ||
11188             plane->cursor.size != size ||
11189             plane->cursor.cntl != cntl) {
11190                 I915_WRITE_FW(CURCNTR(PIPE_A), 0);
11191                 I915_WRITE_FW(CURBASE(PIPE_A), base);
11192                 I915_WRITE_FW(CURSIZE, size);
11193                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
11194                 I915_WRITE_FW(CURCNTR(PIPE_A), cntl);
11195
11196                 plane->cursor.base = base;
11197                 plane->cursor.size = size;
11198                 plane->cursor.cntl = cntl;
11199         } else {
11200                 I915_WRITE_FW(CURPOS(PIPE_A), pos);
11201         }
11202
11203         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
11204 }
11205
11206 static void i845_disable_cursor(struct intel_plane *plane,
11207                                 const struct intel_crtc_state *crtc_state)
11208 {
11209         i845_update_cursor(plane, crtc_state, NULL);
11210 }
11211
11212 static bool i845_cursor_get_hw_state(struct intel_plane *plane,
11213                                      enum pipe *pipe)
11214 {
11215         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11216         enum intel_display_power_domain power_domain;
11217         intel_wakeref_t wakeref;
11218         bool ret;
11219
11220         power_domain = POWER_DOMAIN_PIPE(PIPE_A);
11221         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
11222         if (!wakeref)
11223                 return false;
11224
11225         ret = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
11226
11227         *pipe = PIPE_A;
11228
11229         intel_display_power_put(dev_priv, power_domain, wakeref);
11230
11231         return ret;
11232 }
11233
11234 static unsigned int
11235 i9xx_cursor_max_stride(struct intel_plane *plane,
11236                        u32 pixel_format, u64 modifier,
11237                        unsigned int rotation)
11238 {
11239         return plane->base.dev->mode_config.cursor_width * 4;
11240 }
11241
11242 static u32 i9xx_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state)
11243 {
11244         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
11245         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11246         u32 cntl = 0;
11247
11248         if (INTEL_GEN(dev_priv) >= 11)
11249                 return cntl;
11250
11251         if (crtc_state->gamma_enable)
11252                 cntl = MCURSOR_GAMMA_ENABLE;
11253
11254         if (crtc_state->csc_enable)
11255                 cntl |= MCURSOR_PIPE_CSC_ENABLE;
11256
11257         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
11258                 cntl |= MCURSOR_PIPE_SELECT(crtc->pipe);
11259
11260         return cntl;
11261 }
11262
11263 static u32 i9xx_cursor_ctl(const struct intel_crtc_state *crtc_state,
11264                            const struct intel_plane_state *plane_state)
11265 {
11266         struct drm_i915_private *dev_priv =
11267                 to_i915(plane_state->uapi.plane->dev);
11268         u32 cntl = 0;
11269
11270         if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
11271                 cntl |= MCURSOR_TRICKLE_FEED_DISABLE;
11272
11273         switch (drm_rect_width(&plane_state->uapi.dst)) {
11274         case 64:
11275                 cntl |= MCURSOR_MODE_64_ARGB_AX;
11276                 break;
11277         case 128:
11278                 cntl |= MCURSOR_MODE_128_ARGB_AX;
11279                 break;
11280         case 256:
11281                 cntl |= MCURSOR_MODE_256_ARGB_AX;
11282                 break;
11283         default:
11284                 MISSING_CASE(drm_rect_width(&plane_state->uapi.dst));
11285                 return 0;
11286         }
11287
11288         if (plane_state->hw.rotation & DRM_MODE_ROTATE_180)
11289                 cntl |= MCURSOR_ROTATE_180;
11290
11291         return cntl;
11292 }
11293
11294 static bool i9xx_cursor_size_ok(const struct intel_plane_state *plane_state)
11295 {
11296         struct drm_i915_private *dev_priv =
11297                 to_i915(plane_state->uapi.plane->dev);
11298         int width = drm_rect_width(&plane_state->uapi.dst);
11299         int height = drm_rect_height(&plane_state->uapi.dst);
11300
11301         if (!intel_cursor_size_ok(plane_state))
11302                 return false;
11303
11304         /* Cursor width is limited to a few power-of-two sizes */
11305         switch (width) {
11306         case 256:
11307         case 128:
11308         case 64:
11309                 break;
11310         default:
11311                 return false;
11312         }
11313
11314         /*
11315          * IVB+ have CUR_FBC_CTL which allows an arbitrary cursor
11316          * height from 8 lines up to the cursor width, when the
11317          * cursor is not rotated. Everything else requires square
11318          * cursors.
11319          */
11320         if (HAS_CUR_FBC(dev_priv) &&
11321             plane_state->hw.rotation & DRM_MODE_ROTATE_0) {
11322                 if (height < 8 || height > width)
11323                         return false;
11324         } else {
11325                 if (height != width)
11326                         return false;
11327         }
11328
11329         return true;
11330 }
11331
11332 static int i9xx_check_cursor(struct intel_crtc_state *crtc_state,
11333                              struct intel_plane_state *plane_state)
11334 {
11335         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
11336         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11337         const struct drm_framebuffer *fb = plane_state->hw.fb;
11338         enum pipe pipe = plane->pipe;
11339         int ret;
11340
11341         ret = intel_check_cursor(crtc_state, plane_state);
11342         if (ret)
11343                 return ret;
11344
11345         /* if we want to turn off the cursor ignore width and height */
11346         if (!fb)
11347                 return 0;
11348
11349         /* Check for which cursor types we support */
11350         if (!i9xx_cursor_size_ok(plane_state)) {
11351                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
11352                           drm_rect_width(&plane_state->uapi.dst),
11353                           drm_rect_height(&plane_state->uapi.dst));
11354                 return -EINVAL;
11355         }
11356
11357         WARN_ON(plane_state->uapi.visible &&
11358                 plane_state->color_plane[0].stride != fb->pitches[0]);
11359
11360         if (fb->pitches[0] !=
11361             drm_rect_width(&plane_state->uapi.dst) * fb->format->cpp[0]) {
11362                 DRM_DEBUG_KMS("Invalid cursor stride (%u) (cursor width %d)\n",
11363                               fb->pitches[0],
11364                               drm_rect_width(&plane_state->uapi.dst));
11365                 return -EINVAL;
11366         }
11367
11368         /*
11369          * There's something wrong with the cursor on CHV pipe C.
11370          * If it straddles the left edge of the screen then
11371          * moving it away from the edge or disabling it often
11372          * results in a pipe underrun, and often that can lead to
11373          * dead pipe (constant underrun reported, and it scans
11374          * out just a solid color). To recover from that, the
11375          * display power well must be turned off and on again.
11376          * Refuse the put the cursor into that compromised position.
11377          */
11378         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_C &&
11379             plane_state->uapi.visible && plane_state->uapi.dst.x1 < 0) {
11380                 DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
11381                 return -EINVAL;
11382         }
11383
11384         plane_state->ctl = i9xx_cursor_ctl(crtc_state, plane_state);
11385
11386         return 0;
11387 }
11388
11389 static void i9xx_update_cursor(struct intel_plane *plane,
11390                                const struct intel_crtc_state *crtc_state,
11391                                const struct intel_plane_state *plane_state)
11392 {
11393         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11394         enum pipe pipe = plane->pipe;
11395         u32 cntl = 0, base = 0, pos = 0, fbc_ctl = 0;
11396         unsigned long irqflags;
11397
11398         if (plane_state && plane_state->uapi.visible) {
11399                 unsigned width = drm_rect_width(&plane_state->uapi.dst);
11400                 unsigned height = drm_rect_height(&plane_state->uapi.dst);
11401
11402                 cntl = plane_state->ctl |
11403                         i9xx_cursor_ctl_crtc(crtc_state);
11404
11405                 if (width != height)
11406                         fbc_ctl = CUR_FBC_CTL_EN | (height - 1);
11407
11408                 base = intel_cursor_base(plane_state);
11409                 pos = intel_cursor_position(plane_state);
11410         }
11411
11412         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
11413
11414         /*
11415          * On some platforms writing CURCNTR first will also
11416          * cause CURPOS to be armed by the CURBASE write.
11417          * Without the CURCNTR write the CURPOS write would
11418          * arm itself. Thus we always update CURCNTR before
11419          * CURPOS.
11420          *
11421          * On other platforms CURPOS always requires the
11422          * CURBASE write to arm the update. Additonally
11423          * a write to any of the cursor register will cancel
11424          * an already armed cursor update. Thus leaving out
11425          * the CURBASE write after CURPOS could lead to a
11426          * cursor that doesn't appear to move, or even change
11427          * shape. Thus we always write CURBASE.
11428          *
11429          * The other registers are armed by by the CURBASE write
11430          * except when the plane is getting enabled at which time
11431          * the CURCNTR write arms the update.
11432          */
11433
11434         if (INTEL_GEN(dev_priv) >= 9)
11435                 skl_write_cursor_wm(plane, crtc_state);
11436
11437         if (plane->cursor.base != base ||
11438             plane->cursor.size != fbc_ctl ||
11439             plane->cursor.cntl != cntl) {
11440                 if (HAS_CUR_FBC(dev_priv))
11441                         I915_WRITE_FW(CUR_FBC_CTL(pipe), fbc_ctl);
11442                 I915_WRITE_FW(CURCNTR(pipe), cntl);
11443                 I915_WRITE_FW(CURPOS(pipe), pos);
11444                 I915_WRITE_FW(CURBASE(pipe), base);
11445
11446                 plane->cursor.base = base;
11447                 plane->cursor.size = fbc_ctl;
11448                 plane->cursor.cntl = cntl;
11449         } else {
11450                 I915_WRITE_FW(CURPOS(pipe), pos);
11451                 I915_WRITE_FW(CURBASE(pipe), base);
11452         }
11453
11454         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
11455 }
11456
11457 static void i9xx_disable_cursor(struct intel_plane *plane,
11458                                 const struct intel_crtc_state *crtc_state)
11459 {
11460         i9xx_update_cursor(plane, crtc_state, NULL);
11461 }
11462
11463 static bool i9xx_cursor_get_hw_state(struct intel_plane *plane,
11464                                      enum pipe *pipe)
11465 {
11466         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
11467         enum intel_display_power_domain power_domain;
11468         intel_wakeref_t wakeref;
11469         bool ret;
11470         u32 val;
11471
11472         /*
11473          * Not 100% correct for planes that can move between pipes,
11474          * but that's only the case for gen2-3 which don't have any
11475          * display power wells.
11476          */
11477         power_domain = POWER_DOMAIN_PIPE(plane->pipe);
11478         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
11479         if (!wakeref)
11480                 return false;
11481
11482         val = I915_READ(CURCNTR(plane->pipe));
11483
11484         ret = val & MCURSOR_MODE;
11485
11486         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
11487                 *pipe = plane->pipe;
11488         else
11489                 *pipe = (val & MCURSOR_PIPE_SELECT_MASK) >>
11490                         MCURSOR_PIPE_SELECT_SHIFT;
11491
11492         intel_display_power_put(dev_priv, power_domain, wakeref);
11493
11494         return ret;
11495 }
11496
11497 /* VESA 640x480x72Hz mode to set on the pipe */
11498 static const struct drm_display_mode load_detect_mode = {
11499         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
11500                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
11501 };
11502
11503 struct drm_framebuffer *
11504 intel_framebuffer_create(struct drm_i915_gem_object *obj,
11505                          struct drm_mode_fb_cmd2 *mode_cmd)
11506 {
11507         struct intel_framebuffer *intel_fb;
11508         int ret;
11509
11510         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
11511         if (!intel_fb)
11512                 return ERR_PTR(-ENOMEM);
11513
11514         ret = intel_framebuffer_init(intel_fb, obj, mode_cmd);
11515         if (ret)
11516                 goto err;
11517
11518         return &intel_fb->base;
11519
11520 err:
11521         kfree(intel_fb);
11522         return ERR_PTR(ret);
11523 }
11524
11525 static int intel_modeset_disable_planes(struct drm_atomic_state *state,
11526                                         struct drm_crtc *crtc)
11527 {
11528         struct drm_plane *plane;
11529         struct drm_plane_state *plane_state;
11530         int ret, i;
11531
11532         ret = drm_atomic_add_affected_planes(state, crtc);
11533         if (ret)
11534                 return ret;
11535
11536         for_each_new_plane_in_state(state, plane, plane_state, i) {
11537                 if (plane_state->crtc != crtc)
11538                         continue;
11539
11540                 ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
11541                 if (ret)
11542                         return ret;
11543
11544                 drm_atomic_set_fb_for_plane(plane_state, NULL);
11545         }
11546
11547         return 0;
11548 }
11549
11550 int intel_get_load_detect_pipe(struct drm_connector *connector,
11551                                struct intel_load_detect_pipe *old,
11552                                struct drm_modeset_acquire_ctx *ctx)
11553 {
11554         struct intel_crtc *intel_crtc;
11555         struct intel_encoder *intel_encoder =
11556                 intel_attached_encoder(connector);
11557         struct drm_crtc *possible_crtc;
11558         struct drm_encoder *encoder = &intel_encoder->base;
11559         struct drm_crtc *crtc = NULL;
11560         struct drm_device *dev = encoder->dev;
11561         struct drm_i915_private *dev_priv = to_i915(dev);
11562         struct drm_mode_config *config = &dev->mode_config;
11563         struct drm_atomic_state *state = NULL, *restore_state = NULL;
11564         struct drm_connector_state *connector_state;
11565         struct intel_crtc_state *crtc_state;
11566         int ret, i = -1;
11567
11568         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
11569                       connector->base.id, connector->name,
11570                       encoder->base.id, encoder->name);
11571
11572         old->restore_state = NULL;
11573
11574         WARN_ON(!drm_modeset_is_locked(&config->connection_mutex));
11575
11576         /*
11577          * Algorithm gets a little messy:
11578          *
11579          *   - if the connector already has an assigned crtc, use it (but make
11580          *     sure it's on first)
11581          *
11582          *   - try to find the first unused crtc that can drive this connector,
11583          *     and use that if we find one
11584          */
11585
11586         /* See if we already have a CRTC for this connector */
11587         if (connector->state->crtc) {
11588                 crtc = connector->state->crtc;
11589
11590                 ret = drm_modeset_lock(&crtc->mutex, ctx);
11591                 if (ret)
11592                         goto fail;
11593
11594                 /* Make sure the crtc and connector are running */
11595                 goto found;
11596         }
11597
11598         /* Find an unused one (if possible) */
11599         for_each_crtc(dev, possible_crtc) {
11600                 i++;
11601                 if (!(encoder->possible_crtcs & (1 << i)))
11602                         continue;
11603
11604                 ret = drm_modeset_lock(&possible_crtc->mutex, ctx);
11605                 if (ret)
11606                         goto fail;
11607
11608                 if (possible_crtc->state->enable) {
11609                         drm_modeset_unlock(&possible_crtc->mutex);
11610                         continue;
11611                 }
11612
11613                 crtc = possible_crtc;
11614                 break;
11615         }
11616
11617         /*
11618          * If we didn't find an unused CRTC, don't use any.
11619          */
11620         if (!crtc) {
11621                 DRM_DEBUG_KMS("no pipe available for load-detect\n");
11622                 ret = -ENODEV;
11623                 goto fail;
11624         }
11625
11626 found:
11627         intel_crtc = to_intel_crtc(crtc);
11628
11629         state = drm_atomic_state_alloc(dev);
11630         restore_state = drm_atomic_state_alloc(dev);
11631         if (!state || !restore_state) {
11632                 ret = -ENOMEM;
11633                 goto fail;
11634         }
11635
11636         state->acquire_ctx = ctx;
11637         restore_state->acquire_ctx = ctx;
11638
11639         connector_state = drm_atomic_get_connector_state(state, connector);
11640         if (IS_ERR(connector_state)) {
11641                 ret = PTR_ERR(connector_state);
11642                 goto fail;
11643         }
11644
11645         ret = drm_atomic_set_crtc_for_connector(connector_state, crtc);
11646         if (ret)
11647                 goto fail;
11648
11649         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
11650         if (IS_ERR(crtc_state)) {
11651                 ret = PTR_ERR(crtc_state);
11652                 goto fail;
11653         }
11654
11655         crtc_state->uapi.active = true;
11656
11657         ret = drm_atomic_set_mode_for_crtc(&crtc_state->uapi,
11658                                            &load_detect_mode);
11659         if (ret)
11660                 goto fail;
11661
11662         ret = intel_modeset_disable_planes(state, crtc);
11663         if (ret)
11664                 goto fail;
11665
11666         ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector));
11667         if (!ret)
11668                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, crtc));
11669         if (!ret)
11670                 ret = drm_atomic_add_affected_planes(restore_state, crtc);
11671         if (ret) {
11672                 DRM_DEBUG_KMS("Failed to create a copy of old state to restore: %i\n", ret);
11673                 goto fail;
11674         }
11675
11676         ret = drm_atomic_commit(state);
11677         if (ret) {
11678                 DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
11679                 goto fail;
11680         }
11681
11682         old->restore_state = restore_state;
11683         drm_atomic_state_put(state);
11684
11685         /* let the connector get through one full cycle before testing */
11686         intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
11687         return true;
11688
11689 fail:
11690         if (state) {
11691                 drm_atomic_state_put(state);
11692                 state = NULL;
11693         }
11694         if (restore_state) {
11695                 drm_atomic_state_put(restore_state);
11696                 restore_state = NULL;
11697         }
11698
11699         if (ret == -EDEADLK)
11700                 return ret;
11701
11702         return false;
11703 }
11704
11705 void intel_release_load_detect_pipe(struct drm_connector *connector,
11706                                     struct intel_load_detect_pipe *old,
11707                                     struct drm_modeset_acquire_ctx *ctx)
11708 {
11709         struct intel_encoder *intel_encoder =
11710                 intel_attached_encoder(connector);
11711         struct drm_encoder *encoder = &intel_encoder->base;
11712         struct drm_atomic_state *state = old->restore_state;
11713         int ret;
11714
11715         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
11716                       connector->base.id, connector->name,
11717                       encoder->base.id, encoder->name);
11718
11719         if (!state)
11720                 return;
11721
11722         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
11723         if (ret)
11724                 DRM_DEBUG_KMS("Couldn't release load detect pipe: %i\n", ret);
11725         drm_atomic_state_put(state);
11726 }
11727
11728 static int i9xx_pll_refclk(struct drm_device *dev,
11729                            const struct intel_crtc_state *pipe_config)
11730 {
11731         struct drm_i915_private *dev_priv = to_i915(dev);
11732         u32 dpll = pipe_config->dpll_hw_state.dpll;
11733
11734         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
11735                 return dev_priv->vbt.lvds_ssc_freq;
11736         else if (HAS_PCH_SPLIT(dev_priv))
11737                 return 120000;
11738         else if (!IS_GEN(dev_priv, 2))
11739                 return 96000;
11740         else
11741                 return 48000;
11742 }
11743
11744 /* Returns the clock of the currently programmed mode of the given pipe. */
11745 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
11746                                 struct intel_crtc_state *pipe_config)
11747 {
11748         struct drm_device *dev = crtc->base.dev;
11749         struct drm_i915_private *dev_priv = to_i915(dev);
11750         enum pipe pipe = crtc->pipe;
11751         u32 dpll = pipe_config->dpll_hw_state.dpll;
11752         u32 fp;
11753         struct dpll clock;
11754         int port_clock;
11755         int refclk = i9xx_pll_refclk(dev, pipe_config);
11756
11757         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
11758                 fp = pipe_config->dpll_hw_state.fp0;
11759         else
11760                 fp = pipe_config->dpll_hw_state.fp1;
11761
11762         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
11763         if (IS_PINEVIEW(dev_priv)) {
11764                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
11765                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
11766         } else {
11767                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
11768                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
11769         }
11770
11771         if (!IS_GEN(dev_priv, 2)) {
11772                 if (IS_PINEVIEW(dev_priv))
11773                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
11774                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
11775                 else
11776                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
11777                                DPLL_FPA01_P1_POST_DIV_SHIFT);
11778
11779                 switch (dpll & DPLL_MODE_MASK) {
11780                 case DPLLB_MODE_DAC_SERIAL:
11781                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
11782                                 5 : 10;
11783                         break;
11784                 case DPLLB_MODE_LVDS:
11785                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
11786                                 7 : 14;
11787                         break;
11788                 default:
11789                         DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed "
11790                                   "mode\n", (int)(dpll & DPLL_MODE_MASK));
11791                         return;
11792                 }
11793
11794                 if (IS_PINEVIEW(dev_priv))
11795                         port_clock = pnv_calc_dpll_params(refclk, &clock);
11796                 else
11797                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
11798         } else {
11799                 u32 lvds = IS_I830(dev_priv) ? 0 : I915_READ(LVDS);
11800                 bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN);
11801
11802                 if (is_lvds) {
11803                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
11804                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
11805
11806                         if (lvds & LVDS_CLKB_POWER_UP)
11807                                 clock.p2 = 7;
11808                         else
11809                                 clock.p2 = 14;
11810                 } else {
11811                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
11812                                 clock.p1 = 2;
11813                         else {
11814                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
11815                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
11816                         }
11817                         if (dpll & PLL_P2_DIVIDE_BY_4)
11818                                 clock.p2 = 4;
11819                         else
11820                                 clock.p2 = 2;
11821                 }
11822
11823                 port_clock = i9xx_calc_dpll_params(refclk, &clock);
11824         }
11825
11826         /*
11827          * This value includes pixel_multiplier. We will use
11828          * port_clock to compute adjusted_mode.crtc_clock in the
11829          * encoder's get_config() function.
11830          */
11831         pipe_config->port_clock = port_clock;
11832 }
11833
11834 int intel_dotclock_calculate(int link_freq,
11835                              const struct intel_link_m_n *m_n)
11836 {
11837         /*
11838          * The calculation for the data clock is:
11839          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
11840          * But we want to avoid losing precison if possible, so:
11841          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
11842          *
11843          * and the link clock is simpler:
11844          * link_clock = (m * link_clock) / n
11845          */
11846
11847         if (!m_n->link_n)
11848                 return 0;
11849
11850         return div_u64(mul_u32_u32(m_n->link_m, link_freq), m_n->link_n);
11851 }
11852
11853 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
11854                                    struct intel_crtc_state *pipe_config)
11855 {
11856         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11857
11858         /* read out port_clock from the DPLL */
11859         i9xx_crtc_clock_get(crtc, pipe_config);
11860
11861         /*
11862          * In case there is an active pipe without active ports,
11863          * we may need some idea for the dotclock anyway.
11864          * Calculate one based on the FDI configuration.
11865          */
11866         pipe_config->hw.adjusted_mode.crtc_clock =
11867                 intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
11868                                          &pipe_config->fdi_m_n);
11869 }
11870
11871 static void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
11872                                    struct intel_crtc *crtc)
11873 {
11874         memset(crtc_state, 0, sizeof(*crtc_state));
11875
11876         __drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
11877
11878         crtc_state->cpu_transcoder = INVALID_TRANSCODER;
11879         crtc_state->master_transcoder = INVALID_TRANSCODER;
11880         crtc_state->hsw_workaround_pipe = INVALID_PIPE;
11881         crtc_state->output_format = INTEL_OUTPUT_FORMAT_INVALID;
11882         crtc_state->scaler_state.scaler_id = -1;
11883 }
11884
11885 static struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
11886 {
11887         struct intel_crtc_state *crtc_state;
11888
11889         crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
11890
11891         if (crtc_state)
11892                 intel_crtc_state_reset(crtc_state, crtc);
11893
11894         return crtc_state;
11895 }
11896
11897 /* Returns the currently programmed mode of the given encoder. */
11898 struct drm_display_mode *
11899 intel_encoder_current_mode(struct intel_encoder *encoder)
11900 {
11901         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
11902         struct intel_crtc_state *crtc_state;
11903         struct drm_display_mode *mode;
11904         struct intel_crtc *crtc;
11905         enum pipe pipe;
11906
11907         if (!encoder->get_hw_state(encoder, &pipe))
11908                 return NULL;
11909
11910         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
11911
11912         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
11913         if (!mode)
11914                 return NULL;
11915
11916         crtc_state = intel_crtc_state_alloc(crtc);
11917         if (!crtc_state) {
11918                 kfree(mode);
11919                 return NULL;
11920         }
11921
11922         if (!dev_priv->display.get_pipe_config(crtc, crtc_state)) {
11923                 kfree(crtc_state);
11924                 kfree(mode);
11925                 return NULL;
11926         }
11927
11928         encoder->get_config(encoder, crtc_state);
11929
11930         intel_mode_from_pipe_config(mode, crtc_state);
11931
11932         kfree(crtc_state);
11933
11934         return mode;
11935 }
11936
11937 static void intel_crtc_destroy(struct drm_crtc *crtc)
11938 {
11939         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11940
11941         drm_crtc_cleanup(crtc);
11942         kfree(intel_crtc);
11943 }
11944
11945 /**
11946  * intel_wm_need_update - Check whether watermarks need updating
11947  * @cur: current plane state
11948  * @new: new plane state
11949  *
11950  * Check current plane state versus the new one to determine whether
11951  * watermarks need to be recalculated.
11952  *
11953  * Returns true or false.
11954  */
11955 static bool intel_wm_need_update(const struct intel_plane_state *cur,
11956                                  struct intel_plane_state *new)
11957 {
11958         /* Update watermarks on tiling or size changes. */
11959         if (new->uapi.visible != cur->uapi.visible)
11960                 return true;
11961
11962         if (!cur->hw.fb || !new->hw.fb)
11963                 return false;
11964
11965         if (cur->hw.fb->modifier != new->hw.fb->modifier ||
11966             cur->hw.rotation != new->hw.rotation ||
11967             drm_rect_width(&new->uapi.src) != drm_rect_width(&cur->uapi.src) ||
11968             drm_rect_height(&new->uapi.src) != drm_rect_height(&cur->uapi.src) ||
11969             drm_rect_width(&new->uapi.dst) != drm_rect_width(&cur->uapi.dst) ||
11970             drm_rect_height(&new->uapi.dst) != drm_rect_height(&cur->uapi.dst))
11971                 return true;
11972
11973         return false;
11974 }
11975
11976 static bool needs_scaling(const struct intel_plane_state *state)
11977 {
11978         int src_w = drm_rect_width(&state->uapi.src) >> 16;
11979         int src_h = drm_rect_height(&state->uapi.src) >> 16;
11980         int dst_w = drm_rect_width(&state->uapi.dst);
11981         int dst_h = drm_rect_height(&state->uapi.dst);
11982
11983         return (src_w != dst_w || src_h != dst_h);
11984 }
11985
11986 int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_state,
11987                                     struct intel_crtc_state *crtc_state,
11988                                     const struct intel_plane_state *old_plane_state,
11989                                     struct intel_plane_state *plane_state)
11990 {
11991         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
11992         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
11993         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11994         bool mode_changed = needs_modeset(crtc_state);
11995         bool was_crtc_enabled = old_crtc_state->hw.active;
11996         bool is_crtc_enabled = crtc_state->hw.active;
11997         bool turn_off, turn_on, visible, was_visible;
11998         int ret;
11999
12000         if (INTEL_GEN(dev_priv) >= 9 && plane->id != PLANE_CURSOR) {
12001                 ret = skl_update_scaler_plane(crtc_state, plane_state);
12002                 if (ret)
12003                         return ret;
12004         }
12005
12006         was_visible = old_plane_state->uapi.visible;
12007         visible = plane_state->uapi.visible;
12008
12009         if (!was_crtc_enabled && WARN_ON(was_visible))
12010                 was_visible = false;
12011
12012         /*
12013          * Visibility is calculated as if the crtc was on, but
12014          * after scaler setup everything depends on it being off
12015          * when the crtc isn't active.
12016          *
12017          * FIXME this is wrong for watermarks. Watermarks should also
12018          * be computed as if the pipe would be active. Perhaps move
12019          * per-plane wm computation to the .check_plane() hook, and
12020          * only combine the results from all planes in the current place?
12021          */
12022         if (!is_crtc_enabled) {
12023                 plane_state->uapi.visible = visible = false;
12024                 crtc_state->active_planes &= ~BIT(plane->id);
12025                 crtc_state->data_rate[plane->id] = 0;
12026                 crtc_state->min_cdclk[plane->id] = 0;
12027         }
12028
12029         if (!was_visible && !visible)
12030                 return 0;
12031
12032         turn_off = was_visible && (!visible || mode_changed);
12033         turn_on = visible && (!was_visible || mode_changed);
12034
12035         DRM_DEBUG_ATOMIC("[CRTC:%d:%s] with [PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n",
12036                          crtc->base.base.id, crtc->base.name,
12037                          plane->base.base.id, plane->base.name,
12038                          was_visible, visible,
12039                          turn_off, turn_on, mode_changed);
12040
12041         if (turn_on) {
12042                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
12043                         crtc_state->update_wm_pre = true;
12044
12045                 /* must disable cxsr around plane enable/disable */
12046                 if (plane->id != PLANE_CURSOR)
12047                         crtc_state->disable_cxsr = true;
12048         } else if (turn_off) {
12049                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
12050                         crtc_state->update_wm_post = true;
12051
12052                 /* must disable cxsr around plane enable/disable */
12053                 if (plane->id != PLANE_CURSOR)
12054                         crtc_state->disable_cxsr = true;
12055         } else if (intel_wm_need_update(old_plane_state, plane_state)) {
12056                 if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv)) {
12057                         /* FIXME bollocks */
12058                         crtc_state->update_wm_pre = true;
12059                         crtc_state->update_wm_post = true;
12060                 }
12061         }
12062
12063         if (visible || was_visible)
12064                 crtc_state->fb_bits |= plane->frontbuffer_bit;
12065
12066         /*
12067          * ILK/SNB DVSACNTR/Sprite Enable
12068          * IVB SPR_CTL/Sprite Enable
12069          * "When in Self Refresh Big FIFO mode, a write to enable the
12070          *  plane will be internally buffered and delayed while Big FIFO
12071          *  mode is exiting."
12072          *
12073          * Which means that enabling the sprite can take an extra frame
12074          * when we start in big FIFO mode (LP1+). Thus we need to drop
12075          * down to LP0 and wait for vblank in order to make sure the
12076          * sprite gets enabled on the next vblank after the register write.
12077          * Doing otherwise would risk enabling the sprite one frame after
12078          * we've already signalled flip completion. We can resume LP1+
12079          * once the sprite has been enabled.
12080          *
12081          *
12082          * WaCxSRDisabledForSpriteScaling:ivb
12083          * IVB SPR_SCALE/Scaling Enable
12084          * "Low Power watermarks must be disabled for at least one
12085          *  frame before enabling sprite scaling, and kept disabled
12086          *  until sprite scaling is disabled."
12087          *
12088          * ILK/SNB DVSASCALE/Scaling Enable
12089          * "When in Self Refresh Big FIFO mode, scaling enable will be
12090          *  masked off while Big FIFO mode is exiting."
12091          *
12092          * Despite the w/a only being listed for IVB we assume that
12093          * the ILK/SNB note has similar ramifications, hence we apply
12094          * the w/a on all three platforms.
12095          *
12096          * With experimental results seems this is needed also for primary
12097          * plane, not only sprite plane.
12098          */
12099         if (plane->id != PLANE_CURSOR &&
12100             (IS_GEN_RANGE(dev_priv, 5, 6) ||
12101              IS_IVYBRIDGE(dev_priv)) &&
12102             (turn_on || (!needs_scaling(old_plane_state) &&
12103                          needs_scaling(plane_state))))
12104                 crtc_state->disable_lp_wm = true;
12105
12106         return 0;
12107 }
12108
12109 static bool encoders_cloneable(const struct intel_encoder *a,
12110                                const struct intel_encoder *b)
12111 {
12112         /* masks could be asymmetric, so check both ways */
12113         return a == b || (a->cloneable & (1 << b->type) &&
12114                           b->cloneable & (1 << a->type));
12115 }
12116
12117 static bool check_single_encoder_cloning(struct drm_atomic_state *state,
12118                                          struct intel_crtc *crtc,
12119                                          struct intel_encoder *encoder)
12120 {
12121         struct intel_encoder *source_encoder;
12122         struct drm_connector *connector;
12123         struct drm_connector_state *connector_state;
12124         int i;
12125
12126         for_each_new_connector_in_state(state, connector, connector_state, i) {
12127                 if (connector_state->crtc != &crtc->base)
12128                         continue;
12129
12130                 source_encoder =
12131                         to_intel_encoder(connector_state->best_encoder);
12132                 if (!encoders_cloneable(encoder, source_encoder))
12133                         return false;
12134         }
12135
12136         return true;
12137 }
12138
12139 static int icl_add_linked_planes(struct intel_atomic_state *state)
12140 {
12141         struct intel_plane *plane, *linked;
12142         struct intel_plane_state *plane_state, *linked_plane_state;
12143         int i;
12144
12145         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12146                 linked = plane_state->planar_linked_plane;
12147
12148                 if (!linked)
12149                         continue;
12150
12151                 linked_plane_state = intel_atomic_get_plane_state(state, linked);
12152                 if (IS_ERR(linked_plane_state))
12153                         return PTR_ERR(linked_plane_state);
12154
12155                 WARN_ON(linked_plane_state->planar_linked_plane != plane);
12156                 WARN_ON(linked_plane_state->planar_slave == plane_state->planar_slave);
12157         }
12158
12159         return 0;
12160 }
12161
12162 static int icl_check_nv12_planes(struct intel_crtc_state *crtc_state)
12163 {
12164         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
12165         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12166         struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->uapi.state);
12167         struct intel_plane *plane, *linked;
12168         struct intel_plane_state *plane_state;
12169         int i;
12170
12171         if (INTEL_GEN(dev_priv) < 11)
12172                 return 0;
12173
12174         /*
12175          * Destroy all old plane links and make the slave plane invisible
12176          * in the crtc_state->active_planes mask.
12177          */
12178         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12179                 if (plane->pipe != crtc->pipe || !plane_state->planar_linked_plane)
12180                         continue;
12181
12182                 plane_state->planar_linked_plane = NULL;
12183                 if (plane_state->planar_slave && !plane_state->uapi.visible) {
12184                         crtc_state->active_planes &= ~BIT(plane->id);
12185                         crtc_state->update_planes |= BIT(plane->id);
12186                 }
12187
12188                 plane_state->planar_slave = false;
12189         }
12190
12191         if (!crtc_state->nv12_planes)
12192                 return 0;
12193
12194         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12195                 struct intel_plane_state *linked_state = NULL;
12196
12197                 if (plane->pipe != crtc->pipe ||
12198                     !(crtc_state->nv12_planes & BIT(plane->id)))
12199                         continue;
12200
12201                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, linked) {
12202                         if (!icl_is_nv12_y_plane(linked->id))
12203                                 continue;
12204
12205                         if (crtc_state->active_planes & BIT(linked->id))
12206                                 continue;
12207
12208                         linked_state = intel_atomic_get_plane_state(state, linked);
12209                         if (IS_ERR(linked_state))
12210                                 return PTR_ERR(linked_state);
12211
12212                         break;
12213                 }
12214
12215                 if (!linked_state) {
12216                         DRM_DEBUG_KMS("Need %d free Y planes for planar YUV\n",
12217                                       hweight8(crtc_state->nv12_planes));
12218
12219                         return -EINVAL;
12220                 }
12221
12222                 plane_state->planar_linked_plane = linked;
12223
12224                 linked_state->planar_slave = true;
12225                 linked_state->planar_linked_plane = plane;
12226                 crtc_state->active_planes |= BIT(linked->id);
12227                 crtc_state->update_planes |= BIT(linked->id);
12228                 DRM_DEBUG_KMS("Using %s as Y plane for %s\n", linked->base.name, plane->base.name);
12229
12230                 /* Copy parameters to slave plane */
12231                 linked_state->ctl = plane_state->ctl | PLANE_CTL_YUV420_Y_PLANE;
12232                 linked_state->color_ctl = plane_state->color_ctl;
12233                 memcpy(linked_state->color_plane, plane_state->color_plane,
12234                        sizeof(linked_state->color_plane));
12235
12236                 intel_plane_copy_uapi_to_hw_state(linked_state, plane_state);
12237                 linked_state->uapi.src = plane_state->uapi.src;
12238                 linked_state->uapi.dst = plane_state->uapi.dst;
12239
12240                 if (icl_is_hdr_plane(dev_priv, plane->id)) {
12241                         if (linked->id == PLANE_SPRITE5)
12242                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_7;
12243                         else if (linked->id == PLANE_SPRITE4)
12244                                 plane_state->cus_ctl |= PLANE_CUS_PLANE_6;
12245                         else
12246                                 MISSING_CASE(linked->id);
12247                 }
12248         }
12249
12250         return 0;
12251 }
12252
12253 static bool c8_planes_changed(const struct intel_crtc_state *new_crtc_state)
12254 {
12255         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
12256         struct intel_atomic_state *state =
12257                 to_intel_atomic_state(new_crtc_state->uapi.state);
12258         const struct intel_crtc_state *old_crtc_state =
12259                 intel_atomic_get_old_crtc_state(state, crtc);
12260
12261         return !old_crtc_state->c8_planes != !new_crtc_state->c8_planes;
12262 }
12263
12264 static int icl_add_sync_mode_crtcs(struct intel_crtc_state *crtc_state)
12265 {
12266         struct drm_crtc *crtc = crtc_state->uapi.crtc;
12267         struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->uapi.state);
12268         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
12269         struct drm_connector *master_connector, *connector;
12270         struct drm_connector_state *connector_state;
12271         struct drm_connector_list_iter conn_iter;
12272         struct drm_crtc *master_crtc = NULL;
12273         struct drm_crtc_state *master_crtc_state;
12274         struct intel_crtc_state *master_pipe_config;
12275         int i, tile_group_id;
12276
12277         if (INTEL_GEN(dev_priv) < 11)
12278                 return 0;
12279
12280         /*
12281          * In case of tiled displays there could be one or more slaves but there is
12282          * only one master. Lets make the CRTC used by the connector corresponding
12283          * to the last horizonal and last vertical tile a master/genlock CRTC.
12284          * All the other CRTCs corresponding to other tiles of the same Tile group
12285          * are the slave CRTCs and hold a pointer to their genlock CRTC.
12286          */
12287         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
12288                 if (connector_state->crtc != crtc)
12289                         continue;
12290                 if (!connector->has_tile)
12291                         continue;
12292                 if (crtc_state->hw.mode.hdisplay != connector->tile_h_size ||
12293                     crtc_state->hw.mode.vdisplay != connector->tile_v_size)
12294                         return 0;
12295                 if (connector->tile_h_loc == connector->num_h_tile - 1 &&
12296                     connector->tile_v_loc == connector->num_v_tile - 1)
12297                         continue;
12298                 crtc_state->sync_mode_slaves_mask = 0;
12299                 tile_group_id = connector->tile_group->id;
12300                 drm_connector_list_iter_begin(&dev_priv->drm, &conn_iter);
12301                 drm_for_each_connector_iter(master_connector, &conn_iter) {
12302                         struct drm_connector_state *master_conn_state = NULL;
12303
12304                         if (!master_connector->has_tile)
12305                                 continue;
12306                         if (master_connector->tile_h_loc != master_connector->num_h_tile - 1 ||
12307                             master_connector->tile_v_loc != master_connector->num_v_tile - 1)
12308                                 continue;
12309                         if (master_connector->tile_group->id != tile_group_id)
12310                                 continue;
12311
12312                         master_conn_state = drm_atomic_get_connector_state(&state->base,
12313                                                                            master_connector);
12314                         if (IS_ERR(master_conn_state)) {
12315                                 drm_connector_list_iter_end(&conn_iter);
12316                                 return PTR_ERR(master_conn_state);
12317                         }
12318                         if (master_conn_state->crtc) {
12319                                 master_crtc = master_conn_state->crtc;
12320                                 break;
12321                         }
12322                 }
12323                 drm_connector_list_iter_end(&conn_iter);
12324
12325                 if (!master_crtc) {
12326                         DRM_DEBUG_KMS("Could not find Master CRTC for Slave CRTC %d\n",
12327                                       connector_state->crtc->base.id);
12328                         return -EINVAL;
12329                 }
12330
12331                 master_crtc_state = drm_atomic_get_crtc_state(&state->base,
12332                                                               master_crtc);
12333                 if (IS_ERR(master_crtc_state))
12334                         return PTR_ERR(master_crtc_state);
12335
12336                 master_pipe_config = to_intel_crtc_state(master_crtc_state);
12337                 crtc_state->master_transcoder = master_pipe_config->cpu_transcoder;
12338                 master_pipe_config->sync_mode_slaves_mask |=
12339                         BIT(crtc_state->cpu_transcoder);
12340                 DRM_DEBUG_KMS("Master Transcoder = %s added for Slave CRTC = %d, slave transcoder bitmask = %d\n",
12341                               transcoder_name(crtc_state->master_transcoder),
12342                               crtc_state->uapi.crtc->base.id,
12343                               master_pipe_config->sync_mode_slaves_mask);
12344         }
12345
12346         return 0;
12347 }
12348
12349 static int intel_crtc_atomic_check(struct intel_atomic_state *state,
12350                                    struct intel_crtc *crtc)
12351 {
12352         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12353         struct intel_crtc_state *crtc_state =
12354                 intel_atomic_get_new_crtc_state(state, crtc);
12355         bool mode_changed = needs_modeset(crtc_state);
12356         int ret;
12357
12358         if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv) &&
12359             mode_changed && !crtc_state->hw.active)
12360                 crtc_state->update_wm_post = true;
12361
12362         if (mode_changed && crtc_state->hw.enable &&
12363             dev_priv->display.crtc_compute_clock &&
12364             !WARN_ON(crtc_state->shared_dpll)) {
12365                 ret = dev_priv->display.crtc_compute_clock(crtc, crtc_state);
12366                 if (ret)
12367                         return ret;
12368         }
12369
12370         /*
12371          * May need to update pipe gamma enable bits
12372          * when C8 planes are getting enabled/disabled.
12373          */
12374         if (c8_planes_changed(crtc_state))
12375                 crtc_state->uapi.color_mgmt_changed = true;
12376
12377         if (mode_changed || crtc_state->update_pipe ||
12378             crtc_state->uapi.color_mgmt_changed) {
12379                 ret = intel_color_check(crtc_state);
12380                 if (ret)
12381                         return ret;
12382         }
12383
12384         ret = 0;
12385         if (dev_priv->display.compute_pipe_wm) {
12386                 ret = dev_priv->display.compute_pipe_wm(crtc_state);
12387                 if (ret) {
12388                         DRM_DEBUG_KMS("Target pipe watermarks are invalid\n");
12389                         return ret;
12390                 }
12391         }
12392
12393         if (dev_priv->display.compute_intermediate_wm) {
12394                 if (WARN_ON(!dev_priv->display.compute_pipe_wm))
12395                         return 0;
12396
12397                 /*
12398                  * Calculate 'intermediate' watermarks that satisfy both the
12399                  * old state and the new state.  We can program these
12400                  * immediately.
12401                  */
12402                 ret = dev_priv->display.compute_intermediate_wm(crtc_state);
12403                 if (ret) {
12404                         DRM_DEBUG_KMS("No valid intermediate pipe watermarks are possible\n");
12405                         return ret;
12406                 }
12407         }
12408
12409         if (INTEL_GEN(dev_priv) >= 9) {
12410                 if (mode_changed || crtc_state->update_pipe)
12411                         ret = skl_update_scaler_crtc(crtc_state);
12412                 if (!ret)
12413                         ret = intel_atomic_setup_scalers(dev_priv, crtc,
12414                                                          crtc_state);
12415         }
12416
12417         if (HAS_IPS(dev_priv))
12418                 crtc_state->ips_enabled = hsw_compute_ips_config(crtc_state);
12419
12420         return ret;
12421 }
12422
12423 static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
12424 {
12425         struct intel_connector *connector;
12426         struct drm_connector_list_iter conn_iter;
12427
12428         drm_connector_list_iter_begin(dev, &conn_iter);
12429         for_each_intel_connector_iter(connector, &conn_iter) {
12430                 if (connector->base.state->crtc)
12431                         drm_connector_put(&connector->base);
12432
12433                 if (connector->base.encoder) {
12434                         connector->base.state->best_encoder =
12435                                 connector->base.encoder;
12436                         connector->base.state->crtc =
12437                                 connector->base.encoder->crtc;
12438
12439                         drm_connector_get(&connector->base);
12440                 } else {
12441                         connector->base.state->best_encoder = NULL;
12442                         connector->base.state->crtc = NULL;
12443                 }
12444         }
12445         drm_connector_list_iter_end(&conn_iter);
12446 }
12447
12448 static int
12449 compute_sink_pipe_bpp(const struct drm_connector_state *conn_state,
12450                       struct intel_crtc_state *pipe_config)
12451 {
12452         struct drm_connector *connector = conn_state->connector;
12453         const struct drm_display_info *info = &connector->display_info;
12454         int bpp;
12455
12456         switch (conn_state->max_bpc) {
12457         case 6 ... 7:
12458                 bpp = 6 * 3;
12459                 break;
12460         case 8 ... 9:
12461                 bpp = 8 * 3;
12462                 break;
12463         case 10 ... 11:
12464                 bpp = 10 * 3;
12465                 break;
12466         case 12:
12467                 bpp = 12 * 3;
12468                 break;
12469         default:
12470                 return -EINVAL;
12471         }
12472
12473         if (bpp < pipe_config->pipe_bpp) {
12474                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Limiting display bpp to %d instead of "
12475                               "EDID bpp %d, requested bpp %d, max platform bpp %d\n",
12476                               connector->base.id, connector->name,
12477                               bpp, 3 * info->bpc, 3 * conn_state->max_requested_bpc,
12478                               pipe_config->pipe_bpp);
12479
12480                 pipe_config->pipe_bpp = bpp;
12481         }
12482
12483         return 0;
12484 }
12485
12486 static int
12487 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
12488                           struct intel_crtc_state *pipe_config)
12489 {
12490         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12491         struct drm_atomic_state *state = pipe_config->uapi.state;
12492         struct drm_connector *connector;
12493         struct drm_connector_state *connector_state;
12494         int bpp, i;
12495
12496         if ((IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
12497             IS_CHERRYVIEW(dev_priv)))
12498                 bpp = 10*3;
12499         else if (INTEL_GEN(dev_priv) >= 5)
12500                 bpp = 12*3;
12501         else
12502                 bpp = 8*3;
12503
12504         pipe_config->pipe_bpp = bpp;
12505
12506         /* Clamp display bpp to connector max bpp */
12507         for_each_new_connector_in_state(state, connector, connector_state, i) {
12508                 int ret;
12509
12510                 if (connector_state->crtc != &crtc->base)
12511                         continue;
12512
12513                 ret = compute_sink_pipe_bpp(connector_state, pipe_config);
12514                 if (ret)
12515                         return ret;
12516         }
12517
12518         return 0;
12519 }
12520
12521 static void intel_dump_crtc_timings(const struct drm_display_mode *mode)
12522 {
12523         DRM_DEBUG_KMS("crtc timings: %d %d %d %d %d %d %d %d %d, "
12524                       "type: 0x%x flags: 0x%x\n",
12525                       mode->crtc_clock,
12526                       mode->crtc_hdisplay, mode->crtc_hsync_start,
12527                       mode->crtc_hsync_end, mode->crtc_htotal,
12528                       mode->crtc_vdisplay, mode->crtc_vsync_start,
12529                       mode->crtc_vsync_end, mode->crtc_vtotal,
12530                       mode->type, mode->flags);
12531 }
12532
12533 static inline void
12534 intel_dump_m_n_config(const struct intel_crtc_state *pipe_config,
12535                       const char *id, unsigned int lane_count,
12536                       const struct intel_link_m_n *m_n)
12537 {
12538         DRM_DEBUG_KMS("%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
12539                       id, lane_count,
12540                       m_n->gmch_m, m_n->gmch_n,
12541                       m_n->link_m, m_n->link_n, m_n->tu);
12542 }
12543
12544 static void
12545 intel_dump_infoframe(struct drm_i915_private *dev_priv,
12546                      const union hdmi_infoframe *frame)
12547 {
12548         if ((drm_debug & DRM_UT_KMS) == 0)
12549                 return;
12550
12551         hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, frame);
12552 }
12553
12554 #define OUTPUT_TYPE(x) [INTEL_OUTPUT_ ## x] = #x
12555
12556 static const char * const output_type_str[] = {
12557         OUTPUT_TYPE(UNUSED),
12558         OUTPUT_TYPE(ANALOG),
12559         OUTPUT_TYPE(DVO),
12560         OUTPUT_TYPE(SDVO),
12561         OUTPUT_TYPE(LVDS),
12562         OUTPUT_TYPE(TVOUT),
12563         OUTPUT_TYPE(HDMI),
12564         OUTPUT_TYPE(DP),
12565         OUTPUT_TYPE(EDP),
12566         OUTPUT_TYPE(DSI),
12567         OUTPUT_TYPE(DDI),
12568         OUTPUT_TYPE(DP_MST),
12569 };
12570
12571 #undef OUTPUT_TYPE
12572
12573 static void snprintf_output_types(char *buf, size_t len,
12574                                   unsigned int output_types)
12575 {
12576         char *str = buf;
12577         int i;
12578
12579         str[0] = '\0';
12580
12581         for (i = 0; i < ARRAY_SIZE(output_type_str); i++) {
12582                 int r;
12583
12584                 if ((output_types & BIT(i)) == 0)
12585                         continue;
12586
12587                 r = snprintf(str, len, "%s%s",
12588                              str != buf ? "," : "", output_type_str[i]);
12589                 if (r >= len)
12590                         break;
12591                 str += r;
12592                 len -= r;
12593
12594                 output_types &= ~BIT(i);
12595         }
12596
12597         WARN_ON_ONCE(output_types != 0);
12598 }
12599
12600 static const char * const output_format_str[] = {
12601         [INTEL_OUTPUT_FORMAT_INVALID] = "Invalid",
12602         [INTEL_OUTPUT_FORMAT_RGB] = "RGB",
12603         [INTEL_OUTPUT_FORMAT_YCBCR420] = "YCBCR4:2:0",
12604         [INTEL_OUTPUT_FORMAT_YCBCR444] = "YCBCR4:4:4",
12605 };
12606
12607 static const char *output_formats(enum intel_output_format format)
12608 {
12609         if (format >= ARRAY_SIZE(output_format_str))
12610                 format = INTEL_OUTPUT_FORMAT_INVALID;
12611         return output_format_str[format];
12612 }
12613
12614 static void intel_dump_plane_state(const struct intel_plane_state *plane_state)
12615 {
12616         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
12617         const struct drm_framebuffer *fb = plane_state->hw.fb;
12618         struct drm_format_name_buf format_name;
12619
12620         if (!fb) {
12621                 DRM_DEBUG_KMS("[PLANE:%d:%s] fb: [NOFB], visible: %s\n",
12622                               plane->base.base.id, plane->base.name,
12623                               yesno(plane_state->uapi.visible));
12624                 return;
12625         }
12626
12627         DRM_DEBUG_KMS("[PLANE:%d:%s] fb: [FB:%d] %ux%u format = %s, visible: %s\n",
12628                       plane->base.base.id, plane->base.name,
12629                       fb->base.id, fb->width, fb->height,
12630                       drm_get_format_name(fb->format->format, &format_name),
12631                       yesno(plane_state->uapi.visible));
12632         DRM_DEBUG_KMS("\trotation: 0x%x, scaler: %d\n",
12633                       plane_state->hw.rotation, plane_state->scaler_id);
12634         if (plane_state->uapi.visible)
12635                 DRM_DEBUG_KMS("\tsrc: " DRM_RECT_FP_FMT " dst: " DRM_RECT_FMT "\n",
12636                               DRM_RECT_FP_ARG(&plane_state->uapi.src),
12637                               DRM_RECT_ARG(&plane_state->uapi.dst));
12638 }
12639
12640 static void intel_dump_pipe_config(const struct intel_crtc_state *pipe_config,
12641                                    struct intel_atomic_state *state,
12642                                    const char *context)
12643 {
12644         struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
12645         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12646         const struct intel_plane_state *plane_state;
12647         struct intel_plane *plane;
12648         char buf[64];
12649         int i;
12650
12651         DRM_DEBUG_KMS("[CRTC:%d:%s] enable: %s %s\n",
12652                       crtc->base.base.id, crtc->base.name,
12653                       yesno(pipe_config->hw.enable), context);
12654
12655         if (!pipe_config->hw.enable)
12656                 goto dump_planes;
12657
12658         snprintf_output_types(buf, sizeof(buf), pipe_config->output_types);
12659         DRM_DEBUG_KMS("active: %s, output_types: %s (0x%x), output format: %s\n",
12660                       yesno(pipe_config->hw.active),
12661                       buf, pipe_config->output_types,
12662                       output_formats(pipe_config->output_format));
12663
12664         DRM_DEBUG_KMS("cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
12665                       transcoder_name(pipe_config->cpu_transcoder),
12666                       pipe_config->pipe_bpp, pipe_config->dither);
12667
12668         if (pipe_config->has_pch_encoder)
12669                 intel_dump_m_n_config(pipe_config, "fdi",
12670                                       pipe_config->fdi_lanes,
12671                                       &pipe_config->fdi_m_n);
12672
12673         if (intel_crtc_has_dp_encoder(pipe_config)) {
12674                 intel_dump_m_n_config(pipe_config, "dp m_n",
12675                                 pipe_config->lane_count, &pipe_config->dp_m_n);
12676                 if (pipe_config->has_drrs)
12677                         intel_dump_m_n_config(pipe_config, "dp m2_n2",
12678                                               pipe_config->lane_count,
12679                                               &pipe_config->dp_m2_n2);
12680         }
12681
12682         DRM_DEBUG_KMS("audio: %i, infoframes: %i, infoframes enabled: 0x%x\n",
12683                       pipe_config->has_audio, pipe_config->has_infoframe,
12684                       pipe_config->infoframes.enable);
12685
12686         if (pipe_config->infoframes.enable &
12687             intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GENERAL_CONTROL))
12688                 DRM_DEBUG_KMS("GCP: 0x%x\n", pipe_config->infoframes.gcp);
12689         if (pipe_config->infoframes.enable &
12690             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_AVI))
12691                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.avi);
12692         if (pipe_config->infoframes.enable &
12693             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_SPD))
12694                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.spd);
12695         if (pipe_config->infoframes.enable &
12696             intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_VENDOR))
12697                 intel_dump_infoframe(dev_priv, &pipe_config->infoframes.hdmi);
12698
12699         DRM_DEBUG_KMS("requested mode:\n");
12700         drm_mode_debug_printmodeline(&pipe_config->hw.mode);
12701         DRM_DEBUG_KMS("adjusted mode:\n");
12702         drm_mode_debug_printmodeline(&pipe_config->hw.adjusted_mode);
12703         intel_dump_crtc_timings(&pipe_config->hw.adjusted_mode);
12704         DRM_DEBUG_KMS("port clock: %d, pipe src size: %dx%d, pixel rate %d\n",
12705                       pipe_config->port_clock,
12706                       pipe_config->pipe_src_w, pipe_config->pipe_src_h,
12707                       pipe_config->pixel_rate);
12708
12709         if (INTEL_GEN(dev_priv) >= 9)
12710                 DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
12711                               crtc->num_scalers,
12712                               pipe_config->scaler_state.scaler_users,
12713                               pipe_config->scaler_state.scaler_id);
12714
12715         if (HAS_GMCH(dev_priv))
12716                 DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
12717                               pipe_config->gmch_pfit.control,
12718                               pipe_config->gmch_pfit.pgm_ratios,
12719                               pipe_config->gmch_pfit.lvds_border_bits);
12720         else
12721                 DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s, force thru: %s\n",
12722                               pipe_config->pch_pfit.pos,
12723                               pipe_config->pch_pfit.size,
12724                               enableddisabled(pipe_config->pch_pfit.enabled),
12725                               yesno(pipe_config->pch_pfit.force_thru));
12726
12727         DRM_DEBUG_KMS("ips: %i, double wide: %i\n",
12728                       pipe_config->ips_enabled, pipe_config->double_wide);
12729
12730         intel_dpll_dump_hw_state(dev_priv, &pipe_config->dpll_hw_state);
12731
12732         if (IS_CHERRYVIEW(dev_priv))
12733                 DRM_DEBUG_KMS("cgm_mode: 0x%x gamma_mode: 0x%x gamma_enable: %d csc_enable: %d\n",
12734                               pipe_config->cgm_mode, pipe_config->gamma_mode,
12735                               pipe_config->gamma_enable, pipe_config->csc_enable);
12736         else
12737                 DRM_DEBUG_KMS("csc_mode: 0x%x gamma_mode: 0x%x gamma_enable: %d csc_enable: %d\n",
12738                               pipe_config->csc_mode, pipe_config->gamma_mode,
12739                               pipe_config->gamma_enable, pipe_config->csc_enable);
12740
12741 dump_planes:
12742         if (!state)
12743                 return;
12744
12745         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
12746                 if (plane->pipe == crtc->pipe)
12747                         intel_dump_plane_state(plane_state);
12748         }
12749 }
12750
12751 static bool check_digital_port_conflicts(struct intel_atomic_state *state)
12752 {
12753         struct drm_device *dev = state->base.dev;
12754         struct drm_connector *connector;
12755         struct drm_connector_list_iter conn_iter;
12756         unsigned int used_ports = 0;
12757         unsigned int used_mst_ports = 0;
12758         bool ret = true;
12759
12760         /*
12761          * We're going to peek into connector->state,
12762          * hence connection_mutex must be held.
12763          */
12764         drm_modeset_lock_assert_held(&dev->mode_config.connection_mutex);
12765
12766         /*
12767          * Walk the connector list instead of the encoder
12768          * list to detect the problem on ddi platforms
12769          * where there's just one encoder per digital port.
12770          */
12771         drm_connector_list_iter_begin(dev, &conn_iter);
12772         drm_for_each_connector_iter(connector, &conn_iter) {
12773                 struct drm_connector_state *connector_state;
12774                 struct intel_encoder *encoder;
12775
12776                 connector_state =
12777                         drm_atomic_get_new_connector_state(&state->base,
12778                                                            connector);
12779                 if (!connector_state)
12780                         connector_state = connector->state;
12781
12782                 if (!connector_state->best_encoder)
12783                         continue;
12784
12785                 encoder = to_intel_encoder(connector_state->best_encoder);
12786
12787                 WARN_ON(!connector_state->crtc);
12788
12789                 switch (encoder->type) {
12790                         unsigned int port_mask;
12791                 case INTEL_OUTPUT_DDI:
12792                         if (WARN_ON(!HAS_DDI(to_i915(dev))))
12793                                 break;
12794                         /* else, fall through */
12795                 case INTEL_OUTPUT_DP:
12796                 case INTEL_OUTPUT_HDMI:
12797                 case INTEL_OUTPUT_EDP:
12798                         port_mask = 1 << encoder->port;
12799
12800                         /* the same port mustn't appear more than once */
12801                         if (used_ports & port_mask)
12802                                 ret = false;
12803
12804                         used_ports |= port_mask;
12805                         break;
12806                 case INTEL_OUTPUT_DP_MST:
12807                         used_mst_ports |=
12808                                 1 << encoder->port;
12809                         break;
12810                 default:
12811                         break;
12812                 }
12813         }
12814         drm_connector_list_iter_end(&conn_iter);
12815
12816         /* can't mix MST and SST/HDMI on the same port */
12817         if (used_ports & used_mst_ports)
12818                 return false;
12819
12820         return ret;
12821 }
12822
12823 static void
12824 intel_crtc_copy_uapi_to_hw_state_nomodeset(struct intel_crtc_state *crtc_state)
12825 {
12826         intel_crtc_copy_color_blobs(crtc_state);
12827 }
12828
12829 static void
12830 intel_crtc_copy_uapi_to_hw_state(struct intel_crtc_state *crtc_state)
12831 {
12832         crtc_state->hw.enable = crtc_state->uapi.enable;
12833         crtc_state->hw.active = crtc_state->uapi.active;
12834         crtc_state->hw.mode = crtc_state->uapi.mode;
12835         crtc_state->hw.adjusted_mode = crtc_state->uapi.adjusted_mode;
12836         intel_crtc_copy_uapi_to_hw_state_nomodeset(crtc_state);
12837 }
12838
12839 static void intel_crtc_copy_hw_to_uapi_state(struct intel_crtc_state *crtc_state)
12840 {
12841         crtc_state->uapi.enable = crtc_state->hw.enable;
12842         crtc_state->uapi.active = crtc_state->hw.active;
12843         WARN_ON(drm_atomic_set_mode_for_crtc(&crtc_state->uapi, &crtc_state->hw.mode) < 0);
12844
12845         crtc_state->uapi.adjusted_mode = crtc_state->hw.adjusted_mode;
12846
12847         /* copy color blobs to uapi */
12848         drm_property_replace_blob(&crtc_state->uapi.degamma_lut,
12849                                   crtc_state->hw.degamma_lut);
12850         drm_property_replace_blob(&crtc_state->uapi.gamma_lut,
12851                                   crtc_state->hw.gamma_lut);
12852         drm_property_replace_blob(&crtc_state->uapi.ctm,
12853                                   crtc_state->hw.ctm);
12854 }
12855
12856 static int
12857 intel_crtc_prepare_cleared_state(struct intel_crtc_state *crtc_state)
12858 {
12859         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
12860         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
12861         struct intel_crtc_state *saved_state;
12862
12863         saved_state = intel_crtc_state_alloc(crtc);
12864         if (!saved_state)
12865                 return -ENOMEM;
12866
12867         /* free the old crtc_state->hw members */
12868         intel_crtc_free_hw_state(crtc_state);
12869
12870         /* FIXME: before the switch to atomic started, a new pipe_config was
12871          * kzalloc'd. Code that depends on any field being zero should be
12872          * fixed, so that the crtc_state can be safely duplicated. For now,
12873          * only fields that are know to not cause problems are preserved. */
12874
12875         saved_state->uapi = crtc_state->uapi;
12876         saved_state->scaler_state = crtc_state->scaler_state;
12877         saved_state->shared_dpll = crtc_state->shared_dpll;
12878         saved_state->dpll_hw_state = crtc_state->dpll_hw_state;
12879         memcpy(saved_state->icl_port_dplls, crtc_state->icl_port_dplls,
12880                sizeof(saved_state->icl_port_dplls));
12881         saved_state->crc_enabled = crtc_state->crc_enabled;
12882         if (IS_G4X(dev_priv) ||
12883             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
12884                 saved_state->wm = crtc_state->wm;
12885         /*
12886          * Save the slave bitmask which gets filled for master crtc state during
12887          * slave atomic check call.
12888          */
12889         if (is_trans_port_sync_master(crtc_state))
12890                 saved_state->sync_mode_slaves_mask =
12891                         crtc_state->sync_mode_slaves_mask;
12892
12893         memcpy(crtc_state, saved_state, sizeof(*crtc_state));
12894         kfree(saved_state);
12895
12896         intel_crtc_copy_uapi_to_hw_state(crtc_state);
12897
12898         return 0;
12899 }
12900
12901 static int
12902 intel_modeset_pipe_config(struct intel_crtc_state *pipe_config)
12903 {
12904         struct drm_crtc *crtc = pipe_config->uapi.crtc;
12905         struct drm_atomic_state *state = pipe_config->uapi.state;
12906         struct intel_encoder *encoder;
12907         struct drm_connector *connector;
12908         struct drm_connector_state *connector_state;
12909         int base_bpp, ret;
12910         int i;
12911         bool retry = true;
12912
12913         pipe_config->cpu_transcoder =
12914                 (enum transcoder) to_intel_crtc(crtc)->pipe;
12915
12916         /*
12917          * Sanitize sync polarity flags based on requested ones. If neither
12918          * positive or negative polarity is requested, treat this as meaning
12919          * negative polarity.
12920          */
12921         if (!(pipe_config->hw.adjusted_mode.flags &
12922               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
12923                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
12924
12925         if (!(pipe_config->hw.adjusted_mode.flags &
12926               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
12927                 pipe_config->hw.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
12928
12929         ret = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
12930                                         pipe_config);
12931         if (ret)
12932                 return ret;
12933
12934         base_bpp = pipe_config->pipe_bpp;
12935
12936         /*
12937          * Determine the real pipe dimensions. Note that stereo modes can
12938          * increase the actual pipe size due to the frame doubling and
12939          * insertion of additional space for blanks between the frame. This
12940          * is stored in the crtc timings. We use the requested mode to do this
12941          * computation to clearly distinguish it from the adjusted mode, which
12942          * can be changed by the connectors in the below retry loop.
12943          */
12944         drm_mode_get_hv_timing(&pipe_config->hw.mode,
12945                                &pipe_config->pipe_src_w,
12946                                &pipe_config->pipe_src_h);
12947
12948         for_each_new_connector_in_state(state, connector, connector_state, i) {
12949                 if (connector_state->crtc != crtc)
12950                         continue;
12951
12952                 encoder = to_intel_encoder(connector_state->best_encoder);
12953
12954                 if (!check_single_encoder_cloning(state, to_intel_crtc(crtc), encoder)) {
12955                         DRM_DEBUG_KMS("rejecting invalid cloning configuration\n");
12956                         return -EINVAL;
12957                 }
12958
12959                 /*
12960                  * Determine output_types before calling the .compute_config()
12961                  * hooks so that the hooks can use this information safely.
12962                  */
12963                 if (encoder->compute_output_type)
12964                         pipe_config->output_types |=
12965                                 BIT(encoder->compute_output_type(encoder, pipe_config,
12966                                                                  connector_state));
12967                 else
12968                         pipe_config->output_types |= BIT(encoder->type);
12969         }
12970
12971 encoder_retry:
12972         /* Ensure the port clock defaults are reset when retrying. */
12973         pipe_config->port_clock = 0;
12974         pipe_config->pixel_multiplier = 1;
12975
12976         /* Fill in default crtc timings, allow encoders to overwrite them. */
12977         drm_mode_set_crtcinfo(&pipe_config->hw.adjusted_mode,
12978                               CRTC_STEREO_DOUBLE);
12979
12980         /* Set the crtc_state defaults for trans_port_sync */
12981         pipe_config->master_transcoder = INVALID_TRANSCODER;
12982         ret = icl_add_sync_mode_crtcs(pipe_config);
12983         if (ret) {
12984                 DRM_DEBUG_KMS("Cannot assign Sync Mode CRTCs: %d\n",
12985                               ret);
12986                 return ret;
12987         }
12988
12989         /* Pass our mode to the connectors and the CRTC to give them a chance to
12990          * adjust it according to limitations or connector properties, and also
12991          * a chance to reject the mode entirely.
12992          */
12993         for_each_new_connector_in_state(state, connector, connector_state, i) {
12994                 if (connector_state->crtc != crtc)
12995                         continue;
12996
12997                 encoder = to_intel_encoder(connector_state->best_encoder);
12998                 ret = encoder->compute_config(encoder, pipe_config,
12999                                               connector_state);
13000                 if (ret < 0) {
13001                         if (ret != -EDEADLK)
13002                                 DRM_DEBUG_KMS("Encoder config failure: %d\n",
13003                                               ret);
13004                         return ret;
13005                 }
13006         }
13007
13008         /* Set default port clock if not overwritten by the encoder. Needs to be
13009          * done afterwards in case the encoder adjusts the mode. */
13010         if (!pipe_config->port_clock)
13011                 pipe_config->port_clock = pipe_config->hw.adjusted_mode.crtc_clock
13012                         * pipe_config->pixel_multiplier;
13013
13014         ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
13015         if (ret == -EDEADLK)
13016                 return ret;
13017         if (ret < 0) {
13018                 DRM_DEBUG_KMS("CRTC fixup failed\n");
13019                 return ret;
13020         }
13021
13022         if (ret == RETRY) {
13023                 if (WARN(!retry, "loop in pipe configuration computation\n"))
13024                         return -EINVAL;
13025
13026                 DRM_DEBUG_KMS("CRTC bw constrained, retrying\n");
13027                 retry = false;
13028                 goto encoder_retry;
13029         }
13030
13031         /* Dithering seems to not pass-through bits correctly when it should, so
13032          * only enable it on 6bpc panels and when its not a compliance
13033          * test requesting 6bpc video pattern.
13034          */
13035         pipe_config->dither = (pipe_config->pipe_bpp == 6*3) &&
13036                 !pipe_config->dither_force_disable;
13037         DRM_DEBUG_KMS("hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
13038                       base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
13039
13040         /*
13041          * Make drm_calc_timestamping_constants in
13042          * drm_atomic_helper_update_legacy_modeset_state() happy
13043          */
13044         pipe_config->uapi.adjusted_mode = pipe_config->hw.adjusted_mode;
13045
13046         return 0;
13047 }
13048
13049 bool intel_fuzzy_clock_check(int clock1, int clock2)
13050 {
13051         int diff;
13052
13053         if (clock1 == clock2)
13054                 return true;
13055
13056         if (!clock1 || !clock2)
13057                 return false;
13058
13059         diff = abs(clock1 - clock2);
13060
13061         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
13062                 return true;
13063
13064         return false;
13065 }
13066
13067 static bool
13068 intel_compare_m_n(unsigned int m, unsigned int n,
13069                   unsigned int m2, unsigned int n2,
13070                   bool exact)
13071 {
13072         if (m == m2 && n == n2)
13073                 return true;
13074
13075         if (exact || !m || !n || !m2 || !n2)
13076                 return false;
13077
13078         BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
13079
13080         if (n > n2) {
13081                 while (n > n2) {
13082                         m2 <<= 1;
13083                         n2 <<= 1;
13084                 }
13085         } else if (n < n2) {
13086                 while (n < n2) {
13087                         m <<= 1;
13088                         n <<= 1;
13089                 }
13090         }
13091
13092         if (n != n2)
13093                 return false;
13094
13095         return intel_fuzzy_clock_check(m, m2);
13096 }
13097
13098 static bool
13099 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
13100                        const struct intel_link_m_n *m2_n2,
13101                        bool exact)
13102 {
13103         return m_n->tu == m2_n2->tu &&
13104                 intel_compare_m_n(m_n->gmch_m, m_n->gmch_n,
13105                                   m2_n2->gmch_m, m2_n2->gmch_n, exact) &&
13106                 intel_compare_m_n(m_n->link_m, m_n->link_n,
13107                                   m2_n2->link_m, m2_n2->link_n, exact);
13108 }
13109
13110 static bool
13111 intel_compare_infoframe(const union hdmi_infoframe *a,
13112                         const union hdmi_infoframe *b)
13113 {
13114         return memcmp(a, b, sizeof(*a)) == 0;
13115 }
13116
13117 static void
13118 pipe_config_infoframe_mismatch(struct drm_i915_private *dev_priv,
13119                                bool fastset, const char *name,
13120                                const union hdmi_infoframe *a,
13121                                const union hdmi_infoframe *b)
13122 {
13123         if (fastset) {
13124                 if ((drm_debug & DRM_UT_KMS) == 0)
13125                         return;
13126
13127                 DRM_DEBUG_KMS("fastset mismatch in %s infoframe\n", name);
13128                 DRM_DEBUG_KMS("expected:\n");
13129                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, a);
13130                 DRM_DEBUG_KMS("found:\n");
13131                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, b);
13132         } else {
13133                 DRM_ERROR("mismatch in %s infoframe\n", name);
13134                 DRM_ERROR("expected:\n");
13135                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, a);
13136                 DRM_ERROR("found:\n");
13137                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, b);
13138         }
13139 }
13140
13141 static void __printf(4, 5)
13142 pipe_config_mismatch(bool fastset, const struct intel_crtc *crtc,
13143                      const char *name, const char *format, ...)
13144 {
13145         struct va_format vaf;
13146         va_list args;
13147
13148         va_start(args, format);
13149         vaf.fmt = format;
13150         vaf.va = &args;
13151
13152         if (fastset)
13153                 DRM_DEBUG_KMS("[CRTC:%d:%s] fastset mismatch in %s %pV\n",
13154                               crtc->base.base.id, crtc->base.name, name, &vaf);
13155         else
13156                 DRM_ERROR("[CRTC:%d:%s] mismatch in %s %pV\n",
13157                           crtc->base.base.id, crtc->base.name, name, &vaf);
13158
13159         va_end(args);
13160 }
13161
13162 static bool fastboot_enabled(struct drm_i915_private *dev_priv)
13163 {
13164         if (i915_modparams.fastboot != -1)
13165                 return i915_modparams.fastboot;
13166
13167         /* Enable fastboot by default on Skylake and newer */
13168         if (INTEL_GEN(dev_priv) >= 9)
13169                 return true;
13170
13171         /* Enable fastboot by default on VLV and CHV */
13172         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
13173                 return true;
13174
13175         /* Disabled by default on all others */
13176         return false;
13177 }
13178
13179 static bool
13180 intel_pipe_config_compare(const struct intel_crtc_state *current_config,
13181                           const struct intel_crtc_state *pipe_config,
13182                           bool fastset)
13183 {
13184         struct drm_i915_private *dev_priv = to_i915(current_config->uapi.crtc->dev);
13185         struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
13186         bool ret = true;
13187         u32 bp_gamma = 0;
13188         bool fixup_inherited = fastset &&
13189                 (current_config->hw.mode.private_flags & I915_MODE_FLAG_INHERITED) &&
13190                 !(pipe_config->hw.mode.private_flags & I915_MODE_FLAG_INHERITED);
13191
13192         if (fixup_inherited && !fastboot_enabled(dev_priv)) {
13193                 DRM_DEBUG_KMS("initial modeset and fastboot not set\n");
13194                 ret = false;
13195         }
13196
13197 #define PIPE_CONF_CHECK_X(name) do { \
13198         if (current_config->name != pipe_config->name) { \
13199                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13200                                      "(expected 0x%08x, found 0x%08x)", \
13201                                      current_config->name, \
13202                                      pipe_config->name); \
13203                 ret = false; \
13204         } \
13205 } while (0)
13206
13207 #define PIPE_CONF_CHECK_I(name) do { \
13208         if (current_config->name != pipe_config->name) { \
13209                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13210                                      "(expected %i, found %i)", \
13211                                      current_config->name, \
13212                                      pipe_config->name); \
13213                 ret = false; \
13214         } \
13215 } while (0)
13216
13217 #define PIPE_CONF_CHECK_BOOL(name) do { \
13218         if (current_config->name != pipe_config->name) { \
13219                 pipe_config_mismatch(fastset, crtc,  __stringify(name), \
13220                                      "(expected %s, found %s)", \
13221                                      yesno(current_config->name), \
13222                                      yesno(pipe_config->name)); \
13223                 ret = false; \
13224         } \
13225 } while (0)
13226
13227 /*
13228  * Checks state where we only read out the enabling, but not the entire
13229  * state itself (like full infoframes or ELD for audio). These states
13230  * require a full modeset on bootup to fix up.
13231  */
13232 #define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) do { \
13233         if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \
13234                 PIPE_CONF_CHECK_BOOL(name); \
13235         } else { \
13236                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13237                                      "unable to verify whether state matches exactly, forcing modeset (expected %s, found %s)", \
13238                                      yesno(current_config->name), \
13239                                      yesno(pipe_config->name)); \
13240                 ret = false; \
13241         } \
13242 } while (0)
13243
13244 #define PIPE_CONF_CHECK_P(name) do { \
13245         if (current_config->name != pipe_config->name) { \
13246                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13247                                      "(expected %p, found %p)", \
13248                                      current_config->name, \
13249                                      pipe_config->name); \
13250                 ret = false; \
13251         } \
13252 } while (0)
13253
13254 #define PIPE_CONF_CHECK_M_N(name) do { \
13255         if (!intel_compare_link_m_n(&current_config->name, \
13256                                     &pipe_config->name,\
13257                                     !fastset)) { \
13258                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13259                                      "(expected tu %i gmch %i/%i link %i/%i, " \
13260                                      "found tu %i, gmch %i/%i link %i/%i)", \
13261                                      current_config->name.tu, \
13262                                      current_config->name.gmch_m, \
13263                                      current_config->name.gmch_n, \
13264                                      current_config->name.link_m, \
13265                                      current_config->name.link_n, \
13266                                      pipe_config->name.tu, \
13267                                      pipe_config->name.gmch_m, \
13268                                      pipe_config->name.gmch_n, \
13269                                      pipe_config->name.link_m, \
13270                                      pipe_config->name.link_n); \
13271                 ret = false; \
13272         } \
13273 } while (0)
13274
13275 /* This is required for BDW+ where there is only one set of registers for
13276  * switching between high and low RR.
13277  * This macro can be used whenever a comparison has to be made between one
13278  * hw state and multiple sw state variables.
13279  */
13280 #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) do { \
13281         if (!intel_compare_link_m_n(&current_config->name, \
13282                                     &pipe_config->name, !fastset) && \
13283             !intel_compare_link_m_n(&current_config->alt_name, \
13284                                     &pipe_config->name, !fastset)) { \
13285                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13286                                      "(expected tu %i gmch %i/%i link %i/%i, " \
13287                                      "or tu %i gmch %i/%i link %i/%i, " \
13288                                      "found tu %i, gmch %i/%i link %i/%i)", \
13289                                      current_config->name.tu, \
13290                                      current_config->name.gmch_m, \
13291                                      current_config->name.gmch_n, \
13292                                      current_config->name.link_m, \
13293                                      current_config->name.link_n, \
13294                                      current_config->alt_name.tu, \
13295                                      current_config->alt_name.gmch_m, \
13296                                      current_config->alt_name.gmch_n, \
13297                                      current_config->alt_name.link_m, \
13298                                      current_config->alt_name.link_n, \
13299                                      pipe_config->name.tu, \
13300                                      pipe_config->name.gmch_m, \
13301                                      pipe_config->name.gmch_n, \
13302                                      pipe_config->name.link_m, \
13303                                      pipe_config->name.link_n); \
13304                 ret = false; \
13305         } \
13306 } while (0)
13307
13308 #define PIPE_CONF_CHECK_FLAGS(name, mask) do { \
13309         if ((current_config->name ^ pipe_config->name) & (mask)) { \
13310                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13311                                      "(%x) (expected %i, found %i)", \
13312                                      (mask), \
13313                                      current_config->name & (mask), \
13314                                      pipe_config->name & (mask)); \
13315                 ret = false; \
13316         } \
13317 } while (0)
13318
13319 #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) do { \
13320         if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
13321                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
13322                                      "(expected %i, found %i)", \
13323                                      current_config->name, \
13324                                      pipe_config->name); \
13325                 ret = false; \
13326         } \
13327 } while (0)
13328
13329 #define PIPE_CONF_CHECK_INFOFRAME(name) do { \
13330         if (!intel_compare_infoframe(&current_config->infoframes.name, \
13331                                      &pipe_config->infoframes.name)) { \
13332                 pipe_config_infoframe_mismatch(dev_priv, fastset, __stringify(name), \
13333                                                &current_config->infoframes.name, \
13334                                                &pipe_config->infoframes.name); \
13335                 ret = false; \
13336         } \
13337 } while (0)
13338
13339 #define PIPE_CONF_CHECK_COLOR_LUT(name1, name2, bit_precision) do { \
13340         if (current_config->name1 != pipe_config->name1) { \
13341                 pipe_config_mismatch(fastset, crtc, __stringify(name1), \
13342                                 "(expected %i, found %i, won't compare lut values)", \
13343                                 current_config->name1, \
13344                                 pipe_config->name1); \
13345                 ret = false;\
13346         } else { \
13347                 if (!intel_color_lut_equal(current_config->name2, \
13348                                         pipe_config->name2, pipe_config->name1, \
13349                                         bit_precision)) { \
13350                         pipe_config_mismatch(fastset, crtc, __stringify(name2), \
13351                                         "hw_state doesn't match sw_state"); \
13352                         ret = false; \
13353                 } \
13354         } \
13355 } while (0)
13356
13357 #define PIPE_CONF_QUIRK(quirk) \
13358         ((current_config->quirks | pipe_config->quirks) & (quirk))
13359
13360         PIPE_CONF_CHECK_I(cpu_transcoder);
13361
13362         PIPE_CONF_CHECK_BOOL(has_pch_encoder);
13363         PIPE_CONF_CHECK_I(fdi_lanes);
13364         PIPE_CONF_CHECK_M_N(fdi_m_n);
13365
13366         PIPE_CONF_CHECK_I(lane_count);
13367         PIPE_CONF_CHECK_X(lane_lat_optim_mask);
13368
13369         if (INTEL_GEN(dev_priv) < 8) {
13370                 PIPE_CONF_CHECK_M_N(dp_m_n);
13371
13372                 if (current_config->has_drrs)
13373                         PIPE_CONF_CHECK_M_N(dp_m2_n2);
13374         } else
13375                 PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
13376
13377         PIPE_CONF_CHECK_X(output_types);
13378
13379         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hdisplay);
13380         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_htotal);
13381         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hblank_start);
13382         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hblank_end);
13383         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hsync_start);
13384         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_hsync_end);
13385
13386         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vdisplay);
13387         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vtotal);
13388         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vblank_start);
13389         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vblank_end);
13390         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vsync_start);
13391         PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_vsync_end);
13392
13393         PIPE_CONF_CHECK_I(pixel_multiplier);
13394         PIPE_CONF_CHECK_I(output_format);
13395         PIPE_CONF_CHECK_I(dc3co_exitline);
13396         PIPE_CONF_CHECK_BOOL(has_hdmi_sink);
13397         if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
13398             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
13399                 PIPE_CONF_CHECK_BOOL(limited_color_range);
13400
13401         PIPE_CONF_CHECK_BOOL(hdmi_scrambling);
13402         PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio);
13403         PIPE_CONF_CHECK_BOOL(has_infoframe);
13404         PIPE_CONF_CHECK_BOOL(fec_enable);
13405
13406         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio);
13407
13408         PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13409                               DRM_MODE_FLAG_INTERLACE);
13410
13411         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
13412                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13413                                       DRM_MODE_FLAG_PHSYNC);
13414                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13415                                       DRM_MODE_FLAG_NHSYNC);
13416                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13417                                       DRM_MODE_FLAG_PVSYNC);
13418                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
13419                                       DRM_MODE_FLAG_NVSYNC);
13420         }
13421
13422         PIPE_CONF_CHECK_X(gmch_pfit.control);
13423         /* pfit ratios are autocomputed by the hw on gen4+ */
13424         if (INTEL_GEN(dev_priv) < 4)
13425                 PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios);
13426         PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
13427
13428         /*
13429          * Changing the EDP transcoder input mux
13430          * (A_ONOFF vs. A_ON) requires a full modeset.
13431          */
13432         PIPE_CONF_CHECK_BOOL(pch_pfit.force_thru);
13433
13434         if (!fastset) {
13435                 PIPE_CONF_CHECK_I(pipe_src_w);
13436                 PIPE_CONF_CHECK_I(pipe_src_h);
13437
13438                 PIPE_CONF_CHECK_BOOL(pch_pfit.enabled);
13439                 if (current_config->pch_pfit.enabled) {
13440                         PIPE_CONF_CHECK_X(pch_pfit.pos);
13441                         PIPE_CONF_CHECK_X(pch_pfit.size);
13442                 }
13443
13444                 PIPE_CONF_CHECK_I(scaler_state.scaler_id);
13445                 PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate);
13446
13447                 PIPE_CONF_CHECK_X(gamma_mode);
13448                 if (IS_CHERRYVIEW(dev_priv))
13449                         PIPE_CONF_CHECK_X(cgm_mode);
13450                 else
13451                         PIPE_CONF_CHECK_X(csc_mode);
13452                 PIPE_CONF_CHECK_BOOL(gamma_enable);
13453                 PIPE_CONF_CHECK_BOOL(csc_enable);
13454
13455                 bp_gamma = intel_color_get_gamma_bit_precision(pipe_config);
13456                 if (bp_gamma)
13457                         PIPE_CONF_CHECK_COLOR_LUT(gamma_mode, hw.gamma_lut, bp_gamma);
13458
13459         }
13460
13461         PIPE_CONF_CHECK_BOOL(double_wide);
13462
13463         PIPE_CONF_CHECK_P(shared_dpll);
13464         PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
13465         PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
13466         PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
13467         PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
13468         PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
13469         PIPE_CONF_CHECK_X(dpll_hw_state.spll);
13470         PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
13471         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
13472         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
13473         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0);
13474         PIPE_CONF_CHECK_X(dpll_hw_state.ebb0);
13475         PIPE_CONF_CHECK_X(dpll_hw_state.ebb4);
13476         PIPE_CONF_CHECK_X(dpll_hw_state.pll0);
13477         PIPE_CONF_CHECK_X(dpll_hw_state.pll1);
13478         PIPE_CONF_CHECK_X(dpll_hw_state.pll2);
13479         PIPE_CONF_CHECK_X(dpll_hw_state.pll3);
13480         PIPE_CONF_CHECK_X(dpll_hw_state.pll6);
13481         PIPE_CONF_CHECK_X(dpll_hw_state.pll8);
13482         PIPE_CONF_CHECK_X(dpll_hw_state.pll9);
13483         PIPE_CONF_CHECK_X(dpll_hw_state.pll10);
13484         PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12);
13485         PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl);
13486         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1);
13487         PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl);
13488         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0);
13489         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1);
13490         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf);
13491         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock);
13492         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc);
13493         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias);
13494         PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias);
13495
13496         PIPE_CONF_CHECK_X(dsi_pll.ctrl);
13497         PIPE_CONF_CHECK_X(dsi_pll.div);
13498
13499         if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5)
13500                 PIPE_CONF_CHECK_I(pipe_bpp);
13501
13502         PIPE_CONF_CHECK_CLOCK_FUZZY(hw.adjusted_mode.crtc_clock);
13503         PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
13504
13505         PIPE_CONF_CHECK_I(min_voltage_level);
13506
13507         PIPE_CONF_CHECK_X(infoframes.enable);
13508         PIPE_CONF_CHECK_X(infoframes.gcp);
13509         PIPE_CONF_CHECK_INFOFRAME(avi);
13510         PIPE_CONF_CHECK_INFOFRAME(spd);
13511         PIPE_CONF_CHECK_INFOFRAME(hdmi);
13512         PIPE_CONF_CHECK_INFOFRAME(drm);
13513
13514         PIPE_CONF_CHECK_I(sync_mode_slaves_mask);
13515         PIPE_CONF_CHECK_I(master_transcoder);
13516
13517         PIPE_CONF_CHECK_I(dsc.compression_enable);
13518         PIPE_CONF_CHECK_I(dsc.dsc_split);
13519         PIPE_CONF_CHECK_I(dsc.compressed_bpp);
13520
13521 #undef PIPE_CONF_CHECK_X
13522 #undef PIPE_CONF_CHECK_I
13523 #undef PIPE_CONF_CHECK_BOOL
13524 #undef PIPE_CONF_CHECK_BOOL_INCOMPLETE
13525 #undef PIPE_CONF_CHECK_P
13526 #undef PIPE_CONF_CHECK_FLAGS
13527 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
13528 #undef PIPE_CONF_CHECK_COLOR_LUT
13529 #undef PIPE_CONF_QUIRK
13530
13531         return ret;
13532 }
13533
13534 static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv,
13535                                            const struct intel_crtc_state *pipe_config)
13536 {
13537         if (pipe_config->has_pch_encoder) {
13538                 int fdi_dotclock = intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
13539                                                             &pipe_config->fdi_m_n);
13540                 int dotclock = pipe_config->hw.adjusted_mode.crtc_clock;
13541
13542                 /*
13543                  * FDI already provided one idea for the dotclock.
13544                  * Yell if the encoder disagrees.
13545                  */
13546                 WARN(!intel_fuzzy_clock_check(fdi_dotclock, dotclock),
13547                      "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
13548                      fdi_dotclock, dotclock);
13549         }
13550 }
13551
13552 static void verify_wm_state(struct intel_crtc *crtc,
13553                             struct intel_crtc_state *new_crtc_state)
13554 {
13555         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13556         struct skl_hw_state {
13557                 struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
13558                 struct skl_ddb_entry ddb_uv[I915_MAX_PLANES];
13559                 struct skl_ddb_allocation ddb;
13560                 struct skl_pipe_wm wm;
13561         } *hw;
13562         struct skl_ddb_allocation *sw_ddb;
13563         struct skl_pipe_wm *sw_wm;
13564         struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
13565         const enum pipe pipe = crtc->pipe;
13566         int plane, level, max_level = ilk_wm_max_level(dev_priv);
13567
13568         if (INTEL_GEN(dev_priv) < 9 || !new_crtc_state->hw.active)
13569                 return;
13570
13571         hw = kzalloc(sizeof(*hw), GFP_KERNEL);
13572         if (!hw)
13573                 return;
13574
13575         skl_pipe_wm_get_hw_state(crtc, &hw->wm);
13576         sw_wm = &new_crtc_state->wm.skl.optimal;
13577
13578         skl_pipe_ddb_get_hw_state(crtc, hw->ddb_y, hw->ddb_uv);
13579
13580         skl_ddb_get_hw_state(dev_priv, &hw->ddb);
13581         sw_ddb = &dev_priv->wm.skl_hw.ddb;
13582
13583         if (INTEL_GEN(dev_priv) >= 11 &&
13584             hw->ddb.enabled_slices != sw_ddb->enabled_slices)
13585                 DRM_ERROR("mismatch in DBUF Slices (expected %u, got %u)\n",
13586                           sw_ddb->enabled_slices,
13587                           hw->ddb.enabled_slices);
13588
13589         /* planes */
13590         for_each_universal_plane(dev_priv, pipe, plane) {
13591                 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
13592
13593                 hw_plane_wm = &hw->wm.planes[plane];
13594                 sw_plane_wm = &sw_wm->planes[plane];
13595
13596                 /* Watermarks */
13597                 for (level = 0; level <= max_level; level++) {
13598                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
13599                                                 &sw_plane_wm->wm[level]))
13600                                 continue;
13601
13602                         DRM_ERROR("mismatch in WM pipe %c plane %d level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
13603                                   pipe_name(pipe), plane + 1, level,
13604                                   sw_plane_wm->wm[level].plane_en,
13605                                   sw_plane_wm->wm[level].plane_res_b,
13606                                   sw_plane_wm->wm[level].plane_res_l,
13607                                   hw_plane_wm->wm[level].plane_en,
13608                                   hw_plane_wm->wm[level].plane_res_b,
13609                                   hw_plane_wm->wm[level].plane_res_l);
13610                 }
13611
13612                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
13613                                          &sw_plane_wm->trans_wm)) {
13614                         DRM_ERROR("mismatch in trans WM pipe %c plane %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
13615                                   pipe_name(pipe), plane + 1,
13616                                   sw_plane_wm->trans_wm.plane_en,
13617                                   sw_plane_wm->trans_wm.plane_res_b,
13618                                   sw_plane_wm->trans_wm.plane_res_l,
13619                                   hw_plane_wm->trans_wm.plane_en,
13620                                   hw_plane_wm->trans_wm.plane_res_b,
13621                                   hw_plane_wm->trans_wm.plane_res_l);
13622                 }
13623
13624                 /* DDB */
13625                 hw_ddb_entry = &hw->ddb_y[plane];
13626                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[plane];
13627
13628                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
13629                         DRM_ERROR("mismatch in DDB state pipe %c plane %d (expected (%u,%u), found (%u,%u))\n",
13630                                   pipe_name(pipe), plane + 1,
13631                                   sw_ddb_entry->start, sw_ddb_entry->end,
13632                                   hw_ddb_entry->start, hw_ddb_entry->end);
13633                 }
13634         }
13635
13636         /*
13637          * cursor
13638          * If the cursor plane isn't active, we may not have updated it's ddb
13639          * allocation. In that case since the ddb allocation will be updated
13640          * once the plane becomes visible, we can skip this check
13641          */
13642         if (1) {
13643                 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
13644
13645                 hw_plane_wm = &hw->wm.planes[PLANE_CURSOR];
13646                 sw_plane_wm = &sw_wm->planes[PLANE_CURSOR];
13647
13648                 /* Watermarks */
13649                 for (level = 0; level <= max_level; level++) {
13650                         if (skl_wm_level_equals(&hw_plane_wm->wm[level],
13651                                                 &sw_plane_wm->wm[level]))
13652                                 continue;
13653
13654                         DRM_ERROR("mismatch in WM pipe %c cursor level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
13655                                   pipe_name(pipe), level,
13656                                   sw_plane_wm->wm[level].plane_en,
13657                                   sw_plane_wm->wm[level].plane_res_b,
13658                                   sw_plane_wm->wm[level].plane_res_l,
13659                                   hw_plane_wm->wm[level].plane_en,
13660                                   hw_plane_wm->wm[level].plane_res_b,
13661                                   hw_plane_wm->wm[level].plane_res_l);
13662                 }
13663
13664                 if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
13665                                          &sw_plane_wm->trans_wm)) {
13666                         DRM_ERROR("mismatch in trans WM pipe %c cursor (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
13667                                   pipe_name(pipe),
13668                                   sw_plane_wm->trans_wm.plane_en,
13669                                   sw_plane_wm->trans_wm.plane_res_b,
13670                                   sw_plane_wm->trans_wm.plane_res_l,
13671                                   hw_plane_wm->trans_wm.plane_en,
13672                                   hw_plane_wm->trans_wm.plane_res_b,
13673                                   hw_plane_wm->trans_wm.plane_res_l);
13674                 }
13675
13676                 /* DDB */
13677                 hw_ddb_entry = &hw->ddb_y[PLANE_CURSOR];
13678                 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR];
13679
13680                 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
13681                         DRM_ERROR("mismatch in DDB state pipe %c cursor (expected (%u,%u), found (%u,%u))\n",
13682                                   pipe_name(pipe),
13683                                   sw_ddb_entry->start, sw_ddb_entry->end,
13684                                   hw_ddb_entry->start, hw_ddb_entry->end);
13685                 }
13686         }
13687
13688         kfree(hw);
13689 }
13690
13691 static void
13692 verify_connector_state(struct intel_atomic_state *state,
13693                        struct intel_crtc *crtc)
13694 {
13695         struct drm_connector *connector;
13696         struct drm_connector_state *new_conn_state;
13697         int i;
13698
13699         for_each_new_connector_in_state(&state->base, connector, new_conn_state, i) {
13700                 struct drm_encoder *encoder = connector->encoder;
13701                 struct intel_crtc_state *crtc_state = NULL;
13702
13703                 if (new_conn_state->crtc != &crtc->base)
13704                         continue;
13705
13706                 if (crtc)
13707                         crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
13708
13709                 intel_connector_verify_state(crtc_state, new_conn_state);
13710
13711                 I915_STATE_WARN(new_conn_state->best_encoder != encoder,
13712                      "connector's atomic encoder doesn't match legacy encoder\n");
13713         }
13714 }
13715
13716 static void
13717 verify_encoder_state(struct drm_i915_private *dev_priv, struct intel_atomic_state *state)
13718 {
13719         struct intel_encoder *encoder;
13720         struct drm_connector *connector;
13721         struct drm_connector_state *old_conn_state, *new_conn_state;
13722         int i;
13723
13724         for_each_intel_encoder(&dev_priv->drm, encoder) {
13725                 bool enabled = false, found = false;
13726                 enum pipe pipe;
13727
13728                 DRM_DEBUG_KMS("[ENCODER:%d:%s]\n",
13729                               encoder->base.base.id,
13730                               encoder->base.name);
13731
13732                 for_each_oldnew_connector_in_state(&state->base, connector, old_conn_state,
13733                                                    new_conn_state, i) {
13734                         if (old_conn_state->best_encoder == &encoder->base)
13735                                 found = true;
13736
13737                         if (new_conn_state->best_encoder != &encoder->base)
13738                                 continue;
13739                         found = enabled = true;
13740
13741                         I915_STATE_WARN(new_conn_state->crtc !=
13742                                         encoder->base.crtc,
13743                              "connector's crtc doesn't match encoder crtc\n");
13744                 }
13745
13746                 if (!found)
13747                         continue;
13748
13749                 I915_STATE_WARN(!!encoder->base.crtc != enabled,
13750                      "encoder's enabled state mismatch "
13751                      "(expected %i, found %i)\n",
13752                      !!encoder->base.crtc, enabled);
13753
13754                 if (!encoder->base.crtc) {
13755                         bool active;
13756
13757                         active = encoder->get_hw_state(encoder, &pipe);
13758                         I915_STATE_WARN(active,
13759                              "encoder detached but still enabled on pipe %c.\n",
13760                              pipe_name(pipe));
13761                 }
13762         }
13763 }
13764
13765 static void
13766 verify_crtc_state(struct intel_crtc *crtc,
13767                   struct intel_crtc_state *old_crtc_state,
13768                   struct intel_crtc_state *new_crtc_state)
13769 {
13770         struct drm_device *dev = crtc->base.dev;
13771         struct drm_i915_private *dev_priv = to_i915(dev);
13772         struct intel_encoder *encoder;
13773         struct intel_crtc_state *pipe_config = old_crtc_state;
13774         struct drm_atomic_state *state = old_crtc_state->uapi.state;
13775         bool active;
13776
13777         __drm_atomic_helper_crtc_destroy_state(&old_crtc_state->uapi);
13778         intel_crtc_free_hw_state(old_crtc_state);
13779         intel_crtc_state_reset(old_crtc_state, crtc);
13780         old_crtc_state->uapi.state = state;
13781
13782         DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.base.id, crtc->base.name);
13783
13784         active = dev_priv->display.get_pipe_config(crtc, pipe_config);
13785
13786         /* we keep both pipes enabled on 830 */
13787         if (IS_I830(dev_priv))
13788                 active = new_crtc_state->hw.active;
13789
13790         I915_STATE_WARN(new_crtc_state->hw.active != active,
13791                         "crtc active state doesn't match with hw state "
13792                         "(expected %i, found %i)\n",
13793                         new_crtc_state->hw.active, active);
13794
13795         I915_STATE_WARN(crtc->active != new_crtc_state->hw.active,
13796                         "transitional active state does not match atomic hw state "
13797                         "(expected %i, found %i)\n",
13798                         new_crtc_state->hw.active, crtc->active);
13799
13800         for_each_encoder_on_crtc(dev, &crtc->base, encoder) {
13801                 enum pipe pipe;
13802
13803                 active = encoder->get_hw_state(encoder, &pipe);
13804                 I915_STATE_WARN(active != new_crtc_state->hw.active,
13805                                 "[ENCODER:%i] active %i with crtc active %i\n",
13806                                 encoder->base.base.id, active,
13807                                 new_crtc_state->hw.active);
13808
13809                 I915_STATE_WARN(active && crtc->pipe != pipe,
13810                                 "Encoder connected to wrong pipe %c\n",
13811                                 pipe_name(pipe));
13812
13813                 if (active)
13814                         encoder->get_config(encoder, pipe_config);
13815         }
13816
13817         intel_crtc_compute_pixel_rate(pipe_config);
13818
13819         if (!new_crtc_state->hw.active)
13820                 return;
13821
13822         intel_pipe_config_sanity_check(dev_priv, pipe_config);
13823
13824         if (!intel_pipe_config_compare(new_crtc_state,
13825                                        pipe_config, false)) {
13826                 I915_STATE_WARN(1, "pipe state doesn't match!\n");
13827                 intel_dump_pipe_config(pipe_config, NULL, "[hw state]");
13828                 intel_dump_pipe_config(new_crtc_state, NULL, "[sw state]");
13829         }
13830 }
13831
13832 static void
13833 intel_verify_planes(struct intel_atomic_state *state)
13834 {
13835         struct intel_plane *plane;
13836         const struct intel_plane_state *plane_state;
13837         int i;
13838
13839         for_each_new_intel_plane_in_state(state, plane,
13840                                           plane_state, i)
13841                 assert_plane(plane, plane_state->planar_slave ||
13842                              plane_state->uapi.visible);
13843 }
13844
13845 static void
13846 verify_single_dpll_state(struct drm_i915_private *dev_priv,
13847                          struct intel_shared_dpll *pll,
13848                          struct intel_crtc *crtc,
13849                          struct intel_crtc_state *new_crtc_state)
13850 {
13851         struct intel_dpll_hw_state dpll_hw_state;
13852         unsigned int crtc_mask;
13853         bool active;
13854
13855         memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
13856
13857         DRM_DEBUG_KMS("%s\n", pll->info->name);
13858
13859         active = pll->info->funcs->get_hw_state(dev_priv, pll, &dpll_hw_state);
13860
13861         if (!(pll->info->flags & INTEL_DPLL_ALWAYS_ON)) {
13862                 I915_STATE_WARN(!pll->on && pll->active_mask,
13863                      "pll in active use but not on in sw tracking\n");
13864                 I915_STATE_WARN(pll->on && !pll->active_mask,
13865                      "pll is on but not used by any active crtc\n");
13866                 I915_STATE_WARN(pll->on != active,
13867                      "pll on state mismatch (expected %i, found %i)\n",
13868                      pll->on, active);
13869         }
13870
13871         if (!crtc) {
13872                 I915_STATE_WARN(pll->active_mask & ~pll->state.crtc_mask,
13873                                 "more active pll users than references: %x vs %x\n",
13874                                 pll->active_mask, pll->state.crtc_mask);
13875
13876                 return;
13877         }
13878
13879         crtc_mask = drm_crtc_mask(&crtc->base);
13880
13881         if (new_crtc_state->hw.active)
13882                 I915_STATE_WARN(!(pll->active_mask & crtc_mask),
13883                                 "pll active mismatch (expected pipe %c in active mask 0x%02x)\n",
13884                                 pipe_name(drm_crtc_index(&crtc->base)), pll->active_mask);
13885         else
13886                 I915_STATE_WARN(pll->active_mask & crtc_mask,
13887                                 "pll active mismatch (didn't expect pipe %c in active mask 0x%02x)\n",
13888                                 pipe_name(drm_crtc_index(&crtc->base)), pll->active_mask);
13889
13890         I915_STATE_WARN(!(pll->state.crtc_mask & crtc_mask),
13891                         "pll enabled crtcs mismatch (expected 0x%x in 0x%02x)\n",
13892                         crtc_mask, pll->state.crtc_mask);
13893
13894         I915_STATE_WARN(pll->on && memcmp(&pll->state.hw_state,
13895                                           &dpll_hw_state,
13896                                           sizeof(dpll_hw_state)),
13897                         "pll hw state mismatch\n");
13898 }
13899
13900 static void
13901 verify_shared_dpll_state(struct intel_crtc *crtc,
13902                          struct intel_crtc_state *old_crtc_state,
13903                          struct intel_crtc_state *new_crtc_state)
13904 {
13905         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13906
13907         if (new_crtc_state->shared_dpll)
13908                 verify_single_dpll_state(dev_priv, new_crtc_state->shared_dpll, crtc, new_crtc_state);
13909
13910         if (old_crtc_state->shared_dpll &&
13911             old_crtc_state->shared_dpll != new_crtc_state->shared_dpll) {
13912                 unsigned int crtc_mask = drm_crtc_mask(&crtc->base);
13913                 struct intel_shared_dpll *pll = old_crtc_state->shared_dpll;
13914
13915                 I915_STATE_WARN(pll->active_mask & crtc_mask,
13916                                 "pll active mismatch (didn't expect pipe %c in active mask)\n",
13917                                 pipe_name(drm_crtc_index(&crtc->base)));
13918                 I915_STATE_WARN(pll->state.crtc_mask & crtc_mask,
13919                                 "pll enabled crtcs mismatch (found %x in enabled mask)\n",
13920                                 pipe_name(drm_crtc_index(&crtc->base)));
13921         }
13922 }
13923
13924 static void
13925 intel_modeset_verify_crtc(struct intel_crtc *crtc,
13926                           struct intel_atomic_state *state,
13927                           struct intel_crtc_state *old_crtc_state,
13928                           struct intel_crtc_state *new_crtc_state)
13929 {
13930         if (!needs_modeset(new_crtc_state) && !new_crtc_state->update_pipe)
13931                 return;
13932
13933         verify_wm_state(crtc, new_crtc_state);
13934         verify_connector_state(state, crtc);
13935         verify_crtc_state(crtc, old_crtc_state, new_crtc_state);
13936         verify_shared_dpll_state(crtc, old_crtc_state, new_crtc_state);
13937 }
13938
13939 static void
13940 verify_disabled_dpll_state(struct drm_i915_private *dev_priv)
13941 {
13942         int i;
13943
13944         for (i = 0; i < dev_priv->num_shared_dpll; i++)
13945                 verify_single_dpll_state(dev_priv, &dev_priv->shared_dplls[i], NULL, NULL);
13946 }
13947
13948 static void
13949 intel_modeset_verify_disabled(struct drm_i915_private *dev_priv,
13950                               struct intel_atomic_state *state)
13951 {
13952         verify_encoder_state(dev_priv, state);
13953         verify_connector_state(state, NULL);
13954         verify_disabled_dpll_state(dev_priv);
13955 }
13956
13957 static void
13958 intel_crtc_update_active_timings(const struct intel_crtc_state *crtc_state)
13959 {
13960         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
13961         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
13962         const struct drm_display_mode *adjusted_mode =
13963                 &crtc_state->hw.adjusted_mode;
13964
13965         drm_calc_timestamping_constants(&crtc->base, adjusted_mode);
13966
13967         /*
13968          * The scanline counter increments at the leading edge of hsync.
13969          *
13970          * On most platforms it starts counting from vtotal-1 on the
13971          * first active line. That means the scanline counter value is
13972          * always one less than what we would expect. Ie. just after
13973          * start of vblank, which also occurs at start of hsync (on the
13974          * last active line), the scanline counter will read vblank_start-1.
13975          *
13976          * On gen2 the scanline counter starts counting from 1 instead
13977          * of vtotal-1, so we have to subtract one (or rather add vtotal-1
13978          * to keep the value positive), instead of adding one.
13979          *
13980          * On HSW+ the behaviour of the scanline counter depends on the output
13981          * type. For DP ports it behaves like most other platforms, but on HDMI
13982          * there's an extra 1 line difference. So we need to add two instead of
13983          * one to the value.
13984          *
13985          * On VLV/CHV DSI the scanline counter would appear to increment
13986          * approx. 1/3 of a scanline before start of vblank. Unfortunately
13987          * that means we can't tell whether we're in vblank or not while
13988          * we're on that particular line. We must still set scanline_offset
13989          * to 1 so that the vblank timestamps come out correct when we query
13990          * the scanline counter from within the vblank interrupt handler.
13991          * However if queried just before the start of vblank we'll get an
13992          * answer that's slightly in the future.
13993          */
13994         if (IS_GEN(dev_priv, 2)) {
13995                 int vtotal;
13996
13997                 vtotal = adjusted_mode->crtc_vtotal;
13998                 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
13999                         vtotal /= 2;
14000
14001                 crtc->scanline_offset = vtotal - 1;
14002         } else if (HAS_DDI(dev_priv) &&
14003                    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
14004                 crtc->scanline_offset = 2;
14005         } else {
14006                 crtc->scanline_offset = 1;
14007         }
14008 }
14009
14010 static void intel_modeset_clear_plls(struct intel_atomic_state *state)
14011 {
14012         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14013         struct intel_crtc_state *new_crtc_state;
14014         struct intel_crtc *crtc;
14015         int i;
14016
14017         if (!dev_priv->display.crtc_compute_clock)
14018                 return;
14019
14020         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
14021                 if (!needs_modeset(new_crtc_state))
14022                         continue;
14023
14024                 intel_release_shared_dplls(state, crtc);
14025         }
14026 }
14027
14028 /*
14029  * This implements the workaround described in the "notes" section of the mode
14030  * set sequence documentation. When going from no pipes or single pipe to
14031  * multiple pipes, and planes are enabled after the pipe, we need to wait at
14032  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
14033  */
14034 static int haswell_mode_set_planes_workaround(struct intel_atomic_state *state)
14035 {
14036         struct intel_crtc_state *crtc_state;
14037         struct intel_crtc *crtc;
14038         struct intel_crtc_state *first_crtc_state = NULL;
14039         struct intel_crtc_state *other_crtc_state = NULL;
14040         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
14041         int i;
14042
14043         /* look at all crtc's that are going to be enabled in during modeset */
14044         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
14045                 if (!crtc_state->hw.active ||
14046                     !needs_modeset(crtc_state))
14047                         continue;
14048
14049                 if (first_crtc_state) {
14050                         other_crtc_state = crtc_state;
14051                         break;
14052                 } else {
14053                         first_crtc_state = crtc_state;
14054                         first_pipe = crtc->pipe;
14055                 }
14056         }
14057
14058         /* No workaround needed? */
14059         if (!first_crtc_state)
14060                 return 0;
14061
14062         /* w/a possibly needed, check how many crtc's are already enabled. */
14063         for_each_intel_crtc(state->base.dev, crtc) {
14064                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
14065                 if (IS_ERR(crtc_state))
14066                         return PTR_ERR(crtc_state);
14067
14068                 crtc_state->hsw_workaround_pipe = INVALID_PIPE;
14069
14070                 if (!crtc_state->hw.active ||
14071                     needs_modeset(crtc_state))
14072                         continue;
14073
14074                 /* 2 or more enabled crtcs means no need for w/a */
14075                 if (enabled_pipe != INVALID_PIPE)
14076                         return 0;
14077
14078                 enabled_pipe = crtc->pipe;
14079         }
14080
14081         if (enabled_pipe != INVALID_PIPE)
14082                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
14083         else if (other_crtc_state)
14084                 other_crtc_state->hsw_workaround_pipe = first_pipe;
14085
14086         return 0;
14087 }
14088
14089 static int intel_modeset_checks(struct intel_atomic_state *state)
14090 {
14091         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14092         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14093         struct intel_crtc *crtc;
14094         int ret, i;
14095
14096         /* keep the current setting */
14097         if (!state->cdclk.force_min_cdclk_changed)
14098                 state->cdclk.force_min_cdclk = dev_priv->cdclk.force_min_cdclk;
14099
14100         state->modeset = true;
14101         state->active_pipes = dev_priv->active_pipes;
14102         state->cdclk.logical = dev_priv->cdclk.logical;
14103         state->cdclk.actual = dev_priv->cdclk.actual;
14104
14105         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14106                                             new_crtc_state, i) {
14107                 if (new_crtc_state->hw.active)
14108                         state->active_pipes |= BIT(crtc->pipe);
14109                 else
14110                         state->active_pipes &= ~BIT(crtc->pipe);
14111
14112                 if (old_crtc_state->hw.active != new_crtc_state->hw.active)
14113                         state->active_pipe_changes |= BIT(crtc->pipe);
14114         }
14115
14116         if (state->active_pipe_changes) {
14117                 ret = intel_atomic_lock_global_state(state);
14118                 if (ret)
14119                         return ret;
14120         }
14121
14122         ret = intel_modeset_calc_cdclk(state);
14123         if (ret)
14124                 return ret;
14125
14126         intel_modeset_clear_plls(state);
14127
14128         if (IS_HASWELL(dev_priv))
14129                 return haswell_mode_set_planes_workaround(state);
14130
14131         return 0;
14132 }
14133
14134 /*
14135  * Handle calculation of various watermark data at the end of the atomic check
14136  * phase.  The code here should be run after the per-crtc and per-plane 'check'
14137  * handlers to ensure that all derived state has been updated.
14138  */
14139 static int calc_watermark_data(struct intel_atomic_state *state)
14140 {
14141         struct drm_device *dev = state->base.dev;
14142         struct drm_i915_private *dev_priv = to_i915(dev);
14143
14144         /* Is there platform-specific watermark information to calculate? */
14145         if (dev_priv->display.compute_global_watermarks)
14146                 return dev_priv->display.compute_global_watermarks(state);
14147
14148         return 0;
14149 }
14150
14151 static void intel_crtc_check_fastset(const struct intel_crtc_state *old_crtc_state,
14152                                      struct intel_crtc_state *new_crtc_state)
14153 {
14154         if (!intel_pipe_config_compare(old_crtc_state, new_crtc_state, true))
14155                 return;
14156
14157         new_crtc_state->uapi.mode_changed = false;
14158         new_crtc_state->update_pipe = true;
14159
14160         /*
14161          * If we're not doing the full modeset we want to
14162          * keep the current M/N values as they may be
14163          * sufficiently different to the computed values
14164          * to cause problems.
14165          *
14166          * FIXME: should really copy more fuzzy state here
14167          */
14168         new_crtc_state->fdi_m_n = old_crtc_state->fdi_m_n;
14169         new_crtc_state->dp_m_n = old_crtc_state->dp_m_n;
14170         new_crtc_state->dp_m2_n2 = old_crtc_state->dp_m2_n2;
14171         new_crtc_state->has_drrs = old_crtc_state->has_drrs;
14172 }
14173
14174 static int intel_crtc_add_planes_to_state(struct intel_atomic_state *state,
14175                                           struct intel_crtc *crtc,
14176                                           u8 plane_ids_mask)
14177 {
14178         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14179         struct intel_plane *plane;
14180
14181         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
14182                 struct intel_plane_state *plane_state;
14183
14184                 if ((plane_ids_mask & BIT(plane->id)) == 0)
14185                         continue;
14186
14187                 plane_state = intel_atomic_get_plane_state(state, plane);
14188                 if (IS_ERR(plane_state))
14189                         return PTR_ERR(plane_state);
14190         }
14191
14192         return 0;
14193 }
14194
14195 static bool active_planes_affects_min_cdclk(struct drm_i915_private *dev_priv)
14196 {
14197         /* See {hsw,vlv,ivb}_plane_ratio() */
14198         return IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv) ||
14199                 IS_CHERRYVIEW(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
14200                 IS_IVYBRIDGE(dev_priv);
14201 }
14202
14203 static int intel_atomic_check_planes(struct intel_atomic_state *state,
14204                                      bool *need_modeset)
14205 {
14206         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14207         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14208         struct intel_plane_state *plane_state;
14209         struct intel_plane *plane;
14210         struct intel_crtc *crtc;
14211         int i, ret;
14212
14213         ret = icl_add_linked_planes(state);
14214         if (ret)
14215                 return ret;
14216
14217         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
14218                 ret = intel_plane_atomic_check(state, plane);
14219                 if (ret) {
14220                         DRM_DEBUG_ATOMIC("[PLANE:%d:%s] atomic driver check failed\n",
14221                                          plane->base.base.id, plane->base.name);
14222                         return ret;
14223                 }
14224         }
14225
14226         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14227                                             new_crtc_state, i) {
14228                 u8 old_active_planes, new_active_planes;
14229
14230                 ret = icl_check_nv12_planes(new_crtc_state);
14231                 if (ret)
14232                         return ret;
14233
14234                 /*
14235                  * On some platforms the number of active planes affects
14236                  * the planes' minimum cdclk calculation. Add such planes
14237                  * to the state before we compute the minimum cdclk.
14238                  */
14239                 if (!active_planes_affects_min_cdclk(dev_priv))
14240                         continue;
14241
14242                 old_active_planes = old_crtc_state->active_planes & ~BIT(PLANE_CURSOR);
14243                 new_active_planes = new_crtc_state->active_planes & ~BIT(PLANE_CURSOR);
14244
14245                 if (hweight8(old_active_planes) == hweight8(new_active_planes))
14246                         continue;
14247
14248                 ret = intel_crtc_add_planes_to_state(state, crtc, new_active_planes);
14249                 if (ret)
14250                         return ret;
14251         }
14252
14253         /*
14254          * active_planes bitmask has been updated, and potentially
14255          * affected planes are part of the state. We can now
14256          * compute the minimum cdclk for each plane.
14257          */
14258         for_each_new_intel_plane_in_state(state, plane, plane_state, i)
14259                 *need_modeset |= intel_plane_calc_min_cdclk(state, plane);
14260
14261         return 0;
14262 }
14263
14264 static int intel_atomic_check_crtcs(struct intel_atomic_state *state)
14265 {
14266         struct intel_crtc_state *crtc_state;
14267         struct intel_crtc *crtc;
14268         int i;
14269
14270         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
14271                 int ret = intel_crtc_atomic_check(state, crtc);
14272                 if (ret) {
14273                         DRM_DEBUG_ATOMIC("[CRTC:%d:%s] atomic driver check failed\n",
14274                                          crtc->base.base.id, crtc->base.name);
14275                         return ret;
14276                 }
14277         }
14278
14279         return 0;
14280 }
14281
14282 /**
14283  * intel_atomic_check - validate state object
14284  * @dev: drm device
14285  * @_state: state to validate
14286  */
14287 static int intel_atomic_check(struct drm_device *dev,
14288                               struct drm_atomic_state *_state)
14289 {
14290         struct drm_i915_private *dev_priv = to_i915(dev);
14291         struct intel_atomic_state *state = to_intel_atomic_state(_state);
14292         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14293         struct intel_crtc *crtc;
14294         int ret, i;
14295         bool any_ms = false;
14296
14297         /* Catch I915_MODE_FLAG_INHERITED */
14298         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14299                                             new_crtc_state, i) {
14300                 if (new_crtc_state->hw.mode.private_flags !=
14301                     old_crtc_state->hw.mode.private_flags)
14302                         new_crtc_state->uapi.mode_changed = true;
14303         }
14304
14305         ret = drm_atomic_helper_check_modeset(dev, &state->base);
14306         if (ret)
14307                 goto fail;
14308
14309         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14310                                             new_crtc_state, i) {
14311                 if (!needs_modeset(new_crtc_state)) {
14312                         /* Light copy */
14313                         intel_crtc_copy_uapi_to_hw_state_nomodeset(new_crtc_state);
14314
14315                         continue;
14316                 }
14317
14318                 if (!new_crtc_state->uapi.enable) {
14319                         intel_crtc_copy_uapi_to_hw_state(new_crtc_state);
14320
14321                         any_ms = true;
14322                         continue;
14323                 }
14324
14325                 ret = intel_crtc_prepare_cleared_state(new_crtc_state);
14326                 if (ret)
14327                         goto fail;
14328
14329                 ret = intel_modeset_pipe_config(new_crtc_state);
14330                 if (ret)
14331                         goto fail;
14332
14333                 intel_crtc_check_fastset(old_crtc_state, new_crtc_state);
14334
14335                 if (needs_modeset(new_crtc_state))
14336                         any_ms = true;
14337         }
14338
14339         if (any_ms && !check_digital_port_conflicts(state)) {
14340                 DRM_DEBUG_KMS("rejecting conflicting digital port configuration\n");
14341                 ret = EINVAL;
14342                 goto fail;
14343         }
14344
14345         ret = drm_dp_mst_atomic_check(&state->base);
14346         if (ret)
14347                 goto fail;
14348
14349         any_ms |= state->cdclk.force_min_cdclk_changed;
14350
14351         ret = intel_atomic_check_planes(state, &any_ms);
14352         if (ret)
14353                 goto fail;
14354
14355         if (any_ms) {
14356                 ret = intel_modeset_checks(state);
14357                 if (ret)
14358                         goto fail;
14359         } else {
14360                 state->cdclk.logical = dev_priv->cdclk.logical;
14361         }
14362
14363         ret = intel_atomic_check_crtcs(state);
14364         if (ret)
14365                 goto fail;
14366
14367         intel_fbc_choose_crtc(dev_priv, state);
14368         ret = calc_watermark_data(state);
14369         if (ret)
14370                 goto fail;
14371
14372         ret = intel_bw_atomic_check(state);
14373         if (ret)
14374                 goto fail;
14375
14376         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14377                                             new_crtc_state, i) {
14378                 if (!needs_modeset(new_crtc_state) &&
14379                     !new_crtc_state->update_pipe)
14380                         continue;
14381
14382                 intel_dump_pipe_config(new_crtc_state, state,
14383                                        needs_modeset(new_crtc_state) ?
14384                                        "[modeset]" : "[fastset]");
14385         }
14386
14387         return 0;
14388
14389  fail:
14390         if (ret == -EDEADLK)
14391                 return ret;
14392
14393         /*
14394          * FIXME would probably be nice to know which crtc specifically
14395          * caused the failure, in cases where we can pinpoint it.
14396          */
14397         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14398                                             new_crtc_state, i)
14399                 intel_dump_pipe_config(new_crtc_state, state, "[failed]");
14400
14401         return ret;
14402 }
14403
14404 static int intel_atomic_prepare_commit(struct intel_atomic_state *state)
14405 {
14406         return drm_atomic_helper_prepare_planes(state->base.dev,
14407                                                 &state->base);
14408 }
14409
14410 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
14411 {
14412         struct drm_device *dev = crtc->base.dev;
14413         struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)];
14414
14415         if (!vblank->max_vblank_count)
14416                 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
14417
14418         return crtc->base.funcs->get_vblank_counter(&crtc->base);
14419 }
14420
14421 void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
14422                                   struct intel_crtc_state *crtc_state)
14423 {
14424         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14425
14426         if (!IS_GEN(dev_priv, 2) || crtc_state->active_planes)
14427                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
14428
14429         if (crtc_state->has_pch_encoder) {
14430                 enum pipe pch_transcoder =
14431                         intel_crtc_pch_transcoder(crtc);
14432
14433                 intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
14434         }
14435 }
14436
14437 static void intel_pipe_fastset(const struct intel_crtc_state *old_crtc_state,
14438                                const struct intel_crtc_state *new_crtc_state)
14439 {
14440         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
14441         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
14442
14443         /*
14444          * Update pipe size and adjust fitter if needed: the reason for this is
14445          * that in compute_mode_changes we check the native mode (not the pfit
14446          * mode) to see if we can flip rather than do a full mode set. In the
14447          * fastboot case, we'll flip, but if we don't update the pipesrc and
14448          * pfit state, we'll end up with a big fb scanned out into the wrong
14449          * sized surface.
14450          */
14451         intel_set_pipe_src_size(new_crtc_state);
14452
14453         /* on skylake this is done by detaching scalers */
14454         if (INTEL_GEN(dev_priv) >= 9) {
14455                 skl_detach_scalers(new_crtc_state);
14456
14457                 if (new_crtc_state->pch_pfit.enabled)
14458                         skylake_pfit_enable(new_crtc_state);
14459         } else if (HAS_PCH_SPLIT(dev_priv)) {
14460                 if (new_crtc_state->pch_pfit.enabled)
14461                         ironlake_pfit_enable(new_crtc_state);
14462                 else if (old_crtc_state->pch_pfit.enabled)
14463                         ironlake_pfit_disable(old_crtc_state);
14464         }
14465
14466         if (INTEL_GEN(dev_priv) >= 11)
14467                 icl_set_pipe_chicken(crtc);
14468 }
14469
14470 static void commit_pipe_config(struct intel_atomic_state *state,
14471                                struct intel_crtc_state *old_crtc_state,
14472                                struct intel_crtc_state *new_crtc_state)
14473 {
14474         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
14475         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14476         bool modeset = needs_modeset(new_crtc_state);
14477
14478         /*
14479          * During modesets pipe configuration was programmed as the
14480          * CRTC was enabled.
14481          */
14482         if (!modeset) {
14483                 if (new_crtc_state->uapi.color_mgmt_changed ||
14484                     new_crtc_state->update_pipe)
14485                         intel_color_commit(new_crtc_state);
14486
14487                 if (INTEL_GEN(dev_priv) >= 9)
14488                         skl_detach_scalers(new_crtc_state);
14489
14490                 if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
14491                         bdw_set_pipemisc(new_crtc_state);
14492
14493                 if (new_crtc_state->update_pipe)
14494                         intel_pipe_fastset(old_crtc_state, new_crtc_state);
14495         }
14496
14497         if (dev_priv->display.atomic_update_watermarks)
14498                 dev_priv->display.atomic_update_watermarks(state, crtc);
14499 }
14500
14501 static void intel_update_crtc(struct intel_crtc *crtc,
14502                               struct intel_atomic_state *state,
14503                               struct intel_crtc_state *old_crtc_state,
14504                               struct intel_crtc_state *new_crtc_state)
14505 {
14506         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14507         bool modeset = needs_modeset(new_crtc_state);
14508         struct intel_plane_state *new_plane_state =
14509                 intel_atomic_get_new_plane_state(state,
14510                                                  to_intel_plane(crtc->base.primary));
14511
14512         if (modeset) {
14513                 intel_crtc_update_active_timings(new_crtc_state);
14514
14515                 dev_priv->display.crtc_enable(state, crtc);
14516
14517                 /* vblanks work again, re-enable pipe CRC. */
14518                 intel_crtc_enable_pipe_crc(crtc);
14519         } else {
14520                 if (new_crtc_state->preload_luts &&
14521                     (new_crtc_state->uapi.color_mgmt_changed ||
14522                      new_crtc_state->update_pipe))
14523                         intel_color_load_luts(new_crtc_state);
14524
14525                 intel_pre_plane_update(state, crtc);
14526
14527                 if (new_crtc_state->update_pipe)
14528                         intel_encoders_update_pipe(state, crtc);
14529         }
14530
14531         if (new_crtc_state->update_pipe && !new_crtc_state->enable_fbc)
14532                 intel_fbc_disable(crtc);
14533         else if (new_plane_state)
14534                 intel_fbc_enable(crtc, new_crtc_state, new_plane_state);
14535
14536         /* Perform vblank evasion around commit operation */
14537         intel_pipe_update_start(new_crtc_state);
14538
14539         commit_pipe_config(state, old_crtc_state, new_crtc_state);
14540
14541         if (INTEL_GEN(dev_priv) >= 9)
14542                 skl_update_planes_on_crtc(state, crtc);
14543         else
14544                 i9xx_update_planes_on_crtc(state, crtc);
14545
14546         intel_pipe_update_end(new_crtc_state);
14547
14548         /*
14549          * We usually enable FIFO underrun interrupts as part of the
14550          * CRTC enable sequence during modesets.  But when we inherit a
14551          * valid pipe configuration from the BIOS we need to take care
14552          * of enabling them on the CRTC's first fastset.
14553          */
14554         if (new_crtc_state->update_pipe && !modeset &&
14555             old_crtc_state->hw.mode.private_flags & I915_MODE_FLAG_INHERITED)
14556                 intel_crtc_arm_fifo_underrun(crtc, new_crtc_state);
14557 }
14558
14559 static struct intel_crtc *intel_get_slave_crtc(const struct intel_crtc_state *new_crtc_state)
14560 {
14561         struct drm_i915_private *dev_priv = to_i915(new_crtc_state->uapi.crtc->dev);
14562         enum transcoder slave_transcoder;
14563
14564         WARN_ON(!is_power_of_2(new_crtc_state->sync_mode_slaves_mask));
14565
14566         slave_transcoder = ffs(new_crtc_state->sync_mode_slaves_mask) - 1;
14567         return intel_get_crtc_for_pipe(dev_priv,
14568                                        (enum pipe)slave_transcoder);
14569 }
14570
14571 static void intel_old_crtc_state_disables(struct intel_atomic_state *state,
14572                                           struct intel_crtc_state *old_crtc_state,
14573                                           struct intel_crtc_state *new_crtc_state,
14574                                           struct intel_crtc *crtc)
14575 {
14576         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14577
14578         intel_crtc_disable_planes(state, crtc);
14579
14580         /*
14581          * We need to disable pipe CRC before disabling the pipe,
14582          * or we race against vblank off.
14583          */
14584         intel_crtc_disable_pipe_crc(crtc);
14585
14586         dev_priv->display.crtc_disable(state, crtc);
14587         crtc->active = false;
14588         intel_fbc_disable(crtc);
14589         intel_disable_shared_dpll(old_crtc_state);
14590
14591         /* FIXME unify this for all platforms */
14592         if (!new_crtc_state->hw.active &&
14593             !HAS_GMCH(dev_priv) &&
14594             dev_priv->display.initial_watermarks)
14595                 dev_priv->display.initial_watermarks(state, crtc);
14596 }
14597
14598 static void intel_commit_modeset_disables(struct intel_atomic_state *state)
14599 {
14600         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
14601         struct intel_crtc *crtc;
14602         u32 handled = 0;
14603         int i;
14604
14605         /* Only disable port sync slaves */
14606         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14607                                             new_crtc_state, i) {
14608                 if (!needs_modeset(new_crtc_state))
14609                         continue;
14610
14611                 if (!old_crtc_state->hw.active)
14612                         continue;
14613
14614                 /* In case of Transcoder port Sync master slave CRTCs can be
14615                  * assigned in any order and we need to make sure that
14616                  * slave CRTCs are disabled first and then master CRTC since
14617                  * Slave vblanks are masked till Master Vblanks.
14618                  */
14619                 if (!is_trans_port_sync_slave(old_crtc_state))
14620                         continue;
14621
14622                 intel_pre_plane_update(state, crtc);
14623                 intel_old_crtc_state_disables(state, old_crtc_state,
14624                                               new_crtc_state, crtc);
14625                 handled |= BIT(crtc->pipe);
14626         }
14627
14628         /* Disable everything else left on */
14629         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14630                                             new_crtc_state, i) {
14631                 if (!needs_modeset(new_crtc_state) ||
14632                     (handled & BIT(crtc->pipe)))
14633                         continue;
14634
14635                 intel_pre_plane_update(state, crtc);
14636                 if (old_crtc_state->hw.active)
14637                         intel_old_crtc_state_disables(state, old_crtc_state,
14638                                                       new_crtc_state, crtc);
14639         }
14640 }
14641
14642 static void intel_commit_modeset_enables(struct intel_atomic_state *state)
14643 {
14644         struct intel_crtc *crtc;
14645         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14646         int i;
14647
14648         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
14649                 if (!new_crtc_state->hw.active)
14650                         continue;
14651
14652                 intel_update_crtc(crtc, state, old_crtc_state,
14653                                   new_crtc_state);
14654         }
14655 }
14656
14657 static void intel_crtc_enable_trans_port_sync(struct intel_crtc *crtc,
14658                                               struct intel_atomic_state *state,
14659                                               struct intel_crtc_state *new_crtc_state)
14660 {
14661         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14662
14663         intel_crtc_update_active_timings(new_crtc_state);
14664         dev_priv->display.crtc_enable(state, crtc);
14665         intel_crtc_enable_pipe_crc(crtc);
14666 }
14667
14668 static void intel_set_dp_tp_ctl_normal(struct intel_crtc *crtc,
14669                                        struct intel_atomic_state *state)
14670 {
14671         struct drm_connector *uninitialized_var(conn);
14672         struct drm_connector_state *conn_state;
14673         struct intel_dp *intel_dp;
14674         int i;
14675
14676         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
14677                 if (conn_state->crtc == &crtc->base)
14678                         break;
14679         }
14680         intel_dp = enc_to_intel_dp(&intel_attached_encoder(conn)->base);
14681         intel_dp_stop_link_train(intel_dp);
14682 }
14683
14684 static void intel_post_crtc_enable_updates(struct intel_crtc *crtc,
14685                                            struct intel_atomic_state *state)
14686 {
14687         struct intel_crtc_state *new_crtc_state =
14688                 intel_atomic_get_new_crtc_state(state, crtc);
14689         struct intel_crtc_state *old_crtc_state =
14690                 intel_atomic_get_old_crtc_state(state, crtc);
14691         struct intel_plane_state *new_plane_state =
14692                 intel_atomic_get_new_plane_state(state,
14693                                                  to_intel_plane(crtc->base.primary));
14694         bool modeset = needs_modeset(new_crtc_state);
14695
14696         if (new_crtc_state->update_pipe && !new_crtc_state->enable_fbc)
14697                 intel_fbc_disable(crtc);
14698         else if (new_plane_state)
14699                 intel_fbc_enable(crtc, new_crtc_state, new_plane_state);
14700
14701         /* Perform vblank evasion around commit operation */
14702         intel_pipe_update_start(new_crtc_state);
14703         commit_pipe_config(state, old_crtc_state, new_crtc_state);
14704         skl_update_planes_on_crtc(state, crtc);
14705         intel_pipe_update_end(new_crtc_state);
14706
14707         /*
14708          * We usually enable FIFO underrun interrupts as part of the
14709          * CRTC enable sequence during modesets.  But when we inherit a
14710          * valid pipe configuration from the BIOS we need to take care
14711          * of enabling them on the CRTC's first fastset.
14712          */
14713         if (new_crtc_state->update_pipe && !modeset &&
14714             old_crtc_state->hw.mode.private_flags & I915_MODE_FLAG_INHERITED)
14715                 intel_crtc_arm_fifo_underrun(crtc, new_crtc_state);
14716 }
14717
14718 static void intel_update_trans_port_sync_crtcs(struct intel_crtc *crtc,
14719                                                struct intel_atomic_state *state,
14720                                                struct intel_crtc_state *old_crtc_state,
14721                                                struct intel_crtc_state *new_crtc_state)
14722 {
14723         struct intel_crtc *slave_crtc = intel_get_slave_crtc(new_crtc_state);
14724         struct intel_crtc_state *new_slave_crtc_state =
14725                 intel_atomic_get_new_crtc_state(state, slave_crtc);
14726         struct intel_crtc_state *old_slave_crtc_state =
14727                 intel_atomic_get_old_crtc_state(state, slave_crtc);
14728
14729         WARN_ON(!slave_crtc || !new_slave_crtc_state ||
14730                 !old_slave_crtc_state);
14731
14732         DRM_DEBUG_KMS("Updating Transcoder Port Sync Master CRTC = %d %s and Slave CRTC %d %s\n",
14733                       crtc->base.base.id, crtc->base.name, slave_crtc->base.base.id,
14734                       slave_crtc->base.name);
14735
14736         /* Enable seq for slave with with DP_TP_CTL left Idle until the
14737          * master is ready
14738          */
14739         intel_crtc_enable_trans_port_sync(slave_crtc,
14740                                           state,
14741                                           new_slave_crtc_state);
14742
14743         /* Enable seq for master with with DP_TP_CTL left Idle */
14744         intel_crtc_enable_trans_port_sync(crtc,
14745                                           state,
14746                                           new_crtc_state);
14747
14748         /* Set Slave's DP_TP_CTL to Normal */
14749         intel_set_dp_tp_ctl_normal(slave_crtc,
14750                                    state);
14751
14752         /* Set Master's DP_TP_CTL To Normal */
14753         usleep_range(200, 400);
14754         intel_set_dp_tp_ctl_normal(crtc,
14755                                    state);
14756
14757         /* Now do the post crtc enable for all master and slaves */
14758         intel_post_crtc_enable_updates(slave_crtc,
14759                                        state);
14760         intel_post_crtc_enable_updates(crtc,
14761                                        state);
14762 }
14763
14764 static void skl_commit_modeset_enables(struct intel_atomic_state *state)
14765 {
14766         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
14767         struct intel_crtc *crtc;
14768         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
14769         u8 hw_enabled_slices = dev_priv->wm.skl_hw.ddb.enabled_slices;
14770         u8 required_slices = state->wm_results.ddb.enabled_slices;
14771         struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
14772         u8 dirty_pipes = 0;
14773         int i;
14774
14775         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
14776                 /* ignore allocations for crtc's that have been turned off. */
14777                 if (!needs_modeset(new_crtc_state) && new_crtc_state->hw.active)
14778                         entries[i] = old_crtc_state->wm.skl.ddb;
14779                 if (new_crtc_state->hw.active)
14780                         dirty_pipes |= BIT(crtc->pipe);
14781         }
14782
14783         /* If 2nd DBuf slice required, enable it here */
14784         if (INTEL_GEN(dev_priv) >= 11 && required_slices > hw_enabled_slices)
14785                 icl_dbuf_slices_update(dev_priv, required_slices);
14786
14787         /*
14788          * Whenever the number of active pipes changes, we need to make sure we
14789          * update the pipes in the right order so that their ddb allocations
14790          * never overlap with eachother inbetween CRTC updates. Otherwise we'll
14791          * cause pipe underruns and other bad stuff.
14792          */
14793         while (dirty_pipes) {
14794                 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14795                                                     new_crtc_state, i) {
14796                         enum pipe pipe = crtc->pipe;
14797                         bool modeset = needs_modeset(new_crtc_state);
14798
14799                         if ((dirty_pipes & BIT(pipe)) == 0)
14800                                 continue;
14801
14802                         if (skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
14803                                                         entries,
14804                                                         INTEL_NUM_PIPES(dev_priv), i))
14805                                 continue;
14806
14807                         entries[i] = new_crtc_state->wm.skl.ddb;
14808                         dirty_pipes &= ~BIT(pipe);
14809
14810                         if (modeset && is_trans_port_sync_mode(new_crtc_state)) {
14811                                 if (is_trans_port_sync_master(new_crtc_state))
14812                                         intel_update_trans_port_sync_crtcs(crtc,
14813                                                                            state,
14814                                                                            old_crtc_state,
14815                                                                            new_crtc_state);
14816                                 else
14817                                         continue;
14818                         } else {
14819                                 intel_update_crtc(crtc, state, old_crtc_state,
14820                                                   new_crtc_state);
14821                         }
14822
14823                         /*
14824                          * If this is an already active pipe, it's DDB changed,
14825                          * and this isn't the last pipe that needs updating
14826                          * then we need to wait for a vblank to pass for the
14827                          * new ddb allocation to take effect.
14828                          */
14829                         if (!skl_ddb_entry_equal(&new_crtc_state->wm.skl.ddb,
14830                                                  &old_crtc_state->wm.skl.ddb) &&
14831                             !modeset && dirty_pipes)
14832                                 intel_wait_for_vblank(dev_priv, pipe);
14833                 }
14834         }
14835
14836         /* If 2nd DBuf slice is no more required disable it */
14837         if (INTEL_GEN(dev_priv) >= 11 && required_slices < hw_enabled_slices)
14838                 icl_dbuf_slices_update(dev_priv, required_slices);
14839 }
14840
14841 static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv)
14842 {
14843         struct intel_atomic_state *state, *next;
14844         struct llist_node *freed;
14845
14846         freed = llist_del_all(&dev_priv->atomic_helper.free_list);
14847         llist_for_each_entry_safe(state, next, freed, freed)
14848                 drm_atomic_state_put(&state->base);
14849 }
14850
14851 static void intel_atomic_helper_free_state_worker(struct work_struct *work)
14852 {
14853         struct drm_i915_private *dev_priv =
14854                 container_of(work, typeof(*dev_priv), atomic_helper.free_work);
14855
14856         intel_atomic_helper_free_state(dev_priv);
14857 }
14858
14859 static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state)
14860 {
14861         struct wait_queue_entry wait_fence, wait_reset;
14862         struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev);
14863
14864         init_wait_entry(&wait_fence, 0);
14865         init_wait_entry(&wait_reset, 0);
14866         for (;;) {
14867                 prepare_to_wait(&intel_state->commit_ready.wait,
14868                                 &wait_fence, TASK_UNINTERRUPTIBLE);
14869                 prepare_to_wait(bit_waitqueue(&dev_priv->gt.reset.flags,
14870                                               I915_RESET_MODESET),
14871                                 &wait_reset, TASK_UNINTERRUPTIBLE);
14872
14873
14874                 if (i915_sw_fence_done(&intel_state->commit_ready) ||
14875                     test_bit(I915_RESET_MODESET, &dev_priv->gt.reset.flags))
14876                         break;
14877
14878                 schedule();
14879         }
14880         finish_wait(&intel_state->commit_ready.wait, &wait_fence);
14881         finish_wait(bit_waitqueue(&dev_priv->gt.reset.flags,
14882                                   I915_RESET_MODESET),
14883                     &wait_reset);
14884 }
14885
14886 static void intel_atomic_cleanup_work(struct work_struct *work)
14887 {
14888         struct drm_atomic_state *state =
14889                 container_of(work, struct drm_atomic_state, commit_work);
14890         struct drm_i915_private *i915 = to_i915(state->dev);
14891
14892         drm_atomic_helper_cleanup_planes(&i915->drm, state);
14893         drm_atomic_helper_commit_cleanup_done(state);
14894         drm_atomic_state_put(state);
14895
14896         intel_atomic_helper_free_state(i915);
14897 }
14898
14899 static void intel_atomic_commit_tail(struct intel_atomic_state *state)
14900 {
14901         struct drm_device *dev = state->base.dev;
14902         struct drm_i915_private *dev_priv = to_i915(dev);
14903         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
14904         struct intel_crtc *crtc;
14905         u64 put_domains[I915_MAX_PIPES] = {};
14906         intel_wakeref_t wakeref = 0;
14907         int i;
14908
14909         intel_atomic_commit_fence_wait(state);
14910
14911         drm_atomic_helper_wait_for_dependencies(&state->base);
14912
14913         if (state->modeset)
14914                 wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
14915
14916         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
14917                                             new_crtc_state, i) {
14918                 if (needs_modeset(new_crtc_state) ||
14919                     new_crtc_state->update_pipe) {
14920
14921                         put_domains[crtc->pipe] =
14922                                 modeset_get_crtc_power_domains(new_crtc_state);
14923                 }
14924         }
14925
14926         intel_commit_modeset_disables(state);
14927
14928         /* FIXME: Eventually get rid of our crtc->config pointer */
14929         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
14930                 crtc->config = new_crtc_state;
14931
14932         if (state->modeset) {
14933                 drm_atomic_helper_update_legacy_modeset_state(dev, &state->base);
14934
14935                 intel_set_cdclk_pre_plane_update(dev_priv,
14936                                                  &state->cdclk.actual,
14937                                                  &dev_priv->cdclk.actual,
14938                                                  state->cdclk.pipe);
14939
14940                 /*
14941                  * SKL workaround: bspec recommends we disable the SAGV when we
14942                  * have more then one pipe enabled
14943                  */
14944                 if (!intel_can_enable_sagv(state))
14945                         intel_disable_sagv(dev_priv);
14946
14947                 intel_modeset_verify_disabled(dev_priv, state);
14948         }
14949
14950         /* Complete the events for pipes that have now been disabled */
14951         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
14952                 bool modeset = needs_modeset(new_crtc_state);
14953
14954                 /* Complete events for now disable pipes here. */
14955                 if (modeset && !new_crtc_state->hw.active && new_crtc_state->uapi.event) {
14956                         spin_lock_irq(&dev->event_lock);
14957                         drm_crtc_send_vblank_event(&crtc->base,
14958                                                    new_crtc_state->uapi.event);
14959                         spin_unlock_irq(&dev->event_lock);
14960
14961                         new_crtc_state->uapi.event = NULL;
14962                 }
14963         }
14964
14965         if (state->modeset)
14966                 intel_encoders_update_prepare(state);
14967
14968         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
14969         dev_priv->display.commit_modeset_enables(state);
14970
14971         if (state->modeset) {
14972                 intel_encoders_update_complete(state);
14973
14974                 intel_set_cdclk_post_plane_update(dev_priv,
14975                                                   &state->cdclk.actual,
14976                                                   &dev_priv->cdclk.actual,
14977                                                   state->cdclk.pipe);
14978         }
14979
14980         /* FIXME: We should call drm_atomic_helper_commit_hw_done() here
14981          * already, but still need the state for the delayed optimization. To
14982          * fix this:
14983          * - wrap the optimization/post_plane_update stuff into a per-crtc work.
14984          * - schedule that vblank worker _before_ calling hw_done
14985          * - at the start of commit_tail, cancel it _synchrously
14986          * - switch over to the vblank wait helper in the core after that since
14987          *   we don't need out special handling any more.
14988          */
14989         drm_atomic_helper_wait_for_flip_done(dev, &state->base);
14990
14991         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
14992                 if (new_crtc_state->hw.active &&
14993                     !needs_modeset(new_crtc_state) &&
14994                     !new_crtc_state->preload_luts &&
14995                     (new_crtc_state->uapi.color_mgmt_changed ||
14996                      new_crtc_state->update_pipe))
14997                         intel_color_load_luts(new_crtc_state);
14998         }
14999
15000         /*
15001          * Now that the vblank has passed, we can go ahead and program the
15002          * optimal watermarks on platforms that need two-step watermark
15003          * programming.
15004          *
15005          * TODO: Move this (and other cleanup) to an async worker eventually.
15006          */
15007         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
15008                                             new_crtc_state, i) {
15009                 /*
15010                  * Gen2 reports pipe underruns whenever all planes are disabled.
15011                  * So re-enable underrun reporting after some planes get enabled.
15012                  *
15013                  * We do this before .optimize_watermarks() so that we have a
15014                  * chance of catching underruns with the intermediate watermarks
15015                  * vs. the new plane configuration.
15016                  */
15017                 if (IS_GEN(dev_priv, 2) && planes_enabling(old_crtc_state, new_crtc_state))
15018                         intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
15019
15020                 if (dev_priv->display.optimize_watermarks)
15021                         dev_priv->display.optimize_watermarks(state, crtc);
15022         }
15023
15024         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
15025                 intel_post_plane_update(state, crtc);
15026
15027                 if (put_domains[i])
15028                         modeset_put_power_domains(dev_priv, put_domains[i]);
15029
15030                 intel_modeset_verify_crtc(crtc, state, old_crtc_state, new_crtc_state);
15031         }
15032
15033         /* Underruns don't always raise interrupts, so check manually */
15034         intel_check_cpu_fifo_underruns(dev_priv);
15035         intel_check_pch_fifo_underruns(dev_priv);
15036
15037         if (state->modeset)
15038                 intel_verify_planes(state);
15039
15040         if (state->modeset && intel_can_enable_sagv(state))
15041                 intel_enable_sagv(dev_priv);
15042
15043         drm_atomic_helper_commit_hw_done(&state->base);
15044
15045         if (state->modeset) {
15046                 /* As one of the primary mmio accessors, KMS has a high
15047                  * likelihood of triggering bugs in unclaimed access. After we
15048                  * finish modesetting, see if an error has been flagged, and if
15049                  * so enable debugging for the next modeset - and hope we catch
15050                  * the culprit.
15051                  */
15052                 intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore);
15053                 intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET, wakeref);
15054         }
15055         intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
15056
15057         /*
15058          * Defer the cleanup of the old state to a separate worker to not
15059          * impede the current task (userspace for blocking modesets) that
15060          * are executed inline. For out-of-line asynchronous modesets/flips,
15061          * deferring to a new worker seems overkill, but we would place a
15062          * schedule point (cond_resched()) here anyway to keep latencies
15063          * down.
15064          */
15065         INIT_WORK(&state->base.commit_work, intel_atomic_cleanup_work);
15066         queue_work(system_highpri_wq, &state->base.commit_work);
15067 }
15068
15069 static void intel_atomic_commit_work(struct work_struct *work)
15070 {
15071         struct intel_atomic_state *state =
15072                 container_of(work, struct intel_atomic_state, base.commit_work);
15073
15074         intel_atomic_commit_tail(state);
15075 }
15076
15077 static int __i915_sw_fence_call
15078 intel_atomic_commit_ready(struct i915_sw_fence *fence,
15079                           enum i915_sw_fence_notify notify)
15080 {
15081         struct intel_atomic_state *state =
15082                 container_of(fence, struct intel_atomic_state, commit_ready);
15083
15084         switch (notify) {
15085         case FENCE_COMPLETE:
15086                 /* we do blocking waits in the worker, nothing to do here */
15087                 break;
15088         case FENCE_FREE:
15089                 {
15090                         struct intel_atomic_helper *helper =
15091                                 &to_i915(state->base.dev)->atomic_helper;
15092
15093                         if (llist_add(&state->freed, &helper->free_list))
15094                                 schedule_work(&helper->free_work);
15095                         break;
15096                 }
15097         }
15098
15099         return NOTIFY_DONE;
15100 }
15101
15102 static void intel_atomic_track_fbs(struct intel_atomic_state *state)
15103 {
15104         struct intel_plane_state *old_plane_state, *new_plane_state;
15105         struct intel_plane *plane;
15106         int i;
15107
15108         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
15109                                              new_plane_state, i)
15110                 intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->hw.fb),
15111                                         to_intel_frontbuffer(new_plane_state->hw.fb),
15112                                         plane->frontbuffer_bit);
15113 }
15114
15115 static void assert_global_state_locked(struct drm_i915_private *dev_priv)
15116 {
15117         struct intel_crtc *crtc;
15118
15119         for_each_intel_crtc(&dev_priv->drm, crtc)
15120                 drm_modeset_lock_assert_held(&crtc->base.mutex);
15121 }
15122
15123 static int intel_atomic_commit(struct drm_device *dev,
15124                                struct drm_atomic_state *_state,
15125                                bool nonblock)
15126 {
15127         struct intel_atomic_state *state = to_intel_atomic_state(_state);
15128         struct drm_i915_private *dev_priv = to_i915(dev);
15129         int ret = 0;
15130
15131         state->wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
15132
15133         drm_atomic_state_get(&state->base);
15134         i915_sw_fence_init(&state->commit_ready,
15135                            intel_atomic_commit_ready);
15136
15137         /*
15138          * The intel_legacy_cursor_update() fast path takes care
15139          * of avoiding the vblank waits for simple cursor
15140          * movement and flips. For cursor on/off and size changes,
15141          * we want to perform the vblank waits so that watermark
15142          * updates happen during the correct frames. Gen9+ have
15143          * double buffered watermarks and so shouldn't need this.
15144          *
15145          * Unset state->legacy_cursor_update before the call to
15146          * drm_atomic_helper_setup_commit() because otherwise
15147          * drm_atomic_helper_wait_for_flip_done() is a noop and
15148          * we get FIFO underruns because we didn't wait
15149          * for vblank.
15150          *
15151          * FIXME doing watermarks and fb cleanup from a vblank worker
15152          * (assuming we had any) would solve these problems.
15153          */
15154         if (INTEL_GEN(dev_priv) < 9 && state->base.legacy_cursor_update) {
15155                 struct intel_crtc_state *new_crtc_state;
15156                 struct intel_crtc *crtc;
15157                 int i;
15158
15159                 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
15160                         if (new_crtc_state->wm.need_postvbl_update ||
15161                             new_crtc_state->update_wm_post)
15162                                 state->base.legacy_cursor_update = false;
15163         }
15164
15165         ret = intel_atomic_prepare_commit(state);
15166         if (ret) {
15167                 DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret);
15168                 i915_sw_fence_commit(&state->commit_ready);
15169                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
15170                 return ret;
15171         }
15172
15173         ret = drm_atomic_helper_setup_commit(&state->base, nonblock);
15174         if (!ret)
15175                 ret = drm_atomic_helper_swap_state(&state->base, true);
15176
15177         if (ret) {
15178                 i915_sw_fence_commit(&state->commit_ready);
15179
15180                 drm_atomic_helper_cleanup_planes(dev, &state->base);
15181                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
15182                 return ret;
15183         }
15184         dev_priv->wm.distrust_bios_wm = false;
15185         intel_shared_dpll_swap_state(state);
15186         intel_atomic_track_fbs(state);
15187
15188         if (state->global_state_changed) {
15189                 assert_global_state_locked(dev_priv);
15190
15191                 memcpy(dev_priv->min_cdclk, state->min_cdclk,
15192                        sizeof(state->min_cdclk));
15193                 memcpy(dev_priv->min_voltage_level, state->min_voltage_level,
15194                        sizeof(state->min_voltage_level));
15195                 dev_priv->active_pipes = state->active_pipes;
15196                 dev_priv->cdclk.force_min_cdclk = state->cdclk.force_min_cdclk;
15197
15198                 intel_cdclk_swap_state(state);
15199         }
15200
15201         drm_atomic_state_get(&state->base);
15202         INIT_WORK(&state->base.commit_work, intel_atomic_commit_work);
15203
15204         i915_sw_fence_commit(&state->commit_ready);
15205         if (nonblock && state->modeset) {
15206                 queue_work(dev_priv->modeset_wq, &state->base.commit_work);
15207         } else if (nonblock) {
15208                 queue_work(dev_priv->flip_wq, &state->base.commit_work);
15209         } else {
15210                 if (state->modeset)
15211                         flush_workqueue(dev_priv->modeset_wq);
15212                 intel_atomic_commit_tail(state);
15213         }
15214
15215         return 0;
15216 }
15217
15218 struct wait_rps_boost {
15219         struct wait_queue_entry wait;
15220
15221         struct drm_crtc *crtc;
15222         struct i915_request *request;
15223 };
15224
15225 static int do_rps_boost(struct wait_queue_entry *_wait,
15226                         unsigned mode, int sync, void *key)
15227 {
15228         struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait);
15229         struct i915_request *rq = wait->request;
15230
15231         /*
15232          * If we missed the vblank, but the request is already running it
15233          * is reasonable to assume that it will complete before the next
15234          * vblank without our intervention, so leave RPS alone.
15235          */
15236         if (!i915_request_started(rq))
15237                 intel_rps_boost(rq);
15238         i915_request_put(rq);
15239
15240         drm_crtc_vblank_put(wait->crtc);
15241
15242         list_del(&wait->wait.entry);
15243         kfree(wait);
15244         return 1;
15245 }
15246
15247 static void add_rps_boost_after_vblank(struct drm_crtc *crtc,
15248                                        struct dma_fence *fence)
15249 {
15250         struct wait_rps_boost *wait;
15251
15252         if (!dma_fence_is_i915(fence))
15253                 return;
15254
15255         if (INTEL_GEN(to_i915(crtc->dev)) < 6)
15256                 return;
15257
15258         if (drm_crtc_vblank_get(crtc))
15259                 return;
15260
15261         wait = kmalloc(sizeof(*wait), GFP_KERNEL);
15262         if (!wait) {
15263                 drm_crtc_vblank_put(crtc);
15264                 return;
15265         }
15266
15267         wait->request = to_request(dma_fence_get(fence));
15268         wait->crtc = crtc;
15269
15270         wait->wait.func = do_rps_boost;
15271         wait->wait.flags = 0;
15272
15273         add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait);
15274 }
15275
15276 static int intel_plane_pin_fb(struct intel_plane_state *plane_state)
15277 {
15278         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
15279         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
15280         struct drm_framebuffer *fb = plane_state->hw.fb;
15281         struct i915_vma *vma;
15282
15283         if (plane->id == PLANE_CURSOR &&
15284             INTEL_INFO(dev_priv)->display.cursor_needs_physical) {
15285                 struct drm_i915_gem_object *obj = intel_fb_obj(fb);
15286                 const int align = intel_cursor_alignment(dev_priv);
15287                 int err;
15288
15289                 err = i915_gem_object_attach_phys(obj, align);
15290                 if (err)
15291                         return err;
15292         }
15293
15294         vma = intel_pin_and_fence_fb_obj(fb,
15295                                          &plane_state->view,
15296                                          intel_plane_uses_fence(plane_state),
15297                                          &plane_state->flags);
15298         if (IS_ERR(vma))
15299                 return PTR_ERR(vma);
15300
15301         plane_state->vma = vma;
15302
15303         return 0;
15304 }
15305
15306 static void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
15307 {
15308         struct i915_vma *vma;
15309
15310         vma = fetch_and_zero(&old_plane_state->vma);
15311         if (vma)
15312                 intel_unpin_fb_vma(vma, old_plane_state->flags);
15313 }
15314
15315 static void fb_obj_bump_render_priority(struct drm_i915_gem_object *obj)
15316 {
15317         struct i915_sched_attr attr = {
15318                 .priority = I915_USER_PRIORITY(I915_PRIORITY_DISPLAY),
15319         };
15320
15321         i915_gem_object_wait_priority(obj, 0, &attr);
15322 }
15323
15324 /**
15325  * intel_prepare_plane_fb - Prepare fb for usage on plane
15326  * @plane: drm plane to prepare for
15327  * @_new_plane_state: the plane state being prepared
15328  *
15329  * Prepares a framebuffer for usage on a display plane.  Generally this
15330  * involves pinning the underlying object and updating the frontbuffer tracking
15331  * bits.  Some older platforms need special physical address handling for
15332  * cursor planes.
15333  *
15334  * Returns 0 on success, negative error code on failure.
15335  */
15336 int
15337 intel_prepare_plane_fb(struct drm_plane *plane,
15338                        struct drm_plane_state *_new_plane_state)
15339 {
15340         struct intel_plane_state *new_plane_state =
15341                 to_intel_plane_state(_new_plane_state);
15342         struct intel_atomic_state *intel_state =
15343                 to_intel_atomic_state(new_plane_state->uapi.state);
15344         struct drm_i915_private *dev_priv = to_i915(plane->dev);
15345         struct drm_framebuffer *fb = new_plane_state->hw.fb;
15346         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
15347         struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb);
15348         int ret;
15349
15350         if (old_obj) {
15351                 struct intel_crtc_state *crtc_state =
15352                         intel_atomic_get_new_crtc_state(intel_state,
15353                                                         to_intel_crtc(plane->state->crtc));
15354
15355                 /* Big Hammer, we also need to ensure that any pending
15356                  * MI_WAIT_FOR_EVENT inside a user batch buffer on the
15357                  * current scanout is retired before unpinning the old
15358                  * framebuffer. Note that we rely on userspace rendering
15359                  * into the buffer attached to the pipe they are waiting
15360                  * on. If not, userspace generates a GPU hang with IPEHR
15361                  * point to the MI_WAIT_FOR_EVENT.
15362                  *
15363                  * This should only fail upon a hung GPU, in which case we
15364                  * can safely continue.
15365                  */
15366                 if (needs_modeset(crtc_state)) {
15367                         ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
15368                                                               old_obj->base.resv, NULL,
15369                                                               false, 0,
15370                                                               GFP_KERNEL);
15371                         if (ret < 0)
15372                                 return ret;
15373                 }
15374         }
15375
15376         if (new_plane_state->uapi.fence) { /* explicit fencing */
15377                 ret = i915_sw_fence_await_dma_fence(&intel_state->commit_ready,
15378                                                     new_plane_state->uapi.fence,
15379                                                     I915_FENCE_TIMEOUT,
15380                                                     GFP_KERNEL);
15381                 if (ret < 0)
15382                         return ret;
15383         }
15384
15385         if (!obj)
15386                 return 0;
15387
15388         ret = i915_gem_object_pin_pages(obj);
15389         if (ret)
15390                 return ret;
15391
15392         ret = intel_plane_pin_fb(new_plane_state);
15393
15394         i915_gem_object_unpin_pages(obj);
15395         if (ret)
15396                 return ret;
15397
15398         fb_obj_bump_render_priority(obj);
15399         i915_gem_object_flush_frontbuffer(obj, ORIGIN_DIRTYFB);
15400
15401         if (!new_plane_state->uapi.fence) { /* implicit fencing */
15402                 struct dma_fence *fence;
15403
15404                 ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
15405                                                       obj->base.resv, NULL,
15406                                                       false, I915_FENCE_TIMEOUT,
15407                                                       GFP_KERNEL);
15408                 if (ret < 0)
15409                         return ret;
15410
15411                 fence = dma_resv_get_excl_rcu(obj->base.resv);
15412                 if (fence) {
15413                         add_rps_boost_after_vblank(new_plane_state->hw.crtc,
15414                                                    fence);
15415                         dma_fence_put(fence);
15416                 }
15417         } else {
15418                 add_rps_boost_after_vblank(new_plane_state->hw.crtc,
15419                                            new_plane_state->uapi.fence);
15420         }
15421
15422         /*
15423          * We declare pageflips to be interactive and so merit a small bias
15424          * towards upclocking to deliver the frame on time. By only changing
15425          * the RPS thresholds to sample more regularly and aim for higher
15426          * clocks we can hopefully deliver low power workloads (like kodi)
15427          * that are not quite steady state without resorting to forcing
15428          * maximum clocks following a vblank miss (see do_rps_boost()).
15429          */
15430         if (!intel_state->rps_interactive) {
15431                 intel_rps_mark_interactive(&dev_priv->gt.rps, true);
15432                 intel_state->rps_interactive = true;
15433         }
15434
15435         return 0;
15436 }
15437
15438 /**
15439  * intel_cleanup_plane_fb - Cleans up an fb after plane use
15440  * @plane: drm plane to clean up for
15441  * @_old_plane_state: the state from the previous modeset
15442  *
15443  * Cleans up a framebuffer that has just been removed from a plane.
15444  */
15445 void
15446 intel_cleanup_plane_fb(struct drm_plane *plane,
15447                        struct drm_plane_state *_old_plane_state)
15448 {
15449         struct intel_plane_state *old_plane_state =
15450                 to_intel_plane_state(_old_plane_state);
15451         struct intel_atomic_state *intel_state =
15452                 to_intel_atomic_state(old_plane_state->uapi.state);
15453         struct drm_i915_private *dev_priv = to_i915(plane->dev);
15454
15455         if (intel_state->rps_interactive) {
15456                 intel_rps_mark_interactive(&dev_priv->gt.rps, false);
15457                 intel_state->rps_interactive = false;
15458         }
15459
15460         /* Should only be called after a successful intel_prepare_plane_fb()! */
15461         intel_plane_unpin_fb(old_plane_state);
15462 }
15463
15464 /**
15465  * intel_plane_destroy - destroy a plane
15466  * @plane: plane to destroy
15467  *
15468  * Common destruction function for all types of planes (primary, cursor,
15469  * sprite).
15470  */
15471 void intel_plane_destroy(struct drm_plane *plane)
15472 {
15473         drm_plane_cleanup(plane);
15474         kfree(to_intel_plane(plane));
15475 }
15476
15477 static bool i8xx_plane_format_mod_supported(struct drm_plane *_plane,
15478                                             u32 format, u64 modifier)
15479 {
15480         switch (modifier) {
15481         case DRM_FORMAT_MOD_LINEAR:
15482         case I915_FORMAT_MOD_X_TILED:
15483                 break;
15484         default:
15485                 return false;
15486         }
15487
15488         switch (format) {
15489         case DRM_FORMAT_C8:
15490         case DRM_FORMAT_RGB565:
15491         case DRM_FORMAT_XRGB1555:
15492         case DRM_FORMAT_XRGB8888:
15493                 return modifier == DRM_FORMAT_MOD_LINEAR ||
15494                         modifier == I915_FORMAT_MOD_X_TILED;
15495         default:
15496                 return false;
15497         }
15498 }
15499
15500 static bool i965_plane_format_mod_supported(struct drm_plane *_plane,
15501                                             u32 format, u64 modifier)
15502 {
15503         switch (modifier) {
15504         case DRM_FORMAT_MOD_LINEAR:
15505         case I915_FORMAT_MOD_X_TILED:
15506                 break;
15507         default:
15508                 return false;
15509         }
15510
15511         switch (format) {
15512         case DRM_FORMAT_C8:
15513         case DRM_FORMAT_RGB565:
15514         case DRM_FORMAT_XRGB8888:
15515         case DRM_FORMAT_XBGR8888:
15516         case DRM_FORMAT_ARGB8888:
15517         case DRM_FORMAT_ABGR8888:
15518         case DRM_FORMAT_XRGB2101010:
15519         case DRM_FORMAT_XBGR2101010:
15520         case DRM_FORMAT_ARGB2101010:
15521         case DRM_FORMAT_ABGR2101010:
15522         case DRM_FORMAT_XBGR16161616F:
15523                 return modifier == DRM_FORMAT_MOD_LINEAR ||
15524                         modifier == I915_FORMAT_MOD_X_TILED;
15525         default:
15526                 return false;
15527         }
15528 }
15529
15530 static bool intel_cursor_format_mod_supported(struct drm_plane *_plane,
15531                                               u32 format, u64 modifier)
15532 {
15533         return modifier == DRM_FORMAT_MOD_LINEAR &&
15534                 format == DRM_FORMAT_ARGB8888;
15535 }
15536
15537 static const struct drm_plane_funcs i965_plane_funcs = {
15538         .update_plane = drm_atomic_helper_update_plane,
15539         .disable_plane = drm_atomic_helper_disable_plane,
15540         .destroy = intel_plane_destroy,
15541         .atomic_duplicate_state = intel_plane_duplicate_state,
15542         .atomic_destroy_state = intel_plane_destroy_state,
15543         .format_mod_supported = i965_plane_format_mod_supported,
15544 };
15545
15546 static const struct drm_plane_funcs i8xx_plane_funcs = {
15547         .update_plane = drm_atomic_helper_update_plane,
15548         .disable_plane = drm_atomic_helper_disable_plane,
15549         .destroy = intel_plane_destroy,
15550         .atomic_duplicate_state = intel_plane_duplicate_state,
15551         .atomic_destroy_state = intel_plane_destroy_state,
15552         .format_mod_supported = i8xx_plane_format_mod_supported,
15553 };
15554
15555 static int
15556 intel_legacy_cursor_update(struct drm_plane *_plane,
15557                            struct drm_crtc *_crtc,
15558                            struct drm_framebuffer *fb,
15559                            int crtc_x, int crtc_y,
15560                            unsigned int crtc_w, unsigned int crtc_h,
15561                            u32 src_x, u32 src_y,
15562                            u32 src_w, u32 src_h,
15563                            struct drm_modeset_acquire_ctx *ctx)
15564 {
15565         struct intel_plane *plane = to_intel_plane(_plane);
15566         struct intel_crtc *crtc = to_intel_crtc(_crtc);
15567         struct intel_plane_state *old_plane_state =
15568                 to_intel_plane_state(plane->base.state);
15569         struct intel_plane_state *new_plane_state;
15570         struct intel_crtc_state *crtc_state =
15571                 to_intel_crtc_state(crtc->base.state);
15572         struct intel_crtc_state *new_crtc_state;
15573         int ret;
15574
15575         /*
15576          * When crtc is inactive or there is a modeset pending,
15577          * wait for it to complete in the slowpath
15578          */
15579         if (!crtc_state->hw.active || needs_modeset(crtc_state) ||
15580             crtc_state->update_pipe)
15581                 goto slow;
15582
15583         /*
15584          * Don't do an async update if there is an outstanding commit modifying
15585          * the plane.  This prevents our async update's changes from getting
15586          * overridden by a previous synchronous update's state.
15587          */
15588         if (old_plane_state->uapi.commit &&
15589             !try_wait_for_completion(&old_plane_state->uapi.commit->hw_done))
15590                 goto slow;
15591
15592         /*
15593          * If any parameters change that may affect watermarks,
15594          * take the slowpath. Only changing fb or position should be
15595          * in the fastpath.
15596          */
15597         if (old_plane_state->uapi.crtc != &crtc->base ||
15598             old_plane_state->uapi.src_w != src_w ||
15599             old_plane_state->uapi.src_h != src_h ||
15600             old_plane_state->uapi.crtc_w != crtc_w ||
15601             old_plane_state->uapi.crtc_h != crtc_h ||
15602             !old_plane_state->uapi.fb != !fb)
15603                 goto slow;
15604
15605         new_plane_state = to_intel_plane_state(intel_plane_duplicate_state(&plane->base));
15606         if (!new_plane_state)
15607                 return -ENOMEM;
15608
15609         new_crtc_state = to_intel_crtc_state(intel_crtc_duplicate_state(&crtc->base));
15610         if (!new_crtc_state) {
15611                 ret = -ENOMEM;
15612                 goto out_free;
15613         }
15614
15615         drm_atomic_set_fb_for_plane(&new_plane_state->uapi, fb);
15616
15617         new_plane_state->uapi.src_x = src_x;
15618         new_plane_state->uapi.src_y = src_y;
15619         new_plane_state->uapi.src_w = src_w;
15620         new_plane_state->uapi.src_h = src_h;
15621         new_plane_state->uapi.crtc_x = crtc_x;
15622         new_plane_state->uapi.crtc_y = crtc_y;
15623         new_plane_state->uapi.crtc_w = crtc_w;
15624         new_plane_state->uapi.crtc_h = crtc_h;
15625
15626         ret = intel_plane_atomic_check_with_state(crtc_state, new_crtc_state,
15627                                                   old_plane_state, new_plane_state);
15628         if (ret)
15629                 goto out_free;
15630
15631         ret = intel_plane_pin_fb(new_plane_state);
15632         if (ret)
15633                 goto out_free;
15634
15635         intel_frontbuffer_flush(to_intel_frontbuffer(new_plane_state->hw.fb),
15636                                 ORIGIN_FLIP);
15637         intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->hw.fb),
15638                                 to_intel_frontbuffer(new_plane_state->hw.fb),
15639                                 plane->frontbuffer_bit);
15640
15641         /* Swap plane state */
15642         plane->base.state = &new_plane_state->uapi;
15643
15644         /*
15645          * We cannot swap crtc_state as it may be in use by an atomic commit or
15646          * page flip that's running simultaneously. If we swap crtc_state and
15647          * destroy the old state, we will cause a use-after-free there.
15648          *
15649          * Only update active_planes, which is needed for our internal
15650          * bookkeeping. Either value will do the right thing when updating
15651          * planes atomically. If the cursor was part of the atomic update then
15652          * we would have taken the slowpath.
15653          */
15654         crtc_state->active_planes = new_crtc_state->active_planes;
15655
15656         if (new_plane_state->uapi.visible)
15657                 intel_update_plane(plane, crtc_state, new_plane_state);
15658         else
15659                 intel_disable_plane(plane, crtc_state);
15660
15661         intel_plane_unpin_fb(old_plane_state);
15662
15663 out_free:
15664         if (new_crtc_state)
15665                 intel_crtc_destroy_state(&crtc->base, &new_crtc_state->uapi);
15666         if (ret)
15667                 intel_plane_destroy_state(&plane->base, &new_plane_state->uapi);
15668         else
15669                 intel_plane_destroy_state(&plane->base, &old_plane_state->uapi);
15670         return ret;
15671
15672 slow:
15673         return drm_atomic_helper_update_plane(&plane->base, &crtc->base, fb,
15674                                               crtc_x, crtc_y, crtc_w, crtc_h,
15675                                               src_x, src_y, src_w, src_h, ctx);
15676 }
15677
15678 static const struct drm_plane_funcs intel_cursor_plane_funcs = {
15679         .update_plane = intel_legacy_cursor_update,
15680         .disable_plane = drm_atomic_helper_disable_plane,
15681         .destroy = intel_plane_destroy,
15682         .atomic_duplicate_state = intel_plane_duplicate_state,
15683         .atomic_destroy_state = intel_plane_destroy_state,
15684         .format_mod_supported = intel_cursor_format_mod_supported,
15685 };
15686
15687 static bool i9xx_plane_has_fbc(struct drm_i915_private *dev_priv,
15688                                enum i9xx_plane_id i9xx_plane)
15689 {
15690         if (!HAS_FBC(dev_priv))
15691                 return false;
15692
15693         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
15694                 return i9xx_plane == PLANE_A; /* tied to pipe A */
15695         else if (IS_IVYBRIDGE(dev_priv))
15696                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B ||
15697                         i9xx_plane == PLANE_C;
15698         else if (INTEL_GEN(dev_priv) >= 4)
15699                 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B;
15700         else
15701                 return i9xx_plane == PLANE_A;
15702 }
15703
15704 static struct intel_plane *
15705 intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
15706 {
15707         struct intel_plane *plane;
15708         const struct drm_plane_funcs *plane_funcs;
15709         unsigned int supported_rotations;
15710         unsigned int possible_crtcs;
15711         const u32 *formats;
15712         int num_formats;
15713         int ret, zpos;
15714
15715         if (INTEL_GEN(dev_priv) >= 9)
15716                 return skl_universal_plane_create(dev_priv, pipe,
15717                                                   PLANE_PRIMARY);
15718
15719         plane = intel_plane_alloc();
15720         if (IS_ERR(plane))
15721                 return plane;
15722
15723         plane->pipe = pipe;
15724         /*
15725          * On gen2/3 only plane A can do FBC, but the panel fitter and LVDS
15726          * port is hooked to pipe B. Hence we want plane A feeding pipe B.
15727          */
15728         if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4)
15729                 plane->i9xx_plane = (enum i9xx_plane_id) !pipe;
15730         else
15731                 plane->i9xx_plane = (enum i9xx_plane_id) pipe;
15732         plane->id = PLANE_PRIMARY;
15733         plane->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, plane->id);
15734
15735         plane->has_fbc = i9xx_plane_has_fbc(dev_priv, plane->i9xx_plane);
15736         if (plane->has_fbc) {
15737                 struct intel_fbc *fbc = &dev_priv->fbc;
15738
15739                 fbc->possible_framebuffer_bits |= plane->frontbuffer_bit;
15740         }
15741
15742         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
15743                 formats = vlv_primary_formats;
15744                 num_formats = ARRAY_SIZE(vlv_primary_formats);
15745         } else if (INTEL_GEN(dev_priv) >= 4) {
15746                 /*
15747                  * WaFP16GammaEnabling:ivb
15748                  * "Workaround : When using the 64-bit format, the plane
15749                  *  output on each color channel has one quarter amplitude.
15750                  *  It can be brought up to full amplitude by using pipe
15751                  *  gamma correction or pipe color space conversion to
15752                  *  multiply the plane output by four."
15753                  *
15754                  * There is no dedicated plane gamma for the primary plane,
15755                  * and using the pipe gamma/csc could conflict with other
15756                  * planes, so we choose not to expose fp16 on IVB primary
15757                  * planes. HSW primary planes no longer have this problem.
15758                  */
15759                 if (IS_IVYBRIDGE(dev_priv)) {
15760                         formats = ivb_primary_formats;
15761                         num_formats = ARRAY_SIZE(ivb_primary_formats);
15762                 } else {
15763                         formats = i965_primary_formats;
15764                         num_formats = ARRAY_SIZE(i965_primary_formats);
15765                 }
15766         } else {
15767                 formats = i8xx_primary_formats;
15768                 num_formats = ARRAY_SIZE(i8xx_primary_formats);
15769         }
15770
15771         if (INTEL_GEN(dev_priv) >= 4)
15772                 plane_funcs = &i965_plane_funcs;
15773         else
15774                 plane_funcs = &i8xx_plane_funcs;
15775
15776         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
15777                 plane->min_cdclk = vlv_plane_min_cdclk;
15778         else if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
15779                 plane->min_cdclk = hsw_plane_min_cdclk;
15780         else if (IS_IVYBRIDGE(dev_priv))
15781                 plane->min_cdclk = ivb_plane_min_cdclk;
15782         else
15783                 plane->min_cdclk = i9xx_plane_min_cdclk;
15784
15785         plane->max_stride = i9xx_plane_max_stride;
15786         plane->update_plane = i9xx_update_plane;
15787         plane->disable_plane = i9xx_disable_plane;
15788         plane->get_hw_state = i9xx_plane_get_hw_state;
15789         plane->check_plane = i9xx_plane_check;
15790
15791         possible_crtcs = BIT(pipe);
15792
15793         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
15794                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
15795                                                possible_crtcs, plane_funcs,
15796                                                formats, num_formats,
15797                                                i9xx_format_modifiers,
15798                                                DRM_PLANE_TYPE_PRIMARY,
15799                                                "primary %c", pipe_name(pipe));
15800         else
15801                 ret = drm_universal_plane_init(&dev_priv->drm, &plane->base,
15802                                                possible_crtcs, plane_funcs,
15803                                                formats, num_formats,
15804                                                i9xx_format_modifiers,
15805                                                DRM_PLANE_TYPE_PRIMARY,
15806                                                "plane %c",
15807                                                plane_name(plane->i9xx_plane));
15808         if (ret)
15809                 goto fail;
15810
15811         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
15812                 supported_rotations =
15813                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180 |
15814                         DRM_MODE_REFLECT_X;
15815         } else if (INTEL_GEN(dev_priv) >= 4) {
15816                 supported_rotations =
15817                         DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180;
15818         } else {
15819                 supported_rotations = DRM_MODE_ROTATE_0;
15820         }
15821
15822         if (INTEL_GEN(dev_priv) >= 4)
15823                 drm_plane_create_rotation_property(&plane->base,
15824                                                    DRM_MODE_ROTATE_0,
15825                                                    supported_rotations);
15826
15827         zpos = 0;
15828         drm_plane_create_zpos_immutable_property(&plane->base, zpos);
15829
15830         drm_plane_helper_add(&plane->base, &intel_plane_helper_funcs);
15831
15832         return plane;
15833
15834 fail:
15835         intel_plane_free(plane);
15836
15837         return ERR_PTR(ret);
15838 }
15839
15840 static struct intel_plane *
15841 intel_cursor_plane_create(struct drm_i915_private *dev_priv,
15842                           enum pipe pipe)
15843 {
15844         unsigned int possible_crtcs;
15845         struct intel_plane *cursor;
15846         int ret, zpos;
15847
15848         cursor = intel_plane_alloc();
15849         if (IS_ERR(cursor))
15850                 return cursor;
15851
15852         cursor->pipe = pipe;
15853         cursor->i9xx_plane = (enum i9xx_plane_id) pipe;
15854         cursor->id = PLANE_CURSOR;
15855         cursor->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, cursor->id);
15856
15857         if (IS_I845G(dev_priv) || IS_I865G(dev_priv)) {
15858                 cursor->max_stride = i845_cursor_max_stride;
15859                 cursor->update_plane = i845_update_cursor;
15860                 cursor->disable_plane = i845_disable_cursor;
15861                 cursor->get_hw_state = i845_cursor_get_hw_state;
15862                 cursor->check_plane = i845_check_cursor;
15863         } else {
15864                 cursor->max_stride = i9xx_cursor_max_stride;
15865                 cursor->update_plane = i9xx_update_cursor;
15866                 cursor->disable_plane = i9xx_disable_cursor;
15867                 cursor->get_hw_state = i9xx_cursor_get_hw_state;
15868                 cursor->check_plane = i9xx_check_cursor;
15869         }
15870
15871         cursor->cursor.base = ~0;
15872         cursor->cursor.cntl = ~0;
15873
15874         if (IS_I845G(dev_priv) || IS_I865G(dev_priv) || HAS_CUR_FBC(dev_priv))
15875                 cursor->cursor.size = ~0;
15876
15877         possible_crtcs = BIT(pipe);
15878
15879         ret = drm_universal_plane_init(&dev_priv->drm, &cursor->base,
15880                                        possible_crtcs, &intel_cursor_plane_funcs,
15881                                        intel_cursor_formats,
15882                                        ARRAY_SIZE(intel_cursor_formats),
15883                                        cursor_format_modifiers,
15884                                        DRM_PLANE_TYPE_CURSOR,
15885                                        "cursor %c", pipe_name(pipe));
15886         if (ret)
15887                 goto fail;
15888
15889         if (INTEL_GEN(dev_priv) >= 4)
15890                 drm_plane_create_rotation_property(&cursor->base,
15891                                                    DRM_MODE_ROTATE_0,
15892                                                    DRM_MODE_ROTATE_0 |
15893                                                    DRM_MODE_ROTATE_180);
15894
15895         zpos = RUNTIME_INFO(dev_priv)->num_sprites[pipe] + 1;
15896         drm_plane_create_zpos_immutable_property(&cursor->base, zpos);
15897
15898         drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs);
15899
15900         return cursor;
15901
15902 fail:
15903         intel_plane_free(cursor);
15904
15905         return ERR_PTR(ret);
15906 }
15907
15908 #define INTEL_CRTC_FUNCS \
15909         .gamma_set = drm_atomic_helper_legacy_gamma_set, \
15910         .set_config = drm_atomic_helper_set_config, \
15911         .destroy = intel_crtc_destroy, \
15912         .page_flip = drm_atomic_helper_page_flip, \
15913         .atomic_duplicate_state = intel_crtc_duplicate_state, \
15914         .atomic_destroy_state = intel_crtc_destroy_state, \
15915         .set_crc_source = intel_crtc_set_crc_source, \
15916         .verify_crc_source = intel_crtc_verify_crc_source, \
15917         .get_crc_sources = intel_crtc_get_crc_sources
15918
15919 static const struct drm_crtc_funcs bdw_crtc_funcs = {
15920         INTEL_CRTC_FUNCS,
15921
15922         .get_vblank_counter = g4x_get_vblank_counter,
15923         .enable_vblank = bdw_enable_vblank,
15924         .disable_vblank = bdw_disable_vblank,
15925 };
15926
15927 static const struct drm_crtc_funcs ilk_crtc_funcs = {
15928         INTEL_CRTC_FUNCS,
15929
15930         .get_vblank_counter = g4x_get_vblank_counter,
15931         .enable_vblank = ilk_enable_vblank,
15932         .disable_vblank = ilk_disable_vblank,
15933 };
15934
15935 static const struct drm_crtc_funcs g4x_crtc_funcs = {
15936         INTEL_CRTC_FUNCS,
15937
15938         .get_vblank_counter = g4x_get_vblank_counter,
15939         .enable_vblank = i965_enable_vblank,
15940         .disable_vblank = i965_disable_vblank,
15941 };
15942
15943 static const struct drm_crtc_funcs i965_crtc_funcs = {
15944         INTEL_CRTC_FUNCS,
15945
15946         .get_vblank_counter = i915_get_vblank_counter,
15947         .enable_vblank = i965_enable_vblank,
15948         .disable_vblank = i965_disable_vblank,
15949 };
15950
15951 static const struct drm_crtc_funcs i915gm_crtc_funcs = {
15952         INTEL_CRTC_FUNCS,
15953
15954         .get_vblank_counter = i915_get_vblank_counter,
15955         .enable_vblank = i915gm_enable_vblank,
15956         .disable_vblank = i915gm_disable_vblank,
15957 };
15958
15959 static const struct drm_crtc_funcs i915_crtc_funcs = {
15960         INTEL_CRTC_FUNCS,
15961
15962         .get_vblank_counter = i915_get_vblank_counter,
15963         .enable_vblank = i8xx_enable_vblank,
15964         .disable_vblank = i8xx_disable_vblank,
15965 };
15966
15967 static const struct drm_crtc_funcs i8xx_crtc_funcs = {
15968         INTEL_CRTC_FUNCS,
15969
15970         /* no hw vblank counter */
15971         .enable_vblank = i8xx_enable_vblank,
15972         .disable_vblank = i8xx_disable_vblank,
15973 };
15974
15975 static struct intel_crtc *intel_crtc_alloc(void)
15976 {
15977         struct intel_crtc_state *crtc_state;
15978         struct intel_crtc *crtc;
15979
15980         crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
15981         if (!crtc)
15982                 return ERR_PTR(-ENOMEM);
15983
15984         crtc_state = intel_crtc_state_alloc(crtc);
15985         if (!crtc_state) {
15986                 kfree(crtc);
15987                 return ERR_PTR(-ENOMEM);
15988         }
15989
15990         crtc->base.state = &crtc_state->uapi;
15991         crtc->config = crtc_state;
15992
15993         return crtc;
15994 }
15995
15996 static void intel_crtc_free(struct intel_crtc *crtc)
15997 {
15998         intel_crtc_destroy_state(&crtc->base, crtc->base.state);
15999         kfree(crtc);
16000 }
16001
16002 static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
16003 {
16004         struct intel_plane *primary, *cursor;
16005         const struct drm_crtc_funcs *funcs;
16006         struct intel_crtc *crtc;
16007         int sprite, ret;
16008
16009         crtc = intel_crtc_alloc();
16010         if (IS_ERR(crtc))
16011                 return PTR_ERR(crtc);
16012
16013         crtc->pipe = pipe;
16014         crtc->num_scalers = RUNTIME_INFO(dev_priv)->num_scalers[pipe];
16015
16016         primary = intel_primary_plane_create(dev_priv, pipe);
16017         if (IS_ERR(primary)) {
16018                 ret = PTR_ERR(primary);
16019                 goto fail;
16020         }
16021         crtc->plane_ids_mask |= BIT(primary->id);
16022
16023         for_each_sprite(dev_priv, pipe, sprite) {
16024                 struct intel_plane *plane;
16025
16026                 plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
16027                 if (IS_ERR(plane)) {
16028                         ret = PTR_ERR(plane);
16029                         goto fail;
16030                 }
16031                 crtc->plane_ids_mask |= BIT(plane->id);
16032         }
16033
16034         cursor = intel_cursor_plane_create(dev_priv, pipe);
16035         if (IS_ERR(cursor)) {
16036                 ret = PTR_ERR(cursor);
16037                 goto fail;
16038         }
16039         crtc->plane_ids_mask |= BIT(cursor->id);
16040
16041         if (HAS_GMCH(dev_priv)) {
16042                 if (IS_CHERRYVIEW(dev_priv) ||
16043                     IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
16044                         funcs = &g4x_crtc_funcs;
16045                 else if (IS_GEN(dev_priv, 4))
16046                         funcs = &i965_crtc_funcs;
16047                 else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv))
16048                         funcs = &i915gm_crtc_funcs;
16049                 else if (IS_GEN(dev_priv, 3))
16050                         funcs = &i915_crtc_funcs;
16051                 else
16052                         funcs = &i8xx_crtc_funcs;
16053         } else {
16054                 if (INTEL_GEN(dev_priv) >= 8)
16055                         funcs = &bdw_crtc_funcs;
16056                 else
16057                         funcs = &ilk_crtc_funcs;
16058         }
16059
16060         ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base,
16061                                         &primary->base, &cursor->base,
16062                                         funcs, "pipe %c", pipe_name(pipe));
16063         if (ret)
16064                 goto fail;
16065
16066         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->pipe_to_crtc_mapping) ||
16067                dev_priv->pipe_to_crtc_mapping[pipe] != NULL);
16068         dev_priv->pipe_to_crtc_mapping[pipe] = crtc;
16069
16070         if (INTEL_GEN(dev_priv) < 9) {
16071                 enum i9xx_plane_id i9xx_plane = primary->i9xx_plane;
16072
16073                 BUG_ON(i9xx_plane >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
16074                        dev_priv->plane_to_crtc_mapping[i9xx_plane] != NULL);
16075                 dev_priv->plane_to_crtc_mapping[i9xx_plane] = crtc;
16076         }
16077
16078         intel_color_init(crtc);
16079
16080         WARN_ON(drm_crtc_index(&crtc->base) != crtc->pipe);
16081
16082         return 0;
16083
16084 fail:
16085         intel_crtc_free(crtc);
16086
16087         return ret;
16088 }
16089
16090 int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
16091                                       struct drm_file *file)
16092 {
16093         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
16094         struct drm_crtc *drmmode_crtc;
16095         struct intel_crtc *crtc;
16096
16097         drmmode_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
16098         if (!drmmode_crtc)
16099                 return -ENOENT;
16100
16101         crtc = to_intel_crtc(drmmode_crtc);
16102         pipe_from_crtc_id->pipe = crtc->pipe;
16103
16104         return 0;
16105 }
16106
16107 static u32 intel_encoder_possible_clones(struct intel_encoder *encoder)
16108 {
16109         struct drm_device *dev = encoder->base.dev;
16110         struct intel_encoder *source_encoder;
16111         u32 possible_clones = 0;
16112
16113         for_each_intel_encoder(dev, source_encoder) {
16114                 if (encoders_cloneable(encoder, source_encoder))
16115                         possible_clones |= drm_encoder_mask(&source_encoder->base);
16116         }
16117
16118         return possible_clones;
16119 }
16120
16121 static u32 intel_encoder_possible_crtcs(struct intel_encoder *encoder)
16122 {
16123         struct drm_device *dev = encoder->base.dev;
16124         struct intel_crtc *crtc;
16125         u32 possible_crtcs = 0;
16126
16127         for_each_intel_crtc(dev, crtc) {
16128                 if (encoder->pipe_mask & BIT(crtc->pipe))
16129                         possible_crtcs |= drm_crtc_mask(&crtc->base);
16130         }
16131
16132         return possible_crtcs;
16133 }
16134
16135 static bool ilk_has_edp_a(struct drm_i915_private *dev_priv)
16136 {
16137         if (!IS_MOBILE(dev_priv))
16138                 return false;
16139
16140         if ((I915_READ(DP_A) & DP_DETECTED) == 0)
16141                 return false;
16142
16143         if (IS_GEN(dev_priv, 5) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
16144                 return false;
16145
16146         return true;
16147 }
16148
16149 static bool intel_ddi_crt_present(struct drm_i915_private *dev_priv)
16150 {
16151         if (INTEL_GEN(dev_priv) >= 9)
16152                 return false;
16153
16154         if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
16155                 return false;
16156
16157         if (HAS_PCH_LPT_H(dev_priv) &&
16158             I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
16159                 return false;
16160
16161         /* DDI E can't be used if DDI A requires 4 lanes */
16162         if (I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
16163                 return false;
16164
16165         if (!dev_priv->vbt.int_crt_support)
16166                 return false;
16167
16168         return true;
16169 }
16170
16171 void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv)
16172 {
16173         int pps_num;
16174         int pps_idx;
16175
16176         if (HAS_DDI(dev_priv))
16177                 return;
16178         /*
16179          * This w/a is needed at least on CPT/PPT, but to be sure apply it
16180          * everywhere where registers can be write protected.
16181          */
16182         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
16183                 pps_num = 2;
16184         else
16185                 pps_num = 1;
16186
16187         for (pps_idx = 0; pps_idx < pps_num; pps_idx++) {
16188                 u32 val = I915_READ(PP_CONTROL(pps_idx));
16189
16190                 val = (val & ~PANEL_UNLOCK_MASK) | PANEL_UNLOCK_REGS;
16191                 I915_WRITE(PP_CONTROL(pps_idx), val);
16192         }
16193 }
16194
16195 static void intel_pps_init(struct drm_i915_private *dev_priv)
16196 {
16197         if (HAS_PCH_SPLIT(dev_priv) || IS_GEN9_LP(dev_priv))
16198                 dev_priv->pps_mmio_base = PCH_PPS_BASE;
16199         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
16200                 dev_priv->pps_mmio_base = VLV_PPS_BASE;
16201         else
16202                 dev_priv->pps_mmio_base = PPS_BASE;
16203
16204         intel_pps_unlock_regs_wa(dev_priv);
16205 }
16206
16207 static void intel_setup_outputs(struct drm_i915_private *dev_priv)
16208 {
16209         struct intel_encoder *encoder;
16210         bool dpd_is_edp = false;
16211
16212         intel_pps_init(dev_priv);
16213
16214         if (!HAS_DISPLAY(dev_priv) || !INTEL_DISPLAY_ENABLED(dev_priv))
16215                 return;
16216
16217         if (INTEL_GEN(dev_priv) >= 12) {
16218                 intel_ddi_init(dev_priv, PORT_A);
16219                 intel_ddi_init(dev_priv, PORT_B);
16220                 intel_ddi_init(dev_priv, PORT_D);
16221                 intel_ddi_init(dev_priv, PORT_E);
16222                 intel_ddi_init(dev_priv, PORT_F);
16223                 intel_ddi_init(dev_priv, PORT_G);
16224                 intel_ddi_init(dev_priv, PORT_H);
16225                 intel_ddi_init(dev_priv, PORT_I);
16226                 icl_dsi_init(dev_priv);
16227         } else if (IS_ELKHARTLAKE(dev_priv)) {
16228                 intel_ddi_init(dev_priv, PORT_A);
16229                 intel_ddi_init(dev_priv, PORT_B);
16230                 intel_ddi_init(dev_priv, PORT_C);
16231                 intel_ddi_init(dev_priv, PORT_D);
16232                 icl_dsi_init(dev_priv);
16233         } else if (IS_GEN(dev_priv, 11)) {
16234                 intel_ddi_init(dev_priv, PORT_A);
16235                 intel_ddi_init(dev_priv, PORT_B);
16236                 intel_ddi_init(dev_priv, PORT_C);
16237                 intel_ddi_init(dev_priv, PORT_D);
16238                 intel_ddi_init(dev_priv, PORT_E);
16239                 /*
16240                  * On some ICL SKUs port F is not present. No strap bits for
16241                  * this, so rely on VBT.
16242                  * Work around broken VBTs on SKUs known to have no port F.
16243                  */
16244                 if (IS_ICL_WITH_PORT_F(dev_priv) &&
16245                     intel_bios_is_port_present(dev_priv, PORT_F))
16246                         intel_ddi_init(dev_priv, PORT_F);
16247
16248                 icl_dsi_init(dev_priv);
16249         } else if (IS_GEN9_LP(dev_priv)) {
16250                 /*
16251                  * FIXME: Broxton doesn't support port detection via the
16252                  * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to
16253                  * detect the ports.
16254                  */
16255                 intel_ddi_init(dev_priv, PORT_A);
16256                 intel_ddi_init(dev_priv, PORT_B);
16257                 intel_ddi_init(dev_priv, PORT_C);
16258
16259                 vlv_dsi_init(dev_priv);
16260         } else if (HAS_DDI(dev_priv)) {
16261                 int found;
16262
16263                 if (intel_ddi_crt_present(dev_priv))
16264                         intel_crt_init(dev_priv);
16265
16266                 /*
16267                  * Haswell uses DDI functions to detect digital outputs.
16268                  * On SKL pre-D0 the strap isn't connected, so we assume
16269                  * it's there.
16270                  */
16271                 found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
16272                 /* WaIgnoreDDIAStrap: skl */
16273                 if (found || IS_GEN9_BC(dev_priv))
16274                         intel_ddi_init(dev_priv, PORT_A);
16275
16276                 /* DDI B, C, D, and F detection is indicated by the SFUSE_STRAP
16277                  * register */
16278                 found = I915_READ(SFUSE_STRAP);
16279
16280                 if (found & SFUSE_STRAP_DDIB_DETECTED)
16281                         intel_ddi_init(dev_priv, PORT_B);
16282                 if (found & SFUSE_STRAP_DDIC_DETECTED)
16283                         intel_ddi_init(dev_priv, PORT_C);
16284                 if (found & SFUSE_STRAP_DDID_DETECTED)
16285                         intel_ddi_init(dev_priv, PORT_D);
16286                 if (found & SFUSE_STRAP_DDIF_DETECTED)
16287                         intel_ddi_init(dev_priv, PORT_F);
16288                 /*
16289                  * On SKL we don't have a way to detect DDI-E so we rely on VBT.
16290                  */
16291                 if (IS_GEN9_BC(dev_priv) &&
16292                     intel_bios_is_port_present(dev_priv, PORT_E))
16293                         intel_ddi_init(dev_priv, PORT_E);
16294
16295         } else if (HAS_PCH_SPLIT(dev_priv)) {
16296                 int found;
16297
16298                 /*
16299                  * intel_edp_init_connector() depends on this completing first,
16300                  * to prevent the registration of both eDP and LVDS and the
16301                  * incorrect sharing of the PPS.
16302                  */
16303                 intel_lvds_init(dev_priv);
16304                 intel_crt_init(dev_priv);
16305
16306                 dpd_is_edp = intel_dp_is_port_edp(dev_priv, PORT_D);
16307
16308                 if (ilk_has_edp_a(dev_priv))
16309                         intel_dp_init(dev_priv, DP_A, PORT_A);
16310
16311                 if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) {
16312                         /* PCH SDVOB multiplex with HDMIB */
16313                         found = intel_sdvo_init(dev_priv, PCH_SDVOB, PORT_B);
16314                         if (!found)
16315                                 intel_hdmi_init(dev_priv, PCH_HDMIB, PORT_B);
16316                         if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
16317                                 intel_dp_init(dev_priv, PCH_DP_B, PORT_B);
16318                 }
16319
16320                 if (I915_READ(PCH_HDMIC) & SDVO_DETECTED)
16321                         intel_hdmi_init(dev_priv, PCH_HDMIC, PORT_C);
16322
16323                 if (!dpd_is_edp && I915_READ(PCH_HDMID) & SDVO_DETECTED)
16324                         intel_hdmi_init(dev_priv, PCH_HDMID, PORT_D);
16325
16326                 if (I915_READ(PCH_DP_C) & DP_DETECTED)
16327                         intel_dp_init(dev_priv, PCH_DP_C, PORT_C);
16328
16329                 if (I915_READ(PCH_DP_D) & DP_DETECTED)
16330                         intel_dp_init(dev_priv, PCH_DP_D, PORT_D);
16331         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
16332                 bool has_edp, has_port;
16333
16334                 if (IS_VALLEYVIEW(dev_priv) && dev_priv->vbt.int_crt_support)
16335                         intel_crt_init(dev_priv);
16336
16337                 /*
16338                  * The DP_DETECTED bit is the latched state of the DDC
16339                  * SDA pin at boot. However since eDP doesn't require DDC
16340                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
16341                  * eDP ports may have been muxed to an alternate function.
16342                  * Thus we can't rely on the DP_DETECTED bit alone to detect
16343                  * eDP ports. Consult the VBT as well as DP_DETECTED to
16344                  * detect eDP ports.
16345                  *
16346                  * Sadly the straps seem to be missing sometimes even for HDMI
16347                  * ports (eg. on Voyo V3 - CHT x7-Z8700), so check both strap
16348                  * and VBT for the presence of the port. Additionally we can't
16349                  * trust the port type the VBT declares as we've seen at least
16350                  * HDMI ports that the VBT claim are DP or eDP.
16351                  */
16352                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_B);
16353                 has_port = intel_bios_is_port_present(dev_priv, PORT_B);
16354                 if (I915_READ(VLV_DP_B) & DP_DETECTED || has_port)
16355                         has_edp &= intel_dp_init(dev_priv, VLV_DP_B, PORT_B);
16356                 if ((I915_READ(VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp)
16357                         intel_hdmi_init(dev_priv, VLV_HDMIB, PORT_B);
16358
16359                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_C);
16360                 has_port = intel_bios_is_port_present(dev_priv, PORT_C);
16361                 if (I915_READ(VLV_DP_C) & DP_DETECTED || has_port)
16362                         has_edp &= intel_dp_init(dev_priv, VLV_DP_C, PORT_C);
16363                 if ((I915_READ(VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp)
16364                         intel_hdmi_init(dev_priv, VLV_HDMIC, PORT_C);
16365
16366                 if (IS_CHERRYVIEW(dev_priv)) {
16367                         /*
16368                          * eDP not supported on port D,
16369                          * so no need to worry about it
16370                          */
16371                         has_port = intel_bios_is_port_present(dev_priv, PORT_D);
16372                         if (I915_READ(CHV_DP_D) & DP_DETECTED || has_port)
16373                                 intel_dp_init(dev_priv, CHV_DP_D, PORT_D);
16374                         if (I915_READ(CHV_HDMID) & SDVO_DETECTED || has_port)
16375                                 intel_hdmi_init(dev_priv, CHV_HDMID, PORT_D);
16376                 }
16377
16378                 vlv_dsi_init(dev_priv);
16379         } else if (IS_PINEVIEW(dev_priv)) {
16380                 intel_lvds_init(dev_priv);
16381                 intel_crt_init(dev_priv);
16382         } else if (IS_GEN_RANGE(dev_priv, 3, 4)) {
16383                 bool found = false;
16384
16385                 if (IS_MOBILE(dev_priv))
16386                         intel_lvds_init(dev_priv);
16387
16388                 intel_crt_init(dev_priv);
16389
16390                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
16391                         DRM_DEBUG_KMS("probing SDVOB\n");
16392                         found = intel_sdvo_init(dev_priv, GEN3_SDVOB, PORT_B);
16393                         if (!found && IS_G4X(dev_priv)) {
16394                                 DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
16395                                 intel_hdmi_init(dev_priv, GEN4_HDMIB, PORT_B);
16396                         }
16397
16398                         if (!found && IS_G4X(dev_priv))
16399                                 intel_dp_init(dev_priv, DP_B, PORT_B);
16400                 }
16401
16402                 /* Before G4X SDVOC doesn't have its own detect register */
16403
16404                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
16405                         DRM_DEBUG_KMS("probing SDVOC\n");
16406                         found = intel_sdvo_init(dev_priv, GEN3_SDVOC, PORT_C);
16407                 }
16408
16409                 if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) {
16410
16411                         if (IS_G4X(dev_priv)) {
16412                                 DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
16413                                 intel_hdmi_init(dev_priv, GEN4_HDMIC, PORT_C);
16414                         }
16415                         if (IS_G4X(dev_priv))
16416                                 intel_dp_init(dev_priv, DP_C, PORT_C);
16417                 }
16418
16419                 if (IS_G4X(dev_priv) && (I915_READ(DP_D) & DP_DETECTED))
16420                         intel_dp_init(dev_priv, DP_D, PORT_D);
16421
16422                 if (SUPPORTS_TV(dev_priv))
16423                         intel_tv_init(dev_priv);
16424         } else if (IS_GEN(dev_priv, 2)) {
16425                 if (IS_I85X(dev_priv))
16426                         intel_lvds_init(dev_priv);
16427
16428                 intel_crt_init(dev_priv);
16429                 intel_dvo_init(dev_priv);
16430         }
16431
16432         intel_psr_init(dev_priv);
16433
16434         for_each_intel_encoder(&dev_priv->drm, encoder) {
16435                 encoder->base.possible_crtcs =
16436                         intel_encoder_possible_crtcs(encoder);
16437                 encoder->base.possible_clones =
16438                         intel_encoder_possible_clones(encoder);
16439         }
16440
16441         intel_init_pch_refclk(dev_priv);
16442
16443         drm_helper_move_panel_connectors_to_head(&dev_priv->drm);
16444 }
16445
16446 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
16447 {
16448         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
16449
16450         drm_framebuffer_cleanup(fb);
16451         intel_frontbuffer_put(intel_fb->frontbuffer);
16452
16453         kfree(intel_fb);
16454 }
16455
16456 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
16457                                                 struct drm_file *file,
16458                                                 unsigned int *handle)
16459 {
16460         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
16461
16462         if (obj->userptr.mm) {
16463                 DRM_DEBUG("attempting to use a userptr for a framebuffer, denied\n");
16464                 return -EINVAL;
16465         }
16466
16467         return drm_gem_handle_create(file, &obj->base, handle);
16468 }
16469
16470 static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
16471                                         struct drm_file *file,
16472                                         unsigned flags, unsigned color,
16473                                         struct drm_clip_rect *clips,
16474                                         unsigned num_clips)
16475 {
16476         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
16477
16478         i915_gem_object_flush_if_display(obj);
16479         intel_frontbuffer_flush(to_intel_frontbuffer(fb), ORIGIN_DIRTYFB);
16480
16481         return 0;
16482 }
16483
16484 static const struct drm_framebuffer_funcs intel_fb_funcs = {
16485         .destroy = intel_user_framebuffer_destroy,
16486         .create_handle = intel_user_framebuffer_create_handle,
16487         .dirty = intel_user_framebuffer_dirty,
16488 };
16489
16490 static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
16491                                   struct drm_i915_gem_object *obj,
16492                                   struct drm_mode_fb_cmd2 *mode_cmd)
16493 {
16494         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
16495         struct drm_framebuffer *fb = &intel_fb->base;
16496         u32 max_stride;
16497         unsigned int tiling, stride;
16498         int ret = -EINVAL;
16499         int i;
16500
16501         intel_fb->frontbuffer = intel_frontbuffer_get(obj);
16502         if (!intel_fb->frontbuffer)
16503                 return -ENOMEM;
16504
16505         i915_gem_object_lock(obj);
16506         tiling = i915_gem_object_get_tiling(obj);
16507         stride = i915_gem_object_get_stride(obj);
16508         i915_gem_object_unlock(obj);
16509
16510         if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
16511                 /*
16512                  * If there's a fence, enforce that
16513                  * the fb modifier and tiling mode match.
16514                  */
16515                 if (tiling != I915_TILING_NONE &&
16516                     tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
16517                         DRM_DEBUG_KMS("tiling_mode doesn't match fb modifier\n");
16518                         goto err;
16519                 }
16520         } else {
16521                 if (tiling == I915_TILING_X) {
16522                         mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
16523                 } else if (tiling == I915_TILING_Y) {
16524                         DRM_DEBUG_KMS("No Y tiling for legacy addfb\n");
16525                         goto err;
16526                 }
16527         }
16528
16529         if (!drm_any_plane_has_format(&dev_priv->drm,
16530                                       mode_cmd->pixel_format,
16531                                       mode_cmd->modifier[0])) {
16532                 struct drm_format_name_buf format_name;
16533
16534                 DRM_DEBUG_KMS("unsupported pixel format %s / modifier 0x%llx\n",
16535                               drm_get_format_name(mode_cmd->pixel_format,
16536                                                   &format_name),
16537                               mode_cmd->modifier[0]);
16538                 goto err;
16539         }
16540
16541         /*
16542          * gen2/3 display engine uses the fence if present,
16543          * so the tiling mode must match the fb modifier exactly.
16544          */
16545         if (INTEL_GEN(dev_priv) < 4 &&
16546             tiling != intel_fb_modifier_to_tiling(mode_cmd->modifier[0])) {
16547                 DRM_DEBUG_KMS("tiling_mode must match fb modifier exactly on gen2/3\n");
16548                 goto err;
16549         }
16550
16551         max_stride = intel_fb_max_stride(dev_priv, mode_cmd->pixel_format,
16552                                          mode_cmd->modifier[0]);
16553         if (mode_cmd->pitches[0] > max_stride) {
16554                 DRM_DEBUG_KMS("%s pitch (%u) must be at most %d\n",
16555                               mode_cmd->modifier[0] != DRM_FORMAT_MOD_LINEAR ?
16556                               "tiled" : "linear",
16557                               mode_cmd->pitches[0], max_stride);
16558                 goto err;
16559         }
16560
16561         /*
16562          * If there's a fence, enforce that
16563          * the fb pitch and fence stride match.
16564          */
16565         if (tiling != I915_TILING_NONE && mode_cmd->pitches[0] != stride) {
16566                 DRM_DEBUG_KMS("pitch (%d) must match tiling stride (%d)\n",
16567                               mode_cmd->pitches[0], stride);
16568                 goto err;
16569         }
16570
16571         /* FIXME need to adjust LINOFF/TILEOFF accordingly. */
16572         if (mode_cmd->offsets[0] != 0)
16573                 goto err;
16574
16575         drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd);
16576
16577         for (i = 0; i < fb->format->num_planes; i++) {
16578                 u32 stride_alignment;
16579
16580                 if (mode_cmd->handles[i] != mode_cmd->handles[0]) {
16581                         DRM_DEBUG_KMS("bad plane %d handle\n", i);
16582                         goto err;
16583                 }
16584
16585                 stride_alignment = intel_fb_stride_alignment(fb, i);
16586                 if (fb->pitches[i] & (stride_alignment - 1)) {
16587                         DRM_DEBUG_KMS("plane %d pitch (%d) must be at least %u byte aligned\n",
16588                                       i, fb->pitches[i], stride_alignment);
16589                         goto err;
16590                 }
16591
16592                 if (is_gen12_ccs_plane(fb, i)) {
16593                         int ccs_aux_stride = gen12_ccs_aux_stride(fb, i);
16594
16595                         if (fb->pitches[i] != ccs_aux_stride) {
16596                                 DRM_DEBUG_KMS("ccs aux plane %d pitch (%d) must be %d\n",
16597                                               i,
16598                                               fb->pitches[i], ccs_aux_stride);
16599                                 goto err;
16600                         }
16601                 }
16602
16603                 fb->obj[i] = &obj->base;
16604         }
16605
16606         ret = intel_fill_fb_info(dev_priv, fb);
16607         if (ret)
16608                 goto err;
16609
16610         ret = drm_framebuffer_init(&dev_priv->drm, fb, &intel_fb_funcs);
16611         if (ret) {
16612                 DRM_ERROR("framebuffer init failed %d\n", ret);
16613                 goto err;
16614         }
16615
16616         return 0;
16617
16618 err:
16619         intel_frontbuffer_put(intel_fb->frontbuffer);
16620         return ret;
16621 }
16622
16623 static struct drm_framebuffer *
16624 intel_user_framebuffer_create(struct drm_device *dev,
16625                               struct drm_file *filp,
16626                               const struct drm_mode_fb_cmd2 *user_mode_cmd)
16627 {
16628         struct drm_framebuffer *fb;
16629         struct drm_i915_gem_object *obj;
16630         struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
16631
16632         obj = i915_gem_object_lookup(filp, mode_cmd.handles[0]);
16633         if (!obj)
16634                 return ERR_PTR(-ENOENT);
16635
16636         fb = intel_framebuffer_create(obj, &mode_cmd);
16637         i915_gem_object_put(obj);
16638
16639         return fb;
16640 }
16641
16642 static void intel_atomic_state_free(struct drm_atomic_state *state)
16643 {
16644         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
16645
16646         drm_atomic_state_default_release(state);
16647
16648         i915_sw_fence_fini(&intel_state->commit_ready);
16649
16650         kfree(state);
16651 }
16652
16653 static enum drm_mode_status
16654 intel_mode_valid(struct drm_device *dev,
16655                  const struct drm_display_mode *mode)
16656 {
16657         struct drm_i915_private *dev_priv = to_i915(dev);
16658         int hdisplay_max, htotal_max;
16659         int vdisplay_max, vtotal_max;
16660
16661         /*
16662          * Can't reject DBLSCAN here because Xorg ddxen can add piles
16663          * of DBLSCAN modes to the output's mode list when they detect
16664          * the scaling mode property on the connector. And they don't
16665          * ask the kernel to validate those modes in any way until
16666          * modeset time at which point the client gets a protocol error.
16667          * So in order to not upset those clients we silently ignore the
16668          * DBLSCAN flag on such connectors. For other connectors we will
16669          * reject modes with the DBLSCAN flag in encoder->compute_config().
16670          * And we always reject DBLSCAN modes in connector->mode_valid()
16671          * as we never want such modes on the connector's mode list.
16672          */
16673
16674         if (mode->vscan > 1)
16675                 return MODE_NO_VSCAN;
16676
16677         if (mode->flags & DRM_MODE_FLAG_HSKEW)
16678                 return MODE_H_ILLEGAL;
16679
16680         if (mode->flags & (DRM_MODE_FLAG_CSYNC |
16681                            DRM_MODE_FLAG_NCSYNC |
16682                            DRM_MODE_FLAG_PCSYNC))
16683                 return MODE_HSYNC;
16684
16685         if (mode->flags & (DRM_MODE_FLAG_BCAST |
16686                            DRM_MODE_FLAG_PIXMUX |
16687                            DRM_MODE_FLAG_CLKDIV2))
16688                 return MODE_BAD;
16689
16690         /* Transcoder timing limits */
16691         if (INTEL_GEN(dev_priv) >= 11) {
16692                 hdisplay_max = 16384;
16693                 vdisplay_max = 8192;
16694                 htotal_max = 16384;
16695                 vtotal_max = 8192;
16696         } else if (INTEL_GEN(dev_priv) >= 9 ||
16697                    IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
16698                 hdisplay_max = 8192; /* FDI max 4096 handled elsewhere */
16699                 vdisplay_max = 4096;
16700                 htotal_max = 8192;
16701                 vtotal_max = 8192;
16702         } else if (INTEL_GEN(dev_priv) >= 3) {
16703                 hdisplay_max = 4096;
16704                 vdisplay_max = 4096;
16705                 htotal_max = 8192;
16706                 vtotal_max = 8192;
16707         } else {
16708                 hdisplay_max = 2048;
16709                 vdisplay_max = 2048;
16710                 htotal_max = 4096;
16711                 vtotal_max = 4096;
16712         }
16713
16714         if (mode->hdisplay > hdisplay_max ||
16715             mode->hsync_start > htotal_max ||
16716             mode->hsync_end > htotal_max ||
16717             mode->htotal > htotal_max)
16718                 return MODE_H_ILLEGAL;
16719
16720         if (mode->vdisplay > vdisplay_max ||
16721             mode->vsync_start > vtotal_max ||
16722             mode->vsync_end > vtotal_max ||
16723             mode->vtotal > vtotal_max)
16724                 return MODE_V_ILLEGAL;
16725
16726         if (INTEL_GEN(dev_priv) >= 5) {
16727                 if (mode->hdisplay < 64 ||
16728                     mode->htotal - mode->hdisplay < 32)
16729                         return MODE_H_ILLEGAL;
16730
16731                 if (mode->vtotal - mode->vdisplay < 5)
16732                         return MODE_V_ILLEGAL;
16733         } else {
16734                 if (mode->htotal - mode->hdisplay < 32)
16735                         return MODE_H_ILLEGAL;
16736
16737                 if (mode->vtotal - mode->vdisplay < 3)
16738                         return MODE_V_ILLEGAL;
16739         }
16740
16741         return MODE_OK;
16742 }
16743
16744 enum drm_mode_status
16745 intel_mode_valid_max_plane_size(struct drm_i915_private *dev_priv,
16746                                 const struct drm_display_mode *mode)
16747 {
16748         int plane_width_max, plane_height_max;
16749
16750         /*
16751          * intel_mode_valid() should be
16752          * sufficient on older platforms.
16753          */
16754         if (INTEL_GEN(dev_priv) < 9)
16755                 return MODE_OK;
16756
16757         /*
16758          * Most people will probably want a fullscreen
16759          * plane so let's not advertize modes that are
16760          * too big for that.
16761          */
16762         if (INTEL_GEN(dev_priv) >= 11) {
16763                 plane_width_max = 5120;
16764                 plane_height_max = 4320;
16765         } else {
16766                 plane_width_max = 5120;
16767                 plane_height_max = 4096;
16768         }
16769
16770         if (mode->hdisplay > plane_width_max)
16771                 return MODE_H_ILLEGAL;
16772
16773         if (mode->vdisplay > plane_height_max)
16774                 return MODE_V_ILLEGAL;
16775
16776         return MODE_OK;
16777 }
16778
16779 static const struct drm_mode_config_funcs intel_mode_funcs = {
16780         .fb_create = intel_user_framebuffer_create,
16781         .get_format_info = intel_get_format_info,
16782         .output_poll_changed = intel_fbdev_output_poll_changed,
16783         .mode_valid = intel_mode_valid,
16784         .atomic_check = intel_atomic_check,
16785         .atomic_commit = intel_atomic_commit,
16786         .atomic_state_alloc = intel_atomic_state_alloc,
16787         .atomic_state_clear = intel_atomic_state_clear,
16788         .atomic_state_free = intel_atomic_state_free,
16789 };
16790
16791 /**
16792  * intel_init_display_hooks - initialize the display modesetting hooks
16793  * @dev_priv: device private
16794  */
16795 void intel_init_display_hooks(struct drm_i915_private *dev_priv)
16796 {
16797         intel_init_cdclk_hooks(dev_priv);
16798
16799         if (INTEL_GEN(dev_priv) >= 9) {
16800                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
16801                 dev_priv->display.get_initial_plane_config =
16802                         skylake_get_initial_plane_config;
16803                 dev_priv->display.crtc_compute_clock =
16804                         haswell_crtc_compute_clock;
16805                 dev_priv->display.crtc_enable = haswell_crtc_enable;
16806                 dev_priv->display.crtc_disable = haswell_crtc_disable;
16807         } else if (HAS_DDI(dev_priv)) {
16808                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
16809                 dev_priv->display.get_initial_plane_config =
16810                         i9xx_get_initial_plane_config;
16811                 dev_priv->display.crtc_compute_clock =
16812                         haswell_crtc_compute_clock;
16813                 dev_priv->display.crtc_enable = haswell_crtc_enable;
16814                 dev_priv->display.crtc_disable = haswell_crtc_disable;
16815         } else if (HAS_PCH_SPLIT(dev_priv)) {
16816                 dev_priv->display.get_pipe_config = ironlake_get_pipe_config;
16817                 dev_priv->display.get_initial_plane_config =
16818                         i9xx_get_initial_plane_config;
16819                 dev_priv->display.crtc_compute_clock =
16820                         ironlake_crtc_compute_clock;
16821                 dev_priv->display.crtc_enable = ironlake_crtc_enable;
16822                 dev_priv->display.crtc_disable = ironlake_crtc_disable;
16823         } else if (IS_CHERRYVIEW(dev_priv)) {
16824                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
16825                 dev_priv->display.get_initial_plane_config =
16826                         i9xx_get_initial_plane_config;
16827                 dev_priv->display.crtc_compute_clock = chv_crtc_compute_clock;
16828                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
16829                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
16830         } else if (IS_VALLEYVIEW(dev_priv)) {
16831                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
16832                 dev_priv->display.get_initial_plane_config =
16833                         i9xx_get_initial_plane_config;
16834                 dev_priv->display.crtc_compute_clock = vlv_crtc_compute_clock;
16835                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
16836                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
16837         } else if (IS_G4X(dev_priv)) {
16838                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
16839                 dev_priv->display.get_initial_plane_config =
16840                         i9xx_get_initial_plane_config;
16841                 dev_priv->display.crtc_compute_clock = g4x_crtc_compute_clock;
16842                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
16843                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
16844         } else if (IS_PINEVIEW(dev_priv)) {
16845                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
16846                 dev_priv->display.get_initial_plane_config =
16847                         i9xx_get_initial_plane_config;
16848                 dev_priv->display.crtc_compute_clock = pnv_crtc_compute_clock;
16849                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
16850                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
16851         } else if (!IS_GEN(dev_priv, 2)) {
16852                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
16853                 dev_priv->display.get_initial_plane_config =
16854                         i9xx_get_initial_plane_config;
16855                 dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
16856                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
16857                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
16858         } else {
16859                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
16860                 dev_priv->display.get_initial_plane_config =
16861                         i9xx_get_initial_plane_config;
16862                 dev_priv->display.crtc_compute_clock = i8xx_crtc_compute_clock;
16863                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
16864                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
16865         }
16866
16867         if (IS_GEN(dev_priv, 5)) {
16868                 dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
16869         } else if (IS_GEN(dev_priv, 6)) {
16870                 dev_priv->display.fdi_link_train = gen6_fdi_link_train;
16871         } else if (IS_IVYBRIDGE(dev_priv)) {
16872                 /* FIXME: detect B0+ stepping and use auto training */
16873                 dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
16874         }
16875
16876         if (INTEL_GEN(dev_priv) >= 9)
16877                 dev_priv->display.commit_modeset_enables = skl_commit_modeset_enables;
16878         else
16879                 dev_priv->display.commit_modeset_enables = intel_commit_modeset_enables;
16880
16881 }
16882
16883 void intel_modeset_init_hw(struct drm_i915_private *i915)
16884 {
16885         intel_update_cdclk(i915);
16886         intel_dump_cdclk_state(&i915->cdclk.hw, "Current CDCLK");
16887         i915->cdclk.logical = i915->cdclk.actual = i915->cdclk.hw;
16888 }
16889
16890 /*
16891  * Calculate what we think the watermarks should be for the state we've read
16892  * out of the hardware and then immediately program those watermarks so that
16893  * we ensure the hardware settings match our internal state.
16894  *
16895  * We can calculate what we think WM's should be by creating a duplicate of the
16896  * current state (which was constructed during hardware readout) and running it
16897  * through the atomic check code to calculate new watermark values in the
16898  * state object.
16899  */
16900 static void sanitize_watermarks(struct drm_device *dev)
16901 {
16902         struct drm_i915_private *dev_priv = to_i915(dev);
16903         struct drm_atomic_state *state;
16904         struct intel_atomic_state *intel_state;
16905         struct intel_crtc *crtc;
16906         struct intel_crtc_state *crtc_state;
16907         struct drm_modeset_acquire_ctx ctx;
16908         int ret;
16909         int i;
16910
16911         /* Only supported on platforms that use atomic watermark design */
16912         if (!dev_priv->display.optimize_watermarks)
16913                 return;
16914
16915         /*
16916          * We need to hold connection_mutex before calling duplicate_state so
16917          * that the connector loop is protected.
16918          */
16919         drm_modeset_acquire_init(&ctx, 0);
16920 retry:
16921         ret = drm_modeset_lock_all_ctx(dev, &ctx);
16922         if (ret == -EDEADLK) {
16923                 drm_modeset_backoff(&ctx);
16924                 goto retry;
16925         } else if (WARN_ON(ret)) {
16926                 goto fail;
16927         }
16928
16929         state = drm_atomic_helper_duplicate_state(dev, &ctx);
16930         if (WARN_ON(IS_ERR(state)))
16931                 goto fail;
16932
16933         intel_state = to_intel_atomic_state(state);
16934
16935         /*
16936          * Hardware readout is the only time we don't want to calculate
16937          * intermediate watermarks (since we don't trust the current
16938          * watermarks).
16939          */
16940         if (!HAS_GMCH(dev_priv))
16941                 intel_state->skip_intermediate_wm = true;
16942
16943         ret = intel_atomic_check(dev, state);
16944         if (ret) {
16945                 /*
16946                  * If we fail here, it means that the hardware appears to be
16947                  * programmed in a way that shouldn't be possible, given our
16948                  * understanding of watermark requirements.  This might mean a
16949                  * mistake in the hardware readout code or a mistake in the
16950                  * watermark calculations for a given platform.  Raise a WARN
16951                  * so that this is noticeable.
16952                  *
16953                  * If this actually happens, we'll have to just leave the
16954                  * BIOS-programmed watermarks untouched and hope for the best.
16955                  */
16956                 WARN(true, "Could not determine valid watermarks for inherited state\n");
16957                 goto put_state;
16958         }
16959
16960         /* Write calculated watermark values back */
16961         for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
16962                 crtc_state->wm.need_postvbl_update = true;
16963                 dev_priv->display.optimize_watermarks(intel_state, crtc);
16964
16965                 to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
16966         }
16967
16968 put_state:
16969         drm_atomic_state_put(state);
16970 fail:
16971         drm_modeset_drop_locks(&ctx);
16972         drm_modeset_acquire_fini(&ctx);
16973 }
16974
16975 static void intel_update_fdi_pll_freq(struct drm_i915_private *dev_priv)
16976 {
16977         if (IS_GEN(dev_priv, 5)) {
16978                 u32 fdi_pll_clk =
16979                         I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK;
16980
16981                 dev_priv->fdi_pll_freq = (fdi_pll_clk + 2) * 10000;
16982         } else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv)) {
16983                 dev_priv->fdi_pll_freq = 270000;
16984         } else {
16985                 return;
16986         }
16987
16988         DRM_DEBUG_DRIVER("FDI PLL freq=%d\n", dev_priv->fdi_pll_freq);
16989 }
16990
16991 static int intel_initial_commit(struct drm_device *dev)
16992 {
16993         struct drm_atomic_state *state = NULL;
16994         struct drm_modeset_acquire_ctx ctx;
16995         struct intel_crtc *crtc;
16996         int ret = 0;
16997
16998         state = drm_atomic_state_alloc(dev);
16999         if (!state)
17000                 return -ENOMEM;
17001
17002         drm_modeset_acquire_init(&ctx, 0);
17003
17004 retry:
17005         state->acquire_ctx = &ctx;
17006
17007         for_each_intel_crtc(dev, crtc) {
17008                 struct intel_crtc_state *crtc_state =
17009                         intel_atomic_get_crtc_state(state, crtc);
17010
17011                 if (IS_ERR(crtc_state)) {
17012                         ret = PTR_ERR(crtc_state);
17013                         goto out;
17014                 }
17015
17016                 if (crtc_state->hw.active) {
17017                         ret = drm_atomic_add_affected_planes(state, &crtc->base);
17018                         if (ret)
17019                                 goto out;
17020
17021                         /*
17022                          * FIXME hack to force a LUT update to avoid the
17023                          * plane update forcing the pipe gamma on without
17024                          * having a proper LUT loaded. Remove once we
17025                          * have readout for pipe gamma enable.
17026                          */
17027                         crtc_state->uapi.color_mgmt_changed = true;
17028                 }
17029         }
17030
17031         ret = drm_atomic_commit(state);
17032
17033 out:
17034         if (ret == -EDEADLK) {
17035                 drm_atomic_state_clear(state);
17036                 drm_modeset_backoff(&ctx);
17037                 goto retry;
17038         }
17039
17040         drm_atomic_state_put(state);
17041
17042         drm_modeset_drop_locks(&ctx);
17043         drm_modeset_acquire_fini(&ctx);
17044
17045         return ret;
17046 }
17047
17048 static void intel_mode_config_init(struct drm_i915_private *i915)
17049 {
17050         struct drm_mode_config *mode_config = &i915->drm.mode_config;
17051
17052         drm_mode_config_init(&i915->drm);
17053
17054         mode_config->min_width = 0;
17055         mode_config->min_height = 0;
17056
17057         mode_config->preferred_depth = 24;
17058         mode_config->prefer_shadow = 1;
17059
17060         mode_config->allow_fb_modifiers = true;
17061
17062         mode_config->funcs = &intel_mode_funcs;
17063
17064         /*
17065          * Maximum framebuffer dimensions, chosen to match
17066          * the maximum render engine surface size on gen4+.
17067          */
17068         if (INTEL_GEN(i915) >= 7) {
17069                 mode_config->max_width = 16384;
17070                 mode_config->max_height = 16384;
17071         } else if (INTEL_GEN(i915) >= 4) {
17072                 mode_config->max_width = 8192;
17073                 mode_config->max_height = 8192;
17074         } else if (IS_GEN(i915, 3)) {
17075                 mode_config->max_width = 4096;
17076                 mode_config->max_height = 4096;
17077         } else {
17078                 mode_config->max_width = 2048;
17079                 mode_config->max_height = 2048;
17080         }
17081
17082         if (IS_I845G(i915) || IS_I865G(i915)) {
17083                 mode_config->cursor_width = IS_I845G(i915) ? 64 : 512;
17084                 mode_config->cursor_height = 1023;
17085         } else if (IS_GEN(i915, 2)) {
17086                 mode_config->cursor_width = 64;
17087                 mode_config->cursor_height = 64;
17088         } else {
17089                 mode_config->cursor_width = 256;
17090                 mode_config->cursor_height = 256;
17091         }
17092 }
17093
17094 int intel_modeset_init(struct drm_i915_private *i915)
17095 {
17096         struct drm_device *dev = &i915->drm;
17097         enum pipe pipe;
17098         struct intel_crtc *crtc;
17099         int ret;
17100
17101         i915->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0);
17102         i915->flip_wq = alloc_workqueue("i915_flip", WQ_HIGHPRI |
17103                                         WQ_UNBOUND, WQ_UNBOUND_MAX_ACTIVE);
17104
17105         intel_mode_config_init(i915);
17106
17107         ret = intel_bw_init(i915);
17108         if (ret)
17109                 return ret;
17110
17111         init_llist_head(&i915->atomic_helper.free_list);
17112         INIT_WORK(&i915->atomic_helper.free_work,
17113                   intel_atomic_helper_free_state_worker);
17114
17115         intel_init_quirks(i915);
17116
17117         intel_fbc_init(i915);
17118
17119         intel_init_pm(i915);
17120
17121         intel_panel_sanitize_ssc(i915);
17122
17123         intel_gmbus_setup(i915);
17124
17125         DRM_DEBUG_KMS("%d display pipe%s available.\n",
17126                       INTEL_NUM_PIPES(i915),
17127                       INTEL_NUM_PIPES(i915) > 1 ? "s" : "");
17128
17129         if (HAS_DISPLAY(i915) && INTEL_DISPLAY_ENABLED(i915)) {
17130                 for_each_pipe(i915, pipe) {
17131                         ret = intel_crtc_init(i915, pipe);
17132                         if (ret) {
17133                                 drm_mode_config_cleanup(dev);
17134                                 return ret;
17135                         }
17136                 }
17137         }
17138
17139         intel_shared_dpll_init(dev);
17140         intel_update_fdi_pll_freq(i915);
17141
17142         intel_update_czclk(i915);
17143         intel_modeset_init_hw(i915);
17144
17145         intel_hdcp_component_init(i915);
17146
17147         if (i915->max_cdclk_freq == 0)
17148                 intel_update_max_cdclk(i915);
17149
17150         /* Just disable it once at startup */
17151         intel_vga_disable(i915);
17152         intel_setup_outputs(i915);
17153
17154         drm_modeset_lock_all(dev);
17155         intel_modeset_setup_hw_state(dev, dev->mode_config.acquire_ctx);
17156         drm_modeset_unlock_all(dev);
17157
17158         for_each_intel_crtc(dev, crtc) {
17159                 struct intel_initial_plane_config plane_config = {};
17160
17161                 if (!crtc->active)
17162                         continue;
17163
17164                 /*
17165                  * Note that reserving the BIOS fb up front prevents us
17166                  * from stuffing other stolen allocations like the ring
17167                  * on top.  This prevents some ugliness at boot time, and
17168                  * can even allow for smooth boot transitions if the BIOS
17169                  * fb is large enough for the active pipe configuration.
17170                  */
17171                 i915->display.get_initial_plane_config(crtc, &plane_config);
17172
17173                 /*
17174                  * If the fb is shared between multiple heads, we'll
17175                  * just get the first one.
17176                  */
17177                 intel_find_initial_plane_obj(crtc, &plane_config);
17178         }
17179
17180         /*
17181          * Make sure hardware watermarks really match the state we read out.
17182          * Note that we need to do this after reconstructing the BIOS fb's
17183          * since the watermark calculation done here will use pstate->fb.
17184          */
17185         if (!HAS_GMCH(i915))
17186                 sanitize_watermarks(dev);
17187
17188         /*
17189          * Force all active planes to recompute their states. So that on
17190          * mode_setcrtc after probe, all the intel_plane_state variables
17191          * are already calculated and there is no assert_plane warnings
17192          * during bootup.
17193          */
17194         ret = intel_initial_commit(dev);
17195         if (ret)
17196                 DRM_DEBUG_KMS("Initial commit in probe failed.\n");
17197
17198         return 0;
17199 }
17200
17201 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
17202 {
17203         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
17204         /* 640x480@60Hz, ~25175 kHz */
17205         struct dpll clock = {
17206                 .m1 = 18,
17207                 .m2 = 7,
17208                 .p1 = 13,
17209                 .p2 = 4,
17210                 .n = 2,
17211         };
17212         u32 dpll, fp;
17213         int i;
17214
17215         WARN_ON(i9xx_calc_dpll_params(48000, &clock) != 25154);
17216
17217         DRM_DEBUG_KMS("enabling pipe %c due to force quirk (vco=%d dot=%d)\n",
17218                       pipe_name(pipe), clock.vco, clock.dot);
17219
17220         fp = i9xx_dpll_compute_fp(&clock);
17221         dpll = DPLL_DVO_2X_MODE |
17222                 DPLL_VGA_MODE_DIS |
17223                 ((clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT) |
17224                 PLL_P2_DIVIDE_BY_4 |
17225                 PLL_REF_INPUT_DREFCLK |
17226                 DPLL_VCO_ENABLE;
17227
17228         I915_WRITE(FP0(pipe), fp);
17229         I915_WRITE(FP1(pipe), fp);
17230
17231         I915_WRITE(HTOTAL(pipe), (640 - 1) | ((800 - 1) << 16));
17232         I915_WRITE(HBLANK(pipe), (640 - 1) | ((800 - 1) << 16));
17233         I915_WRITE(HSYNC(pipe), (656 - 1) | ((752 - 1) << 16));
17234         I915_WRITE(VTOTAL(pipe), (480 - 1) | ((525 - 1) << 16));
17235         I915_WRITE(VBLANK(pipe), (480 - 1) | ((525 - 1) << 16));
17236         I915_WRITE(VSYNC(pipe), (490 - 1) | ((492 - 1) << 16));
17237         I915_WRITE(PIPESRC(pipe), ((640 - 1) << 16) | (480 - 1));
17238
17239         /*
17240          * Apparently we need to have VGA mode enabled prior to changing
17241          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
17242          * dividers, even though the register value does change.
17243          */
17244         I915_WRITE(DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS);
17245         I915_WRITE(DPLL(pipe), dpll);
17246
17247         /* Wait for the clocks to stabilize. */
17248         POSTING_READ(DPLL(pipe));
17249         udelay(150);
17250
17251         /* The pixel multiplier can only be updated once the
17252          * DPLL is enabled and the clocks are stable.
17253          *
17254          * So write it again.
17255          */
17256         I915_WRITE(DPLL(pipe), dpll);
17257
17258         /* We do this three times for luck */
17259         for (i = 0; i < 3 ; i++) {
17260                 I915_WRITE(DPLL(pipe), dpll);
17261                 POSTING_READ(DPLL(pipe));
17262                 udelay(150); /* wait for warmup */
17263         }
17264
17265         I915_WRITE(PIPECONF(pipe), PIPECONF_ENABLE | PIPECONF_PROGRESSIVE);
17266         POSTING_READ(PIPECONF(pipe));
17267
17268         intel_wait_for_pipe_scanline_moving(crtc);
17269 }
17270
17271 void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
17272 {
17273         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
17274
17275         DRM_DEBUG_KMS("disabling pipe %c due to force quirk\n",
17276                       pipe_name(pipe));
17277
17278         WARN_ON(I915_READ(DSPCNTR(PLANE_A)) & DISPLAY_PLANE_ENABLE);
17279         WARN_ON(I915_READ(DSPCNTR(PLANE_B)) & DISPLAY_PLANE_ENABLE);
17280         WARN_ON(I915_READ(DSPCNTR(PLANE_C)) & DISPLAY_PLANE_ENABLE);
17281         WARN_ON(I915_READ(CURCNTR(PIPE_A)) & MCURSOR_MODE);
17282         WARN_ON(I915_READ(CURCNTR(PIPE_B)) & MCURSOR_MODE);
17283
17284         I915_WRITE(PIPECONF(pipe), 0);
17285         POSTING_READ(PIPECONF(pipe));
17286
17287         intel_wait_for_pipe_scanline_stopped(crtc);
17288
17289         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
17290         POSTING_READ(DPLL(pipe));
17291 }
17292
17293 static void
17294 intel_sanitize_plane_mapping(struct drm_i915_private *dev_priv)
17295 {
17296         struct intel_crtc *crtc;
17297
17298         if (INTEL_GEN(dev_priv) >= 4)
17299                 return;
17300
17301         for_each_intel_crtc(&dev_priv->drm, crtc) {
17302                 struct intel_plane *plane =
17303                         to_intel_plane(crtc->base.primary);
17304                 struct intel_crtc *plane_crtc;
17305                 enum pipe pipe;
17306
17307                 if (!plane->get_hw_state(plane, &pipe))
17308                         continue;
17309
17310                 if (pipe == crtc->pipe)
17311                         continue;
17312
17313                 DRM_DEBUG_KMS("[PLANE:%d:%s] attached to the wrong pipe, disabling plane\n",
17314                               plane->base.base.id, plane->base.name);
17315
17316                 plane_crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
17317                 intel_plane_disable_noatomic(plane_crtc, plane);
17318         }
17319 }
17320
17321 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
17322 {
17323         struct drm_device *dev = crtc->base.dev;
17324         struct intel_encoder *encoder;
17325
17326         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
17327                 return true;
17328
17329         return false;
17330 }
17331
17332 static struct intel_connector *intel_encoder_find_connector(struct intel_encoder *encoder)
17333 {
17334         struct drm_device *dev = encoder->base.dev;
17335         struct intel_connector *connector;
17336
17337         for_each_connector_on_encoder(dev, &encoder->base, connector)
17338                 return connector;
17339
17340         return NULL;
17341 }
17342
17343 static bool has_pch_trancoder(struct drm_i915_private *dev_priv,
17344                               enum pipe pch_transcoder)
17345 {
17346         return HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
17347                 (HAS_PCH_LPT_H(dev_priv) && pch_transcoder == PIPE_A);
17348 }
17349
17350 static void intel_sanitize_frame_start_delay(const struct intel_crtc_state *crtc_state)
17351 {
17352         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
17353         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
17354         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
17355
17356         if (INTEL_GEN(dev_priv) >= 9 ||
17357             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
17358                 i915_reg_t reg = CHICKEN_TRANS(cpu_transcoder);
17359                 u32 val;
17360
17361                 if (transcoder_is_dsi(cpu_transcoder))
17362                         return;
17363
17364                 val = I915_READ(reg);
17365                 val &= ~HSW_FRAME_START_DELAY_MASK;
17366                 val |= HSW_FRAME_START_DELAY(0);
17367                 I915_WRITE(reg, val);
17368         } else {
17369                 i915_reg_t reg = PIPECONF(cpu_transcoder);
17370                 u32 val;
17371
17372                 val = I915_READ(reg);
17373                 val &= ~PIPECONF_FRAME_START_DELAY_MASK;
17374                 val |= PIPECONF_FRAME_START_DELAY(0);
17375                 I915_WRITE(reg, val);
17376         }
17377
17378         if (!crtc_state->has_pch_encoder)
17379                 return;
17380
17381         if (HAS_PCH_IBX(dev_priv)) {
17382                 i915_reg_t reg = PCH_TRANSCONF(crtc->pipe);
17383                 u32 val;
17384
17385                 val = I915_READ(reg);
17386                 val &= ~TRANS_FRAME_START_DELAY_MASK;
17387                 val |= TRANS_FRAME_START_DELAY(0);
17388                 I915_WRITE(reg, val);
17389         } else {
17390                 enum pipe pch_transcoder = intel_crtc_pch_transcoder(crtc);
17391                 i915_reg_t reg = TRANS_CHICKEN2(pch_transcoder);
17392                 u32 val;
17393
17394                 val = I915_READ(reg);
17395                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
17396                 val |= TRANS_CHICKEN2_FRAME_START_DELAY(0);
17397                 I915_WRITE(reg, val);
17398         }
17399 }
17400
17401 static void intel_sanitize_crtc(struct intel_crtc *crtc,
17402                                 struct drm_modeset_acquire_ctx *ctx)
17403 {
17404         struct drm_device *dev = crtc->base.dev;
17405         struct drm_i915_private *dev_priv = to_i915(dev);
17406         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
17407
17408         if (crtc_state->hw.active) {
17409                 struct intel_plane *plane;
17410
17411                 /* Clear any frame start delays used for debugging left by the BIOS */
17412                 intel_sanitize_frame_start_delay(crtc_state);
17413
17414                 /* Disable everything but the primary plane */
17415                 for_each_intel_plane_on_crtc(dev, crtc, plane) {
17416                         const struct intel_plane_state *plane_state =
17417                                 to_intel_plane_state(plane->base.state);
17418
17419                         if (plane_state->uapi.visible &&
17420                             plane->base.type != DRM_PLANE_TYPE_PRIMARY)
17421                                 intel_plane_disable_noatomic(crtc, plane);
17422                 }
17423
17424                 /*
17425                  * Disable any background color set by the BIOS, but enable the
17426                  * gamma and CSC to match how we program our planes.
17427                  */
17428                 if (INTEL_GEN(dev_priv) >= 9)
17429                         I915_WRITE(SKL_BOTTOM_COLOR(crtc->pipe),
17430                                    SKL_BOTTOM_COLOR_GAMMA_ENABLE |
17431                                    SKL_BOTTOM_COLOR_CSC_ENABLE);
17432         }
17433
17434         /* Adjust the state of the output pipe according to whether we
17435          * have active connectors/encoders. */
17436         if (crtc_state->hw.active && !intel_crtc_has_encoders(crtc))
17437                 intel_crtc_disable_noatomic(crtc, ctx);
17438
17439         if (crtc_state->hw.active || HAS_GMCH(dev_priv)) {
17440                 /*
17441                  * We start out with underrun reporting disabled to avoid races.
17442                  * For correct bookkeeping mark this on active crtcs.
17443                  *
17444                  * Also on gmch platforms we dont have any hardware bits to
17445                  * disable the underrun reporting. Which means we need to start
17446                  * out with underrun reporting disabled also on inactive pipes,
17447                  * since otherwise we'll complain about the garbage we read when
17448                  * e.g. coming up after runtime pm.
17449                  *
17450                  * No protection against concurrent access is required - at
17451                  * worst a fifo underrun happens which also sets this to false.
17452                  */
17453                 crtc->cpu_fifo_underrun_disabled = true;
17454                 /*
17455                  * We track the PCH trancoder underrun reporting state
17456                  * within the crtc. With crtc for pipe A housing the underrun
17457                  * reporting state for PCH transcoder A, crtc for pipe B housing
17458                  * it for PCH transcoder B, etc. LPT-H has only PCH transcoder A,
17459                  * and marking underrun reporting as disabled for the non-existing
17460                  * PCH transcoders B and C would prevent enabling the south
17461                  * error interrupt (see cpt_can_enable_serr_int()).
17462                  */
17463                 if (has_pch_trancoder(dev_priv, crtc->pipe))
17464                         crtc->pch_fifo_underrun_disabled = true;
17465         }
17466 }
17467
17468 static bool has_bogus_dpll_config(const struct intel_crtc_state *crtc_state)
17469 {
17470         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
17471
17472         /*
17473          * Some SNB BIOSen (eg. ASUS K53SV) are known to misprogram
17474          * the hardware when a high res displays plugged in. DPLL P
17475          * divider is zero, and the pipe timings are bonkers. We'll
17476          * try to disable everything in that case.
17477          *
17478          * FIXME would be nice to be able to sanitize this state
17479          * without several WARNs, but for now let's take the easy
17480          * road.
17481          */
17482         return IS_GEN(dev_priv, 6) &&
17483                 crtc_state->hw.active &&
17484                 crtc_state->shared_dpll &&
17485                 crtc_state->port_clock == 0;
17486 }
17487
17488 static void intel_sanitize_encoder(struct intel_encoder *encoder)
17489 {
17490         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
17491         struct intel_connector *connector;
17492         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
17493         struct intel_crtc_state *crtc_state = crtc ?
17494                 to_intel_crtc_state(crtc->base.state) : NULL;
17495
17496         /* We need to check both for a crtc link (meaning that the
17497          * encoder is active and trying to read from a pipe) and the
17498          * pipe itself being active. */
17499         bool has_active_crtc = crtc_state &&
17500                 crtc_state->hw.active;
17501
17502         if (crtc_state && has_bogus_dpll_config(crtc_state)) {
17503                 DRM_DEBUG_KMS("BIOS has misprogrammed the hardware. Disabling pipe %c\n",
17504                               pipe_name(crtc->pipe));
17505                 has_active_crtc = false;
17506         }
17507
17508         connector = intel_encoder_find_connector(encoder);
17509         if (connector && !has_active_crtc) {
17510                 DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n",
17511                               encoder->base.base.id,
17512                               encoder->base.name);
17513
17514                 /* Connector is active, but has no active pipe. This is
17515                  * fallout from our resume register restoring. Disable
17516                  * the encoder manually again. */
17517                 if (crtc_state) {
17518                         struct drm_encoder *best_encoder;
17519
17520                         DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n",
17521                                       encoder->base.base.id,
17522                                       encoder->base.name);
17523
17524                         /* avoid oopsing in case the hooks consult best_encoder */
17525                         best_encoder = connector->base.state->best_encoder;
17526                         connector->base.state->best_encoder = &encoder->base;
17527
17528                         if (encoder->disable)
17529                                 encoder->disable(encoder, crtc_state,
17530                                                  connector->base.state);
17531                         if (encoder->post_disable)
17532                                 encoder->post_disable(encoder, crtc_state,
17533                                                       connector->base.state);
17534
17535                         connector->base.state->best_encoder = best_encoder;
17536                 }
17537                 encoder->base.crtc = NULL;
17538
17539                 /* Inconsistent output/port/pipe state happens presumably due to
17540                  * a bug in one of the get_hw_state functions. Or someplace else
17541                  * in our code, like the register restore mess on resume. Clamp
17542                  * things to off as a safer default. */
17543
17544                 connector->base.dpms = DRM_MODE_DPMS_OFF;
17545                 connector->base.encoder = NULL;
17546         }
17547
17548         /* notify opregion of the sanitized encoder state */
17549         intel_opregion_notify_encoder(encoder, connector && has_active_crtc);
17550
17551         if (INTEL_GEN(dev_priv) >= 11)
17552                 icl_sanitize_encoder_pll_mapping(encoder);
17553 }
17554
17555 /* FIXME read out full plane state for all planes */
17556 static void readout_plane_state(struct drm_i915_private *dev_priv)
17557 {
17558         struct intel_plane *plane;
17559         struct intel_crtc *crtc;
17560
17561         for_each_intel_plane(&dev_priv->drm, plane) {
17562                 struct intel_plane_state *plane_state =
17563                         to_intel_plane_state(plane->base.state);
17564                 struct intel_crtc_state *crtc_state;
17565                 enum pipe pipe = PIPE_A;
17566                 bool visible;
17567
17568                 visible = plane->get_hw_state(plane, &pipe);
17569
17570                 crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
17571                 crtc_state = to_intel_crtc_state(crtc->base.state);
17572
17573                 intel_set_plane_visible(crtc_state, plane_state, visible);
17574
17575                 DRM_DEBUG_KMS("[PLANE:%d:%s] hw state readout: %s, pipe %c\n",
17576                               plane->base.base.id, plane->base.name,
17577                               enableddisabled(visible), pipe_name(pipe));
17578         }
17579
17580         for_each_intel_crtc(&dev_priv->drm, crtc) {
17581                 struct intel_crtc_state *crtc_state =
17582                         to_intel_crtc_state(crtc->base.state);
17583
17584                 fixup_active_planes(crtc_state);
17585         }
17586 }
17587
17588 static void intel_modeset_readout_hw_state(struct drm_device *dev)
17589 {
17590         struct drm_i915_private *dev_priv = to_i915(dev);
17591         enum pipe pipe;
17592         struct intel_crtc *crtc;
17593         struct intel_encoder *encoder;
17594         struct intel_connector *connector;
17595         struct drm_connector_list_iter conn_iter;
17596         int i;
17597
17598         dev_priv->active_pipes = 0;
17599
17600         for_each_intel_crtc(dev, crtc) {
17601                 struct intel_crtc_state *crtc_state =
17602                         to_intel_crtc_state(crtc->base.state);
17603
17604                 __drm_atomic_helper_crtc_destroy_state(&crtc_state->uapi);
17605                 intel_crtc_free_hw_state(crtc_state);
17606                 intel_crtc_state_reset(crtc_state, crtc);
17607
17608                 crtc_state->hw.active = crtc_state->hw.enable =
17609                         dev_priv->display.get_pipe_config(crtc, crtc_state);
17610
17611                 crtc->base.enabled = crtc_state->hw.enable;
17612                 crtc->active = crtc_state->hw.active;
17613
17614                 if (crtc_state->hw.active)
17615                         dev_priv->active_pipes |= BIT(crtc->pipe);
17616
17617                 DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n",
17618                               crtc->base.base.id, crtc->base.name,
17619                               enableddisabled(crtc_state->hw.active));
17620         }
17621
17622         readout_plane_state(dev_priv);
17623
17624         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
17625                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
17626
17627                 pll->on = pll->info->funcs->get_hw_state(dev_priv, pll,
17628                                                         &pll->state.hw_state);
17629
17630                 if (IS_ELKHARTLAKE(dev_priv) && pll->on &&
17631                     pll->info->id == DPLL_ID_EHL_DPLL4) {
17632                         pll->wakeref = intel_display_power_get(dev_priv,
17633                                                                POWER_DOMAIN_DPLL_DC_OFF);
17634                 }
17635
17636                 pll->state.crtc_mask = 0;
17637                 for_each_intel_crtc(dev, crtc) {
17638                         struct intel_crtc_state *crtc_state =
17639                                 to_intel_crtc_state(crtc->base.state);
17640
17641                         if (crtc_state->hw.active &&
17642                             crtc_state->shared_dpll == pll)
17643                                 pll->state.crtc_mask |= 1 << crtc->pipe;
17644                 }
17645                 pll->active_mask = pll->state.crtc_mask;
17646
17647                 DRM_DEBUG_KMS("%s hw state readout: crtc_mask 0x%08x, on %i\n",
17648                               pll->info->name, pll->state.crtc_mask, pll->on);
17649         }
17650
17651         for_each_intel_encoder(dev, encoder) {
17652                 pipe = 0;
17653
17654                 if (encoder->get_hw_state(encoder, &pipe)) {
17655                         struct intel_crtc_state *crtc_state;
17656
17657                         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
17658                         crtc_state = to_intel_crtc_state(crtc->base.state);
17659
17660                         encoder->base.crtc = &crtc->base;
17661                         encoder->get_config(encoder, crtc_state);
17662                 } else {
17663                         encoder->base.crtc = NULL;
17664                 }
17665
17666                 DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
17667                               encoder->base.base.id, encoder->base.name,
17668                               enableddisabled(encoder->base.crtc),
17669                               pipe_name(pipe));
17670         }
17671
17672         drm_connector_list_iter_begin(dev, &conn_iter);
17673         for_each_intel_connector_iter(connector, &conn_iter) {
17674                 if (connector->get_hw_state(connector)) {
17675                         struct intel_crtc_state *crtc_state;
17676                         struct intel_crtc *crtc;
17677
17678                         connector->base.dpms = DRM_MODE_DPMS_ON;
17679
17680                         encoder = connector->encoder;
17681                         connector->base.encoder = &encoder->base;
17682
17683                         crtc = to_intel_crtc(encoder->base.crtc);
17684                         crtc_state = crtc ? to_intel_crtc_state(crtc->base.state) : NULL;
17685
17686                         if (crtc_state && crtc_state->hw.active) {
17687                                 /*
17688                                  * This has to be done during hardware readout
17689                                  * because anything calling .crtc_disable may
17690                                  * rely on the connector_mask being accurate.
17691                                  */
17692                                 crtc_state->uapi.connector_mask |=
17693                                         drm_connector_mask(&connector->base);
17694                                 crtc_state->uapi.encoder_mask |=
17695                                         drm_encoder_mask(&encoder->base);
17696                         }
17697                 } else {
17698                         connector->base.dpms = DRM_MODE_DPMS_OFF;
17699                         connector->base.encoder = NULL;
17700                 }
17701                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n",
17702                               connector->base.base.id, connector->base.name,
17703                               enableddisabled(connector->base.encoder));
17704         }
17705         drm_connector_list_iter_end(&conn_iter);
17706
17707         for_each_intel_crtc(dev, crtc) {
17708                 struct intel_bw_state *bw_state =
17709                         to_intel_bw_state(dev_priv->bw_obj.state);
17710                 struct intel_crtc_state *crtc_state =
17711                         to_intel_crtc_state(crtc->base.state);
17712                 struct intel_plane *plane;
17713                 int min_cdclk = 0;
17714
17715                 if (crtc_state->hw.active) {
17716                         struct drm_display_mode *mode = &crtc_state->hw.mode;
17717
17718                         intel_mode_from_pipe_config(&crtc_state->hw.adjusted_mode,
17719                                                     crtc_state);
17720
17721                         *mode = crtc_state->hw.adjusted_mode;
17722                         mode->hdisplay = crtc_state->pipe_src_w;
17723                         mode->vdisplay = crtc_state->pipe_src_h;
17724
17725                         /*
17726                          * The initial mode needs to be set in order to keep
17727                          * the atomic core happy. It wants a valid mode if the
17728                          * crtc's enabled, so we do the above call.
17729                          *
17730                          * But we don't set all the derived state fully, hence
17731                          * set a flag to indicate that a full recalculation is
17732                          * needed on the next commit.
17733                          */
17734                         mode->private_flags = I915_MODE_FLAG_INHERITED;
17735
17736                         intel_crtc_compute_pixel_rate(crtc_state);
17737
17738                         intel_crtc_update_active_timings(crtc_state);
17739
17740                         intel_crtc_copy_hw_to_uapi_state(crtc_state);
17741                 }
17742
17743                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
17744                         const struct intel_plane_state *plane_state =
17745                                 to_intel_plane_state(plane->base.state);
17746
17747                         /*
17748                          * FIXME don't have the fb yet, so can't
17749                          * use intel_plane_data_rate() :(
17750                          */
17751                         if (plane_state->uapi.visible)
17752                                 crtc_state->data_rate[plane->id] =
17753                                         4 * crtc_state->pixel_rate;
17754                         /*
17755                          * FIXME don't have the fb yet, so can't
17756                          * use plane->min_cdclk() :(
17757                          */
17758                         if (plane_state->uapi.visible && plane->min_cdclk) {
17759                                 if (crtc_state->double_wide ||
17760                                     INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
17761                                         crtc_state->min_cdclk[plane->id] =
17762                                                 DIV_ROUND_UP(crtc_state->pixel_rate, 2);
17763                                 else
17764                                         crtc_state->min_cdclk[plane->id] =
17765                                                 crtc_state->pixel_rate;
17766                         }
17767                         DRM_DEBUG_KMS("[PLANE:%d:%s] min_cdclk %d kHz\n",
17768                                       plane->base.base.id, plane->base.name,
17769                                       crtc_state->min_cdclk[plane->id]);
17770                 }
17771
17772                 if (crtc_state->hw.active) {
17773                         min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
17774                         if (WARN_ON(min_cdclk < 0))
17775                                 min_cdclk = 0;
17776                 }
17777
17778                 dev_priv->min_cdclk[crtc->pipe] = min_cdclk;
17779                 dev_priv->min_voltage_level[crtc->pipe] =
17780                         crtc_state->min_voltage_level;
17781
17782                 intel_bw_crtc_update(bw_state, crtc_state);
17783
17784                 intel_pipe_config_sanity_check(dev_priv, crtc_state);
17785         }
17786 }
17787
17788 static void
17789 get_encoder_power_domains(struct drm_i915_private *dev_priv)
17790 {
17791         struct intel_encoder *encoder;
17792
17793         for_each_intel_encoder(&dev_priv->drm, encoder) {
17794                 struct intel_crtc_state *crtc_state;
17795
17796                 if (!encoder->get_power_domains)
17797                         continue;
17798
17799                 /*
17800                  * MST-primary and inactive encoders don't have a crtc state
17801                  * and neither of these require any power domain references.
17802                  */
17803                 if (!encoder->base.crtc)
17804                         continue;
17805
17806                 crtc_state = to_intel_crtc_state(encoder->base.crtc->state);
17807                 encoder->get_power_domains(encoder, crtc_state);
17808         }
17809 }
17810
17811 static void intel_early_display_was(struct drm_i915_private *dev_priv)
17812 {
17813         /* Display WA #1185 WaDisableDARBFClkGating:cnl,glk */
17814         if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
17815                 I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
17816                            DARBF_GATING_DIS);
17817
17818         if (IS_HASWELL(dev_priv)) {
17819                 /*
17820                  * WaRsPkgCStateDisplayPMReq:hsw
17821                  * System hang if this isn't done before disabling all planes!
17822                  */
17823                 I915_WRITE(CHICKEN_PAR1_1,
17824                            I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
17825         }
17826 }
17827
17828 static void ibx_sanitize_pch_hdmi_port(struct drm_i915_private *dev_priv,
17829                                        enum port port, i915_reg_t hdmi_reg)
17830 {
17831         u32 val = I915_READ(hdmi_reg);
17832
17833         if (val & SDVO_ENABLE ||
17834             (val & SDVO_PIPE_SEL_MASK) == SDVO_PIPE_SEL(PIPE_A))
17835                 return;
17836
17837         DRM_DEBUG_KMS("Sanitizing transcoder select for HDMI %c\n",
17838                       port_name(port));
17839
17840         val &= ~SDVO_PIPE_SEL_MASK;
17841         val |= SDVO_PIPE_SEL(PIPE_A);
17842
17843         I915_WRITE(hdmi_reg, val);
17844 }
17845
17846 static void ibx_sanitize_pch_dp_port(struct drm_i915_private *dev_priv,
17847                                      enum port port, i915_reg_t dp_reg)
17848 {
17849         u32 val = I915_READ(dp_reg);
17850
17851         if (val & DP_PORT_EN ||
17852             (val & DP_PIPE_SEL_MASK) == DP_PIPE_SEL(PIPE_A))
17853                 return;
17854
17855         DRM_DEBUG_KMS("Sanitizing transcoder select for DP %c\n",
17856                       port_name(port));
17857
17858         val &= ~DP_PIPE_SEL_MASK;
17859         val |= DP_PIPE_SEL(PIPE_A);
17860
17861         I915_WRITE(dp_reg, val);
17862 }
17863
17864 static void ibx_sanitize_pch_ports(struct drm_i915_private *dev_priv)
17865 {
17866         /*
17867          * The BIOS may select transcoder B on some of the PCH
17868          * ports even it doesn't enable the port. This would trip
17869          * assert_pch_dp_disabled() and assert_pch_hdmi_disabled().
17870          * Sanitize the transcoder select bits to prevent that. We
17871          * assume that the BIOS never actually enabled the port,
17872          * because if it did we'd actually have to toggle the port
17873          * on and back off to make the transcoder A select stick
17874          * (see. intel_dp_link_down(), intel_disable_hdmi(),
17875          * intel_disable_sdvo()).
17876          */
17877         ibx_sanitize_pch_dp_port(dev_priv, PORT_B, PCH_DP_B);
17878         ibx_sanitize_pch_dp_port(dev_priv, PORT_C, PCH_DP_C);
17879         ibx_sanitize_pch_dp_port(dev_priv, PORT_D, PCH_DP_D);
17880
17881         /* PCH SDVOB multiplex with HDMIB */
17882         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_B, PCH_HDMIB);
17883         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_C, PCH_HDMIC);
17884         ibx_sanitize_pch_hdmi_port(dev_priv, PORT_D, PCH_HDMID);
17885 }
17886
17887 /* Scan out the current hw modeset state,
17888  * and sanitizes it to the current state
17889  */
17890 static void
17891 intel_modeset_setup_hw_state(struct drm_device *dev,
17892                              struct drm_modeset_acquire_ctx *ctx)
17893 {
17894         struct drm_i915_private *dev_priv = to_i915(dev);
17895         struct intel_encoder *encoder;
17896         struct intel_crtc *crtc;
17897         intel_wakeref_t wakeref;
17898         int i;
17899
17900         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
17901
17902         intel_early_display_was(dev_priv);
17903         intel_modeset_readout_hw_state(dev);
17904
17905         /* HW state is read out, now we need to sanitize this mess. */
17906
17907         /* Sanitize the TypeC port mode upfront, encoders depend on this */
17908         for_each_intel_encoder(dev, encoder) {
17909                 enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
17910
17911                 /* We need to sanitize only the MST primary port. */
17912                 if (encoder->type != INTEL_OUTPUT_DP_MST &&
17913                     intel_phy_is_tc(dev_priv, phy))
17914                         intel_tc_port_sanitize(enc_to_dig_port(&encoder->base));
17915         }
17916
17917         get_encoder_power_domains(dev_priv);
17918
17919         if (HAS_PCH_IBX(dev_priv))
17920                 ibx_sanitize_pch_ports(dev_priv);
17921
17922         /*
17923          * intel_sanitize_plane_mapping() may need to do vblank
17924          * waits, so we need vblank interrupts restored beforehand.
17925          */
17926         for_each_intel_crtc(&dev_priv->drm, crtc) {
17927                 struct intel_crtc_state *crtc_state =
17928                         to_intel_crtc_state(crtc->base.state);
17929
17930                 drm_crtc_vblank_reset(&crtc->base);
17931
17932                 if (crtc_state->hw.active)
17933                         intel_crtc_vblank_on(crtc_state);
17934         }
17935
17936         intel_sanitize_plane_mapping(dev_priv);
17937
17938         for_each_intel_encoder(dev, encoder)
17939                 intel_sanitize_encoder(encoder);
17940
17941         for_each_intel_crtc(&dev_priv->drm, crtc) {
17942                 struct intel_crtc_state *crtc_state =
17943                         to_intel_crtc_state(crtc->base.state);
17944
17945                 intel_sanitize_crtc(crtc, ctx);
17946                 intel_dump_pipe_config(crtc_state, NULL, "[setup_hw_state]");
17947         }
17948
17949         intel_modeset_update_connector_atomic_state(dev);
17950
17951         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
17952                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
17953
17954                 if (!pll->on || pll->active_mask)
17955                         continue;
17956
17957                 DRM_DEBUG_KMS("%s enabled but not in use, disabling\n",
17958                               pll->info->name);
17959
17960                 pll->info->funcs->disable(dev_priv, pll);
17961                 pll->on = false;
17962         }
17963
17964         if (IS_G4X(dev_priv)) {
17965                 g4x_wm_get_hw_state(dev_priv);
17966                 g4x_wm_sanitize(dev_priv);
17967         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
17968                 vlv_wm_get_hw_state(dev_priv);
17969                 vlv_wm_sanitize(dev_priv);
17970         } else if (INTEL_GEN(dev_priv) >= 9) {
17971                 skl_wm_get_hw_state(dev_priv);
17972         } else if (HAS_PCH_SPLIT(dev_priv)) {
17973                 ilk_wm_get_hw_state(dev_priv);
17974         }
17975
17976         for_each_intel_crtc(dev, crtc) {
17977                 struct intel_crtc_state *crtc_state =
17978                         to_intel_crtc_state(crtc->base.state);
17979                 u64 put_domains;
17980
17981                 put_domains = modeset_get_crtc_power_domains(crtc_state);
17982                 if (WARN_ON(put_domains))
17983                         modeset_put_power_domains(dev_priv, put_domains);
17984         }
17985
17986         intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref);
17987 }
17988
17989 void intel_display_resume(struct drm_device *dev)
17990 {
17991         struct drm_i915_private *dev_priv = to_i915(dev);
17992         struct drm_atomic_state *state = dev_priv->modeset_restore_state;
17993         struct drm_modeset_acquire_ctx ctx;
17994         int ret;
17995
17996         dev_priv->modeset_restore_state = NULL;
17997         if (state)
17998                 state->acquire_ctx = &ctx;
17999
18000         drm_modeset_acquire_init(&ctx, 0);
18001
18002         while (1) {
18003                 ret = drm_modeset_lock_all_ctx(dev, &ctx);
18004                 if (ret != -EDEADLK)
18005                         break;
18006
18007                 drm_modeset_backoff(&ctx);
18008         }
18009
18010         if (!ret)
18011                 ret = __intel_display_resume(dev, state, &ctx);
18012
18013         intel_enable_ipc(dev_priv);
18014         drm_modeset_drop_locks(&ctx);
18015         drm_modeset_acquire_fini(&ctx);
18016
18017         if (ret)
18018                 DRM_ERROR("Restoring old state failed with %i\n", ret);
18019         if (state)
18020                 drm_atomic_state_put(state);
18021 }
18022
18023 static void intel_hpd_poll_fini(struct drm_i915_private *i915)
18024 {
18025         struct intel_connector *connector;
18026         struct drm_connector_list_iter conn_iter;
18027
18028         /* Kill all the work that may have been queued by hpd. */
18029         drm_connector_list_iter_begin(&i915->drm, &conn_iter);
18030         for_each_intel_connector_iter(connector, &conn_iter) {
18031                 if (connector->modeset_retry_work.func)
18032                         cancel_work_sync(&connector->modeset_retry_work);
18033                 if (connector->hdcp.shim) {
18034                         cancel_delayed_work_sync(&connector->hdcp.check_work);
18035                         cancel_work_sync(&connector->hdcp.prop_work);
18036                 }
18037         }
18038         drm_connector_list_iter_end(&conn_iter);
18039 }
18040
18041 void intel_modeset_driver_remove(struct drm_i915_private *i915)
18042 {
18043         flush_workqueue(i915->flip_wq);
18044         flush_workqueue(i915->modeset_wq);
18045
18046         flush_work(&i915->atomic_helper.free_work);
18047         WARN_ON(!llist_empty(&i915->atomic_helper.free_list));
18048
18049         /*
18050          * Interrupts and polling as the first thing to avoid creating havoc.
18051          * Too much stuff here (turning of connectors, ...) would
18052          * experience fancy races otherwise.
18053          */
18054         intel_irq_uninstall(i915);
18055
18056         /*
18057          * Due to the hpd irq storm handling the hotplug work can re-arm the
18058          * poll handlers. Hence disable polling after hpd handling is shut down.
18059          */
18060         intel_hpd_poll_fini(i915);
18061
18062         /*
18063          * MST topology needs to be suspended so we don't have any calls to
18064          * fbdev after it's finalized. MST will be destroyed later as part of
18065          * drm_mode_config_cleanup()
18066          */
18067         intel_dp_mst_suspend(i915);
18068
18069         /* poll work can call into fbdev, hence clean that up afterwards */
18070         intel_fbdev_fini(i915);
18071
18072         intel_unregister_dsm_handler();
18073
18074         intel_fbc_global_disable(i915);
18075
18076         /* flush any delayed tasks or pending work */
18077         flush_scheduled_work();
18078
18079         intel_hdcp_component_fini(i915);
18080
18081         drm_mode_config_cleanup(&i915->drm);
18082
18083         intel_overlay_cleanup(i915);
18084
18085         intel_gmbus_teardown(i915);
18086
18087         destroy_workqueue(i915->flip_wq);
18088         destroy_workqueue(i915->modeset_wq);
18089
18090         intel_fbc_cleanup_cfb(i915);
18091 }
18092
18093 #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
18094
18095 struct intel_display_error_state {
18096
18097         u32 power_well_driver;
18098
18099         struct intel_cursor_error_state {
18100                 u32 control;
18101                 u32 position;
18102                 u32 base;
18103                 u32 size;
18104         } cursor[I915_MAX_PIPES];
18105
18106         struct intel_pipe_error_state {
18107                 bool power_domain_on;
18108                 u32 source;
18109                 u32 stat;
18110         } pipe[I915_MAX_PIPES];
18111
18112         struct intel_plane_error_state {
18113                 u32 control;
18114                 u32 stride;
18115                 u32 size;
18116                 u32 pos;
18117                 u32 addr;
18118                 u32 surface;
18119                 u32 tile_offset;
18120         } plane[I915_MAX_PIPES];
18121
18122         struct intel_transcoder_error_state {
18123                 bool available;
18124                 bool power_domain_on;
18125                 enum transcoder cpu_transcoder;
18126
18127                 u32 conf;
18128
18129                 u32 htotal;
18130                 u32 hblank;
18131                 u32 hsync;
18132                 u32 vtotal;
18133                 u32 vblank;
18134                 u32 vsync;
18135         } transcoder[5];
18136 };
18137
18138 struct intel_display_error_state *
18139 intel_display_capture_error_state(struct drm_i915_private *dev_priv)
18140 {
18141         struct intel_display_error_state *error;
18142         int transcoders[] = {
18143                 TRANSCODER_A,
18144                 TRANSCODER_B,
18145                 TRANSCODER_C,
18146                 TRANSCODER_D,
18147                 TRANSCODER_EDP,
18148         };
18149         int i;
18150
18151         BUILD_BUG_ON(ARRAY_SIZE(transcoders) != ARRAY_SIZE(error->transcoder));
18152
18153         if (!HAS_DISPLAY(dev_priv) || !INTEL_DISPLAY_ENABLED(dev_priv))
18154                 return NULL;
18155
18156         error = kzalloc(sizeof(*error), GFP_ATOMIC);
18157         if (error == NULL)
18158                 return NULL;
18159
18160         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
18161                 error->power_well_driver = I915_READ(HSW_PWR_WELL_CTL2);
18162
18163         for_each_pipe(dev_priv, i) {
18164                 error->pipe[i].power_domain_on =
18165                         __intel_display_power_is_enabled(dev_priv,
18166                                                          POWER_DOMAIN_PIPE(i));
18167                 if (!error->pipe[i].power_domain_on)
18168                         continue;
18169
18170                 error->cursor[i].control = I915_READ(CURCNTR(i));
18171                 error->cursor[i].position = I915_READ(CURPOS(i));
18172                 error->cursor[i].base = I915_READ(CURBASE(i));
18173
18174                 error->plane[i].control = I915_READ(DSPCNTR(i));
18175                 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
18176                 if (INTEL_GEN(dev_priv) <= 3) {
18177                         error->plane[i].size = I915_READ(DSPSIZE(i));
18178                         error->plane[i].pos = I915_READ(DSPPOS(i));
18179                 }
18180                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
18181                         error->plane[i].addr = I915_READ(DSPADDR(i));
18182                 if (INTEL_GEN(dev_priv) >= 4) {
18183                         error->plane[i].surface = I915_READ(DSPSURF(i));
18184                         error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
18185                 }
18186
18187                 error->pipe[i].source = I915_READ(PIPESRC(i));
18188
18189                 if (HAS_GMCH(dev_priv))
18190                         error->pipe[i].stat = I915_READ(PIPESTAT(i));
18191         }
18192
18193         for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
18194                 enum transcoder cpu_transcoder = transcoders[i];
18195
18196                 if (!INTEL_INFO(dev_priv)->trans_offsets[cpu_transcoder])
18197                         continue;
18198
18199                 error->transcoder[i].available = true;
18200                 error->transcoder[i].power_domain_on =
18201                         __intel_display_power_is_enabled(dev_priv,
18202                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder));
18203                 if (!error->transcoder[i].power_domain_on)
18204                         continue;
18205
18206                 error->transcoder[i].cpu_transcoder = cpu_transcoder;
18207
18208                 error->transcoder[i].conf = I915_READ(PIPECONF(cpu_transcoder));
18209                 error->transcoder[i].htotal = I915_READ(HTOTAL(cpu_transcoder));
18210                 error->transcoder[i].hblank = I915_READ(HBLANK(cpu_transcoder));
18211                 error->transcoder[i].hsync = I915_READ(HSYNC(cpu_transcoder));
18212                 error->transcoder[i].vtotal = I915_READ(VTOTAL(cpu_transcoder));
18213                 error->transcoder[i].vblank = I915_READ(VBLANK(cpu_transcoder));
18214                 error->transcoder[i].vsync = I915_READ(VSYNC(cpu_transcoder));
18215         }
18216
18217         return error;
18218 }
18219
18220 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
18221
18222 void
18223 intel_display_print_error_state(struct drm_i915_error_state_buf *m,
18224                                 struct intel_display_error_state *error)
18225 {
18226         struct drm_i915_private *dev_priv = m->i915;
18227         int i;
18228
18229         if (!error)
18230                 return;
18231
18232         err_printf(m, "Num Pipes: %d\n", INTEL_NUM_PIPES(dev_priv));
18233         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
18234                 err_printf(m, "PWR_WELL_CTL2: %08x\n",
18235                            error->power_well_driver);
18236         for_each_pipe(dev_priv, i) {
18237                 err_printf(m, "Pipe [%d]:\n", i);
18238                 err_printf(m, "  Power: %s\n",
18239                            onoff(error->pipe[i].power_domain_on));
18240                 err_printf(m, "  SRC: %08x\n", error->pipe[i].source);
18241                 err_printf(m, "  STAT: %08x\n", error->pipe[i].stat);
18242
18243                 err_printf(m, "Plane [%d]:\n", i);
18244                 err_printf(m, "  CNTR: %08x\n", error->plane[i].control);
18245                 err_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
18246                 if (INTEL_GEN(dev_priv) <= 3) {
18247                         err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
18248                         err_printf(m, "  POS: %08x\n", error->plane[i].pos);
18249                 }
18250                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
18251                         err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
18252                 if (INTEL_GEN(dev_priv) >= 4) {
18253                         err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
18254                         err_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
18255                 }
18256
18257                 err_printf(m, "Cursor [%d]:\n", i);
18258                 err_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
18259                 err_printf(m, "  POS: %08x\n", error->cursor[i].position);
18260                 err_printf(m, "  BASE: %08x\n", error->cursor[i].base);
18261         }
18262
18263         for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
18264                 if (!error->transcoder[i].available)
18265                         continue;
18266
18267                 err_printf(m, "CPU transcoder: %s\n",
18268                            transcoder_name(error->transcoder[i].cpu_transcoder));
18269                 err_printf(m, "  Power: %s\n",
18270                            onoff(error->transcoder[i].power_domain_on));
18271                 err_printf(m, "  CONF: %08x\n", error->transcoder[i].conf);
18272                 err_printf(m, "  HTOTAL: %08x\n", error->transcoder[i].htotal);
18273                 err_printf(m, "  HBLANK: %08x\n", error->transcoder[i].hblank);
18274                 err_printf(m, "  HSYNC: %08x\n", error->transcoder[i].hsync);
18275                 err_printf(m, "  VTOTAL: %08x\n", error->transcoder[i].vtotal);
18276                 err_printf(m, "  VBLANK: %08x\n", error->transcoder[i].vblank);
18277                 err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
18278         }
18279 }
18280
18281 #endif