drivers/gpu/drm/i915/display/intel_bw.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include <drm/drm_atomic_state_helper.h>
   7
   8 #include "i915_drv.h"
   9 #include "i915_reg.h"
  10 #include "i915_utils.h"
  11 #include "intel_atomic.h"
  12 #include "intel_bw.h"
  13 #include "intel_cdclk.h"
  14 #include "intel_display_core.h"
  15 #include "intel_display_types.h"
  16 #include "skl_watermark.h"
  17 #include "intel_mchbar_regs.h"
  18 #include "intel_pcode.h"
  19
  20 /* Parameters for Qclk Geyserville (QGV) */
  21 struct intel_qgv_point {
  22         u16 dclk, t_rp, t_rdpre, t_rc, t_ras, t_rcd;
  23 };
  24
  25 struct intel_psf_gv_point {
  26         u8 clk; /* clock in multiples of 16.6666 MHz */
  27 };
  28
  29 struct intel_qgv_info {
  30         struct intel_qgv_point points[I915_NUM_QGV_POINTS];
  31         struct intel_psf_gv_point psf_points[I915_NUM_PSF_GV_POINTS];
  32         u8 num_points;
  33         u8 num_psf_points;
  34         u8 t_bl;
  35         u8 max_numchannels;
  36         u8 channel_width;
  37         u8 deinterleave;
  38 };
  39
  40 static int dg1_mchbar_read_qgv_point_info(struct drm_i915_private *dev_priv,
  41                                           struct intel_qgv_point *sp,
  42                                           int point)
  43 {
  44         u32 dclk_ratio, dclk_reference;
  45         u32 val;
  46
  47         val = intel_uncore_read(&dev_priv->uncore, SA_PERF_STATUS_0_0_0_MCHBAR_PC);
  48         dclk_ratio = REG_FIELD_GET(DG1_QCLK_RATIO_MASK, val);
  49         if (val & DG1_QCLK_REFERENCE)
  50                 dclk_reference = 6; /* 6 * 16.666 MHz = 100 MHz */
  51         else
  52                 dclk_reference = 8; /* 8 * 16.666 MHz = 133 MHz */
  53         sp->dclk = DIV_ROUND_UP((16667 * dclk_ratio * dclk_reference) + 500, 1000);
  54
  55         val = intel_uncore_read(&dev_priv->uncore, SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU);
  56         if (val & DG1_GEAR_TYPE)
  57                 sp->dclk *= 2;
  58
  59         if (sp->dclk == 0)
  60                 return -EINVAL;
  61
  62         val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR);
  63         sp->t_rp = REG_FIELD_GET(DG1_DRAM_T_RP_MASK, val);
  64         sp->t_rdpre = REG_FIELD_GET(DG1_DRAM_T_RDPRE_MASK, val);
  65
  66         val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH);
  67         sp->t_rcd = REG_FIELD_GET(DG1_DRAM_T_RCD_MASK, val);
  68         sp->t_ras = REG_FIELD_GET(DG1_DRAM_T_RAS_MASK, val);
  69
  70         sp->t_rc = sp->t_rp + sp->t_ras;
  71
  72         return 0;
  73 }
  74
  75 static int icl_pcode_read_qgv_point_info(struct drm_i915_private *dev_priv,
  76                                          struct intel_qgv_point *sp,
  77                                          int point)
  78 {
  79         u32 val = 0, val2 = 0;
  80         u16 dclk;
  81         int ret;
  82
  83         ret = snb_pcode_read(&dev_priv->uncore, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
  84                              ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point),
  85                              &val, &val2);
  86         if (ret)
  87                 return ret;
  88
  89         dclk = val & 0xffff;
  90         sp->dclk = DIV_ROUND_UP((16667 * dclk) + (DISPLAY_VER(dev_priv) > 11 ? 500 : 0), 1000);
  91         sp->t_rp = (val & 0xff0000) >> 16;
  92         sp->t_rcd = (val & 0xff000000) >> 24;
  93
  94         sp->t_rdpre = val2 & 0xff;
  95         sp->t_ras = (val2 & 0xff00) >> 8;
  96
  97         sp->t_rc = sp->t_rp + sp->t_ras;
  98
  99         return 0;
 100 }
 101
 102 static int adls_pcode_read_psf_gv_point_info(struct drm_i915_private *dev_priv,
 103                                             struct intel_psf_gv_point *points)
 104 {
 105         u32 val = 0;
 106         int ret;
 107         int i;
 108
 109         ret = snb_pcode_read(&dev_priv->uncore, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
 110                              ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, &val, NULL);
 111         if (ret)
 112                 return ret;
 113
 114         for (i = 0; i < I915_NUM_PSF_GV_POINTS; i++) {
 115                 points[i].clk = val & 0xff;
 116                 val >>= 8;
 117         }
 118
 119         return 0;
 120 }
 121
 122 static u16 icl_qgv_points_mask(struct drm_i915_private *i915)
 123 {
 124         unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points;
 125         unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
 126         u16 qgv_points = 0, psf_points = 0;
 127
 128         /*
 129          * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects
 130          * it with failure if we try masking any unadvertised points.
 131          * So need to operate only with those returned from PCode.
 132          */
 133         if (num_qgv_points > 0)
 134                 qgv_points = GENMASK(num_qgv_points - 1, 0);
 135
 136         if (num_psf_gv_points > 0)
 137                 psf_points = GENMASK(num_psf_gv_points - 1, 0);
 138
 139         return ICL_PCODE_REQ_QGV_PT(qgv_points) | ADLS_PCODE_REQ_PSF_PT(psf_points);
 140 }
 141
 142 static bool is_sagv_enabled(struct drm_i915_private *i915, u16 points_mask)
 143 {
 144         return !is_power_of_2(~points_mask & icl_qgv_points_mask(i915) &
 145                               ICL_PCODE_REQ_QGV_PT_MASK);
 146 }
 147
 148 int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv,
 149                                   u32 points_mask)
 150 {
 151         int ret;
 152
 153         if (DISPLAY_VER(dev_priv) >= 14)
 154                 return 0;
 155
 156         /* bspec says to keep retrying for at least 1 ms */
 157         ret = skl_pcode_request(&dev_priv->uncore, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG,
 158                                 points_mask,
 159                                 ICL_PCODE_REP_QGV_MASK | ADLS_PCODE_REP_PSF_MASK,
 160                                 ICL_PCODE_REP_QGV_SAFE | ADLS_PCODE_REP_PSF_SAFE,
 161                                 1);
 162
 163         if (ret < 0) {
 164                 drm_err(&dev_priv->drm, "Failed to disable qgv points (%d) points: 0x%x\n", ret, points_mask);
 165                 return ret;
 166         }
 167
 168         dev_priv->display.sagv.status = is_sagv_enabled(dev_priv, points_mask) ?
 169                 I915_SAGV_ENABLED : I915_SAGV_DISABLED;
 170
 171         return 0;
 172 }
 173
 174 static int mtl_read_qgv_point_info(struct drm_i915_private *dev_priv,
 175                                    struct intel_qgv_point *sp, int point)
 176 {
 177         u32 val, val2;
 178         u16 dclk;
 179
 180         val = intel_uncore_read(&dev_priv->uncore,
 181                                 MTL_MEM_SS_INFO_QGV_POINT_LOW(point));
 182         val2 = intel_uncore_read(&dev_priv->uncore,
 183                                  MTL_MEM_SS_INFO_QGV_POINT_HIGH(point));
 184         dclk = REG_FIELD_GET(MTL_DCLK_MASK, val);
 185         sp->dclk = DIV_ROUND_CLOSEST(16667 * dclk, 1000);
 186         sp->t_rp = REG_FIELD_GET(MTL_TRP_MASK, val);
 187         sp->t_rcd = REG_FIELD_GET(MTL_TRCD_MASK, val);
 188
 189         sp->t_rdpre = REG_FIELD_GET(MTL_TRDPRE_MASK, val2);
 190         sp->t_ras = REG_FIELD_GET(MTL_TRAS_MASK, val2);
 191
 192         sp->t_rc = sp->t_rp + sp->t_ras;
 193
 194         return 0;
 195 }
 196
 197 static int
 198 intel_read_qgv_point_info(struct drm_i915_private *dev_priv,
 199                           struct intel_qgv_point *sp,
 200                           int point)
 201 {
 202         if (DISPLAY_VER(dev_priv) >= 14)
 203                 return mtl_read_qgv_point_info(dev_priv, sp, point);
 204         else if (IS_DG1(dev_priv))
 205                 return dg1_mchbar_read_qgv_point_info(dev_priv, sp, point);
 206         else
 207                 return icl_pcode_read_qgv_point_info(dev_priv, sp, point);
 208 }
 209
 210 static int icl_get_qgv_points(struct drm_i915_private *dev_priv,
 211                               struct intel_qgv_info *qi,
 212                               bool is_y_tile)
 213 {
 214         const struct dram_info *dram_info = &dev_priv->dram_info;
 215         int i, ret;
 216
 217         qi->num_points = dram_info->num_qgv_points;
 218         qi->num_psf_points = dram_info->num_psf_gv_points;
 219
 220         if (DISPLAY_VER(dev_priv) >= 14) {
 221                 switch (dram_info->type) {
 222                 case INTEL_DRAM_DDR4:
 223                         qi->t_bl = 4;
 224                         qi->max_numchannels = 2;
 225                         qi->channel_width = 64;
 226                         qi->deinterleave = 2;
 227                         break;
 228                 case INTEL_DRAM_DDR5:
 229                         qi->t_bl = 8;
 230                         qi->max_numchannels = 4;
 231                         qi->channel_width = 32;
 232                         qi->deinterleave = 2;
 233                         break;
 234                 case INTEL_DRAM_LPDDR4:
 235                 case INTEL_DRAM_LPDDR5:
 236                         qi->t_bl = 16;
 237                         qi->max_numchannels = 8;
 238                         qi->channel_width = 16;
 239                         qi->deinterleave = 4;
 240                         break;
 241                 default:
 242                         MISSING_CASE(dram_info->type);
 243                         return -EINVAL;
 244                 }
 245         } else if (DISPLAY_VER(dev_priv) >= 12) {
 246                 switch (dram_info->type) {
 247                 case INTEL_DRAM_DDR4:
 248                         qi->t_bl = is_y_tile ? 8 : 4;
 249                         qi->max_numchannels = 2;
 250                         qi->channel_width = 64;
 251                         qi->deinterleave = is_y_tile ? 1 : 2;
 252                         break;
 253                 case INTEL_DRAM_DDR5:
 254                         qi->t_bl = is_y_tile ? 16 : 8;
 255                         qi->max_numchannels = 4;
 256                         qi->channel_width = 32;
 257                         qi->deinterleave = is_y_tile ? 1 : 2;
 258                         break;
 259                 case INTEL_DRAM_LPDDR4:
 260                         if (IS_ROCKETLAKE(dev_priv)) {
 261                                 qi->t_bl = 8;
 262                                 qi->max_numchannels = 4;
 263                                 qi->channel_width = 32;
 264                                 qi->deinterleave = 2;
 265                                 break;
 266                         }
 267                         fallthrough;
 268                 case INTEL_DRAM_LPDDR5:
 269                         qi->t_bl = 16;
 270                         qi->max_numchannels = 8;
 271                         qi->channel_width = 16;
 272                         qi->deinterleave = is_y_tile ? 2 : 4;
 273                         break;
 274                 default:
 275                         qi->t_bl = 16;
 276                         qi->max_numchannels = 1;
 277                         break;
 278                 }
 279         } else if (DISPLAY_VER(dev_priv) == 11) {
 280                 qi->t_bl = dev_priv->dram_info.type == INTEL_DRAM_DDR4 ? 4 : 8;
 281                 qi->max_numchannels = 1;
 282         }
 283
 284         if (drm_WARN_ON(&dev_priv->drm,
 285                         qi->num_points > ARRAY_SIZE(qi->points)))
 286                 qi->num_points = ARRAY_SIZE(qi->points);
 287
 288         for (i = 0; i < qi->num_points; i++) {
 289                 struct intel_qgv_point *sp = &qi->points[i];
 290
 291                 ret = intel_read_qgv_point_info(dev_priv, sp, i);
 292                 if (ret)
 293                         return ret;
 294
 295                 drm_dbg_kms(&dev_priv->drm,
 296                             "QGV %d: DCLK=%d tRP=%d tRDPRE=%d tRAS=%d tRCD=%d tRC=%d\n",
 297                             i, sp->dclk, sp->t_rp, sp->t_rdpre, sp->t_ras,
 298                             sp->t_rcd, sp->t_rc);
 299         }
 300
 301         if (qi->num_psf_points > 0) {
 302                 ret = adls_pcode_read_psf_gv_point_info(dev_priv, qi->psf_points);
 303                 if (ret) {
 304                         drm_err(&dev_priv->drm, "Failed to read PSF point data; PSF points will not be considered in bandwidth calculations.\n");
 305                         qi->num_psf_points = 0;
 306                 }
 307
 308                 for (i = 0; i < qi->num_psf_points; i++)
 309                         drm_dbg_kms(&dev_priv->drm,
 310                                     "PSF GV %d: CLK=%d \n",
 311                                     i, qi->psf_points[i].clk);
 312         }
 313
 314         return 0;
 315 }
 316
 317 static int adl_calc_psf_bw(int clk)
 318 {
 319         /*
 320          * clk is multiples of 16.666MHz (100/6)
 321          * According to BSpec PSF GV bandwidth is
 322          * calculated as BW = 64 * clk * 16.666Mhz
 323          */
 324         return DIV_ROUND_CLOSEST(64 * clk * 100, 6);
 325 }
 326
 327 static int icl_sagv_max_dclk(const struct intel_qgv_info *qi)
 328 {
 329         u16 dclk = 0;
 330         int i;
 331
 332         for (i = 0; i < qi->num_points; i++)
 333                 dclk = max(dclk, qi->points[i].dclk);
 334
 335         return dclk;
 336 }
 337
 338 struct intel_sa_info {
 339         u16 displayrtids;
 340         u8 deburst, deprogbwlimit, derating;
 341 };
 342
 343 static const struct intel_sa_info icl_sa_info = {
 344         .deburst = 8,
 345         .deprogbwlimit = 25, /* GB/s */
 346         .displayrtids = 128,
 347         .derating = 10,
 348 };
 349
 350 static const struct intel_sa_info tgl_sa_info = {
 351         .deburst = 16,
 352         .deprogbwlimit = 34, /* GB/s */
 353         .displayrtids = 256,
 354         .derating = 10,
 355 };
 356
 357 static const struct intel_sa_info rkl_sa_info = {
 358         .deburst = 8,
 359         .deprogbwlimit = 20, /* GB/s */
 360         .displayrtids = 128,
 361         .derating = 10,
 362 };
 363
 364 static const struct intel_sa_info adls_sa_info = {
 365         .deburst = 16,
 366         .deprogbwlimit = 38, /* GB/s */
 367         .displayrtids = 256,
 368         .derating = 10,
 369 };
 370
 371 static const struct intel_sa_info adlp_sa_info = {
 372         .deburst = 16,
 373         .deprogbwlimit = 38, /* GB/s */
 374         .displayrtids = 256,
 375         .derating = 20,
 376 };
 377
 378 static const struct intel_sa_info mtl_sa_info = {
 379         .deburst = 32,
 380         .deprogbwlimit = 38, /* GB/s */
 381         .displayrtids = 256,
 382         .derating = 10,
 383 };
 384
 385 static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa)
 386 {
 387         struct intel_qgv_info qi = {};
 388         bool is_y_tile = true; /* assume y tile may be used */
 389         int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels);
 390         int ipqdepth, ipqdepthpch = 16;
 391         int dclk_max;
 392         int maxdebw;
 393         int num_groups = ARRAY_SIZE(dev_priv->display.bw.max);
 394         int i, ret;
 395
 396         ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile);
 397         if (ret) {
 398                 drm_dbg_kms(&dev_priv->drm,
 399                             "Failed to get memory subsystem information, ignoring bandwidth limits");
 400                 return ret;
 401         }
 402
 403         dclk_max = icl_sagv_max_dclk(&qi);
 404         maxdebw = min(sa->deprogbwlimit * 1000, dclk_max * 16 * 6 / 10);
 405         ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
 406         qi.deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
 407
 408         for (i = 0; i < num_groups; i++) {
 409                 struct intel_bw_info *bi = &dev_priv->display.bw.max[i];
 410                 int clpchgroup;
 411                 int j;
 412
 413                 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
 414                 bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1;
 415
 416                 bi->num_qgv_points = qi.num_points;
 417                 bi->num_psf_gv_points = qi.num_psf_points;
 418
 419                 for (j = 0; j < qi.num_points; j++) {
 420                         const struct intel_qgv_point *sp = &qi.points[j];
 421                         int ct, bw;
 422
 423                         /*
 424                          * Max row cycle time
 425                          *
 426                          * FIXME what is the logic behind the
 427                          * assumed burst length?
 428                          */
 429                         ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
 430                                    (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
 431                         bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
 432
 433                         bi->deratedbw[j] = min(maxdebw,
 434                                                bw * (100 - sa->derating) / 100);
 435
 436                         drm_dbg_kms(&dev_priv->drm,
 437                                     "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
 438                                     i, j, bi->num_planes, bi->deratedbw[j]);
 439                 }
 440         }
 441         /*
 442          * In case if SAGV is disabled in BIOS, we always get 1
 443          * SAGV point, but we can't send PCode commands to restrict it
 444          * as it will fail and pointless anyway.
 445          */
 446         if (qi.num_points == 1)
 447                 dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 448         else
 449                 dev_priv->display.sagv.status = I915_SAGV_ENABLED;
 450
 451         return 0;
 452 }
 453
 454 static int tgl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa)
 455 {
 456         struct intel_qgv_info qi = {};
 457         const struct dram_info *dram_info = &dev_priv->dram_info;
 458         bool is_y_tile = true; /* assume y tile may be used */
 459         int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels);
 460         int ipqdepth, ipqdepthpch = 16;
 461         int dclk_max;
 462         int maxdebw, peakbw;
 463         int clperchgroup;
 464         int num_groups = ARRAY_SIZE(dev_priv->display.bw.max);
 465         int i, ret;
 466
 467         ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile);
 468         if (ret) {
 469                 drm_dbg_kms(&dev_priv->drm,
 470                             "Failed to get memory subsystem information, ignoring bandwidth limits");
 471                 return ret;
 472         }
 473
 474         if (DISPLAY_VER(dev_priv) < 14 &&
 475             (dram_info->type == INTEL_DRAM_LPDDR4 || dram_info->type == INTEL_DRAM_LPDDR5))
 476                 num_channels *= 2;
 477
 478         qi.deinterleave = qi.deinterleave ? : DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
 479
 480         if (num_channels < qi.max_numchannels && DISPLAY_VER(dev_priv) >= 12)
 481                 qi.deinterleave = max(DIV_ROUND_UP(qi.deinterleave, 2), 1);
 482
 483         if (DISPLAY_VER(dev_priv) > 11 && num_channels > qi.max_numchannels)
 484                 drm_warn(&dev_priv->drm, "Number of channels exceeds max number of channels.");
 485         if (qi.max_numchannels != 0)
 486                 num_channels = min_t(u8, num_channels, qi.max_numchannels);
 487
 488         dclk_max = icl_sagv_max_dclk(&qi);
 489
 490         peakbw = num_channels * DIV_ROUND_UP(qi.channel_width, 8) * dclk_max;
 491         maxdebw = min(sa->deprogbwlimit * 1000, peakbw * 6 / 10); /* 60% */
 492
 493         ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
 494         /*
 495          * clperchgroup = 4kpagespermempage * clperchperblock,
 496          * clperchperblock = 8 / num_channels * interleave
 497          */
 498         clperchgroup = 4 * DIV_ROUND_UP(8, num_channels) * qi.deinterleave;
 499
 500         for (i = 0; i < num_groups; i++) {
 501                 struct intel_bw_info *bi = &dev_priv->display.bw.max[i];
 502                 struct intel_bw_info *bi_next;
 503                 int clpchgroup;
 504                 int j;
 505
 506                 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
 507
 508                 if (i < num_groups - 1) {
 509                         bi_next = &dev_priv->display.bw.max[i + 1];
 510
 511                         if (clpchgroup < clperchgroup)
 512                                 bi_next->num_planes = (ipqdepth - clpchgroup) /
 513                                                        clpchgroup + 1;
 514                         else
 515                                 bi_next->num_planes = 0;
 516                 }
 517
 518                 bi->num_qgv_points = qi.num_points;
 519                 bi->num_psf_gv_points = qi.num_psf_points;
 520
 521                 for (j = 0; j < qi.num_points; j++) {
 522                         const struct intel_qgv_point *sp = &qi.points[j];
 523                         int ct, bw;
 524
 525                         /*
 526                          * Max row cycle time
 527                          *
 528                          * FIXME what is the logic behind the
 529                          * assumed burst length?
 530                          */
 531                         ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
 532                                    (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
 533                         bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
 534
 535                         bi->deratedbw[j] = min(maxdebw,
 536                                                bw * (100 - sa->derating) / 100);
 537                         bi->peakbw[j] = DIV_ROUND_CLOSEST(sp->dclk *
 538                                                           num_channels *
 539                                                           qi.channel_width, 8);
 540
 541                         drm_dbg_kms(&dev_priv->drm,
 542                                     "BW%d / QGV %d: num_planes=%d deratedbw=%u peakbw: %u\n",
 543                                     i, j, bi->num_planes, bi->deratedbw[j],
 544                                     bi->peakbw[j]);
 545                 }
 546
 547                 for (j = 0; j < qi.num_psf_points; j++) {
 548                         const struct intel_psf_gv_point *sp = &qi.psf_points[j];
 549
 550                         bi->psf_bw[j] = adl_calc_psf_bw(sp->clk);
 551
 552                         drm_dbg_kms(&dev_priv->drm,
 553                                     "BW%d / PSF GV %d: num_planes=%d bw=%u\n",
 554                                     i, j, bi->num_planes, bi->psf_bw[j]);
 555                 }
 556         }
 557
 558         /*
 559          * In case if SAGV is disabled in BIOS, we always get 1
 560          * SAGV point, but we can't send PCode commands to restrict it
 561          * as it will fail and pointless anyway.
 562          */
 563         if (qi.num_points == 1)
 564                 dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 565         else
 566                 dev_priv->display.sagv.status = I915_SAGV_ENABLED;
 567
 568         return 0;
 569 }
 570
 571 static void dg2_get_bw_info(struct drm_i915_private *i915)
 572 {
 573         unsigned int deratedbw = IS_DG2_G11(i915) ? 38000 : 50000;
 574         int num_groups = ARRAY_SIZE(i915->display.bw.max);
 575         int i;
 576
 577         /*
 578          * DG2 doesn't have SAGV or QGV points, just a constant max bandwidth
 579          * that doesn't depend on the number of planes enabled. So fill all the
 580          * plane group with constant bw information for uniformity with other
 581          * platforms. DG2-G10 platforms have a constant 50 GB/s bandwidth,
 582          * whereas DG2-G11 platforms have 38 GB/s.
 583          */
 584         for (i = 0; i < num_groups; i++) {
 585                 struct intel_bw_info *bi = &i915->display.bw.max[i];
 586
 587                 bi->num_planes = 1;
 588                 /* Need only one dummy QGV point per group */
 589                 bi->num_qgv_points = 1;
 590                 bi->deratedbw[0] = deratedbw;
 591         }
 592
 593         i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 594 }
 595
 596 static unsigned int icl_max_bw_index(struct drm_i915_private *dev_priv,
 597                                      int num_planes, int qgv_point)
 598 {
 599         int i;
 600
 601         /*
 602          * Let's return max bw for 0 planes
 603          */
 604         num_planes = max(1, num_planes);
 605
 606         for (i = 0; i < ARRAY_SIZE(dev_priv->display.bw.max); i++) {
 607                 const struct intel_bw_info *bi =
 608                         &dev_priv->display.bw.max[i];
 609
 610                 /*
 611                  * Pcode will not expose all QGV points when
 612                  * SAGV is forced to off/min/med/max.
 613                  */
 614                 if (qgv_point >= bi->num_qgv_points)
 615                         return UINT_MAX;
 616
 617                 if (num_planes >= bi->num_planes)
 618                         return i;
 619         }
 620
 621         return UINT_MAX;
 622 }
 623
 624 static unsigned int tgl_max_bw_index(struct drm_i915_private *dev_priv,
 625                                      int num_planes, int qgv_point)
 626 {
 627         int i;
 628
 629         /*
 630          * Let's return max bw for 0 planes
 631          */
 632         num_planes = max(1, num_planes);
 633
 634         for (i = ARRAY_SIZE(dev_priv->display.bw.max) - 1; i >= 0; i--) {
 635                 const struct intel_bw_info *bi =
 636                         &dev_priv->display.bw.max[i];
 637
 638                 /*
 639                  * Pcode will not expose all QGV points when
 640                  * SAGV is forced to off/min/med/max.
 641                  */
 642                 if (qgv_point >= bi->num_qgv_points)
 643                         return UINT_MAX;
 644
 645                 if (num_planes <= bi->num_planes)
 646                         return i;
 647         }
 648
 649         return 0;
 650 }
 651
 652 static unsigned int adl_psf_bw(struct drm_i915_private *dev_priv,
 653                                int psf_gv_point)
 654 {
 655         const struct intel_bw_info *bi =
 656                         &dev_priv->display.bw.max[0];
 657
 658         return bi->psf_bw[psf_gv_point];
 659 }
 660
 661 void intel_bw_init_hw(struct drm_i915_private *dev_priv)
 662 {
 663         if (!HAS_DISPLAY(dev_priv))
 664                 return;
 665
 666         if (DISPLAY_VER(dev_priv) >= 14)
 667                 tgl_get_bw_info(dev_priv, &mtl_sa_info);
 668         else if (IS_DG2(dev_priv))
 669                 dg2_get_bw_info(dev_priv);
 670         else if (IS_ALDERLAKE_P(dev_priv))
 671                 tgl_get_bw_info(dev_priv, &adlp_sa_info);
 672         else if (IS_ALDERLAKE_S(dev_priv))
 673                 tgl_get_bw_info(dev_priv, &adls_sa_info);
 674         else if (IS_ROCKETLAKE(dev_priv))
 675                 tgl_get_bw_info(dev_priv, &rkl_sa_info);
 676         else if (DISPLAY_VER(dev_priv) == 12)
 677                 tgl_get_bw_info(dev_priv, &tgl_sa_info);
 678         else if (DISPLAY_VER(dev_priv) == 11)
 679                 icl_get_bw_info(dev_priv, &icl_sa_info);
 680 }
 681
 682 static unsigned int intel_bw_crtc_num_active_planes(const struct intel_crtc_state *crtc_state)
 683 {
 684         /*
 685          * We assume cursors are small enough
 686          * to not not cause bandwidth problems.
 687          */
 688         return hweight8(crtc_state->active_planes & ~BIT(PLANE_CURSOR));
 689 }
 690
 691 static unsigned int intel_bw_crtc_data_rate(const struct intel_crtc_state *crtc_state)
 692 {
 693         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 694         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 695         unsigned int data_rate = 0;
 696         enum plane_id plane_id;
 697
 698         for_each_plane_id_on_crtc(crtc, plane_id) {
 699                 /*
 700                  * We assume cursors are small enough
 701                  * to not not cause bandwidth problems.
 702                  */
 703                 if (plane_id == PLANE_CURSOR)
 704                         continue;
 705
 706                 data_rate += crtc_state->data_rate[plane_id];
 707
 708                 if (DISPLAY_VER(i915) < 11)
 709                         data_rate += crtc_state->data_rate_y[plane_id];
 710         }
 711
 712         return data_rate;
 713 }
 714
 715 /* "Maximum Pipe Read Bandwidth" */
 716 static int intel_bw_crtc_min_cdclk(const struct intel_crtc_state *crtc_state)
 717 {
 718         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 719         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 720
 721         if (DISPLAY_VER(i915) < 12)
 722                 return 0;
 723
 724         return DIV_ROUND_UP_ULL(mul_u32_u32(intel_bw_crtc_data_rate(crtc_state), 10), 512);
 725 }
 726
 727 void intel_bw_crtc_update(struct intel_bw_state *bw_state,
 728                           const struct intel_crtc_state *crtc_state)
 729 {
 730         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 731         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 732
 733         bw_state->data_rate[crtc->pipe] =
 734                 intel_bw_crtc_data_rate(crtc_state);
 735         bw_state->num_active_planes[crtc->pipe] =
 736                 intel_bw_crtc_num_active_planes(crtc_state);
 737
 738         drm_dbg_kms(&i915->drm, "pipe %c data rate %u num active planes %u\n",
 739                     pipe_name(crtc->pipe),
 740                     bw_state->data_rate[crtc->pipe],
 741                     bw_state->num_active_planes[crtc->pipe]);
 742 }
 743
 744 static unsigned int intel_bw_num_active_planes(struct drm_i915_private *dev_priv,
 745                                                const struct intel_bw_state *bw_state)
 746 {
 747         unsigned int num_active_planes = 0;
 748         enum pipe pipe;
 749
 750         for_each_pipe(dev_priv, pipe)
 751                 num_active_planes += bw_state->num_active_planes[pipe];
 752
 753         return num_active_planes;
 754 }
 755
 756 static unsigned int intel_bw_data_rate(struct drm_i915_private *dev_priv,
 757                                        const struct intel_bw_state *bw_state)
 758 {
 759         unsigned int data_rate = 0;
 760         enum pipe pipe;
 761
 762         for_each_pipe(dev_priv, pipe)
 763                 data_rate += bw_state->data_rate[pipe];
 764
 765         if (DISPLAY_VER(dev_priv) >= 13 && i915_vtd_active(dev_priv))
 766                 data_rate = DIV_ROUND_UP(data_rate * 105, 100);
 767
 768         return data_rate;
 769 }
 770
 771 struct intel_bw_state *
 772 intel_atomic_get_old_bw_state(struct intel_atomic_state *state)
 773 {
 774         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 775         struct intel_global_state *bw_state;
 776
 777         bw_state = intel_atomic_get_old_global_obj_state(state, &dev_priv->display.bw.obj);
 778
 779         return to_intel_bw_state(bw_state);
 780 }
 781
 782 struct intel_bw_state *
 783 intel_atomic_get_new_bw_state(struct intel_atomic_state *state)
 784 {
 785         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 786         struct intel_global_state *bw_state;
 787
 788         bw_state = intel_atomic_get_new_global_obj_state(state, &dev_priv->display.bw.obj);
 789
 790         return to_intel_bw_state(bw_state);
 791 }
 792
 793 struct intel_bw_state *
 794 intel_atomic_get_bw_state(struct intel_atomic_state *state)
 795 {
 796         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 797         struct intel_global_state *bw_state;
 798
 799         bw_state = intel_atomic_get_global_obj_state(state, &dev_priv->display.bw.obj);
 800         if (IS_ERR(bw_state))
 801                 return ERR_CAST(bw_state);
 802
 803         return to_intel_bw_state(bw_state);
 804 }
 805
 806 static int mtl_find_qgv_points(struct drm_i915_private *i915,
 807                                unsigned int data_rate,
 808                                unsigned int num_active_planes,
 809                                struct intel_bw_state *new_bw_state)
 810 {
 811         unsigned int best_rate = UINT_MAX;
 812         unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
 813         unsigned int qgv_peak_bw  = 0;
 814         int i;
 815         int ret;
 816
 817         ret = intel_atomic_lock_global_state(&new_bw_state->base);
 818         if (ret)
 819                 return ret;
 820
 821         /*
 822          * If SAGV cannot be enabled, disable the pcode SAGV by passing all 1's
 823          * for qgv peak bw in PM Demand request. So assign UINT_MAX if SAGV is
 824          * not enabled. PM Demand code will clamp the value for the register
 825          */
 826         if (!intel_can_enable_sagv(i915, new_bw_state)) {
 827                 new_bw_state->qgv_point_peakbw = U16_MAX;
 828                 drm_dbg_kms(&i915->drm, "No SAGV, use UINT_MAX as peak bw.");
 829                 return 0;
 830         }
 831
 832         /*
 833          * Find the best QGV point by comparing the data_rate with max data rate
 834          * offered per plane group
 835          */
 836         for (i = 0; i < num_qgv_points; i++) {
 837                 unsigned int bw_index =
 838                         tgl_max_bw_index(i915, num_active_planes, i);
 839                 unsigned int max_data_rate;
 840
 841                 if (bw_index >= ARRAY_SIZE(i915->display.bw.max))
 842                         continue;
 843
 844                 max_data_rate = i915->display.bw.max[bw_index].deratedbw[i];
 845
 846                 if (max_data_rate < data_rate)
 847                         continue;
 848
 849                 if (max_data_rate - data_rate < best_rate) {
 850                         best_rate = max_data_rate - data_rate;
 851                         qgv_peak_bw = i915->display.bw.max[bw_index].peakbw[i];
 852                 }
 853
 854                 drm_dbg_kms(&i915->drm, "QGV point %d: max bw %d required %d qgv_peak_bw: %d\n",
 855                             i, max_data_rate, data_rate, qgv_peak_bw);
 856         }
 857
 858         drm_dbg_kms(&i915->drm, "Matching peaks QGV bw: %d for required data rate: %d\n",
 859                     qgv_peak_bw, data_rate);
 860
 861         /*
 862          * The display configuration cannot be supported if no QGV point
 863          * satisfying the required data rate is found
 864          */
 865         if (qgv_peak_bw == 0) {
 866                 drm_dbg_kms(&i915->drm, "No QGV points for bw %d for display configuration(%d active planes).\n",
 867                             data_rate, num_active_planes);
 868                 return -EINVAL;
 869         }
 870
 871         /* MTL PM DEMAND expects QGV BW parameter in multiples of 100 mbps */
 872         new_bw_state->qgv_point_peakbw = DIV_ROUND_CLOSEST(qgv_peak_bw, 100);
 873
 874         return 0;
 875 }
 876
 877 static int icl_find_qgv_points(struct drm_i915_private *i915,
 878                                unsigned int data_rate,
 879                                unsigned int num_active_planes,
 880                                const struct intel_bw_state *old_bw_state,
 881                                struct intel_bw_state *new_bw_state)
 882 {
 883         unsigned int max_bw_point = 0;
 884         unsigned int max_bw = 0;
 885         unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points;
 886         unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
 887         u16 psf_points = 0;
 888         u16 qgv_points = 0;
 889         int i;
 890         int ret;
 891
 892         ret = intel_atomic_lock_global_state(&new_bw_state->base);
 893         if (ret)
 894                 return ret;
 895
 896         for (i = 0; i < num_qgv_points; i++) {
 897                 unsigned int idx;
 898                 unsigned int max_data_rate;
 899
 900                 if (DISPLAY_VER(i915) > 11)
 901                         idx = tgl_max_bw_index(i915, num_active_planes, i);
 902                 else
 903                         idx = icl_max_bw_index(i915, num_active_planes, i);
 904
 905                 if (idx >= ARRAY_SIZE(i915->display.bw.max))
 906                         continue;
 907
 908                 max_data_rate = i915->display.bw.max[idx].deratedbw[i];
 909
 910                 /*
 911                  * We need to know which qgv point gives us
 912                  * maximum bandwidth in order to disable SAGV
 913                  * if we find that we exceed SAGV block time
 914                  * with watermarks. By that moment we already
 915                  * have those, as it is calculated earlier in
 916                  * intel_atomic_check,
 917                  */
 918                 if (max_data_rate > max_bw) {
 919                         max_bw_point = i;
 920                         max_bw = max_data_rate;
 921                 }
 922                 if (max_data_rate >= data_rate)
 923                         qgv_points |= BIT(i);
 924
 925                 drm_dbg_kms(&i915->drm, "QGV point %d: max bw %d required %d\n",
 926                             i, max_data_rate, data_rate);
 927         }
 928
 929         for (i = 0; i < num_psf_gv_points; i++) {
 930                 unsigned int max_data_rate = adl_psf_bw(i915, i);
 931
 932                 if (max_data_rate >= data_rate)
 933                         psf_points |= BIT(i);
 934
 935                 drm_dbg_kms(&i915->drm, "PSF GV point %d: max bw %d"
 936                             " required %d\n",
 937                             i, max_data_rate, data_rate);
 938         }
 939
 940         /*
 941          * BSpec states that we always should have at least one allowed point
 942          * left, so if we couldn't - simply reject the configuration for obvious
 943          * reasons.
 944          */
 945         if (qgv_points == 0) {
 946                 drm_dbg_kms(&i915->drm, "No QGV points provide sufficient memory"
 947                             " bandwidth %d for display configuration(%d active planes).\n",
 948                             data_rate, num_active_planes);
 949                 return -EINVAL;
 950         }
 951
 952         if (num_psf_gv_points > 0 && psf_points == 0) {
 953                 drm_dbg_kms(&i915->drm, "No PSF GV points provide sufficient memory"
 954                             " bandwidth %d for display configuration(%d active planes).\n",
 955                             data_rate, num_active_planes);
 956                 return -EINVAL;
 957         }
 958
 959         /*
 960          * Leave only single point with highest bandwidth, if
 961          * we can't enable SAGV due to the increased memory latency it may
 962          * cause.
 963          */
 964         if (!intel_can_enable_sagv(i915, new_bw_state)) {
 965                 qgv_points = BIT(max_bw_point);
 966                 drm_dbg_kms(&i915->drm, "No SAGV, using single QGV point %d\n",
 967                             max_bw_point);
 968         }
 969
 970         /*
 971          * We store the ones which need to be masked as that is what PCode
 972          * actually accepts as a parameter.
 973          */
 974         new_bw_state->qgv_points_mask =
 975                 ~(ICL_PCODE_REQ_QGV_PT(qgv_points) |
 976                   ADLS_PCODE_REQ_PSF_PT(psf_points)) &
 977                 icl_qgv_points_mask(i915);
 978
 979         /*
 980          * If the actual mask had changed we need to make sure that
 981          * the commits are serialized(in case this is a nomodeset, nonblocking)
 982          */
 983         if (new_bw_state->qgv_points_mask != old_bw_state->qgv_points_mask) {
 984                 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
 985                 if (ret)
 986                         return ret;
 987         }
 988
 989         return 0;
 990 }
 991
 992 static int intel_bw_check_qgv_points(struct drm_i915_private *i915,
 993                                      const struct intel_bw_state *old_bw_state,
 994                                      struct intel_bw_state *new_bw_state)
 995 {
 996         unsigned int data_rate = intel_bw_data_rate(i915, new_bw_state);
 997         unsigned int num_active_planes =
 998                         intel_bw_num_active_planes(i915, new_bw_state);
 999
1000         data_rate = DIV_ROUND_UP(data_rate, 1000);
1001
1002         if (DISPLAY_VER(i915) >= 14)
1003                 return mtl_find_qgv_points(i915, data_rate, num_active_planes,
1004                                            new_bw_state);
1005         else
1006                 return icl_find_qgv_points(i915, data_rate, num_active_planes,
1007                                            old_bw_state, new_bw_state);
1008 }
1009
1010 static bool intel_bw_state_changed(struct drm_i915_private *i915,
1011                                    const struct intel_bw_state *old_bw_state,
1012                                    const struct intel_bw_state *new_bw_state)
1013 {
1014         enum pipe pipe;
1015
1016         for_each_pipe(i915, pipe) {
1017                 const struct intel_dbuf_bw *old_crtc_bw =
1018                         &old_bw_state->dbuf_bw[pipe];
1019                 const struct intel_dbuf_bw *new_crtc_bw =
1020                         &new_bw_state->dbuf_bw[pipe];
1021                 enum dbuf_slice slice;
1022
1023                 for_each_dbuf_slice(i915, slice) {
1024                         if (old_crtc_bw->max_bw[slice] != new_crtc_bw->max_bw[slice] ||
1025                             old_crtc_bw->active_planes[slice] != new_crtc_bw->active_planes[slice])
1026                                 return true;
1027                 }
1028
1029                 if (old_bw_state->min_cdclk[pipe] != new_bw_state->min_cdclk[pipe])
1030                         return true;
1031         }
1032
1033         return false;
1034 }
1035
1036 static void skl_plane_calc_dbuf_bw(struct intel_bw_state *bw_state,
1037                                    struct intel_crtc *crtc,
1038                                    enum plane_id plane_id,
1039                                    const struct skl_ddb_entry *ddb,
1040                                    unsigned int data_rate)
1041 {
1042         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1043         struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe];
1044         unsigned int dbuf_mask = skl_ddb_dbuf_slice_mask(i915, ddb);
1045         enum dbuf_slice slice;
1046
1047         /*
1048          * The arbiter can only really guarantee an
1049          * equal share of the total bw to each plane.
1050          */
1051         for_each_dbuf_slice_in_mask(i915, slice, dbuf_mask) {
1052                 crtc_bw->max_bw[slice] = max(crtc_bw->max_bw[slice], data_rate);
1053                 crtc_bw->active_planes[slice] |= BIT(plane_id);
1054         }
1055 }
1056
1057 static void skl_crtc_calc_dbuf_bw(struct intel_bw_state *bw_state,
1058                                   const struct intel_crtc_state *crtc_state)
1059 {
1060         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1061         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1062         struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe];
1063         enum plane_id plane_id;
1064
1065         memset(crtc_bw, 0, sizeof(*crtc_bw));
1066
1067         if (!crtc_state->hw.active)
1068                 return;
1069
1070         for_each_plane_id_on_crtc(crtc, plane_id) {
1071                 /*
1072                  * We assume cursors are small enough
1073                  * to not cause bandwidth problems.
1074                  */
1075                 if (plane_id == PLANE_CURSOR)
1076                         continue;
1077
1078                 skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id,
1079                                        &crtc_state->wm.skl.plane_ddb[plane_id],
1080                                        crtc_state->data_rate[plane_id]);
1081
1082                 if (DISPLAY_VER(i915) < 11)
1083                         skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id,
1084                                                &crtc_state->wm.skl.plane_ddb_y[plane_id],
1085                                                crtc_state->data_rate[plane_id]);
1086         }
1087 }
1088
1089 /* "Maximum Data Buffer Bandwidth" */
1090 static int
1091 intel_bw_dbuf_min_cdclk(struct drm_i915_private *i915,
1092                         const struct intel_bw_state *bw_state)
1093 {
1094         unsigned int total_max_bw = 0;
1095         enum dbuf_slice slice;
1096
1097         for_each_dbuf_slice(i915, slice) {
1098                 int num_active_planes = 0;
1099                 unsigned int max_bw = 0;
1100                 enum pipe pipe;
1101
1102                 /*
1103                  * The arbiter can only really guarantee an
1104                  * equal share of the total bw to each plane.
1105                  */
1106                 for_each_pipe(i915, pipe) {
1107                         const struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[pipe];
1108
1109                         max_bw = max(crtc_bw->max_bw[slice], max_bw);
1110                         num_active_planes += hweight8(crtc_bw->active_planes[slice]);
1111                 }
1112                 max_bw *= num_active_planes;
1113
1114                 total_max_bw = max(total_max_bw, max_bw);
1115         }
1116
1117         return DIV_ROUND_UP(total_max_bw, 64);
1118 }
1119
1120 int intel_bw_min_cdclk(struct drm_i915_private *i915,
1121                        const struct intel_bw_state *bw_state)
1122 {
1123         enum pipe pipe;
1124         int min_cdclk;
1125
1126         min_cdclk = intel_bw_dbuf_min_cdclk(i915, bw_state);
1127
1128         for_each_pipe(i915, pipe)
1129                 min_cdclk = max(bw_state->min_cdclk[pipe], min_cdclk);
1130
1131         return min_cdclk;
1132 }
1133
1134 int intel_bw_calc_min_cdclk(struct intel_atomic_state *state,
1135                             bool *need_cdclk_calc)
1136 {
1137         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
1138         struct intel_bw_state *new_bw_state = NULL;
1139         const struct intel_bw_state *old_bw_state = NULL;
1140         const struct intel_cdclk_state *cdclk_state;
1141         const struct intel_crtc_state *crtc_state;
1142         int old_min_cdclk, new_min_cdclk;
1143         struct intel_crtc *crtc;
1144         int i;
1145
1146         if (DISPLAY_VER(dev_priv) < 9)
1147                 return 0;
1148
1149         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
1150                 new_bw_state = intel_atomic_get_bw_state(state);
1151                 if (IS_ERR(new_bw_state))
1152                         return PTR_ERR(new_bw_state);
1153
1154                 old_bw_state = intel_atomic_get_old_bw_state(state);
1155
1156                 skl_crtc_calc_dbuf_bw(new_bw_state, crtc_state);
1157
1158                 new_bw_state->min_cdclk[crtc->pipe] =
1159                         intel_bw_crtc_min_cdclk(crtc_state);
1160         }
1161
1162         if (!old_bw_state)
1163                 return 0;
1164
1165         if (intel_bw_state_changed(dev_priv, old_bw_state, new_bw_state)) {
1166                 int ret = intel_atomic_lock_global_state(&new_bw_state->base);
1167                 if (ret)
1168                         return ret;
1169         }
1170
1171         old_min_cdclk = intel_bw_min_cdclk(dev_priv, old_bw_state);
1172         new_min_cdclk = intel_bw_min_cdclk(dev_priv, new_bw_state);
1173
1174         /*
1175          * No need to check against the cdclk state if
1176          * the min cdclk doesn't increase.
1177          *
1178          * Ie. we only ever increase the cdclk due to bandwidth
1179          * requirements. This can reduce back and forth
1180          * display blinking due to constant cdclk changes.
1181          */
1182         if (new_min_cdclk <= old_min_cdclk)
1183                 return 0;
1184
1185         cdclk_state = intel_atomic_get_cdclk_state(state);
1186         if (IS_ERR(cdclk_state))
1187                 return PTR_ERR(cdclk_state);
1188
1189         /*
1190          * No need to recalculate the cdclk state if
1191          * the min cdclk doesn't increase.
1192          *
1193          * Ie. we only ever increase the cdclk due to bandwidth
1194          * requirements. This can reduce back and forth
1195          * display blinking due to constant cdclk changes.
1196          */
1197         if (new_min_cdclk <= cdclk_state->bw_min_cdclk)
1198                 return 0;
1199
1200         drm_dbg_kms(&dev_priv->drm,
1201                     "new bandwidth min cdclk (%d kHz) > old min cdclk (%d kHz)\n",
1202                     new_min_cdclk, cdclk_state->bw_min_cdclk);
1203         *need_cdclk_calc = true;
1204
1205         return 0;
1206 }
1207
1208 static int intel_bw_check_data_rate(struct intel_atomic_state *state, bool *changed)
1209 {
1210         struct drm_i915_private *i915 = to_i915(state->base.dev);
1211         const struct intel_crtc_state *new_crtc_state, *old_crtc_state;
1212         struct intel_crtc *crtc;
1213         int i;
1214
1215         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
1216                                             new_crtc_state, i) {
1217                 unsigned int old_data_rate =
1218                         intel_bw_crtc_data_rate(old_crtc_state);
1219                 unsigned int new_data_rate =
1220                         intel_bw_crtc_data_rate(new_crtc_state);
1221                 unsigned int old_active_planes =
1222                         intel_bw_crtc_num_active_planes(old_crtc_state);
1223                 unsigned int new_active_planes =
1224                         intel_bw_crtc_num_active_planes(new_crtc_state);
1225                 struct intel_bw_state *new_bw_state;
1226
1227                 /*
1228                  * Avoid locking the bw state when
1229                  * nothing significant has changed.
1230                  */
1231                 if (old_data_rate == new_data_rate &&
1232                     old_active_planes == new_active_planes)
1233                         continue;
1234
1235                 new_bw_state = intel_atomic_get_bw_state(state);
1236                 if (IS_ERR(new_bw_state))
1237                         return PTR_ERR(new_bw_state);
1238
1239                 new_bw_state->data_rate[crtc->pipe] = new_data_rate;
1240                 new_bw_state->num_active_planes[crtc->pipe] = new_active_planes;
1241
1242                 *changed = true;
1243
1244                 drm_dbg_kms(&i915->drm,
1245                             "[CRTC:%d:%s] data rate %u num active planes %u\n",
1246                             crtc->base.base.id, crtc->base.name,
1247                             new_bw_state->data_rate[crtc->pipe],
1248                             new_bw_state->num_active_planes[crtc->pipe]);
1249         }
1250
1251         return 0;
1252 }
1253
1254 int intel_bw_atomic_check(struct intel_atomic_state *state)
1255 {
1256         bool changed = false;
1257         struct drm_i915_private *i915 = to_i915(state->base.dev);
1258         struct intel_bw_state *new_bw_state;
1259         const struct intel_bw_state *old_bw_state;
1260         int ret;
1261
1262         /* FIXME earlier gens need some checks too */
1263         if (DISPLAY_VER(i915) < 11)
1264                 return 0;
1265
1266         ret = intel_bw_check_data_rate(state, &changed);
1267         if (ret)
1268                 return ret;
1269
1270         old_bw_state = intel_atomic_get_old_bw_state(state);
1271         new_bw_state = intel_atomic_get_new_bw_state(state);
1272
1273         if (new_bw_state &&
1274             intel_can_enable_sagv(i915, old_bw_state) !=
1275             intel_can_enable_sagv(i915, new_bw_state))
1276                 changed = true;
1277
1278         /*
1279          * If none of our inputs (data rates, number of active
1280          * planes, SAGV yes/no) changed then nothing to do here.
1281          */
1282         if (!changed)
1283                 return 0;
1284
1285         ret = intel_bw_check_qgv_points(i915, old_bw_state, new_bw_state);
1286         if (ret)
1287                 return ret;
1288
1289         return 0;
1290 }
1291
1292 static struct intel_global_state *
1293 intel_bw_duplicate_state(struct intel_global_obj *obj)
1294 {
1295         struct intel_bw_state *state;
1296
1297         state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
1298         if (!state)
1299                 return NULL;
1300
1301         return &state->base;
1302 }
1303
1304 static void intel_bw_destroy_state(struct intel_global_obj *obj,
1305                                    struct intel_global_state *state)
1306 {
1307         kfree(state);
1308 }
1309
1310 static const struct intel_global_state_funcs intel_bw_funcs = {
1311         .atomic_duplicate_state = intel_bw_duplicate_state,
1312         .atomic_destroy_state = intel_bw_destroy_state,
1313 };
1314
1315 int intel_bw_init(struct drm_i915_private *dev_priv)
1316 {
1317         struct intel_bw_state *state;
1318
1319         state = kzalloc(sizeof(*state), GFP_KERNEL);
1320         if (!state)
1321                 return -ENOMEM;
1322
1323         intel_atomic_global_obj_init(dev_priv, &dev_priv->display.bw.obj,
1324                                      &state->base, &intel_bw_funcs);
1325
1326         return 0;
1327 }