drivers/gpu/drm/i915/display/intel_bw.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include <drm/drm_atomic_state_helper.h>
   7
   8 #include "i915_reg.h"
   9 #include "i915_utils.h"
  10 #include "intel_atomic.h"
  11 #include "intel_bw.h"
  12 #include "intel_cdclk.h"
  13 #include "intel_display_types.h"
  14 #include "intel_mchbar_regs.h"
  15 #include "intel_pcode.h"
  16 #include "intel_pm.h"
  17
  18 /* Parameters for Qclk Geyserville (QGV) */
  19 struct intel_qgv_point {
  20         u16 dclk, t_rp, t_rdpre, t_rc, t_ras, t_rcd;
  21 };
  22
  23 struct intel_psf_gv_point {
  24         u8 clk; /* clock in multiples of 16.6666 MHz */
  25 };
  26
  27 struct intel_qgv_info {
  28         struct intel_qgv_point points[I915_NUM_QGV_POINTS];
  29         struct intel_psf_gv_point psf_points[I915_NUM_PSF_GV_POINTS];
  30         u8 num_points;
  31         u8 num_psf_points;
  32         u8 t_bl;
  33         u8 max_numchannels;
  34         u8 channel_width;
  35         u8 deinterleave;
  36 };
  37
  38 static int dg1_mchbar_read_qgv_point_info(struct drm_i915_private *dev_priv,
  39                                           struct intel_qgv_point *sp,
  40                                           int point)
  41 {
  42         u32 dclk_ratio, dclk_reference;
  43         u32 val;
  44
  45         val = intel_uncore_read(&dev_priv->uncore, SA_PERF_STATUS_0_0_0_MCHBAR_PC);
  46         dclk_ratio = REG_FIELD_GET(DG1_QCLK_RATIO_MASK, val);
  47         if (val & DG1_QCLK_REFERENCE)
  48                 dclk_reference = 6; /* 6 * 16.666 MHz = 100 MHz */
  49         else
  50                 dclk_reference = 8; /* 8 * 16.666 MHz = 133 MHz */
  51         sp->dclk = DIV_ROUND_UP((16667 * dclk_ratio * dclk_reference) + 500, 1000);
  52
  53         val = intel_uncore_read(&dev_priv->uncore, SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU);
  54         if (val & DG1_GEAR_TYPE)
  55                 sp->dclk *= 2;
  56
  57         if (sp->dclk == 0)
  58                 return -EINVAL;
  59
  60         val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR);
  61         sp->t_rp = REG_FIELD_GET(DG1_DRAM_T_RP_MASK, val);
  62         sp->t_rdpre = REG_FIELD_GET(DG1_DRAM_T_RDPRE_MASK, val);
  63
  64         val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH);
  65         sp->t_rcd = REG_FIELD_GET(DG1_DRAM_T_RCD_MASK, val);
  66         sp->t_ras = REG_FIELD_GET(DG1_DRAM_T_RAS_MASK, val);
  67
  68         sp->t_rc = sp->t_rp + sp->t_ras;
  69
  70         return 0;
  71 }
  72
  73 static int icl_pcode_read_qgv_point_info(struct drm_i915_private *dev_priv,
  74                                          struct intel_qgv_point *sp,
  75                                          int point)
  76 {
  77         u32 val = 0, val2 = 0;
  78         u16 dclk;
  79         int ret;
  80
  81         ret = snb_pcode_read(dev_priv, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
  82                              ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point),
  83                              &val, &val2);
  84         if (ret)
  85                 return ret;
  86
  87         dclk = val & 0xffff;
  88         sp->dclk = DIV_ROUND_UP((16667 * dclk) + (DISPLAY_VER(dev_priv) > 11 ? 500 : 0), 1000);
  89         sp->t_rp = (val & 0xff0000) >> 16;
  90         sp->t_rcd = (val & 0xff000000) >> 24;
  91
  92         sp->t_rdpre = val2 & 0xff;
  93         sp->t_ras = (val2 & 0xff00) >> 8;
  94
  95         sp->t_rc = sp->t_rp + sp->t_ras;
  96
  97         return 0;
  98 }
  99
 100 static int adls_pcode_read_psf_gv_point_info(struct drm_i915_private *dev_priv,
 101                                             struct intel_psf_gv_point *points)
 102 {
 103         u32 val = 0;
 104         int ret;
 105         int i;
 106
 107         ret = snb_pcode_read(dev_priv, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
 108                              ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, &val, NULL);
 109         if (ret)
 110                 return ret;
 111
 112         for (i = 0; i < I915_NUM_PSF_GV_POINTS; i++) {
 113                 points[i].clk = val & 0xff;
 114                 val >>= 8;
 115         }
 116
 117         return 0;
 118 }
 119
 120 int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv,
 121                                   u32 points_mask)
 122 {
 123         int ret;
 124
 125         /* bspec says to keep retrying for at least 1 ms */
 126         ret = skl_pcode_request(dev_priv, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG,
 127                                 points_mask,
 128                                 ICL_PCODE_REP_QGV_MASK | ADLS_PCODE_REP_PSF_MASK,
 129                                 ICL_PCODE_REP_QGV_SAFE | ADLS_PCODE_REP_PSF_SAFE,
 130                                 1);
 131
 132         if (ret < 0) {
 133                 drm_err(&dev_priv->drm, "Failed to disable qgv points (%d) points: 0x%x\n", ret, points_mask);
 134                 return ret;
 135         }
 136
 137         return 0;
 138 }
 139
 140 static int icl_get_qgv_points(struct drm_i915_private *dev_priv,
 141                               struct intel_qgv_info *qi,
 142                               bool is_y_tile)
 143 {
 144         const struct dram_info *dram_info = &dev_priv->dram_info;
 145         int i, ret;
 146
 147         qi->num_points = dram_info->num_qgv_points;
 148         qi->num_psf_points = dram_info->num_psf_gv_points;
 149
 150         if (DISPLAY_VER(dev_priv) >= 12)
 151                 switch (dram_info->type) {
 152                 case INTEL_DRAM_DDR4:
 153                         qi->t_bl = is_y_tile ? 8 : 4;
 154                         qi->max_numchannels = 2;
 155                         qi->channel_width = 64;
 156                         qi->deinterleave = is_y_tile ? 1 : 2;
 157                         break;
 158                 case INTEL_DRAM_DDR5:
 159                         qi->t_bl = is_y_tile ? 16 : 8;
 160                         qi->max_numchannels = 4;
 161                         qi->channel_width = 32;
 162                         qi->deinterleave = is_y_tile ? 1 : 2;
 163                         break;
 164                 case INTEL_DRAM_LPDDR4:
 165                         if (IS_ROCKETLAKE(dev_priv)) {
 166                                 qi->t_bl = 8;
 167                                 qi->max_numchannels = 4;
 168                                 qi->channel_width = 32;
 169                                 qi->deinterleave = 2;
 170                                 break;
 171                         }
 172                         fallthrough;
 173                 case INTEL_DRAM_LPDDR5:
 174                         qi->t_bl = 16;
 175                         qi->max_numchannels = 8;
 176                         qi->channel_width = 16;
 177                         qi->deinterleave = is_y_tile ? 2 : 4;
 178                         break;
 179                 default:
 180                         qi->t_bl = 16;
 181                         qi->max_numchannels = 1;
 182                         break;
 183                 }
 184         else if (DISPLAY_VER(dev_priv) == 11) {
 185                 qi->t_bl = dev_priv->dram_info.type == INTEL_DRAM_DDR4 ? 4 : 8;
 186                 qi->max_numchannels = 1;
 187         }
 188
 189         if (drm_WARN_ON(&dev_priv->drm,
 190                         qi->num_points > ARRAY_SIZE(qi->points)))
 191                 qi->num_points = ARRAY_SIZE(qi->points);
 192
 193         for (i = 0; i < qi->num_points; i++) {
 194                 struct intel_qgv_point *sp = &qi->points[i];
 195
 196                 if (IS_DG1(dev_priv))
 197                         ret = dg1_mchbar_read_qgv_point_info(dev_priv, sp, i);
 198                 else
 199                         ret = icl_pcode_read_qgv_point_info(dev_priv, sp, i);
 200
 201                 if (ret)
 202                         return ret;
 203
 204                 drm_dbg_kms(&dev_priv->drm,
 205                             "QGV %d: DCLK=%d tRP=%d tRDPRE=%d tRAS=%d tRCD=%d tRC=%d\n",
 206                             i, sp->dclk, sp->t_rp, sp->t_rdpre, sp->t_ras,
 207                             sp->t_rcd, sp->t_rc);
 208         }
 209
 210         if (qi->num_psf_points > 0) {
 211                 ret = adls_pcode_read_psf_gv_point_info(dev_priv, qi->psf_points);
 212                 if (ret) {
 213                         drm_err(&dev_priv->drm, "Failed to read PSF point data; PSF points will not be considered in bandwidth calculations.\n");
 214                         qi->num_psf_points = 0;
 215                 }
 216
 217                 for (i = 0; i < qi->num_psf_points; i++)
 218                         drm_dbg_kms(&dev_priv->drm,
 219                                     "PSF GV %d: CLK=%d \n",
 220                                     i, qi->psf_points[i].clk);
 221         }
 222
 223         return 0;
 224 }
 225
 226 static int adl_calc_psf_bw(int clk)
 227 {
 228         /*
 229          * clk is multiples of 16.666MHz (100/6)
 230          * According to BSpec PSF GV bandwidth is
 231          * calculated as BW = 64 * clk * 16.666Mhz
 232          */
 233         return DIV_ROUND_CLOSEST(64 * clk * 100, 6);
 234 }
 235
 236 static int icl_sagv_max_dclk(const struct intel_qgv_info *qi)
 237 {
 238         u16 dclk = 0;
 239         int i;
 240
 241         for (i = 0; i < qi->num_points; i++)
 242                 dclk = max(dclk, qi->points[i].dclk);
 243
 244         return dclk;
 245 }
 246
 247 struct intel_sa_info {
 248         u16 displayrtids;
 249         u8 deburst, deprogbwlimit, derating;
 250 };
 251
 252 static const struct intel_sa_info icl_sa_info = {
 253         .deburst = 8,
 254         .deprogbwlimit = 25, /* GB/s */
 255         .displayrtids = 128,
 256         .derating = 10,
 257 };
 258
 259 static const struct intel_sa_info tgl_sa_info = {
 260         .deburst = 16,
 261         .deprogbwlimit = 34, /* GB/s */
 262         .displayrtids = 256,
 263         .derating = 10,
 264 };
 265
 266 static const struct intel_sa_info rkl_sa_info = {
 267         .deburst = 8,
 268         .deprogbwlimit = 20, /* GB/s */
 269         .displayrtids = 128,
 270         .derating = 10,
 271 };
 272
 273 static const struct intel_sa_info adls_sa_info = {
 274         .deburst = 16,
 275         .deprogbwlimit = 38, /* GB/s */
 276         .displayrtids = 256,
 277         .derating = 10,
 278 };
 279
 280 static const struct intel_sa_info adlp_sa_info = {
 281         .deburst = 16,
 282         .deprogbwlimit = 38, /* GB/s */
 283         .displayrtids = 256,
 284         .derating = 20,
 285 };
 286
 287 static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa)
 288 {
 289         struct intel_qgv_info qi = {};
 290         bool is_y_tile = true; /* assume y tile may be used */
 291         int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels);
 292         int ipqdepth, ipqdepthpch = 16;
 293         int dclk_max;
 294         int maxdebw;
 295         int num_groups = ARRAY_SIZE(dev_priv->max_bw);
 296         int i, ret;
 297
 298         ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile);
 299         if (ret) {
 300                 drm_dbg_kms(&dev_priv->drm,
 301                             "Failed to get memory subsystem information, ignoring bandwidth limits");
 302                 return ret;
 303         }
 304
 305         dclk_max = icl_sagv_max_dclk(&qi);
 306         maxdebw = min(sa->deprogbwlimit * 1000, dclk_max * 16 * 6 / 10);
 307         ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
 308         qi.deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
 309
 310         for (i = 0; i < num_groups; i++) {
 311                 struct intel_bw_info *bi = &dev_priv->max_bw[i];
 312                 int clpchgroup;
 313                 int j;
 314
 315                 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
 316                 bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1;
 317
 318                 bi->num_qgv_points = qi.num_points;
 319                 bi->num_psf_gv_points = qi.num_psf_points;
 320
 321                 for (j = 0; j < qi.num_points; j++) {
 322                         const struct intel_qgv_point *sp = &qi.points[j];
 323                         int ct, bw;
 324
 325                         /*
 326                          * Max row cycle time
 327                          *
 328                          * FIXME what is the logic behind the
 329                          * assumed burst length?
 330                          */
 331                         ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
 332                                    (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
 333                         bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
 334
 335                         bi->deratedbw[j] = min(maxdebw,
 336                                                bw * (100 - sa->derating) / 100);
 337
 338                         drm_dbg_kms(&dev_priv->drm,
 339                                     "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
 340                                     i, j, bi->num_planes, bi->deratedbw[j]);
 341                 }
 342         }
 343         /*
 344          * In case if SAGV is disabled in BIOS, we always get 1
 345          * SAGV point, but we can't send PCode commands to restrict it
 346          * as it will fail and pointless anyway.
 347          */
 348         if (qi.num_points == 1)
 349                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
 350         else
 351                 dev_priv->sagv_status = I915_SAGV_ENABLED;
 352
 353         return 0;
 354 }
 355
 356 static int tgl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa)
 357 {
 358         struct intel_qgv_info qi = {};
 359         const struct dram_info *dram_info = &dev_priv->dram_info;
 360         bool is_y_tile = true; /* assume y tile may be used */
 361         int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels);
 362         int ipqdepth, ipqdepthpch = 16;
 363         int dclk_max;
 364         int maxdebw, peakbw;
 365         int clperchgroup;
 366         int num_groups = ARRAY_SIZE(dev_priv->max_bw);
 367         int i, ret;
 368
 369         ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile);
 370         if (ret) {
 371                 drm_dbg_kms(&dev_priv->drm,
 372                             "Failed to get memory subsystem information, ignoring bandwidth limits");
 373                 return ret;
 374         }
 375
 376         if (dram_info->type == INTEL_DRAM_LPDDR4 || dram_info->type == INTEL_DRAM_LPDDR5)
 377                 num_channels *= 2;
 378
 379         qi.deinterleave = qi.deinterleave ? : DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
 380
 381         if (num_channels < qi.max_numchannels && DISPLAY_VER(dev_priv) >= 12)
 382                 qi.deinterleave = max(DIV_ROUND_UP(qi.deinterleave, 2), 1);
 383
 384         if (DISPLAY_VER(dev_priv) > 11 && num_channels > qi.max_numchannels)
 385                 drm_warn(&dev_priv->drm, "Number of channels exceeds max number of channels.");
 386         if (qi.max_numchannels != 0)
 387                 num_channels = min_t(u8, num_channels, qi.max_numchannels);
 388
 389         dclk_max = icl_sagv_max_dclk(&qi);
 390
 391         peakbw = num_channels * DIV_ROUND_UP(qi.channel_width, 8) * dclk_max;
 392         maxdebw = min(sa->deprogbwlimit * 1000, peakbw * 6 / 10); /* 60% */
 393
 394         ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
 395         /*
 396          * clperchgroup = 4kpagespermempage * clperchperblock,
 397          * clperchperblock = 8 / num_channels * interleave
 398          */
 399         clperchgroup = 4 * DIV_ROUND_UP(8, num_channels) * qi.deinterleave;
 400
 401         for (i = 0; i < num_groups; i++) {
 402                 struct intel_bw_info *bi = &dev_priv->max_bw[i];
 403                 struct intel_bw_info *bi_next;
 404                 int clpchgroup;
 405                 int j;
 406
 407                 if (i < num_groups - 1)
 408                         bi_next = &dev_priv->max_bw[i + 1];
 409
 410                 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
 411
 412                 if (i < num_groups - 1 && clpchgroup < clperchgroup)
 413                         bi_next->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1;
 414                 else
 415                         bi_next->num_planes = 0;
 416
 417                 bi->num_qgv_points = qi.num_points;
 418                 bi->num_psf_gv_points = qi.num_psf_points;
 419
 420                 for (j = 0; j < qi.num_points; j++) {
 421                         const struct intel_qgv_point *sp = &qi.points[j];
 422                         int ct, bw;
 423
 424                         /*
 425                          * Max row cycle time
 426                          *
 427                          * FIXME what is the logic behind the
 428                          * assumed burst length?
 429                          */
 430                         ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
 431                                    (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
 432                         bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
 433
 434                         bi->deratedbw[j] = min(maxdebw,
 435                                                bw * (100 - sa->derating) / 100);
 436
 437                         drm_dbg_kms(&dev_priv->drm,
 438                                     "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
 439                                     i, j, bi->num_planes, bi->deratedbw[j]);
 440                 }
 441
 442                 for (j = 0; j < qi.num_psf_points; j++) {
 443                         const struct intel_psf_gv_point *sp = &qi.psf_points[j];
 444
 445                         bi->psf_bw[j] = adl_calc_psf_bw(sp->clk);
 446
 447                         drm_dbg_kms(&dev_priv->drm,
 448                                     "BW%d / PSF GV %d: num_planes=%d bw=%u\n",
 449                                     i, j, bi->num_planes, bi->psf_bw[j]);
 450                 }
 451         }
 452
 453         /*
 454          * In case if SAGV is disabled in BIOS, we always get 1
 455          * SAGV point, but we can't send PCode commands to restrict it
 456          * as it will fail and pointless anyway.
 457          */
 458         if (qi.num_points == 1)
 459                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
 460         else
 461                 dev_priv->sagv_status = I915_SAGV_ENABLED;
 462
 463         return 0;
 464 }
 465
 466 static void dg2_get_bw_info(struct drm_i915_private *i915)
 467 {
 468         unsigned int deratedbw = IS_DG2_G11(i915) ? 38000 : 50000;
 469         int num_groups = ARRAY_SIZE(i915->max_bw);
 470         int i;
 471
 472         /*
 473          * DG2 doesn't have SAGV or QGV points, just a constant max bandwidth
 474          * that doesn't depend on the number of planes enabled. So fill all the
 475          * plane group with constant bw information for uniformity with other
 476          * platforms. DG2-G10 platforms have a constant 50 GB/s bandwidth,
 477          * whereas DG2-G11 platforms have 38 GB/s.
 478          */
 479         for (i = 0; i < num_groups; i++) {
 480                 struct intel_bw_info *bi = &i915->max_bw[i];
 481
 482                 bi->num_planes = 1;
 483                 /* Need only one dummy QGV point per group */
 484                 bi->num_qgv_points = 1;
 485                 bi->deratedbw[0] = deratedbw;
 486         }
 487
 488         i915->sagv_status = I915_SAGV_NOT_CONTROLLED;
 489 }
 490
 491 static unsigned int icl_max_bw(struct drm_i915_private *dev_priv,
 492                                int num_planes, int qgv_point)
 493 {
 494         int i;
 495
 496         /*
 497          * Let's return max bw for 0 planes
 498          */
 499         num_planes = max(1, num_planes);
 500
 501         for (i = 0; i < ARRAY_SIZE(dev_priv->max_bw); i++) {
 502                 const struct intel_bw_info *bi =
 503                         &dev_priv->max_bw[i];
 504
 505                 /*
 506                  * Pcode will not expose all QGV points when
 507                  * SAGV is forced to off/min/med/max.
 508                  */
 509                 if (qgv_point >= bi->num_qgv_points)
 510                         return UINT_MAX;
 511
 512                 if (num_planes >= bi->num_planes)
 513                         return bi->deratedbw[qgv_point];
 514         }
 515
 516         return 0;
 517 }
 518
 519 static unsigned int tgl_max_bw(struct drm_i915_private *dev_priv,
 520                                int num_planes, int qgv_point)
 521 {
 522         int i;
 523
 524         /*
 525          * Let's return max bw for 0 planes
 526          */
 527         num_planes = max(1, num_planes);
 528
 529         for (i = ARRAY_SIZE(dev_priv->max_bw) - 1; i >= 0; i--) {
 530                 const struct intel_bw_info *bi =
 531                         &dev_priv->max_bw[i];
 532
 533                 /*
 534                  * Pcode will not expose all QGV points when
 535                  * SAGV is forced to off/min/med/max.
 536                  */
 537                 if (qgv_point >= bi->num_qgv_points)
 538                         return UINT_MAX;
 539
 540                 if (num_planes <= bi->num_planes)
 541                         return bi->deratedbw[qgv_point];
 542         }
 543
 544         return dev_priv->max_bw[0].deratedbw[qgv_point];
 545 }
 546
 547 static unsigned int adl_psf_bw(struct drm_i915_private *dev_priv,
 548                                int psf_gv_point)
 549 {
 550         const struct intel_bw_info *bi =
 551                         &dev_priv->max_bw[0];
 552
 553         return bi->psf_bw[psf_gv_point];
 554 }
 555
 556 void intel_bw_init_hw(struct drm_i915_private *dev_priv)
 557 {
 558         if (!HAS_DISPLAY(dev_priv))
 559                 return;
 560
 561         if (IS_DG2(dev_priv))
 562                 dg2_get_bw_info(dev_priv);
 563         else if (IS_ALDERLAKE_P(dev_priv))
 564                 tgl_get_bw_info(dev_priv, &adlp_sa_info);
 565         else if (IS_ALDERLAKE_S(dev_priv))
 566                 tgl_get_bw_info(dev_priv, &adls_sa_info);
 567         else if (IS_ROCKETLAKE(dev_priv))
 568                 tgl_get_bw_info(dev_priv, &rkl_sa_info);
 569         else if (DISPLAY_VER(dev_priv) == 12)
 570                 tgl_get_bw_info(dev_priv, &tgl_sa_info);
 571         else if (DISPLAY_VER(dev_priv) == 11)
 572                 icl_get_bw_info(dev_priv, &icl_sa_info);
 573 }
 574
 575 static unsigned int intel_bw_crtc_num_active_planes(const struct intel_crtc_state *crtc_state)
 576 {
 577         /*
 578          * We assume cursors are small enough
 579          * to not not cause bandwidth problems.
 580          */
 581         return hweight8(crtc_state->active_planes & ~BIT(PLANE_CURSOR));
 582 }
 583
 584 static unsigned int intel_bw_crtc_data_rate(const struct intel_crtc_state *crtc_state)
 585 {
 586         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 587         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 588         unsigned int data_rate = 0;
 589         enum plane_id plane_id;
 590
 591         for_each_plane_id_on_crtc(crtc, plane_id) {
 592                 /*
 593                  * We assume cursors are small enough
 594                  * to not not cause bandwidth problems.
 595                  */
 596                 if (plane_id == PLANE_CURSOR)
 597                         continue;
 598
 599                 data_rate += crtc_state->data_rate[plane_id];
 600
 601                 if (DISPLAY_VER(i915) < 11)
 602                         data_rate += crtc_state->data_rate_y[plane_id];
 603         }
 604
 605         return data_rate;
 606 }
 607
 608 /* "Maximum Pipe Read Bandwidth" */
 609 static int intel_bw_crtc_min_cdclk(const struct intel_crtc_state *crtc_state)
 610 {
 611         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 612         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 613
 614         if (DISPLAY_VER(i915) < 12)
 615                 return 0;
 616
 617         return DIV_ROUND_UP_ULL(mul_u32_u32(intel_bw_crtc_data_rate(crtc_state), 10), 512);
 618 }
 619
 620 void intel_bw_crtc_update(struct intel_bw_state *bw_state,
 621                           const struct intel_crtc_state *crtc_state)
 622 {
 623         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 624         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 625
 626         bw_state->data_rate[crtc->pipe] =
 627                 intel_bw_crtc_data_rate(crtc_state);
 628         bw_state->num_active_planes[crtc->pipe] =
 629                 intel_bw_crtc_num_active_planes(crtc_state);
 630
 631         drm_dbg_kms(&i915->drm, "pipe %c data rate %u num active planes %u\n",
 632                     pipe_name(crtc->pipe),
 633                     bw_state->data_rate[crtc->pipe],
 634                     bw_state->num_active_planes[crtc->pipe]);
 635 }
 636
 637 static unsigned int intel_bw_num_active_planes(struct drm_i915_private *dev_priv,
 638                                                const struct intel_bw_state *bw_state)
 639 {
 640         unsigned int num_active_planes = 0;
 641         enum pipe pipe;
 642
 643         for_each_pipe(dev_priv, pipe)
 644                 num_active_planes += bw_state->num_active_planes[pipe];
 645
 646         return num_active_planes;
 647 }
 648
 649 static unsigned int intel_bw_data_rate(struct drm_i915_private *dev_priv,
 650                                        const struct intel_bw_state *bw_state)
 651 {
 652         unsigned int data_rate = 0;
 653         enum pipe pipe;
 654
 655         for_each_pipe(dev_priv, pipe)
 656                 data_rate += bw_state->data_rate[pipe];
 657
 658         if (DISPLAY_VER(dev_priv) >= 13 && i915_vtd_active(dev_priv))
 659                 data_rate = DIV_ROUND_UP(data_rate * 105, 100);
 660
 661         return data_rate;
 662 }
 663
 664 struct intel_bw_state *
 665 intel_atomic_get_old_bw_state(struct intel_atomic_state *state)
 666 {
 667         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 668         struct intel_global_state *bw_state;
 669
 670         bw_state = intel_atomic_get_old_global_obj_state(state, &dev_priv->bw_obj);
 671
 672         return to_intel_bw_state(bw_state);
 673 }
 674
 675 struct intel_bw_state *
 676 intel_atomic_get_new_bw_state(struct intel_atomic_state *state)
 677 {
 678         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 679         struct intel_global_state *bw_state;
 680
 681         bw_state = intel_atomic_get_new_global_obj_state(state, &dev_priv->bw_obj);
 682
 683         return to_intel_bw_state(bw_state);
 684 }
 685
 686 struct intel_bw_state *
 687 intel_atomic_get_bw_state(struct intel_atomic_state *state)
 688 {
 689         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 690         struct intel_global_state *bw_state;
 691
 692         bw_state = intel_atomic_get_global_obj_state(state, &dev_priv->bw_obj);
 693         if (IS_ERR(bw_state))
 694                 return ERR_CAST(bw_state);
 695
 696         return to_intel_bw_state(bw_state);
 697 }
 698
 699 static bool intel_bw_state_changed(struct drm_i915_private *i915,
 700                                    const struct intel_bw_state *old_bw_state,
 701                                    const struct intel_bw_state *new_bw_state)
 702 {
 703         enum pipe pipe;
 704
 705         for_each_pipe(i915, pipe) {
 706                 const struct intel_dbuf_bw *old_crtc_bw =
 707                         &old_bw_state->dbuf_bw[pipe];
 708                 const struct intel_dbuf_bw *new_crtc_bw =
 709                         &new_bw_state->dbuf_bw[pipe];
 710                 enum dbuf_slice slice;
 711
 712                 for_each_dbuf_slice(i915, slice) {
 713                         if (old_crtc_bw->max_bw[slice] != new_crtc_bw->max_bw[slice] ||
 714                             old_crtc_bw->active_planes[slice] != new_crtc_bw->active_planes[slice])
 715                                 return true;
 716                 }
 717
 718                 if (old_bw_state->min_cdclk[pipe] != new_bw_state->min_cdclk[pipe])
 719                         return true;
 720         }
 721
 722         return false;
 723 }
 724
 725 static void skl_plane_calc_dbuf_bw(struct intel_bw_state *bw_state,
 726                                    struct intel_crtc *crtc,
 727                                    enum plane_id plane_id,
 728                                    const struct skl_ddb_entry *ddb,
 729                                    unsigned int data_rate)
 730 {
 731         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 732         struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe];
 733         unsigned int dbuf_mask = skl_ddb_dbuf_slice_mask(i915, ddb);
 734         enum dbuf_slice slice;
 735
 736         /*
 737          * The arbiter can only really guarantee an
 738          * equal share of the total bw to each plane.
 739          */
 740         for_each_dbuf_slice_in_mask(i915, slice, dbuf_mask) {
 741                 crtc_bw->max_bw[slice] = max(crtc_bw->max_bw[slice], data_rate);
 742                 crtc_bw->active_planes[slice] |= BIT(plane_id);
 743         }
 744 }
 745
 746 static void skl_crtc_calc_dbuf_bw(struct intel_bw_state *bw_state,
 747                                   const struct intel_crtc_state *crtc_state)
 748 {
 749         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 750         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 751         struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe];
 752         enum plane_id plane_id;
 753
 754         memset(crtc_bw, 0, sizeof(*crtc_bw));
 755
 756         if (!crtc_state->hw.active)
 757                 return;
 758
 759         for_each_plane_id_on_crtc(crtc, plane_id) {
 760                 /*
 761                  * We assume cursors are small enough
 762                  * to not cause bandwidth problems.
 763                  */
 764                 if (plane_id == PLANE_CURSOR)
 765                         continue;
 766
 767                 skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id,
 768                                        &crtc_state->wm.skl.plane_ddb[plane_id],
 769                                        crtc_state->data_rate[plane_id]);
 770
 771                 if (DISPLAY_VER(i915) < 11)
 772                         skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id,
 773                                                &crtc_state->wm.skl.plane_ddb_y[plane_id],
 774                                                crtc_state->data_rate[plane_id]);
 775         }
 776 }
 777
 778 /* "Maximum Data Buffer Bandwidth" */
 779 static int
 780 intel_bw_dbuf_min_cdclk(struct drm_i915_private *i915,
 781                         const struct intel_bw_state *bw_state)
 782 {
 783         unsigned int total_max_bw = 0;
 784         enum dbuf_slice slice;
 785
 786         for_each_dbuf_slice(i915, slice) {
 787                 int num_active_planes = 0;
 788                 unsigned int max_bw = 0;
 789                 enum pipe pipe;
 790
 791                 /*
 792                  * The arbiter can only really guarantee an
 793                  * equal share of the total bw to each plane.
 794                  */
 795                 for_each_pipe(i915, pipe) {
 796                         const struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[pipe];
 797
 798                         max_bw = max(crtc_bw->max_bw[slice], max_bw);
 799                         num_active_planes += hweight8(crtc_bw->active_planes[slice]);
 800                 }
 801                 max_bw *= num_active_planes;
 802
 803                 total_max_bw = max(total_max_bw, max_bw);
 804         }
 805
 806         return DIV_ROUND_UP(total_max_bw, 64);
 807 }
 808
 809 int intel_bw_min_cdclk(struct drm_i915_private *i915,
 810                        const struct intel_bw_state *bw_state)
 811 {
 812         enum pipe pipe;
 813         int min_cdclk;
 814
 815         min_cdclk = intel_bw_dbuf_min_cdclk(i915, bw_state);
 816
 817         for_each_pipe(i915, pipe)
 818                 min_cdclk = max(bw_state->min_cdclk[pipe], min_cdclk);
 819
 820         return min_cdclk;
 821 }
 822
 823 int intel_bw_calc_min_cdclk(struct intel_atomic_state *state,
 824                             bool *need_cdclk_calc)
 825 {
 826         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 827         struct intel_bw_state *new_bw_state = NULL;
 828         const struct intel_bw_state *old_bw_state = NULL;
 829         const struct intel_cdclk_state *cdclk_state;
 830         const struct intel_crtc_state *crtc_state;
 831         int old_min_cdclk, new_min_cdclk;
 832         struct intel_crtc *crtc;
 833         int i;
 834
 835         if (DISPLAY_VER(dev_priv) < 9)
 836                 return 0;
 837
 838         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
 839                 new_bw_state = intel_atomic_get_bw_state(state);
 840                 if (IS_ERR(new_bw_state))
 841                         return PTR_ERR(new_bw_state);
 842
 843                 old_bw_state = intel_atomic_get_old_bw_state(state);
 844
 845                 skl_crtc_calc_dbuf_bw(new_bw_state, crtc_state);
 846
 847                 new_bw_state->min_cdclk[crtc->pipe] =
 848                         intel_bw_crtc_min_cdclk(crtc_state);
 849         }
 850
 851         if (!old_bw_state)
 852                 return 0;
 853
 854         if (intel_bw_state_changed(dev_priv, old_bw_state, new_bw_state)) {
 855                 int ret = intel_atomic_lock_global_state(&new_bw_state->base);
 856                 if (ret)
 857                         return ret;
 858         }
 859
 860         old_min_cdclk = intel_bw_min_cdclk(dev_priv, old_bw_state);
 861         new_min_cdclk = intel_bw_min_cdclk(dev_priv, new_bw_state);
 862
 863         /*
 864          * No need to check against the cdclk state if
 865          * the min cdclk doesn't increase.
 866          *
 867          * Ie. we only ever increase the cdclk due to bandwidth
 868          * requirements. This can reduce back and forth
 869          * display blinking due to constant cdclk changes.
 870          */
 871         if (new_min_cdclk <= old_min_cdclk)
 872                 return 0;
 873
 874         cdclk_state = intel_atomic_get_cdclk_state(state);
 875         if (IS_ERR(cdclk_state))
 876                 return PTR_ERR(cdclk_state);
 877
 878         /*
 879          * No need to recalculate the cdclk state if
 880          * the min cdclk doesn't increase.
 881          *
 882          * Ie. we only ever increase the cdclk due to bandwidth
 883          * requirements. This can reduce back and forth
 884          * display blinking due to constant cdclk changes.
 885          */
 886         if (new_min_cdclk <= cdclk_state->bw_min_cdclk)
 887                 return 0;
 888
 889         drm_dbg_kms(&dev_priv->drm,
 890                     "new bandwidth min cdclk (%d kHz) > old min cdclk (%d kHz)\n",
 891                     new_min_cdclk, cdclk_state->bw_min_cdclk);
 892         *need_cdclk_calc = true;
 893
 894         return 0;
 895 }
 896
 897 static u16 icl_qgv_points_mask(struct drm_i915_private *i915)
 898 {
 899         unsigned int num_psf_gv_points = i915->max_bw[0].num_psf_gv_points;
 900         unsigned int num_qgv_points = i915->max_bw[0].num_qgv_points;
 901         u16 qgv_points = 0, psf_points = 0;
 902
 903         /*
 904          * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects
 905          * it with failure if we try masking any unadvertised points.
 906          * So need to operate only with those returned from PCode.
 907          */
 908         if (num_qgv_points > 0)
 909                 qgv_points = GENMASK(num_qgv_points - 1, 0);
 910
 911         if (num_psf_gv_points > 0)
 912                 psf_points = GENMASK(num_psf_gv_points - 1, 0);
 913
 914         return ICL_PCODE_REQ_QGV_PT(qgv_points) | ADLS_PCODE_REQ_PSF_PT(psf_points);
 915 }
 916
 917 static int intel_bw_check_data_rate(struct intel_atomic_state *state, bool *changed)
 918 {
 919         struct drm_i915_private *i915 = to_i915(state->base.dev);
 920         const struct intel_crtc_state *new_crtc_state, *old_crtc_state;
 921         struct intel_crtc *crtc;
 922         int i;
 923
 924         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
 925                                             new_crtc_state, i) {
 926                 unsigned int old_data_rate =
 927                         intel_bw_crtc_data_rate(old_crtc_state);
 928                 unsigned int new_data_rate =
 929                         intel_bw_crtc_data_rate(new_crtc_state);
 930                 unsigned int old_active_planes =
 931                         intel_bw_crtc_num_active_planes(old_crtc_state);
 932                 unsigned int new_active_planes =
 933                         intel_bw_crtc_num_active_planes(new_crtc_state);
 934                 struct intel_bw_state *new_bw_state;
 935
 936                 /*
 937                  * Avoid locking the bw state when
 938                  * nothing significant has changed.
 939                  */
 940                 if (old_data_rate == new_data_rate &&
 941                     old_active_planes == new_active_planes)
 942                         continue;
 943
 944                 new_bw_state = intel_atomic_get_bw_state(state);
 945                 if (IS_ERR(new_bw_state))
 946                         return PTR_ERR(new_bw_state);
 947
 948                 new_bw_state->data_rate[crtc->pipe] = new_data_rate;
 949                 new_bw_state->num_active_planes[crtc->pipe] = new_active_planes;
 950
 951                 *changed = true;
 952
 953                 drm_dbg_kms(&i915->drm,
 954                             "[CRTC:%d:%s] data rate %u num active planes %u\n",
 955                             crtc->base.base.id, crtc->base.name,
 956                             new_bw_state->data_rate[crtc->pipe],
 957                             new_bw_state->num_active_planes[crtc->pipe]);
 958         }
 959
 960         return 0;
 961 }
 962
 963 int intel_bw_atomic_check(struct intel_atomic_state *state)
 964 {
 965         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 966         const struct intel_bw_state *old_bw_state;
 967         struct intel_bw_state *new_bw_state;
 968         unsigned int data_rate;
 969         unsigned int num_active_planes;
 970         int i, ret;
 971         u16 qgv_points = 0, psf_points = 0;
 972         unsigned int max_bw_point = 0, max_bw = 0;
 973         unsigned int num_qgv_points = dev_priv->max_bw[0].num_qgv_points;
 974         unsigned int num_psf_gv_points = dev_priv->max_bw[0].num_psf_gv_points;
 975         bool changed = false;
 976
 977         /* FIXME earlier gens need some checks too */
 978         if (DISPLAY_VER(dev_priv) < 11)
 979                 return 0;
 980
 981         ret = intel_bw_check_data_rate(state, &changed);
 982         if (ret)
 983                 return ret;
 984
 985         old_bw_state = intel_atomic_get_old_bw_state(state);
 986         new_bw_state = intel_atomic_get_new_bw_state(state);
 987
 988         if (new_bw_state &&
 989             intel_can_enable_sagv(dev_priv, old_bw_state) !=
 990             intel_can_enable_sagv(dev_priv, new_bw_state))
 991                 changed = true;
 992
 993         /*
 994          * If none of our inputs (data rates, number of active
 995          * planes, SAGV yes/no) changed then nothing to do here.
 996          */
 997         if (!changed)
 998                 return 0;
 999
1000         ret = intel_atomic_lock_global_state(&new_bw_state->base);
1001         if (ret)
1002                 return ret;
1003
1004         data_rate = intel_bw_data_rate(dev_priv, new_bw_state);
1005         data_rate = DIV_ROUND_UP(data_rate, 1000);
1006
1007         num_active_planes = intel_bw_num_active_planes(dev_priv, new_bw_state);
1008
1009         for (i = 0; i < num_qgv_points; i++) {
1010                 unsigned int max_data_rate;
1011
1012                 if (DISPLAY_VER(dev_priv) > 11)
1013                         max_data_rate = tgl_max_bw(dev_priv, num_active_planes, i);
1014                 else
1015                         max_data_rate = icl_max_bw(dev_priv, num_active_planes, i);
1016                 /*
1017                  * We need to know which qgv point gives us
1018                  * maximum bandwidth in order to disable SAGV
1019                  * if we find that we exceed SAGV block time
1020                  * with watermarks. By that moment we already
1021                  * have those, as it is calculated earlier in
1022                  * intel_atomic_check,
1023                  */
1024                 if (max_data_rate > max_bw) {
1025                         max_bw_point = i;
1026                         max_bw = max_data_rate;
1027                 }
1028                 if (max_data_rate >= data_rate)
1029                         qgv_points |= BIT(i);
1030
1031                 drm_dbg_kms(&dev_priv->drm, "QGV point %d: max bw %d required %d\n",
1032                             i, max_data_rate, data_rate);
1033         }
1034
1035         for (i = 0; i < num_psf_gv_points; i++) {
1036                 unsigned int max_data_rate = adl_psf_bw(dev_priv, i);
1037
1038                 if (max_data_rate >= data_rate)
1039                         psf_points |= BIT(i);
1040
1041                 drm_dbg_kms(&dev_priv->drm, "PSF GV point %d: max bw %d"
1042                             " required %d\n",
1043                             i, max_data_rate, data_rate);
1044         }
1045
1046         /*
1047          * BSpec states that we always should have at least one allowed point
1048          * left, so if we couldn't - simply reject the configuration for obvious
1049          * reasons.
1050          */
1051         if (qgv_points == 0) {
1052                 drm_dbg_kms(&dev_priv->drm, "No QGV points provide sufficient memory"
1053                             " bandwidth %d for display configuration(%d active planes).\n",
1054                             data_rate, num_active_planes);
1055                 return -EINVAL;
1056         }
1057
1058         if (num_psf_gv_points > 0 && psf_points == 0) {
1059                 drm_dbg_kms(&dev_priv->drm, "No PSF GV points provide sufficient memory"
1060                             " bandwidth %d for display configuration(%d active planes).\n",
1061                             data_rate, num_active_planes);
1062                 return -EINVAL;
1063         }
1064
1065         /*
1066          * Leave only single point with highest bandwidth, if
1067          * we can't enable SAGV due to the increased memory latency it may
1068          * cause.
1069          */
1070         if (!intel_can_enable_sagv(dev_priv, new_bw_state)) {
1071                 qgv_points = BIT(max_bw_point);
1072                 drm_dbg_kms(&dev_priv->drm, "No SAGV, using single QGV point %d\n",
1073                             max_bw_point);
1074         }
1075
1076         /*
1077          * We store the ones which need to be masked as that is what PCode
1078          * actually accepts as a parameter.
1079          */
1080         new_bw_state->qgv_points_mask =
1081                 ~(ICL_PCODE_REQ_QGV_PT(qgv_points) |
1082                   ADLS_PCODE_REQ_PSF_PT(psf_points)) &
1083                 icl_qgv_points_mask(dev_priv);
1084
1085         /*
1086          * If the actual mask had changed we need to make sure that
1087          * the commits are serialized(in case this is a nomodeset, nonblocking)
1088          */
1089         if (new_bw_state->qgv_points_mask != old_bw_state->qgv_points_mask) {
1090                 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
1091                 if (ret)
1092                         return ret;
1093         }
1094
1095         return 0;
1096 }
1097
1098 static struct intel_global_state *
1099 intel_bw_duplicate_state(struct intel_global_obj *obj)
1100 {
1101         struct intel_bw_state *state;
1102
1103         state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
1104         if (!state)
1105                 return NULL;
1106
1107         return &state->base;
1108 }
1109
1110 static void intel_bw_destroy_state(struct intel_global_obj *obj,
1111                                    struct intel_global_state *state)
1112 {
1113         kfree(state);
1114 }
1115
1116 static const struct intel_global_state_funcs intel_bw_funcs = {
1117         .atomic_duplicate_state = intel_bw_duplicate_state,
1118         .atomic_destroy_state = intel_bw_destroy_state,
1119 };
1120
1121 int intel_bw_init(struct drm_i915_private *dev_priv)
1122 {
1123         struct intel_bw_state *state;
1124
1125         state = kzalloc(sizeof(*state), GFP_KERNEL);
1126         if (!state)
1127                 return -ENOMEM;
1128
1129         intel_atomic_global_obj_init(dev_priv, &dev_priv->bw_obj,
1130                                      &state->base, &intel_bw_funcs);
1131
1132         return 0;
1133 }