1 // SPDX-License-Identifier: MIT
3 * Copyright © 2019 Intel Corporation
6 #include <drm/drm_atomic_state_helper.h>
10 #include "i915_utils.h"
11 #include "intel_atomic.h"
13 #include "intel_cdclk.h"
14 #include "intel_display_core.h"
15 #include "intel_display_types.h"
16 #include "skl_watermark.h"
17 #include "intel_mchbar_regs.h"
18 #include "intel_pcode.h"
20 /* Parameters for Qclk Geyserville (QGV) */
21 struct intel_qgv_point {
22 u16 dclk, t_rp, t_rdpre, t_rc, t_ras, t_rcd;
25 struct intel_psf_gv_point {
26 u8 clk; /* clock in multiples of 16.6666 MHz */
29 struct intel_qgv_info {
30 struct intel_qgv_point points[I915_NUM_QGV_POINTS];
31 struct intel_psf_gv_point psf_points[I915_NUM_PSF_GV_POINTS];
40 static int dg1_mchbar_read_qgv_point_info(struct drm_i915_private *dev_priv,
41 struct intel_qgv_point *sp,
44 u32 dclk_ratio, dclk_reference;
47 val = intel_uncore_read(&dev_priv->uncore, SA_PERF_STATUS_0_0_0_MCHBAR_PC);
48 dclk_ratio = REG_FIELD_GET(DG1_QCLK_RATIO_MASK, val);
49 if (val & DG1_QCLK_REFERENCE)
50 dclk_reference = 6; /* 6 * 16.666 MHz = 100 MHz */
52 dclk_reference = 8; /* 8 * 16.666 MHz = 133 MHz */
53 sp->dclk = DIV_ROUND_UP((16667 * dclk_ratio * dclk_reference) + 500, 1000);
55 val = intel_uncore_read(&dev_priv->uncore, SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU);
56 if (val & DG1_GEAR_TYPE)
62 val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR);
63 sp->t_rp = REG_FIELD_GET(DG1_DRAM_T_RP_MASK, val);
64 sp->t_rdpre = REG_FIELD_GET(DG1_DRAM_T_RDPRE_MASK, val);
66 val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH);
67 sp->t_rcd = REG_FIELD_GET(DG1_DRAM_T_RCD_MASK, val);
68 sp->t_ras = REG_FIELD_GET(DG1_DRAM_T_RAS_MASK, val);
70 sp->t_rc = sp->t_rp + sp->t_ras;
75 static int icl_pcode_read_qgv_point_info(struct drm_i915_private *dev_priv,
76 struct intel_qgv_point *sp,
79 u32 val = 0, val2 = 0;
83 ret = snb_pcode_read(&dev_priv->uncore, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
84 ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point),
90 sp->dclk = DIV_ROUND_UP((16667 * dclk) + (DISPLAY_VER(dev_priv) > 11 ? 500 : 0), 1000);
91 sp->t_rp = (val & 0xff0000) >> 16;
92 sp->t_rcd = (val & 0xff000000) >> 24;
94 sp->t_rdpre = val2 & 0xff;
95 sp->t_ras = (val2 & 0xff00) >> 8;
97 sp->t_rc = sp->t_rp + sp->t_ras;
102 static int adls_pcode_read_psf_gv_point_info(struct drm_i915_private *dev_priv,
103 struct intel_psf_gv_point *points)
109 ret = snb_pcode_read(&dev_priv->uncore, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
110 ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, &val, NULL);
114 for (i = 0; i < I915_NUM_PSF_GV_POINTS; i++) {
115 points[i].clk = val & 0xff;
122 static u16 icl_qgv_points_mask(struct drm_i915_private *i915)
124 unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points;
125 unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
126 u16 qgv_points = 0, psf_points = 0;
129 * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects
130 * it with failure if we try masking any unadvertised points.
131 * So need to operate only with those returned from PCode.
133 if (num_qgv_points > 0)
134 qgv_points = GENMASK(num_qgv_points - 1, 0);
136 if (num_psf_gv_points > 0)
137 psf_points = GENMASK(num_psf_gv_points - 1, 0);
139 return ICL_PCODE_REQ_QGV_PT(qgv_points) | ADLS_PCODE_REQ_PSF_PT(psf_points);
142 static bool is_sagv_enabled(struct drm_i915_private *i915, u16 points_mask)
144 return !is_power_of_2(~points_mask & icl_qgv_points_mask(i915) &
145 ICL_PCODE_REQ_QGV_PT_MASK);
148 int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv,
153 if (DISPLAY_VER(dev_priv) >= 14)
156 /* bspec says to keep retrying for at least 1 ms */
157 ret = skl_pcode_request(&dev_priv->uncore, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG,
159 ICL_PCODE_REP_QGV_MASK | ADLS_PCODE_REP_PSF_MASK,
160 ICL_PCODE_REP_QGV_SAFE | ADLS_PCODE_REP_PSF_SAFE,
164 drm_err(&dev_priv->drm, "Failed to disable qgv points (%d) points: 0x%x\n", ret, points_mask);
168 dev_priv->display.sagv.status = is_sagv_enabled(dev_priv, points_mask) ?
169 I915_SAGV_ENABLED : I915_SAGV_DISABLED;
174 static int mtl_read_qgv_point_info(struct drm_i915_private *dev_priv,
175 struct intel_qgv_point *sp, int point)
180 val = intel_uncore_read(&dev_priv->uncore,
181 MTL_MEM_SS_INFO_QGV_POINT_LOW(point));
182 val2 = intel_uncore_read(&dev_priv->uncore,
183 MTL_MEM_SS_INFO_QGV_POINT_HIGH(point));
184 dclk = REG_FIELD_GET(MTL_DCLK_MASK, val);
185 sp->dclk = DIV_ROUND_CLOSEST(16667 * dclk, 1000);
186 sp->t_rp = REG_FIELD_GET(MTL_TRP_MASK, val);
187 sp->t_rcd = REG_FIELD_GET(MTL_TRCD_MASK, val);
189 sp->t_rdpre = REG_FIELD_GET(MTL_TRDPRE_MASK, val2);
190 sp->t_ras = REG_FIELD_GET(MTL_TRAS_MASK, val2);
192 sp->t_rc = sp->t_rp + sp->t_ras;
198 intel_read_qgv_point_info(struct drm_i915_private *dev_priv,
199 struct intel_qgv_point *sp,
202 if (DISPLAY_VER(dev_priv) >= 14)
203 return mtl_read_qgv_point_info(dev_priv, sp, point);
204 else if (IS_DG1(dev_priv))
205 return dg1_mchbar_read_qgv_point_info(dev_priv, sp, point);
207 return icl_pcode_read_qgv_point_info(dev_priv, sp, point);
210 static int icl_get_qgv_points(struct drm_i915_private *dev_priv,
211 struct intel_qgv_info *qi,
214 const struct dram_info *dram_info = &dev_priv->dram_info;
217 qi->num_points = dram_info->num_qgv_points;
218 qi->num_psf_points = dram_info->num_psf_gv_points;
220 if (DISPLAY_VER(dev_priv) >= 14) {
221 switch (dram_info->type) {
222 case INTEL_DRAM_DDR4:
224 qi->max_numchannels = 2;
225 qi->channel_width = 64;
226 qi->deinterleave = 2;
228 case INTEL_DRAM_DDR5:
230 qi->max_numchannels = 4;
231 qi->channel_width = 32;
232 qi->deinterleave = 2;
234 case INTEL_DRAM_LPDDR4:
235 case INTEL_DRAM_LPDDR5:
237 qi->max_numchannels = 8;
238 qi->channel_width = 16;
239 qi->deinterleave = 4;
242 MISSING_CASE(dram_info->type);
245 } else if (DISPLAY_VER(dev_priv) >= 12) {
246 switch (dram_info->type) {
247 case INTEL_DRAM_DDR4:
248 qi->t_bl = is_y_tile ? 8 : 4;
249 qi->max_numchannels = 2;
250 qi->channel_width = 64;
251 qi->deinterleave = is_y_tile ? 1 : 2;
253 case INTEL_DRAM_DDR5:
254 qi->t_bl = is_y_tile ? 16 : 8;
255 qi->max_numchannels = 4;
256 qi->channel_width = 32;
257 qi->deinterleave = is_y_tile ? 1 : 2;
259 case INTEL_DRAM_LPDDR4:
260 if (IS_ROCKETLAKE(dev_priv)) {
262 qi->max_numchannels = 4;
263 qi->channel_width = 32;
264 qi->deinterleave = 2;
268 case INTEL_DRAM_LPDDR5:
270 qi->max_numchannels = 8;
271 qi->channel_width = 16;
272 qi->deinterleave = is_y_tile ? 2 : 4;
276 qi->max_numchannels = 1;
279 } else if (DISPLAY_VER(dev_priv) == 11) {
280 qi->t_bl = dev_priv->dram_info.type == INTEL_DRAM_DDR4 ? 4 : 8;
281 qi->max_numchannels = 1;
284 if (drm_WARN_ON(&dev_priv->drm,
285 qi->num_points > ARRAY_SIZE(qi->points)))
286 qi->num_points = ARRAY_SIZE(qi->points);
288 for (i = 0; i < qi->num_points; i++) {
289 struct intel_qgv_point *sp = &qi->points[i];
291 ret = intel_read_qgv_point_info(dev_priv, sp, i);
295 drm_dbg_kms(&dev_priv->drm,
296 "QGV %d: DCLK=%d tRP=%d tRDPRE=%d tRAS=%d tRCD=%d tRC=%d\n",
297 i, sp->dclk, sp->t_rp, sp->t_rdpre, sp->t_ras,
298 sp->t_rcd, sp->t_rc);
301 if (qi->num_psf_points > 0) {
302 ret = adls_pcode_read_psf_gv_point_info(dev_priv, qi->psf_points);
304 drm_err(&dev_priv->drm, "Failed to read PSF point data; PSF points will not be considered in bandwidth calculations.\n");
305 qi->num_psf_points = 0;
308 for (i = 0; i < qi->num_psf_points; i++)
309 drm_dbg_kms(&dev_priv->drm,
310 "PSF GV %d: CLK=%d \n",
311 i, qi->psf_points[i].clk);
317 static int adl_calc_psf_bw(int clk)
320 * clk is multiples of 16.666MHz (100/6)
321 * According to BSpec PSF GV bandwidth is
322 * calculated as BW = 64 * clk * 16.666Mhz
324 return DIV_ROUND_CLOSEST(64 * clk * 100, 6);
327 static int icl_sagv_max_dclk(const struct intel_qgv_info *qi)
332 for (i = 0; i < qi->num_points; i++)
333 dclk = max(dclk, qi->points[i].dclk);
338 struct intel_sa_info {
340 u8 deburst, deprogbwlimit, derating;
343 static const struct intel_sa_info icl_sa_info = {
345 .deprogbwlimit = 25, /* GB/s */
350 static const struct intel_sa_info tgl_sa_info = {
352 .deprogbwlimit = 34, /* GB/s */
357 static const struct intel_sa_info rkl_sa_info = {
359 .deprogbwlimit = 20, /* GB/s */
364 static const struct intel_sa_info adls_sa_info = {
366 .deprogbwlimit = 38, /* GB/s */
371 static const struct intel_sa_info adlp_sa_info = {
373 .deprogbwlimit = 38, /* GB/s */
378 static const struct intel_sa_info mtl_sa_info = {
380 .deprogbwlimit = 38, /* GB/s */
385 static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa)
387 struct intel_qgv_info qi = {};
388 bool is_y_tile = true; /* assume y tile may be used */
389 int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels);
390 int ipqdepth, ipqdepthpch = 16;
393 int num_groups = ARRAY_SIZE(dev_priv->display.bw.max);
396 ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile);
398 drm_dbg_kms(&dev_priv->drm,
399 "Failed to get memory subsystem information, ignoring bandwidth limits");
403 dclk_max = icl_sagv_max_dclk(&qi);
404 maxdebw = min(sa->deprogbwlimit * 1000, dclk_max * 16 * 6 / 10);
405 ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
406 qi.deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
408 for (i = 0; i < num_groups; i++) {
409 struct intel_bw_info *bi = &dev_priv->display.bw.max[i];
413 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
414 bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1;
416 bi->num_qgv_points = qi.num_points;
417 bi->num_psf_gv_points = qi.num_psf_points;
419 for (j = 0; j < qi.num_points; j++) {
420 const struct intel_qgv_point *sp = &qi.points[j];
426 * FIXME what is the logic behind the
427 * assumed burst length?
429 ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
430 (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
431 bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
433 bi->deratedbw[j] = min(maxdebw,
434 bw * (100 - sa->derating) / 100);
436 drm_dbg_kms(&dev_priv->drm,
437 "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
438 i, j, bi->num_planes, bi->deratedbw[j]);
442 * In case if SAGV is disabled in BIOS, we always get 1
443 * SAGV point, but we can't send PCode commands to restrict it
444 * as it will fail and pointless anyway.
446 if (qi.num_points == 1)
447 dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
449 dev_priv->display.sagv.status = I915_SAGV_ENABLED;
454 static int tgl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa)
456 struct intel_qgv_info qi = {};
457 const struct dram_info *dram_info = &dev_priv->dram_info;
458 bool is_y_tile = true; /* assume y tile may be used */
459 int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels);
460 int ipqdepth, ipqdepthpch = 16;
464 int num_groups = ARRAY_SIZE(dev_priv->display.bw.max);
467 ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile);
469 drm_dbg_kms(&dev_priv->drm,
470 "Failed to get memory subsystem information, ignoring bandwidth limits");
474 if (DISPLAY_VER(dev_priv) < 14 &&
475 (dram_info->type == INTEL_DRAM_LPDDR4 || dram_info->type == INTEL_DRAM_LPDDR5))
478 qi.deinterleave = qi.deinterleave ? : DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
480 if (num_channels < qi.max_numchannels && DISPLAY_VER(dev_priv) >= 12)
481 qi.deinterleave = max(DIV_ROUND_UP(qi.deinterleave, 2), 1);
483 if (DISPLAY_VER(dev_priv) > 11 && num_channels > qi.max_numchannels)
484 drm_warn(&dev_priv->drm, "Number of channels exceeds max number of channels.");
485 if (qi.max_numchannels != 0)
486 num_channels = min_t(u8, num_channels, qi.max_numchannels);
488 dclk_max = icl_sagv_max_dclk(&qi);
490 peakbw = num_channels * DIV_ROUND_UP(qi.channel_width, 8) * dclk_max;
491 maxdebw = min(sa->deprogbwlimit * 1000, peakbw * 6 / 10); /* 60% */
493 ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
495 * clperchgroup = 4kpagespermempage * clperchperblock,
496 * clperchperblock = 8 / num_channels * interleave
498 clperchgroup = 4 * DIV_ROUND_UP(8, num_channels) * qi.deinterleave;
500 for (i = 0; i < num_groups; i++) {
501 struct intel_bw_info *bi = &dev_priv->display.bw.max[i];
502 struct intel_bw_info *bi_next;
506 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
508 if (i < num_groups - 1) {
509 bi_next = &dev_priv->display.bw.max[i + 1];
511 if (clpchgroup < clperchgroup)
512 bi_next->num_planes = (ipqdepth - clpchgroup) /
515 bi_next->num_planes = 0;
518 bi->num_qgv_points = qi.num_points;
519 bi->num_psf_gv_points = qi.num_psf_points;
521 for (j = 0; j < qi.num_points; j++) {
522 const struct intel_qgv_point *sp = &qi.points[j];
528 * FIXME what is the logic behind the
529 * assumed burst length?
531 ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
532 (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
533 bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
535 bi->deratedbw[j] = min(maxdebw,
536 bw * (100 - sa->derating) / 100);
537 bi->peakbw[j] = DIV_ROUND_CLOSEST(sp->dclk *
539 qi.channel_width, 8);
541 drm_dbg_kms(&dev_priv->drm,
542 "BW%d / QGV %d: num_planes=%d deratedbw=%u peakbw: %u\n",
543 i, j, bi->num_planes, bi->deratedbw[j],
547 for (j = 0; j < qi.num_psf_points; j++) {
548 const struct intel_psf_gv_point *sp = &qi.psf_points[j];
550 bi->psf_bw[j] = adl_calc_psf_bw(sp->clk);
552 drm_dbg_kms(&dev_priv->drm,
553 "BW%d / PSF GV %d: num_planes=%d bw=%u\n",
554 i, j, bi->num_planes, bi->psf_bw[j]);
559 * In case if SAGV is disabled in BIOS, we always get 1
560 * SAGV point, but we can't send PCode commands to restrict it
561 * as it will fail and pointless anyway.
563 if (qi.num_points == 1)
564 dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
566 dev_priv->display.sagv.status = I915_SAGV_ENABLED;
571 static void dg2_get_bw_info(struct drm_i915_private *i915)
573 unsigned int deratedbw = IS_DG2_G11(i915) ? 38000 : 50000;
574 int num_groups = ARRAY_SIZE(i915->display.bw.max);
578 * DG2 doesn't have SAGV or QGV points, just a constant max bandwidth
579 * that doesn't depend on the number of planes enabled. So fill all the
580 * plane group with constant bw information for uniformity with other
581 * platforms. DG2-G10 platforms have a constant 50 GB/s bandwidth,
582 * whereas DG2-G11 platforms have 38 GB/s.
584 for (i = 0; i < num_groups; i++) {
585 struct intel_bw_info *bi = &i915->display.bw.max[i];
588 /* Need only one dummy QGV point per group */
589 bi->num_qgv_points = 1;
590 bi->deratedbw[0] = deratedbw;
593 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
596 static unsigned int icl_max_bw_index(struct drm_i915_private *dev_priv,
597 int num_planes, int qgv_point)
602 * Let's return max bw for 0 planes
604 num_planes = max(1, num_planes);
606 for (i = 0; i < ARRAY_SIZE(dev_priv->display.bw.max); i++) {
607 const struct intel_bw_info *bi =
608 &dev_priv->display.bw.max[i];
611 * Pcode will not expose all QGV points when
612 * SAGV is forced to off/min/med/max.
614 if (qgv_point >= bi->num_qgv_points)
617 if (num_planes >= bi->num_planes)
624 static unsigned int tgl_max_bw_index(struct drm_i915_private *dev_priv,
625 int num_planes, int qgv_point)
630 * Let's return max bw for 0 planes
632 num_planes = max(1, num_planes);
634 for (i = ARRAY_SIZE(dev_priv->display.bw.max) - 1; i >= 0; i--) {
635 const struct intel_bw_info *bi =
636 &dev_priv->display.bw.max[i];
639 * Pcode will not expose all QGV points when
640 * SAGV is forced to off/min/med/max.
642 if (qgv_point >= bi->num_qgv_points)
645 if (num_planes <= bi->num_planes)
652 static unsigned int adl_psf_bw(struct drm_i915_private *dev_priv,
655 const struct intel_bw_info *bi =
656 &dev_priv->display.bw.max[0];
658 return bi->psf_bw[psf_gv_point];
661 void intel_bw_init_hw(struct drm_i915_private *dev_priv)
663 if (!HAS_DISPLAY(dev_priv))
666 if (DISPLAY_VER(dev_priv) >= 14)
667 tgl_get_bw_info(dev_priv, &mtl_sa_info);
668 else if (IS_DG2(dev_priv))
669 dg2_get_bw_info(dev_priv);
670 else if (IS_ALDERLAKE_P(dev_priv))
671 tgl_get_bw_info(dev_priv, &adlp_sa_info);
672 else if (IS_ALDERLAKE_S(dev_priv))
673 tgl_get_bw_info(dev_priv, &adls_sa_info);
674 else if (IS_ROCKETLAKE(dev_priv))
675 tgl_get_bw_info(dev_priv, &rkl_sa_info);
676 else if (DISPLAY_VER(dev_priv) == 12)
677 tgl_get_bw_info(dev_priv, &tgl_sa_info);
678 else if (DISPLAY_VER(dev_priv) == 11)
679 icl_get_bw_info(dev_priv, &icl_sa_info);
682 static unsigned int intel_bw_crtc_num_active_planes(const struct intel_crtc_state *crtc_state)
685 * We assume cursors are small enough
686 * to not not cause bandwidth problems.
688 return hweight8(crtc_state->active_planes & ~BIT(PLANE_CURSOR));
691 static unsigned int intel_bw_crtc_data_rate(const struct intel_crtc_state *crtc_state)
693 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
694 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
695 unsigned int data_rate = 0;
696 enum plane_id plane_id;
698 for_each_plane_id_on_crtc(crtc, plane_id) {
700 * We assume cursors are small enough
701 * to not not cause bandwidth problems.
703 if (plane_id == PLANE_CURSOR)
706 data_rate += crtc_state->data_rate[plane_id];
708 if (DISPLAY_VER(i915) < 11)
709 data_rate += crtc_state->data_rate_y[plane_id];
715 /* "Maximum Pipe Read Bandwidth" */
716 static int intel_bw_crtc_min_cdclk(const struct intel_crtc_state *crtc_state)
718 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
719 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
721 if (DISPLAY_VER(i915) < 12)
724 return DIV_ROUND_UP_ULL(mul_u32_u32(intel_bw_crtc_data_rate(crtc_state), 10), 512);
727 void intel_bw_crtc_update(struct intel_bw_state *bw_state,
728 const struct intel_crtc_state *crtc_state)
730 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
731 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
733 bw_state->data_rate[crtc->pipe] =
734 intel_bw_crtc_data_rate(crtc_state);
735 bw_state->num_active_planes[crtc->pipe] =
736 intel_bw_crtc_num_active_planes(crtc_state);
738 drm_dbg_kms(&i915->drm, "pipe %c data rate %u num active planes %u\n",
739 pipe_name(crtc->pipe),
740 bw_state->data_rate[crtc->pipe],
741 bw_state->num_active_planes[crtc->pipe]);
744 static unsigned int intel_bw_num_active_planes(struct drm_i915_private *dev_priv,
745 const struct intel_bw_state *bw_state)
747 unsigned int num_active_planes = 0;
750 for_each_pipe(dev_priv, pipe)
751 num_active_planes += bw_state->num_active_planes[pipe];
753 return num_active_planes;
756 static unsigned int intel_bw_data_rate(struct drm_i915_private *dev_priv,
757 const struct intel_bw_state *bw_state)
759 unsigned int data_rate = 0;
762 for_each_pipe(dev_priv, pipe)
763 data_rate += bw_state->data_rate[pipe];
765 if (DISPLAY_VER(dev_priv) >= 13 && i915_vtd_active(dev_priv))
766 data_rate = DIV_ROUND_UP(data_rate * 105, 100);
771 struct intel_bw_state *
772 intel_atomic_get_old_bw_state(struct intel_atomic_state *state)
774 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
775 struct intel_global_state *bw_state;
777 bw_state = intel_atomic_get_old_global_obj_state(state, &dev_priv->display.bw.obj);
779 return to_intel_bw_state(bw_state);
782 struct intel_bw_state *
783 intel_atomic_get_new_bw_state(struct intel_atomic_state *state)
785 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
786 struct intel_global_state *bw_state;
788 bw_state = intel_atomic_get_new_global_obj_state(state, &dev_priv->display.bw.obj);
790 return to_intel_bw_state(bw_state);
793 struct intel_bw_state *
794 intel_atomic_get_bw_state(struct intel_atomic_state *state)
796 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
797 struct intel_global_state *bw_state;
799 bw_state = intel_atomic_get_global_obj_state(state, &dev_priv->display.bw.obj);
800 if (IS_ERR(bw_state))
801 return ERR_CAST(bw_state);
803 return to_intel_bw_state(bw_state);
806 static int mtl_find_qgv_points(struct drm_i915_private *i915,
807 unsigned int data_rate,
808 unsigned int num_active_planes,
809 struct intel_bw_state *new_bw_state)
811 unsigned int best_rate = UINT_MAX;
812 unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
813 unsigned int qgv_peak_bw = 0;
817 ret = intel_atomic_lock_global_state(&new_bw_state->base);
822 * If SAGV cannot be enabled, disable the pcode SAGV by passing all 1's
823 * for qgv peak bw in PM Demand request. So assign UINT_MAX if SAGV is
824 * not enabled. PM Demand code will clamp the value for the register
826 if (!intel_can_enable_sagv(i915, new_bw_state)) {
827 new_bw_state->qgv_point_peakbw = U16_MAX;
828 drm_dbg_kms(&i915->drm, "No SAGV, use UINT_MAX as peak bw.");
833 * Find the best QGV point by comparing the data_rate with max data rate
834 * offered per plane group
836 for (i = 0; i < num_qgv_points; i++) {
837 unsigned int bw_index =
838 tgl_max_bw_index(i915, num_active_planes, i);
839 unsigned int max_data_rate;
841 if (bw_index >= ARRAY_SIZE(i915->display.bw.max))
844 max_data_rate = i915->display.bw.max[bw_index].deratedbw[i];
846 if (max_data_rate < data_rate)
849 if (max_data_rate - data_rate < best_rate) {
850 best_rate = max_data_rate - data_rate;
851 qgv_peak_bw = i915->display.bw.max[bw_index].peakbw[i];
854 drm_dbg_kms(&i915->drm, "QGV point %d: max bw %d required %d qgv_peak_bw: %d\n",
855 i, max_data_rate, data_rate, qgv_peak_bw);
858 drm_dbg_kms(&i915->drm, "Matching peaks QGV bw: %d for required data rate: %d\n",
859 qgv_peak_bw, data_rate);
862 * The display configuration cannot be supported if no QGV point
863 * satisfying the required data rate is found
865 if (qgv_peak_bw == 0) {
866 drm_dbg_kms(&i915->drm, "No QGV points for bw %d for display configuration(%d active planes).\n",
867 data_rate, num_active_planes);
871 /* MTL PM DEMAND expects QGV BW parameter in multiples of 100 mbps */
872 new_bw_state->qgv_point_peakbw = DIV_ROUND_CLOSEST(qgv_peak_bw, 100);
877 static int icl_find_qgv_points(struct drm_i915_private *i915,
878 unsigned int data_rate,
879 unsigned int num_active_planes,
880 const struct intel_bw_state *old_bw_state,
881 struct intel_bw_state *new_bw_state)
883 unsigned int max_bw_point = 0;
884 unsigned int max_bw = 0;
885 unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points;
886 unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
892 ret = intel_atomic_lock_global_state(&new_bw_state->base);
896 for (i = 0; i < num_qgv_points; i++) {
898 unsigned int max_data_rate;
900 if (DISPLAY_VER(i915) > 11)
901 idx = tgl_max_bw_index(i915, num_active_planes, i);
903 idx = icl_max_bw_index(i915, num_active_planes, i);
905 if (idx >= ARRAY_SIZE(i915->display.bw.max))
908 max_data_rate = i915->display.bw.max[idx].deratedbw[i];
911 * We need to know which qgv point gives us
912 * maximum bandwidth in order to disable SAGV
913 * if we find that we exceed SAGV block time
914 * with watermarks. By that moment we already
915 * have those, as it is calculated earlier in
916 * intel_atomic_check,
918 if (max_data_rate > max_bw) {
920 max_bw = max_data_rate;
922 if (max_data_rate >= data_rate)
923 qgv_points |= BIT(i);
925 drm_dbg_kms(&i915->drm, "QGV point %d: max bw %d required %d\n",
926 i, max_data_rate, data_rate);
929 for (i = 0; i < num_psf_gv_points; i++) {
930 unsigned int max_data_rate = adl_psf_bw(i915, i);
932 if (max_data_rate >= data_rate)
933 psf_points |= BIT(i);
935 drm_dbg_kms(&i915->drm, "PSF GV point %d: max bw %d"
937 i, max_data_rate, data_rate);
941 * BSpec states that we always should have at least one allowed point
942 * left, so if we couldn't - simply reject the configuration for obvious
945 if (qgv_points == 0) {
946 drm_dbg_kms(&i915->drm, "No QGV points provide sufficient memory"
947 " bandwidth %d for display configuration(%d active planes).\n",
948 data_rate, num_active_planes);
952 if (num_psf_gv_points > 0 && psf_points == 0) {
953 drm_dbg_kms(&i915->drm, "No PSF GV points provide sufficient memory"
954 " bandwidth %d for display configuration(%d active planes).\n",
955 data_rate, num_active_planes);
960 * Leave only single point with highest bandwidth, if
961 * we can't enable SAGV due to the increased memory latency it may
964 if (!intel_can_enable_sagv(i915, new_bw_state)) {
965 qgv_points = BIT(max_bw_point);
966 drm_dbg_kms(&i915->drm, "No SAGV, using single QGV point %d\n",
971 * We store the ones which need to be masked as that is what PCode
972 * actually accepts as a parameter.
974 new_bw_state->qgv_points_mask =
975 ~(ICL_PCODE_REQ_QGV_PT(qgv_points) |
976 ADLS_PCODE_REQ_PSF_PT(psf_points)) &
977 icl_qgv_points_mask(i915);
980 * If the actual mask had changed we need to make sure that
981 * the commits are serialized(in case this is a nomodeset, nonblocking)
983 if (new_bw_state->qgv_points_mask != old_bw_state->qgv_points_mask) {
984 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
992 static int intel_bw_check_qgv_points(struct drm_i915_private *i915,
993 const struct intel_bw_state *old_bw_state,
994 struct intel_bw_state *new_bw_state)
996 unsigned int data_rate = intel_bw_data_rate(i915, new_bw_state);
997 unsigned int num_active_planes =
998 intel_bw_num_active_planes(i915, new_bw_state);
1000 data_rate = DIV_ROUND_UP(data_rate, 1000);
1002 if (DISPLAY_VER(i915) >= 14)
1003 return mtl_find_qgv_points(i915, data_rate, num_active_planes,
1006 return icl_find_qgv_points(i915, data_rate, num_active_planes,
1007 old_bw_state, new_bw_state);
1010 static bool intel_bw_state_changed(struct drm_i915_private *i915,
1011 const struct intel_bw_state *old_bw_state,
1012 const struct intel_bw_state *new_bw_state)
1016 for_each_pipe(i915, pipe) {
1017 const struct intel_dbuf_bw *old_crtc_bw =
1018 &old_bw_state->dbuf_bw[pipe];
1019 const struct intel_dbuf_bw *new_crtc_bw =
1020 &new_bw_state->dbuf_bw[pipe];
1021 enum dbuf_slice slice;
1023 for_each_dbuf_slice(i915, slice) {
1024 if (old_crtc_bw->max_bw[slice] != new_crtc_bw->max_bw[slice] ||
1025 old_crtc_bw->active_planes[slice] != new_crtc_bw->active_planes[slice])
1029 if (old_bw_state->min_cdclk[pipe] != new_bw_state->min_cdclk[pipe])
1036 static void skl_plane_calc_dbuf_bw(struct intel_bw_state *bw_state,
1037 struct intel_crtc *crtc,
1038 enum plane_id plane_id,
1039 const struct skl_ddb_entry *ddb,
1040 unsigned int data_rate)
1042 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1043 struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe];
1044 unsigned int dbuf_mask = skl_ddb_dbuf_slice_mask(i915, ddb);
1045 enum dbuf_slice slice;
1048 * The arbiter can only really guarantee an
1049 * equal share of the total bw to each plane.
1051 for_each_dbuf_slice_in_mask(i915, slice, dbuf_mask) {
1052 crtc_bw->max_bw[slice] = max(crtc_bw->max_bw[slice], data_rate);
1053 crtc_bw->active_planes[slice] |= BIT(plane_id);
1057 static void skl_crtc_calc_dbuf_bw(struct intel_bw_state *bw_state,
1058 const struct intel_crtc_state *crtc_state)
1060 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1061 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1062 struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe];
1063 enum plane_id plane_id;
1065 memset(crtc_bw, 0, sizeof(*crtc_bw));
1067 if (!crtc_state->hw.active)
1070 for_each_plane_id_on_crtc(crtc, plane_id) {
1072 * We assume cursors are small enough
1073 * to not cause bandwidth problems.
1075 if (plane_id == PLANE_CURSOR)
1078 skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id,
1079 &crtc_state->wm.skl.plane_ddb[plane_id],
1080 crtc_state->data_rate[plane_id]);
1082 if (DISPLAY_VER(i915) < 11)
1083 skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id,
1084 &crtc_state->wm.skl.plane_ddb_y[plane_id],
1085 crtc_state->data_rate[plane_id]);
1089 /* "Maximum Data Buffer Bandwidth" */
1091 intel_bw_dbuf_min_cdclk(struct drm_i915_private *i915,
1092 const struct intel_bw_state *bw_state)
1094 unsigned int total_max_bw = 0;
1095 enum dbuf_slice slice;
1097 for_each_dbuf_slice(i915, slice) {
1098 int num_active_planes = 0;
1099 unsigned int max_bw = 0;
1103 * The arbiter can only really guarantee an
1104 * equal share of the total bw to each plane.
1106 for_each_pipe(i915, pipe) {
1107 const struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[pipe];
1109 max_bw = max(crtc_bw->max_bw[slice], max_bw);
1110 num_active_planes += hweight8(crtc_bw->active_planes[slice]);
1112 max_bw *= num_active_planes;
1114 total_max_bw = max(total_max_bw, max_bw);
1117 return DIV_ROUND_UP(total_max_bw, 64);
1120 int intel_bw_min_cdclk(struct drm_i915_private *i915,
1121 const struct intel_bw_state *bw_state)
1126 min_cdclk = intel_bw_dbuf_min_cdclk(i915, bw_state);
1128 for_each_pipe(i915, pipe)
1129 min_cdclk = max(bw_state->min_cdclk[pipe], min_cdclk);
1134 int intel_bw_calc_min_cdclk(struct intel_atomic_state *state,
1135 bool *need_cdclk_calc)
1137 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
1138 struct intel_bw_state *new_bw_state = NULL;
1139 const struct intel_bw_state *old_bw_state = NULL;
1140 const struct intel_cdclk_state *cdclk_state;
1141 const struct intel_crtc_state *crtc_state;
1142 int old_min_cdclk, new_min_cdclk;
1143 struct intel_crtc *crtc;
1146 if (DISPLAY_VER(dev_priv) < 9)
1149 for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
1150 new_bw_state = intel_atomic_get_bw_state(state);
1151 if (IS_ERR(new_bw_state))
1152 return PTR_ERR(new_bw_state);
1154 old_bw_state = intel_atomic_get_old_bw_state(state);
1156 skl_crtc_calc_dbuf_bw(new_bw_state, crtc_state);
1158 new_bw_state->min_cdclk[crtc->pipe] =
1159 intel_bw_crtc_min_cdclk(crtc_state);
1165 if (intel_bw_state_changed(dev_priv, old_bw_state, new_bw_state)) {
1166 int ret = intel_atomic_lock_global_state(&new_bw_state->base);
1171 old_min_cdclk = intel_bw_min_cdclk(dev_priv, old_bw_state);
1172 new_min_cdclk = intel_bw_min_cdclk(dev_priv, new_bw_state);
1175 * No need to check against the cdclk state if
1176 * the min cdclk doesn't increase.
1178 * Ie. we only ever increase the cdclk due to bandwidth
1179 * requirements. This can reduce back and forth
1180 * display blinking due to constant cdclk changes.
1182 if (new_min_cdclk <= old_min_cdclk)
1185 cdclk_state = intel_atomic_get_cdclk_state(state);
1186 if (IS_ERR(cdclk_state))
1187 return PTR_ERR(cdclk_state);
1190 * No need to recalculate the cdclk state if
1191 * the min cdclk doesn't increase.
1193 * Ie. we only ever increase the cdclk due to bandwidth
1194 * requirements. This can reduce back and forth
1195 * display blinking due to constant cdclk changes.
1197 if (new_min_cdclk <= cdclk_state->bw_min_cdclk)
1200 drm_dbg_kms(&dev_priv->drm,
1201 "new bandwidth min cdclk (%d kHz) > old min cdclk (%d kHz)\n",
1202 new_min_cdclk, cdclk_state->bw_min_cdclk);
1203 *need_cdclk_calc = true;
1208 static int intel_bw_check_data_rate(struct intel_atomic_state *state, bool *changed)
1210 struct drm_i915_private *i915 = to_i915(state->base.dev);
1211 const struct intel_crtc_state *new_crtc_state, *old_crtc_state;
1212 struct intel_crtc *crtc;
1215 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
1216 new_crtc_state, i) {
1217 unsigned int old_data_rate =
1218 intel_bw_crtc_data_rate(old_crtc_state);
1219 unsigned int new_data_rate =
1220 intel_bw_crtc_data_rate(new_crtc_state);
1221 unsigned int old_active_planes =
1222 intel_bw_crtc_num_active_planes(old_crtc_state);
1223 unsigned int new_active_planes =
1224 intel_bw_crtc_num_active_planes(new_crtc_state);
1225 struct intel_bw_state *new_bw_state;
1228 * Avoid locking the bw state when
1229 * nothing significant has changed.
1231 if (old_data_rate == new_data_rate &&
1232 old_active_planes == new_active_planes)
1235 new_bw_state = intel_atomic_get_bw_state(state);
1236 if (IS_ERR(new_bw_state))
1237 return PTR_ERR(new_bw_state);
1239 new_bw_state->data_rate[crtc->pipe] = new_data_rate;
1240 new_bw_state->num_active_planes[crtc->pipe] = new_active_planes;
1244 drm_dbg_kms(&i915->drm,
1245 "[CRTC:%d:%s] data rate %u num active planes %u\n",
1246 crtc->base.base.id, crtc->base.name,
1247 new_bw_state->data_rate[crtc->pipe],
1248 new_bw_state->num_active_planes[crtc->pipe]);
1254 int intel_bw_atomic_check(struct intel_atomic_state *state)
1256 bool changed = false;
1257 struct drm_i915_private *i915 = to_i915(state->base.dev);
1258 struct intel_bw_state *new_bw_state;
1259 const struct intel_bw_state *old_bw_state;
1262 /* FIXME earlier gens need some checks too */
1263 if (DISPLAY_VER(i915) < 11)
1266 ret = intel_bw_check_data_rate(state, &changed);
1270 old_bw_state = intel_atomic_get_old_bw_state(state);
1271 new_bw_state = intel_atomic_get_new_bw_state(state);
1274 intel_can_enable_sagv(i915, old_bw_state) !=
1275 intel_can_enable_sagv(i915, new_bw_state))
1279 * If none of our inputs (data rates, number of active
1280 * planes, SAGV yes/no) changed then nothing to do here.
1285 ret = intel_bw_check_qgv_points(i915, old_bw_state, new_bw_state);
1292 static struct intel_global_state *
1293 intel_bw_duplicate_state(struct intel_global_obj *obj)
1295 struct intel_bw_state *state;
1297 state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
1301 return &state->base;
1304 static void intel_bw_destroy_state(struct intel_global_obj *obj,
1305 struct intel_global_state *state)
1310 static const struct intel_global_state_funcs intel_bw_funcs = {
1311 .atomic_duplicate_state = intel_bw_duplicate_state,
1312 .atomic_destroy_state = intel_bw_destroy_state,
1315 int intel_bw_init(struct drm_i915_private *dev_priv)
1317 struct intel_bw_state *state;
1319 state = kzalloc(sizeof(*state), GFP_KERNEL);
1323 intel_atomic_global_obj_init(dev_priv, &dev_priv->display.bw.obj,
1324 &state->base, &intel_bw_funcs);