drivers/gpu/drm/i915/gt/intel_rps.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include <drm/i915_drm.h>
   7
   8 #include "i915_drv.h"
   9 #include "intel_breadcrumbs.h"
  10 #include "intel_gt.h"
  11 #include "intel_gt_clock_utils.h"
  12 #include "intel_gt_irq.h"
  13 #include "intel_gt_pm_irq.h"
  14 #include "intel_rps.h"
  15 #include "intel_sideband.h"
  16 #include "../../../platform/x86/intel_ips.h"
  17
  18 #define BUSY_MAX_EI     20u /* ms */
  19
  20 /*
  21  * Lock protecting IPS related data structures
  22  */
  23 static DEFINE_SPINLOCK(mchdev_lock);
  24
  25 static struct intel_gt *rps_to_gt(struct intel_rps *rps)
  26 {
  27         return container_of(rps, struct intel_gt, rps);
  28 }
  29
  30 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)
  31 {
  32         return rps_to_gt(rps)->i915;
  33 }
  34
  35 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)
  36 {
  37         return rps_to_gt(rps)->uncore;
  38 }
  39
  40 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)
  41 {
  42         return mask & ~rps->pm_intrmsk_mbz;
  43 }
  44
  45 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val)
  46 {
  47         intel_uncore_write_fw(uncore, reg, val);
  48 }
  49
  50 static void rps_timer(struct timer_list *t)
  51 {
  52         struct intel_rps *rps = from_timer(rps, t, timer);
  53         struct intel_engine_cs *engine;
  54         ktime_t dt, last, timestamp;
  55         enum intel_engine_id id;
  56         s64 max_busy[3] = {};
  57
  58         timestamp = 0;
  59         for_each_engine(engine, rps_to_gt(rps), id) {
  60                 s64 busy;
  61                 int i;
  62
  63                 dt = intel_engine_get_busy_time(engine, &timestamp);
  64                 last = engine->stats.rps;
  65                 engine->stats.rps = dt;
  66
  67                 busy = ktime_to_ns(ktime_sub(dt, last));
  68                 for (i = 0; i < ARRAY_SIZE(max_busy); i++) {
  69                         if (busy > max_busy[i])
  70                                 swap(busy, max_busy[i]);
  71                 }
  72         }
  73         last = rps->pm_timestamp;
  74         rps->pm_timestamp = timestamp;
  75
  76         if (intel_rps_is_active(rps)) {
  77                 s64 busy;
  78                 int i;
  79
  80                 dt = ktime_sub(timestamp, last);
  81
  82                 /*
  83                  * Our goal is to evaluate each engine independently, so we run
  84                  * at the lowest clocks required to sustain the heaviest
  85                  * workload. However, a task may be split into sequential
  86                  * dependent operations across a set of engines, such that
  87                  * the independent contributions do not account for high load,
  88                  * but overall the task is GPU bound. For example, consider
  89                  * video decode on vcs followed by colour post-processing
  90                  * on vecs, followed by general post-processing on rcs.
  91                  * Since multi-engines being active does imply a single
  92                  * continuous workload across all engines, we hedge our
  93                  * bets by only contributing a factor of the distributed
  94                  * load into our busyness calculation.
  95                  */
  96                 busy = max_busy[0];
  97                 for (i = 1; i < ARRAY_SIZE(max_busy); i++) {
  98                         if (!max_busy[i])
  99                                 break;
 100
 101                         busy += div_u64(max_busy[i], 1 << i);
 102                 }
 103                 GT_TRACE(rps_to_gt(rps),
 104                          "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n",
 105                          busy, (int)div64_u64(100 * busy, dt),
 106                          max_busy[0], max_busy[1], max_busy[2],
 107                          rps->pm_interval);
 108
 109                 if (100 * busy > rps->power.up_threshold * dt &&
 110                     rps->cur_freq < rps->max_freq_softlimit) {
 111                         rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD;
 112                         rps->pm_interval = 1;
 113                         schedule_work(&rps->work);
 114                 } else if (100 * busy < rps->power.down_threshold * dt &&
 115                            rps->cur_freq > rps->min_freq_softlimit) {
 116                         rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD;
 117                         rps->pm_interval = 1;
 118                         schedule_work(&rps->work);
 119                 } else {
 120                         rps->last_adj = 0;
 121                 }
 122
 123                 mod_timer(&rps->timer,
 124                           jiffies + msecs_to_jiffies(rps->pm_interval));
 125                 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI);
 126         }
 127 }
 128
 129 static void rps_start_timer(struct intel_rps *rps)
 130 {
 131         rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
 132         rps->pm_interval = 1;
 133         mod_timer(&rps->timer, jiffies + 1);
 134 }
 135
 136 static void rps_stop_timer(struct intel_rps *rps)
 137 {
 138         del_timer_sync(&rps->timer);
 139         rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
 140         cancel_work_sync(&rps->work);
 141 }
 142
 143 static u32 rps_pm_mask(struct intel_rps *rps, u8 val)
 144 {
 145         u32 mask = 0;
 146
 147         /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */
 148         if (val > rps->min_freq_softlimit)
 149                 mask |= (GEN6_PM_RP_UP_EI_EXPIRED |
 150                          GEN6_PM_RP_DOWN_THRESHOLD |
 151                          GEN6_PM_RP_DOWN_TIMEOUT);
 152
 153         if (val < rps->max_freq_softlimit)
 154                 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
 155
 156         mask &= rps->pm_events;
 157
 158         return rps_pm_sanitize_mask(rps, ~mask);
 159 }
 160
 161 static void rps_reset_ei(struct intel_rps *rps)
 162 {
 163         memset(&rps->ei, 0, sizeof(rps->ei));
 164 }
 165
 166 static void rps_enable_interrupts(struct intel_rps *rps)
 167 {
 168         struct intel_gt *gt = rps_to_gt(rps);
 169
 170         GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n",
 171                  rps->pm_events, rps_pm_mask(rps, rps->last_freq));
 172
 173         rps_reset_ei(rps);
 174
 175         spin_lock_irq(&gt->irq_lock);
 176         gen6_gt_pm_enable_irq(gt, rps->pm_events);
 177         spin_unlock_irq(&gt->irq_lock);
 178
 179         intel_uncore_write(gt->uncore,
 180                            GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq));
 181 }
 182
 183 static void gen6_rps_reset_interrupts(struct intel_rps *rps)
 184 {
 185         gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS);
 186 }
 187
 188 static void gen11_rps_reset_interrupts(struct intel_rps *rps)
 189 {
 190         while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM))
 191                 ;
 192 }
 193
 194 static void rps_reset_interrupts(struct intel_rps *rps)
 195 {
 196         struct intel_gt *gt = rps_to_gt(rps);
 197
 198         spin_lock_irq(&gt->irq_lock);
 199         if (GRAPHICS_VER(gt->i915) >= 11)
 200                 gen11_rps_reset_interrupts(rps);
 201         else
 202                 gen6_rps_reset_interrupts(rps);
 203
 204         rps->pm_iir = 0;
 205         spin_unlock_irq(&gt->irq_lock);
 206 }
 207
 208 static void rps_disable_interrupts(struct intel_rps *rps)
 209 {
 210         struct intel_gt *gt = rps_to_gt(rps);
 211
 212         intel_uncore_write(gt->uncore,
 213                            GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u));
 214
 215         spin_lock_irq(&gt->irq_lock);
 216         gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
 217         spin_unlock_irq(&gt->irq_lock);
 218
 219         intel_synchronize_irq(gt->i915);
 220
 221         /*
 222          * Now that we will not be generating any more work, flush any
 223          * outstanding tasks. As we are called on the RPS idle path,
 224          * we will reset the GPU to minimum frequencies, so the current
 225          * state of the worker can be discarded.
 226          */
 227         cancel_work_sync(&rps->work);
 228
 229         rps_reset_interrupts(rps);
 230         GT_TRACE(gt, "interrupts:off\n");
 231 }
 232
 233 static const struct cparams {
 234         u16 i;
 235         u16 t;
 236         u16 m;
 237         u16 c;
 238 } cparams[] = {
 239         { 1, 1333, 301, 28664 },
 240         { 1, 1066, 294, 24460 },
 241         { 1, 800, 294, 25192 },
 242         { 0, 1333, 276, 27605 },
 243         { 0, 1066, 276, 27605 },
 244         { 0, 800, 231, 23784 },
 245 };
 246
 247 static void gen5_rps_init(struct intel_rps *rps)
 248 {
 249         struct drm_i915_private *i915 = rps_to_i915(rps);
 250         struct intel_uncore *uncore = rps_to_uncore(rps);
 251         u8 fmax, fmin, fstart;
 252         u32 rgvmodectl;
 253         int c_m, i;
 254
 255         if (i915->fsb_freq <= 3200)
 256                 c_m = 0;
 257         else if (i915->fsb_freq <= 4800)
 258                 c_m = 1;
 259         else
 260                 c_m = 2;
 261
 262         for (i = 0; i < ARRAY_SIZE(cparams); i++) {
 263                 if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) {
 264                         rps->ips.m = cparams[i].m;
 265                         rps->ips.c = cparams[i].c;
 266                         break;
 267                 }
 268         }
 269
 270         rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
 271
 272         /* Set up min, max, and cur for interrupt handling */
 273         fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
 274         fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
 275         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
 276                 MEMMODE_FSTART_SHIFT;
 277         drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n",
 278                 fmax, fmin, fstart);
 279
 280         rps->min_freq = fmax;
 281         rps->efficient_freq = fstart;
 282         rps->max_freq = fmin;
 283 }
 284
 285 static unsigned long
 286 __ips_chipset_val(struct intel_ips *ips)
 287 {
 288         struct intel_uncore *uncore =
 289                 rps_to_uncore(container_of(ips, struct intel_rps, ips));
 290         unsigned long now = jiffies_to_msecs(jiffies), dt;
 291         unsigned long result;
 292         u64 total, delta;
 293
 294         lockdep_assert_held(&mchdev_lock);
 295
 296         /*
 297          * Prevent division-by-zero if we are asking too fast.
 298          * Also, we don't get interesting results if we are polling
 299          * faster than once in 10ms, so just return the saved value
 300          * in such cases.
 301          */
 302         dt = now - ips->last_time1;
 303         if (dt <= 10)
 304                 return ips->chipset_power;
 305
 306         /* FIXME: handle per-counter overflow */
 307         total = intel_uncore_read(uncore, DMIEC);
 308         total += intel_uncore_read(uncore, DDREC);
 309         total += intel_uncore_read(uncore, CSIEC);
 310
 311         delta = total - ips->last_count1;
 312
 313         result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10);
 314
 315         ips->last_count1 = total;
 316         ips->last_time1 = now;
 317
 318         ips->chipset_power = result;
 319
 320         return result;
 321 }
 322
 323 static unsigned long ips_mch_val(struct intel_uncore *uncore)
 324 {
 325         unsigned int m, x, b;
 326         u32 tsfs;
 327
 328         tsfs = intel_uncore_read(uncore, TSFS);
 329         x = intel_uncore_read8(uncore, TR1);
 330
 331         b = tsfs & TSFS_INTR_MASK;
 332         m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT;
 333
 334         return m * x / 127 - b;
 335 }
 336
 337 static int _pxvid_to_vd(u8 pxvid)
 338 {
 339         if (pxvid == 0)
 340                 return 0;
 341
 342         if (pxvid >= 8 && pxvid < 31)
 343                 pxvid = 31;
 344
 345         return (pxvid + 2) * 125;
 346 }
 347
 348 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)
 349 {
 350         const int vd = _pxvid_to_vd(pxvid);
 351
 352         if (INTEL_INFO(i915)->is_mobile)
 353                 return max(vd - 1125, 0);
 354
 355         return vd;
 356 }
 357
 358 static void __gen5_ips_update(struct intel_ips *ips)
 359 {
 360         struct intel_uncore *uncore =
 361                 rps_to_uncore(container_of(ips, struct intel_rps, ips));
 362         u64 now, delta, dt;
 363         u32 count;
 364
 365         lockdep_assert_held(&mchdev_lock);
 366
 367         now = ktime_get_raw_ns();
 368         dt = now - ips->last_time2;
 369         do_div(dt, NSEC_PER_MSEC);
 370
 371         /* Don't divide by 0 */
 372         if (dt <= 10)
 373                 return;
 374
 375         count = intel_uncore_read(uncore, GFXEC);
 376         delta = count - ips->last_count2;
 377
 378         ips->last_count2 = count;
 379         ips->last_time2 = now;
 380
 381         /* More magic constants... */
 382         ips->gfx_power = div_u64(delta * 1181, dt * 10);
 383 }
 384
 385 static void gen5_rps_update(struct intel_rps *rps)
 386 {
 387         spin_lock_irq(&mchdev_lock);
 388         __gen5_ips_update(&rps->ips);
 389         spin_unlock_irq(&mchdev_lock);
 390 }
 391
 392 static unsigned int gen5_invert_freq(struct intel_rps *rps,
 393                                      unsigned int val)
 394 {
 395         /* Invert the frequency bin into an ips delay */
 396         val = rps->max_freq - val;
 397         val = rps->min_freq + val;
 398
 399         return val;
 400 }
 401
 402 static int __gen5_rps_set(struct intel_rps *rps, u8 val)
 403 {
 404         struct intel_uncore *uncore = rps_to_uncore(rps);
 405         u16 rgvswctl;
 406
 407         lockdep_assert_held(&mchdev_lock);
 408
 409         rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
 410         if (rgvswctl & MEMCTL_CMD_STS) {
 411                 DRM_DEBUG("gpu busy, RCS change rejected\n");
 412                 return -EBUSY; /* still busy with another command */
 413         }
 414
 415         /* Invert the frequency bin into an ips delay */
 416         val = gen5_invert_freq(rps, val);
 417
 418         rgvswctl =
 419                 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
 420                 (val << MEMCTL_FREQ_SHIFT) |
 421                 MEMCTL_SFCAVM;
 422         intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
 423         intel_uncore_posting_read16(uncore, MEMSWCTL);
 424
 425         rgvswctl |= MEMCTL_CMD_STS;
 426         intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
 427
 428         return 0;
 429 }
 430
 431 static int gen5_rps_set(struct intel_rps *rps, u8 val)
 432 {
 433         int err;
 434
 435         spin_lock_irq(&mchdev_lock);
 436         err = __gen5_rps_set(rps, val);
 437         spin_unlock_irq(&mchdev_lock);
 438
 439         return err;
 440 }
 441
 442 static unsigned long intel_pxfreq(u32 vidfreq)
 443 {
 444         int div = (vidfreq & 0x3f0000) >> 16;
 445         int post = (vidfreq & 0x3000) >> 12;
 446         int pre = (vidfreq & 0x7);
 447
 448         if (!pre)
 449                 return 0;
 450
 451         return div * 133333 / (pre << post);
 452 }
 453
 454 static unsigned int init_emon(struct intel_uncore *uncore)
 455 {
 456         u8 pxw[16];
 457         int i;
 458
 459         /* Disable to program */
 460         intel_uncore_write(uncore, ECR, 0);
 461         intel_uncore_posting_read(uncore, ECR);
 462
 463         /* Program energy weights for various events */
 464         intel_uncore_write(uncore, SDEW, 0x15040d00);
 465         intel_uncore_write(uncore, CSIEW0, 0x007f0000);
 466         intel_uncore_write(uncore, CSIEW1, 0x1e220004);
 467         intel_uncore_write(uncore, CSIEW2, 0x04000004);
 468
 469         for (i = 0; i < 5; i++)
 470                 intel_uncore_write(uncore, PEW(i), 0);
 471         for (i = 0; i < 3; i++)
 472                 intel_uncore_write(uncore, DEW(i), 0);
 473
 474         /* Program P-state weights to account for frequency power adjustment */
 475         for (i = 0; i < 16; i++) {
 476                 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i));
 477                 unsigned int freq = intel_pxfreq(pxvidfreq);
 478                 unsigned int vid =
 479                         (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
 480                 unsigned int val;
 481
 482                 val = vid * vid * freq / 1000 * 255;
 483                 val /= 127 * 127 * 900;
 484
 485                 pxw[i] = val;
 486         }
 487         /* Render standby states get 0 weight */
 488         pxw[14] = 0;
 489         pxw[15] = 0;
 490
 491         for (i = 0; i < 4; i++) {
 492                 intel_uncore_write(uncore, PXW(i),
 493                                    pxw[i * 4 + 0] << 24 |
 494                                    pxw[i * 4 + 1] << 16 |
 495                                    pxw[i * 4 + 2] <<  8 |
 496                                    pxw[i * 4 + 3] <<  0);
 497         }
 498
 499         /* Adjust magic regs to magic values (more experimental results) */
 500         intel_uncore_write(uncore, OGW0, 0);
 501         intel_uncore_write(uncore, OGW1, 0);
 502         intel_uncore_write(uncore, EG0, 0x00007f00);
 503         intel_uncore_write(uncore, EG1, 0x0000000e);
 504         intel_uncore_write(uncore, EG2, 0x000e0000);
 505         intel_uncore_write(uncore, EG3, 0x68000300);
 506         intel_uncore_write(uncore, EG4, 0x42000000);
 507         intel_uncore_write(uncore, EG5, 0x00140031);
 508         intel_uncore_write(uncore, EG6, 0);
 509         intel_uncore_write(uncore, EG7, 0);
 510
 511         for (i = 0; i < 8; i++)
 512                 intel_uncore_write(uncore, PXWL(i), 0);
 513
 514         /* Enable PMON + select events */
 515         intel_uncore_write(uncore, ECR, 0x80000019);
 516
 517         return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK;
 518 }
 519
 520 static bool gen5_rps_enable(struct intel_rps *rps)
 521 {
 522         struct drm_i915_private *i915 = rps_to_i915(rps);
 523         struct intel_uncore *uncore = rps_to_uncore(rps);
 524         u8 fstart, vstart;
 525         u32 rgvmodectl;
 526
 527         spin_lock_irq(&mchdev_lock);
 528
 529         rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
 530
 531         /* Enable temp reporting */
 532         intel_uncore_write16(uncore, PMMISC,
 533                              intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN);
 534         intel_uncore_write16(uncore, TSC1,
 535                              intel_uncore_read16(uncore, TSC1) | TSE);
 536
 537         /* 100ms RC evaluation intervals */
 538         intel_uncore_write(uncore, RCUPEI, 100000);
 539         intel_uncore_write(uncore, RCDNEI, 100000);
 540
 541         /* Set max/min thresholds to 90ms and 80ms respectively */
 542         intel_uncore_write(uncore, RCBMAXAVG, 90000);
 543         intel_uncore_write(uncore, RCBMINAVG, 80000);
 544
 545         intel_uncore_write(uncore, MEMIHYST, 1);
 546
 547         /* Set up min, max, and cur for interrupt handling */
 548         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
 549                 MEMMODE_FSTART_SHIFT;
 550
 551         vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
 552                   PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
 553
 554         intel_uncore_write(uncore,
 555                            MEMINTREN,
 556                            MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
 557
 558         intel_uncore_write(uncore, VIDSTART, vstart);
 559         intel_uncore_posting_read(uncore, VIDSTART);
 560
 561         rgvmodectl |= MEMMODE_SWMODE_EN;
 562         intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
 563
 564         if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
 565                              MEMCTL_CMD_STS) == 0, 10))
 566                 drm_err(&uncore->i915->drm,
 567                         "stuck trying to change perf mode\n");
 568         mdelay(1);
 569
 570         __gen5_rps_set(rps, rps->cur_freq);
 571
 572         rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC);
 573         rps->ips.last_count1 += intel_uncore_read(uncore, DDREC);
 574         rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC);
 575         rps->ips.last_time1 = jiffies_to_msecs(jiffies);
 576
 577         rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
 578         rps->ips.last_time2 = ktime_get_raw_ns();
 579
 580         spin_lock(&i915->irq_lock);
 581         ilk_enable_display_irq(i915, DE_PCU_EVENT);
 582         spin_unlock(&i915->irq_lock);
 583
 584         spin_unlock_irq(&mchdev_lock);
 585
 586         rps->ips.corr = init_emon(uncore);
 587
 588         return true;
 589 }
 590
 591 static void gen5_rps_disable(struct intel_rps *rps)
 592 {
 593         struct drm_i915_private *i915 = rps_to_i915(rps);
 594         struct intel_uncore *uncore = rps_to_uncore(rps);
 595         u16 rgvswctl;
 596
 597         spin_lock_irq(&mchdev_lock);
 598
 599         spin_lock(&i915->irq_lock);
 600         ilk_disable_display_irq(i915, DE_PCU_EVENT);
 601         spin_unlock(&i915->irq_lock);
 602
 603         rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
 604
 605         /* Ack interrupts, disable EFC interrupt */
 606         intel_uncore_write(uncore, MEMINTREN,
 607                            intel_uncore_read(uncore, MEMINTREN) &
 608                            ~MEMINT_EVAL_CHG_EN);
 609         intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
 610
 611         /* Go back to the starting frequency */
 612         __gen5_rps_set(rps, rps->idle_freq);
 613         mdelay(1);
 614         rgvswctl |= MEMCTL_CMD_STS;
 615         intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
 616         mdelay(1);
 617
 618         spin_unlock_irq(&mchdev_lock);
 619 }
 620
 621 static u32 rps_limits(struct intel_rps *rps, u8 val)
 622 {
 623         u32 limits;
 624
 625         /*
 626          * Only set the down limit when we've reached the lowest level to avoid
 627          * getting more interrupts, otherwise leave this clear. This prevents a
 628          * race in the hw when coming out of rc6: There's a tiny window where
 629          * the hw runs at the minimal clock before selecting the desired
 630          * frequency, if the down threshold expires in that window we will not
 631          * receive a down interrupt.
 632          */
 633         if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
 634                 limits = rps->max_freq_softlimit << 23;
 635                 if (val <= rps->min_freq_softlimit)
 636                         limits |= rps->min_freq_softlimit << 14;
 637         } else {
 638                 limits = rps->max_freq_softlimit << 24;
 639                 if (val <= rps->min_freq_softlimit)
 640                         limits |= rps->min_freq_softlimit << 16;
 641         }
 642
 643         return limits;
 644 }
 645
 646 static void rps_set_power(struct intel_rps *rps, int new_power)
 647 {
 648         struct intel_gt *gt = rps_to_gt(rps);
 649         struct intel_uncore *uncore = gt->uncore;
 650         u32 threshold_up = 0, threshold_down = 0; /* in % */
 651         u32 ei_up = 0, ei_down = 0;
 652
 653         lockdep_assert_held(&rps->power.mutex);
 654
 655         if (new_power == rps->power.mode)
 656                 return;
 657
 658         threshold_up = 95;
 659         threshold_down = 85;
 660
 661         /* Note the units here are not exactly 1us, but 1280ns. */
 662         switch (new_power) {
 663         case LOW_POWER:
 664                 ei_up = 16000;
 665                 ei_down = 32000;
 666                 break;
 667
 668         case BETWEEN:
 669                 ei_up = 13000;
 670                 ei_down = 32000;
 671                 break;
 672
 673         case HIGH_POWER:
 674                 ei_up = 10000;
 675                 ei_down = 32000;
 676                 break;
 677         }
 678
 679         /* When byt can survive without system hang with dynamic
 680          * sw freq adjustments, this restriction can be lifted.
 681          */
 682         if (IS_VALLEYVIEW(gt->i915))
 683                 goto skip_hw_write;
 684
 685         GT_TRACE(gt,
 686                  "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n",
 687                  new_power, threshold_up, ei_up, threshold_down, ei_down);
 688
 689         set(uncore, GEN6_RP_UP_EI,
 690             intel_gt_ns_to_pm_interval(gt, ei_up * 1000));
 691         set(uncore, GEN6_RP_UP_THRESHOLD,
 692             intel_gt_ns_to_pm_interval(gt, ei_up * threshold_up * 10));
 693
 694         set(uncore, GEN6_RP_DOWN_EI,
 695             intel_gt_ns_to_pm_interval(gt, ei_down * 1000));
 696         set(uncore, GEN6_RP_DOWN_THRESHOLD,
 697             intel_gt_ns_to_pm_interval(gt, ei_down * threshold_down * 10));
 698
 699         set(uncore, GEN6_RP_CONTROL,
 700             (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
 701             GEN6_RP_MEDIA_HW_NORMAL_MODE |
 702             GEN6_RP_MEDIA_IS_GFX |
 703             GEN6_RP_ENABLE |
 704             GEN6_RP_UP_BUSY_AVG |
 705             GEN6_RP_DOWN_IDLE_AVG);
 706
 707 skip_hw_write:
 708         rps->power.mode = new_power;
 709         rps->power.up_threshold = threshold_up;
 710         rps->power.down_threshold = threshold_down;
 711 }
 712
 713 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)
 714 {
 715         int new_power;
 716
 717         new_power = rps->power.mode;
 718         switch (rps->power.mode) {
 719         case LOW_POWER:
 720                 if (val > rps->efficient_freq + 1 &&
 721                     val > rps->cur_freq)
 722                         new_power = BETWEEN;
 723                 break;
 724
 725         case BETWEEN:
 726                 if (val <= rps->efficient_freq &&
 727                     val < rps->cur_freq)
 728                         new_power = LOW_POWER;
 729                 else if (val >= rps->rp0_freq &&
 730                          val > rps->cur_freq)
 731                         new_power = HIGH_POWER;
 732                 break;
 733
 734         case HIGH_POWER:
 735                 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
 736                     val < rps->cur_freq)
 737                         new_power = BETWEEN;
 738                 break;
 739         }
 740         /* Max/min bins are special */
 741         if (val <= rps->min_freq_softlimit)
 742                 new_power = LOW_POWER;
 743         if (val >= rps->max_freq_softlimit)
 744                 new_power = HIGH_POWER;
 745
 746         mutex_lock(&rps->power.mutex);
 747         if (rps->power.interactive)
 748                 new_power = HIGH_POWER;
 749         rps_set_power(rps, new_power);
 750         mutex_unlock(&rps->power.mutex);
 751 }
 752
 753 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)
 754 {
 755         GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n", yesno(interactive));
 756
 757         mutex_lock(&rps->power.mutex);
 758         if (interactive) {
 759                 if (!rps->power.interactive++ && intel_rps_is_active(rps))
 760                         rps_set_power(rps, HIGH_POWER);
 761         } else {
 762                 GEM_BUG_ON(!rps->power.interactive);
 763                 rps->power.interactive--;
 764         }
 765         mutex_unlock(&rps->power.mutex);
 766 }
 767
 768 static int gen6_rps_set(struct intel_rps *rps, u8 val)
 769 {
 770         struct intel_uncore *uncore = rps_to_uncore(rps);
 771         struct drm_i915_private *i915 = rps_to_i915(rps);
 772         u32 swreq;
 773
 774         if (GRAPHICS_VER(i915) >= 9)
 775                 swreq = GEN9_FREQUENCY(val);
 776         else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
 777                 swreq = HSW_FREQUENCY(val);
 778         else
 779                 swreq = (GEN6_FREQUENCY(val) |
 780                          GEN6_OFFSET(0) |
 781                          GEN6_AGGRESSIVE_TURBO);
 782         set(uncore, GEN6_RPNSWREQ, swreq);
 783
 784         GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n",
 785                  val, intel_gpu_freq(rps, val), swreq);
 786
 787         return 0;
 788 }
 789
 790 static int vlv_rps_set(struct intel_rps *rps, u8 val)
 791 {
 792         struct drm_i915_private *i915 = rps_to_i915(rps);
 793         int err;
 794
 795         vlv_punit_get(i915);
 796         err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val);
 797         vlv_punit_put(i915);
 798
 799         GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n",
 800                  val, intel_gpu_freq(rps, val));
 801
 802         return err;
 803 }
 804
 805 static int rps_set(struct intel_rps *rps, u8 val, bool update)
 806 {
 807         struct drm_i915_private *i915 = rps_to_i915(rps);
 808         int err;
 809
 810         if (val == rps->last_freq)
 811                 return 0;
 812
 813         if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
 814                 err = vlv_rps_set(rps, val);
 815         else if (GRAPHICS_VER(i915) >= 6)
 816                 err = gen6_rps_set(rps, val);
 817         else
 818                 err = gen5_rps_set(rps, val);
 819         if (err)
 820                 return err;
 821
 822         if (update && GRAPHICS_VER(i915) >= 6)
 823                 gen6_rps_set_thresholds(rps, val);
 824         rps->last_freq = val;
 825
 826         return 0;
 827 }
 828
 829 void intel_rps_unpark(struct intel_rps *rps)
 830 {
 831         if (!intel_rps_is_enabled(rps))
 832                 return;
 833
 834         GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq);
 835
 836         /*
 837          * Use the user's desired frequency as a guide, but for better
 838          * performance, jump directly to RPe as our starting frequency.
 839          */
 840         mutex_lock(&rps->lock);
 841
 842         intel_rps_set_active(rps);
 843         intel_rps_set(rps,
 844                       clamp(rps->cur_freq,
 845                             rps->min_freq_softlimit,
 846                             rps->max_freq_softlimit));
 847
 848         mutex_unlock(&rps->lock);
 849
 850         rps->pm_iir = 0;
 851         if (intel_rps_has_interrupts(rps))
 852                 rps_enable_interrupts(rps);
 853         if (intel_rps_uses_timer(rps))
 854                 rps_start_timer(rps);
 855
 856         if (GRAPHICS_VER(rps_to_i915(rps)) == 5)
 857                 gen5_rps_update(rps);
 858 }
 859
 860 void intel_rps_park(struct intel_rps *rps)
 861 {
 862         int adj;
 863
 864         GEM_BUG_ON(atomic_read(&rps->num_waiters));
 865
 866         if (!intel_rps_clear_active(rps))
 867                 return;
 868
 869         if (intel_rps_uses_timer(rps))
 870                 rps_stop_timer(rps);
 871         if (intel_rps_has_interrupts(rps))
 872                 rps_disable_interrupts(rps);
 873
 874         if (rps->last_freq <= rps->idle_freq)
 875                 return;
 876
 877         /*
 878          * The punit delays the write of the frequency and voltage until it
 879          * determines the GPU is awake. During normal usage we don't want to
 880          * waste power changing the frequency if the GPU is sleeping (rc6).
 881          * However, the GPU and driver is now idle and we do not want to delay
 882          * switching to minimum voltage (reducing power whilst idle) as we do
 883          * not expect to be woken in the near future and so must flush the
 884          * change by waking the device.
 885          *
 886          * We choose to take the media powerwell (either would do to trick the
 887          * punit into committing the voltage change) as that takes a lot less
 888          * power than the render powerwell.
 889          */
 890         intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA);
 891         rps_set(rps, rps->idle_freq, false);
 892         intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA);
 893
 894         /*
 895          * Since we will try and restart from the previously requested
 896          * frequency on unparking, treat this idle point as a downclock
 897          * interrupt and reduce the frequency for resume. If we park/unpark
 898          * more frequently than the rps worker can run, we will not respond
 899          * to any EI and never see a change in frequency.
 900          *
 901          * (Note we accommodate Cherryview's limitation of only using an
 902          * even bin by applying it to all.)
 903          */
 904         adj = rps->last_adj;
 905         if (adj < 0)
 906                 adj *= 2;
 907         else /* CHV needs even encode values */
 908                 adj = -2;
 909         rps->last_adj = adj;
 910         rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq);
 911         if (rps->cur_freq < rps->efficient_freq) {
 912                 rps->cur_freq = rps->efficient_freq;
 913                 rps->last_adj = 0;
 914         }
 915
 916         GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq);
 917 }
 918
 919 void intel_rps_boost(struct i915_request *rq)
 920 {
 921         if (i915_request_signaled(rq) || i915_request_has_waitboost(rq))
 922                 return;
 923
 924         /* Serializes with i915_request_retire() */
 925         if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) {
 926                 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps;
 927
 928                 if (atomic_fetch_inc(&rps->num_waiters))
 929                         return;
 930
 931                 if (!intel_rps_is_active(rps))
 932                         return;
 933
 934                 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
 935                          rq->fence.context, rq->fence.seqno);
 936
 937                 if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
 938                         schedule_work(&rps->work);
 939
 940                 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */
 941         }
 942 }
 943
 944 int intel_rps_set(struct intel_rps *rps, u8 val)
 945 {
 946         int err;
 947
 948         lockdep_assert_held(&rps->lock);
 949         GEM_BUG_ON(val > rps->max_freq);
 950         GEM_BUG_ON(val < rps->min_freq);
 951
 952         if (intel_rps_is_active(rps)) {
 953                 err = rps_set(rps, val, true);
 954                 if (err)
 955                         return err;
 956
 957                 /*
 958                  * Make sure we continue to get interrupts
 959                  * until we hit the minimum or maximum frequencies.
 960                  */
 961                 if (intel_rps_has_interrupts(rps)) {
 962                         struct intel_uncore *uncore = rps_to_uncore(rps);
 963
 964                         set(uncore,
 965                             GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val));
 966
 967                         set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val));
 968                 }
 969         }
 970
 971         rps->cur_freq = val;
 972         return 0;
 973 }
 974
 975 static void gen6_rps_init(struct intel_rps *rps)
 976 {
 977         struct drm_i915_private *i915 = rps_to_i915(rps);
 978         struct intel_uncore *uncore = rps_to_uncore(rps);
 979
 980         /* All of these values are in units of 50MHz */
 981
 982         /* static values from HW: RP0 > RP1 > RPn (min_freq) */
 983         if (IS_GEN9_LP(i915)) {
 984                 u32 rp_state_cap = intel_uncore_read(uncore, BXT_RP_STATE_CAP);
 985
 986                 rps->rp0_freq = (rp_state_cap >> 16) & 0xff;
 987                 rps->rp1_freq = (rp_state_cap >>  8) & 0xff;
 988                 rps->min_freq = (rp_state_cap >>  0) & 0xff;
 989         } else {
 990                 u32 rp_state_cap = intel_uncore_read(uncore, GEN6_RP_STATE_CAP);
 991
 992                 rps->rp0_freq = (rp_state_cap >>  0) & 0xff;
 993                 rps->rp1_freq = (rp_state_cap >>  8) & 0xff;
 994                 rps->min_freq = (rp_state_cap >> 16) & 0xff;
 995         }
 996
 997         /* hw_max = RP0 until we check for overclocking */
 998         rps->max_freq = rps->rp0_freq;
 999
1000         rps->efficient_freq = rps->rp1_freq;
1001         if (IS_HASWELL(i915) || IS_BROADWELL(i915) ||
1002             IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 10) {
1003                 u32 ddcc_status = 0;
1004
1005                 if (sandybridge_pcode_read(i915,
1006                                            HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
1007                                            &ddcc_status, NULL) == 0)
1008                         rps->efficient_freq =
1009                                 clamp_t(u8,
1010                                         (ddcc_status >> 8) & 0xff,
1011                                         rps->min_freq,
1012                                         rps->max_freq);
1013         }
1014
1015         if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 10) {
1016                 /* Store the frequency values in 16.66 MHZ units, which is
1017                  * the natural hardware unit for SKL
1018                  */
1019                 rps->rp0_freq *= GEN9_FREQ_SCALER;
1020                 rps->rp1_freq *= GEN9_FREQ_SCALER;
1021                 rps->min_freq *= GEN9_FREQ_SCALER;
1022                 rps->max_freq *= GEN9_FREQ_SCALER;
1023                 rps->efficient_freq *= GEN9_FREQ_SCALER;
1024         }
1025 }
1026
1027 static bool rps_reset(struct intel_rps *rps)
1028 {
1029         struct drm_i915_private *i915 = rps_to_i915(rps);
1030
1031         /* force a reset */
1032         rps->power.mode = -1;
1033         rps->last_freq = -1;
1034
1035         if (rps_set(rps, rps->min_freq, true)) {
1036                 drm_err(&i915->drm, "Failed to reset RPS to initial values\n");
1037                 return false;
1038         }
1039
1040         rps->cur_freq = rps->min_freq;
1041         return true;
1042 }
1043
1044 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
1045 static bool gen9_rps_enable(struct intel_rps *rps)
1046 {
1047         struct intel_gt *gt = rps_to_gt(rps);
1048         struct intel_uncore *uncore = gt->uncore;
1049
1050         /* Program defaults and thresholds for RPS */
1051         if (GRAPHICS_VER(gt->i915) == 9)
1052                 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1053                                       GEN9_FREQUENCY(rps->rp1_freq));
1054
1055         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa);
1056
1057         rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1058
1059         return rps_reset(rps);
1060 }
1061
1062 static bool gen8_rps_enable(struct intel_rps *rps)
1063 {
1064         struct intel_uncore *uncore = rps_to_uncore(rps);
1065
1066         intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1067                               HSW_FREQUENCY(rps->rp1_freq));
1068
1069         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1070
1071         rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1072
1073         return rps_reset(rps);
1074 }
1075
1076 static bool gen6_rps_enable(struct intel_rps *rps)
1077 {
1078         struct intel_uncore *uncore = rps_to_uncore(rps);
1079
1080         /* Power down if completely idle for over 50ms */
1081         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000);
1082         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1083
1084         rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1085                           GEN6_PM_RP_DOWN_THRESHOLD |
1086                           GEN6_PM_RP_DOWN_TIMEOUT);
1087
1088         return rps_reset(rps);
1089 }
1090
1091 static int chv_rps_max_freq(struct intel_rps *rps)
1092 {
1093         struct drm_i915_private *i915 = rps_to_i915(rps);
1094         struct intel_gt *gt = rps_to_gt(rps);
1095         u32 val;
1096
1097         val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1098
1099         switch (gt->info.sseu.eu_total) {
1100         case 8:
1101                 /* (2 * 4) config */
1102                 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT;
1103                 break;
1104         case 12:
1105                 /* (2 * 6) config */
1106                 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT;
1107                 break;
1108         case 16:
1109                 /* (2 * 8) config */
1110         default:
1111                 /* Setting (2 * 8) Min RP0 for any other combination */
1112                 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT;
1113                 break;
1114         }
1115
1116         return val & FB_GFX_FREQ_FUSE_MASK;
1117 }
1118
1119 static int chv_rps_rpe_freq(struct intel_rps *rps)
1120 {
1121         struct drm_i915_private *i915 = rps_to_i915(rps);
1122         u32 val;
1123
1124         val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG);
1125         val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT;
1126
1127         return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
1128 }
1129
1130 static int chv_rps_guar_freq(struct intel_rps *rps)
1131 {
1132         struct drm_i915_private *i915 = rps_to_i915(rps);
1133         u32 val;
1134
1135         val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1136
1137         return val & FB_GFX_FREQ_FUSE_MASK;
1138 }
1139
1140 static u32 chv_rps_min_freq(struct intel_rps *rps)
1141 {
1142         struct drm_i915_private *i915 = rps_to_i915(rps);
1143         u32 val;
1144
1145         val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE);
1146         val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT;
1147
1148         return val & FB_GFX_FREQ_FUSE_MASK;
1149 }
1150
1151 static bool chv_rps_enable(struct intel_rps *rps)
1152 {
1153         struct intel_uncore *uncore = rps_to_uncore(rps);
1154         struct drm_i915_private *i915 = rps_to_i915(rps);
1155         u32 val;
1156
1157         /* 1: Program defaults and thresholds for RPS*/
1158         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1159         intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1160         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1161         intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1162         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1163
1164         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1165
1166         /* 2: Enable RPS */
1167         intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1168                               GEN6_RP_MEDIA_HW_NORMAL_MODE |
1169                               GEN6_RP_MEDIA_IS_GFX |
1170                               GEN6_RP_ENABLE |
1171                               GEN6_RP_UP_BUSY_AVG |
1172                               GEN6_RP_DOWN_IDLE_AVG);
1173
1174         rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1175                           GEN6_PM_RP_DOWN_THRESHOLD |
1176                           GEN6_PM_RP_DOWN_TIMEOUT);
1177
1178         /* Setting Fixed Bias */
1179         vlv_punit_get(i915);
1180
1181         val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
1182         vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1183
1184         val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1185
1186         vlv_punit_put(i915);
1187
1188         /* RPS code assumes GPLL is used */
1189         drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1190                       "GPLL not enabled\n");
1191
1192         drm_dbg(&i915->drm, "GPLL enabled? %s\n", yesno(val & GPLLENABLE));
1193         drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1194
1195         return rps_reset(rps);
1196 }
1197
1198 static int vlv_rps_guar_freq(struct intel_rps *rps)
1199 {
1200         struct drm_i915_private *i915 = rps_to_i915(rps);
1201         u32 val, rp1;
1202
1203         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1204
1205         rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK;
1206         rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
1207
1208         return rp1;
1209 }
1210
1211 static int vlv_rps_max_freq(struct intel_rps *rps)
1212 {
1213         struct drm_i915_private *i915 = rps_to_i915(rps);
1214         u32 val, rp0;
1215
1216         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1217
1218         rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
1219         /* Clamp to max */
1220         rp0 = min_t(u32, rp0, 0xea);
1221
1222         return rp0;
1223 }
1224
1225 static int vlv_rps_rpe_freq(struct intel_rps *rps)
1226 {
1227         struct drm_i915_private *i915 = rps_to_i915(rps);
1228         u32 val, rpe;
1229
1230         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
1231         rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
1232         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
1233         rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
1234
1235         return rpe;
1236 }
1237
1238 static int vlv_rps_min_freq(struct intel_rps *rps)
1239 {
1240         struct drm_i915_private *i915 = rps_to_i915(rps);
1241         u32 val;
1242
1243         val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff;
1244         /*
1245          * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
1246          * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
1247          * a BYT-M B0 the above register contains 0xbf. Moreover when setting
1248          * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
1249          * to make sure it matches what Punit accepts.
1250          */
1251         return max_t(u32, val, 0xc0);
1252 }
1253
1254 static bool vlv_rps_enable(struct intel_rps *rps)
1255 {
1256         struct intel_uncore *uncore = rps_to_uncore(rps);
1257         struct drm_i915_private *i915 = rps_to_i915(rps);
1258         u32 val;
1259
1260         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1261         intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1262         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1263         intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1264         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1265
1266         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1267
1268         intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1269                               GEN6_RP_MEDIA_TURBO |
1270                               GEN6_RP_MEDIA_HW_NORMAL_MODE |
1271                               GEN6_RP_MEDIA_IS_GFX |
1272                               GEN6_RP_ENABLE |
1273                               GEN6_RP_UP_BUSY_AVG |
1274                               GEN6_RP_DOWN_IDLE_CONT);
1275
1276         /* WaGsvRC0ResidencyMethod:vlv */
1277         rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED;
1278
1279         vlv_punit_get(i915);
1280
1281         /* Setting Fixed Bias */
1282         val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
1283         vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1284
1285         val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1286
1287         vlv_punit_put(i915);
1288
1289         /* RPS code assumes GPLL is used */
1290         drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1291                       "GPLL not enabled\n");
1292
1293         drm_dbg(&i915->drm, "GPLL enabled? %s\n", yesno(val & GPLLENABLE));
1294         drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1295
1296         return rps_reset(rps);
1297 }
1298
1299 static unsigned long __ips_gfx_val(struct intel_ips *ips)
1300 {
1301         struct intel_rps *rps = container_of(ips, typeof(*rps), ips);
1302         struct intel_uncore *uncore = rps_to_uncore(rps);
1303         unsigned int t, state1, state2;
1304         u32 pxvid, ext_v;
1305         u64 corr, corr2;
1306
1307         lockdep_assert_held(&mchdev_lock);
1308
1309         pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq));
1310         pxvid = (pxvid >> 24) & 0x7f;
1311         ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid);
1312
1313         state1 = ext_v;
1314
1315         /* Revel in the empirically derived constants */
1316
1317         /* Correction factor in 1/100000 units */
1318         t = ips_mch_val(uncore);
1319         if (t > 80)
1320                 corr = t * 2349 + 135940;
1321         else if (t >= 50)
1322                 corr = t * 964 + 29317;
1323         else /* < 50 */
1324                 corr = t * 301 + 1004;
1325
1326         corr = div_u64(corr * 150142 * state1, 10000) - 78642;
1327         corr2 = div_u64(corr, 100000) * ips->corr;
1328
1329         state2 = div_u64(corr2 * state1, 10000);
1330         state2 /= 100; /* convert to mW */
1331
1332         __gen5_ips_update(ips);
1333
1334         return ips->gfx_power + state2;
1335 }
1336
1337 static bool has_busy_stats(struct intel_rps *rps)
1338 {
1339         struct intel_engine_cs *engine;
1340         enum intel_engine_id id;
1341
1342         for_each_engine(engine, rps_to_gt(rps), id) {
1343                 if (!intel_engine_supports_stats(engine))
1344                         return false;
1345         }
1346
1347         return true;
1348 }
1349
1350 void intel_rps_enable(struct intel_rps *rps)
1351 {
1352         struct drm_i915_private *i915 = rps_to_i915(rps);
1353         struct intel_uncore *uncore = rps_to_uncore(rps);
1354         bool enabled = false;
1355
1356         if (!HAS_RPS(i915))
1357                 return;
1358
1359         intel_gt_check_clock_frequency(rps_to_gt(rps));
1360
1361         intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
1362         if (rps->max_freq <= rps->min_freq)
1363                 /* leave disabled, no room for dynamic reclocking */;
1364         else if (IS_CHERRYVIEW(i915))
1365                 enabled = chv_rps_enable(rps);
1366         else if (IS_VALLEYVIEW(i915))
1367                 enabled = vlv_rps_enable(rps);
1368         else if (GRAPHICS_VER(i915) >= 9)
1369                 enabled = gen9_rps_enable(rps);
1370         else if (GRAPHICS_VER(i915) >= 8)
1371                 enabled = gen8_rps_enable(rps);
1372         else if (GRAPHICS_VER(i915) >= 6)
1373                 enabled = gen6_rps_enable(rps);
1374         else if (IS_IRONLAKE_M(i915))
1375                 enabled = gen5_rps_enable(rps);
1376         else
1377                 MISSING_CASE(GRAPHICS_VER(i915));
1378         intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
1379         if (!enabled)
1380                 return;
1381
1382         GT_TRACE(rps_to_gt(rps),
1383                  "min:%x, max:%x, freq:[%d, %d]\n",
1384                  rps->min_freq, rps->max_freq,
1385                  intel_gpu_freq(rps, rps->min_freq),
1386                  intel_gpu_freq(rps, rps->max_freq));
1387
1388         GEM_BUG_ON(rps->max_freq < rps->min_freq);
1389         GEM_BUG_ON(rps->idle_freq > rps->max_freq);
1390
1391         GEM_BUG_ON(rps->efficient_freq < rps->min_freq);
1392         GEM_BUG_ON(rps->efficient_freq > rps->max_freq);
1393
1394         if (has_busy_stats(rps))
1395                 intel_rps_set_timer(rps);
1396         else if (GRAPHICS_VER(i915) >= 6)
1397                 intel_rps_set_interrupts(rps);
1398         else
1399                 /* Ironlake currently uses intel_ips.ko */ {}
1400
1401         intel_rps_set_enabled(rps);
1402 }
1403
1404 static void gen6_rps_disable(struct intel_rps *rps)
1405 {
1406         set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0);
1407 }
1408
1409 void intel_rps_disable(struct intel_rps *rps)
1410 {
1411         struct drm_i915_private *i915 = rps_to_i915(rps);
1412
1413         intel_rps_clear_enabled(rps);
1414         intel_rps_clear_interrupts(rps);
1415         intel_rps_clear_timer(rps);
1416
1417         if (GRAPHICS_VER(i915) >= 6)
1418                 gen6_rps_disable(rps);
1419         else if (IS_IRONLAKE_M(i915))
1420                 gen5_rps_disable(rps);
1421 }
1422
1423 static int byt_gpu_freq(struct intel_rps *rps, int val)
1424 {
1425         /*
1426          * N = val - 0xb7
1427          * Slow = Fast = GPLL ref * N
1428          */
1429         return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
1430 }
1431
1432 static int byt_freq_opcode(struct intel_rps *rps, int val)
1433 {
1434         return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
1435 }
1436
1437 static int chv_gpu_freq(struct intel_rps *rps, int val)
1438 {
1439         /*
1440          * N = val / 2
1441          * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
1442          */
1443         return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
1444 }
1445
1446 static int chv_freq_opcode(struct intel_rps *rps, int val)
1447 {
1448         /* CHV needs even values */
1449         return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
1450 }
1451
1452 int intel_gpu_freq(struct intel_rps *rps, int val)
1453 {
1454         struct drm_i915_private *i915 = rps_to_i915(rps);
1455
1456         if (GRAPHICS_VER(i915) >= 9)
1457                 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
1458                                          GEN9_FREQ_SCALER);
1459         else if (IS_CHERRYVIEW(i915))
1460                 return chv_gpu_freq(rps, val);
1461         else if (IS_VALLEYVIEW(i915))
1462                 return byt_gpu_freq(rps, val);
1463         else if (GRAPHICS_VER(i915) >= 6)
1464                 return val * GT_FREQUENCY_MULTIPLIER;
1465         else
1466                 return val;
1467 }
1468
1469 int intel_freq_opcode(struct intel_rps *rps, int val)
1470 {
1471         struct drm_i915_private *i915 = rps_to_i915(rps);
1472
1473         if (GRAPHICS_VER(i915) >= 9)
1474                 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
1475                                          GT_FREQUENCY_MULTIPLIER);
1476         else if (IS_CHERRYVIEW(i915))
1477                 return chv_freq_opcode(rps, val);
1478         else if (IS_VALLEYVIEW(i915))
1479                 return byt_freq_opcode(rps, val);
1480         else if (GRAPHICS_VER(i915) >= 6)
1481                 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
1482         else
1483                 return val;
1484 }
1485
1486 static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
1487 {
1488         struct drm_i915_private *i915 = rps_to_i915(rps);
1489
1490         rps->gpll_ref_freq =
1491                 vlv_get_cck_clock(i915, "GPLL ref",
1492                                   CCK_GPLL_CLOCK_CONTROL,
1493                                   i915->czclk_freq);
1494
1495         drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n",
1496                 rps->gpll_ref_freq);
1497 }
1498
1499 static void vlv_rps_init(struct intel_rps *rps)
1500 {
1501         struct drm_i915_private *i915 = rps_to_i915(rps);
1502         u32 val;
1503
1504         vlv_iosf_sb_get(i915,
1505                         BIT(VLV_IOSF_SB_PUNIT) |
1506                         BIT(VLV_IOSF_SB_NC) |
1507                         BIT(VLV_IOSF_SB_CCK));
1508
1509         vlv_init_gpll_ref_freq(rps);
1510
1511         val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1512         switch ((val >> 6) & 3) {
1513         case 0:
1514         case 1:
1515                 i915->mem_freq = 800;
1516                 break;
1517         case 2:
1518                 i915->mem_freq = 1066;
1519                 break;
1520         case 3:
1521                 i915->mem_freq = 1333;
1522                 break;
1523         }
1524         drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq);
1525
1526         rps->max_freq = vlv_rps_max_freq(rps);
1527         rps->rp0_freq = rps->max_freq;
1528         drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1529                 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1530
1531         rps->efficient_freq = vlv_rps_rpe_freq(rps);
1532         drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1533                 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1534
1535         rps->rp1_freq = vlv_rps_guar_freq(rps);
1536         drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
1537                 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1538
1539         rps->min_freq = vlv_rps_min_freq(rps);
1540         drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1541                 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1542
1543         vlv_iosf_sb_put(i915,
1544                         BIT(VLV_IOSF_SB_PUNIT) |
1545                         BIT(VLV_IOSF_SB_NC) |
1546                         BIT(VLV_IOSF_SB_CCK));
1547 }
1548
1549 static void chv_rps_init(struct intel_rps *rps)
1550 {
1551         struct drm_i915_private *i915 = rps_to_i915(rps);
1552         u32 val;
1553
1554         vlv_iosf_sb_get(i915,
1555                         BIT(VLV_IOSF_SB_PUNIT) |
1556                         BIT(VLV_IOSF_SB_NC) |
1557                         BIT(VLV_IOSF_SB_CCK));
1558
1559         vlv_init_gpll_ref_freq(rps);
1560
1561         val = vlv_cck_read(i915, CCK_FUSE_REG);
1562
1563         switch ((val >> 2) & 0x7) {
1564         case 3:
1565                 i915->mem_freq = 2000;
1566                 break;
1567         default:
1568                 i915->mem_freq = 1600;
1569                 break;
1570         }
1571         drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq);
1572
1573         rps->max_freq = chv_rps_max_freq(rps);
1574         rps->rp0_freq = rps->max_freq;
1575         drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1576                 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1577
1578         rps->efficient_freq = chv_rps_rpe_freq(rps);
1579         drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1580                 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1581
1582         rps->rp1_freq = chv_rps_guar_freq(rps);
1583         drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n",
1584                 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1585
1586         rps->min_freq = chv_rps_min_freq(rps);
1587         drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1588                 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1589
1590         vlv_iosf_sb_put(i915,
1591                         BIT(VLV_IOSF_SB_PUNIT) |
1592                         BIT(VLV_IOSF_SB_NC) |
1593                         BIT(VLV_IOSF_SB_CCK));
1594
1595         drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq |
1596                                    rps->rp1_freq | rps->min_freq) & 1,
1597                       "Odd GPU freq values\n");
1598 }
1599
1600 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)
1601 {
1602         ei->ktime = ktime_get_raw();
1603         ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT);
1604         ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT);
1605 }
1606
1607 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)
1608 {
1609         struct intel_uncore *uncore = rps_to_uncore(rps);
1610         const struct intel_rps_ei *prev = &rps->ei;
1611         struct intel_rps_ei now;
1612         u32 events = 0;
1613
1614         if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
1615                 return 0;
1616
1617         vlv_c0_read(uncore, &now);
1618
1619         if (prev->ktime) {
1620                 u64 time, c0;
1621                 u32 render, media;
1622
1623                 time = ktime_us_delta(now.ktime, prev->ktime);
1624
1625                 time *= rps_to_i915(rps)->czclk_freq;
1626
1627                 /* Workload can be split between render + media,
1628                  * e.g. SwapBuffers being blitted in X after being rendered in
1629                  * mesa. To account for this we need to combine both engines
1630                  * into our activity counter.
1631                  */
1632                 render = now.render_c0 - prev->render_c0;
1633                 media = now.media_c0 - prev->media_c0;
1634                 c0 = max(render, media);
1635                 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1636
1637                 if (c0 > time * rps->power.up_threshold)
1638                         events = GEN6_PM_RP_UP_THRESHOLD;
1639                 else if (c0 < time * rps->power.down_threshold)
1640                         events = GEN6_PM_RP_DOWN_THRESHOLD;
1641         }
1642
1643         rps->ei = now;
1644         return events;
1645 }
1646
1647 static void rps_work(struct work_struct *work)
1648 {
1649         struct intel_rps *rps = container_of(work, typeof(*rps), work);
1650         struct intel_gt *gt = rps_to_gt(rps);
1651         struct drm_i915_private *i915 = rps_to_i915(rps);
1652         bool client_boost = false;
1653         int new_freq, adj, min, max;
1654         u32 pm_iir = 0;
1655
1656         spin_lock_irq(&gt->irq_lock);
1657         pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events;
1658         client_boost = atomic_read(&rps->num_waiters);
1659         spin_unlock_irq(&gt->irq_lock);
1660
1661         /* Make sure we didn't queue anything we're not going to process. */
1662         if (!pm_iir && !client_boost)
1663                 goto out;
1664
1665         mutex_lock(&rps->lock);
1666         if (!intel_rps_is_active(rps)) {
1667                 mutex_unlock(&rps->lock);
1668                 return;
1669         }
1670
1671         pm_iir |= vlv_wa_c0_ei(rps, pm_iir);
1672
1673         adj = rps->last_adj;
1674         new_freq = rps->cur_freq;
1675         min = rps->min_freq_softlimit;
1676         max = rps->max_freq_softlimit;
1677         if (client_boost)
1678                 max = rps->max_freq;
1679
1680         GT_TRACE(gt,
1681                  "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n",
1682                  pm_iir, yesno(client_boost),
1683                  adj, new_freq, min, max);
1684
1685         if (client_boost && new_freq < rps->boost_freq) {
1686                 new_freq = rps->boost_freq;
1687                 adj = 0;
1688         } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1689                 if (adj > 0)
1690                         adj *= 2;
1691                 else /* CHV needs even encode values */
1692                         adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1;
1693
1694                 if (new_freq >= rps->max_freq_softlimit)
1695                         adj = 0;
1696         } else if (client_boost) {
1697                 adj = 0;
1698         } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1699                 if (rps->cur_freq > rps->efficient_freq)
1700                         new_freq = rps->efficient_freq;
1701                 else if (rps->cur_freq > rps->min_freq_softlimit)
1702                         new_freq = rps->min_freq_softlimit;
1703                 adj = 0;
1704         } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1705                 if (adj < 0)
1706                         adj *= 2;
1707                 else /* CHV needs even encode values */
1708                         adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1;
1709
1710                 if (new_freq <= rps->min_freq_softlimit)
1711                         adj = 0;
1712         } else { /* unknown event */
1713                 adj = 0;
1714         }
1715
1716         /*
1717          * sysfs frequency limits may have snuck in while
1718          * servicing the interrupt
1719          */
1720         new_freq += adj;
1721         new_freq = clamp_t(int, new_freq, min, max);
1722
1723         if (intel_rps_set(rps, new_freq)) {
1724                 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n");
1725                 adj = 0;
1726         }
1727         rps->last_adj = adj;
1728
1729         mutex_unlock(&rps->lock);
1730
1731 out:
1732         spin_lock_irq(&gt->irq_lock);
1733         gen6_gt_pm_unmask_irq(gt, rps->pm_events);
1734         spin_unlock_irq(&gt->irq_lock);
1735 }
1736
1737 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1738 {
1739         struct intel_gt *gt = rps_to_gt(rps);
1740         const u32 events = rps->pm_events & pm_iir;
1741
1742         lockdep_assert_held(&gt->irq_lock);
1743
1744         if (unlikely(!events))
1745                 return;
1746
1747         GT_TRACE(gt, "irq events:%x\n", events);
1748
1749         gen6_gt_pm_mask_irq(gt, events);
1750
1751         rps->pm_iir |= events;
1752         schedule_work(&rps->work);
1753 }
1754
1755 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1756 {
1757         struct intel_gt *gt = rps_to_gt(rps);
1758         u32 events;
1759
1760         events = pm_iir & rps->pm_events;
1761         if (events) {
1762                 spin_lock(&gt->irq_lock);
1763
1764                 GT_TRACE(gt, "irq events:%x\n", events);
1765
1766                 gen6_gt_pm_mask_irq(gt, events);
1767                 rps->pm_iir |= events;
1768
1769                 schedule_work(&rps->work);
1770                 spin_unlock(&gt->irq_lock);
1771         }
1772
1773         if (GRAPHICS_VER(gt->i915) >= 8)
1774                 return;
1775
1776         if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1777                 intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10);
1778
1779         if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1780                 DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
1781 }
1782
1783 void gen5_rps_irq_handler(struct intel_rps *rps)
1784 {
1785         struct intel_uncore *uncore = rps_to_uncore(rps);
1786         u32 busy_up, busy_down, max_avg, min_avg;
1787         u8 new_freq;
1788
1789         spin_lock(&mchdev_lock);
1790
1791         intel_uncore_write16(uncore,
1792                              MEMINTRSTS,
1793                              intel_uncore_read(uncore, MEMINTRSTS));
1794
1795         intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
1796         busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
1797         busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
1798         max_avg = intel_uncore_read(uncore, RCBMAXAVG);
1799         min_avg = intel_uncore_read(uncore, RCBMINAVG);
1800
1801         /* Handle RCS change request from hw */
1802         new_freq = rps->cur_freq;
1803         if (busy_up > max_avg)
1804                 new_freq++;
1805         else if (busy_down < min_avg)
1806                 new_freq--;
1807         new_freq = clamp(new_freq,
1808                          rps->min_freq_softlimit,
1809                          rps->max_freq_softlimit);
1810
1811         if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq))
1812                 rps->cur_freq = new_freq;
1813
1814         spin_unlock(&mchdev_lock);
1815 }
1816
1817 void intel_rps_init_early(struct intel_rps *rps)
1818 {
1819         mutex_init(&rps->lock);
1820         mutex_init(&rps->power.mutex);
1821
1822         INIT_WORK(&rps->work, rps_work);
1823         timer_setup(&rps->timer, rps_timer, 0);
1824
1825         atomic_set(&rps->num_waiters, 0);
1826 }
1827
1828 void intel_rps_init(struct intel_rps *rps)
1829 {
1830         struct drm_i915_private *i915 = rps_to_i915(rps);
1831
1832         if (IS_CHERRYVIEW(i915))
1833                 chv_rps_init(rps);
1834         else if (IS_VALLEYVIEW(i915))
1835                 vlv_rps_init(rps);
1836         else if (GRAPHICS_VER(i915) >= 6)
1837                 gen6_rps_init(rps);
1838         else if (IS_IRONLAKE_M(i915))
1839                 gen5_rps_init(rps);
1840
1841         /* Derive initial user preferences/limits from the hardware limits */
1842         rps->max_freq_softlimit = rps->max_freq;
1843         rps->min_freq_softlimit = rps->min_freq;
1844
1845         /* After setting max-softlimit, find the overclock max freq */
1846         if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {
1847                 u32 params = 0;
1848
1849                 sandybridge_pcode_read(i915, GEN6_READ_OC_PARAMS,
1850                                        &params, NULL);
1851                 if (params & BIT(31)) { /* OC supported */
1852                         drm_dbg(&i915->drm,
1853                                 "Overclocking supported, max: %dMHz, overclock: %dMHz\n",
1854                                 (rps->max_freq & 0xff) * 50,
1855                                 (params & 0xff) * 50);
1856                         rps->max_freq = params & 0xff;
1857                 }
1858         }
1859
1860         /* Finally allow us to boost to max by default */
1861         rps->boost_freq = rps->max_freq;
1862         rps->idle_freq = rps->min_freq;
1863
1864         /* Start in the middle, from here we will autotune based on workload */
1865         rps->cur_freq = rps->efficient_freq;
1866
1867         rps->pm_intrmsk_mbz = 0;
1868
1869         /*
1870          * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
1871          * if GEN6_PM_UP_EI_EXPIRED is masked.
1872          *
1873          * TODO: verify if this can be reproduced on VLV,CHV.
1874          */
1875         if (GRAPHICS_VER(i915) <= 7)
1876                 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
1877
1878         if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11)
1879                 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
1880 }
1881
1882 void intel_rps_sanitize(struct intel_rps *rps)
1883 {
1884         if (GRAPHICS_VER(rps_to_i915(rps)) >= 6)
1885                 rps_disable_interrupts(rps);
1886 }
1887
1888 u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat)
1889 {
1890         struct drm_i915_private *i915 = rps_to_i915(rps);
1891         u32 cagf;
1892
1893         if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
1894                 cagf = (rpstat >> 8) & 0xff;
1895         else if (GRAPHICS_VER(i915) >= 9)
1896                 cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
1897         else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
1898                 cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
1899         else if (GRAPHICS_VER(i915) >= 6)
1900                 cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
1901         else
1902                 cagf = gen5_invert_freq(rps, (rpstat & MEMSTAT_PSTATE_MASK) >>
1903                                         MEMSTAT_PSTATE_SHIFT);
1904
1905         return cagf;
1906 }
1907
1908 static u32 read_cagf(struct intel_rps *rps)
1909 {
1910         struct drm_i915_private *i915 = rps_to_i915(rps);
1911         struct intel_uncore *uncore = rps_to_uncore(rps);
1912         u32 freq;
1913
1914         if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
1915                 vlv_punit_get(i915);
1916                 freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1917                 vlv_punit_put(i915);
1918         } else if (GRAPHICS_VER(i915) >= 6) {
1919                 freq = intel_uncore_read(uncore, GEN6_RPSTAT1);
1920         } else {
1921                 freq = intel_uncore_read(uncore, MEMSTAT_ILK);
1922         }
1923
1924         return intel_rps_get_cagf(rps, freq);
1925 }
1926
1927 u32 intel_rps_read_actual_frequency(struct intel_rps *rps)
1928 {
1929         struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
1930         intel_wakeref_t wakeref;
1931         u32 freq = 0;
1932
1933         with_intel_runtime_pm_if_in_use(rpm, wakeref)
1934                 freq = intel_gpu_freq(rps, read_cagf(rps));
1935
1936         return freq;
1937 }
1938
1939 /* External interface for intel_ips.ko */
1940
1941 static struct drm_i915_private __rcu *ips_mchdev;
1942
1943 /**
1944  * Tells the intel_ips driver that the i915 driver is now loaded, if
1945  * IPS got loaded first.
1946  *
1947  * This awkward dance is so that neither module has to depend on the
1948  * other in order for IPS to do the appropriate communication of
1949  * GPU turbo limits to i915.
1950  */
1951 static void
1952 ips_ping_for_i915_load(void)
1953 {
1954         void (*link)(void);
1955
1956         link = symbol_get(ips_link_to_i915_driver);
1957         if (link) {
1958                 link();
1959                 symbol_put(ips_link_to_i915_driver);
1960         }
1961 }
1962
1963 void intel_rps_driver_register(struct intel_rps *rps)
1964 {
1965         struct intel_gt *gt = rps_to_gt(rps);
1966
1967         /*
1968          * We only register the i915 ips part with intel-ips once everything is
1969          * set up, to avoid intel-ips sneaking in and reading bogus values.
1970          */
1971         if (GRAPHICS_VER(gt->i915) == 5) {
1972                 GEM_BUG_ON(ips_mchdev);
1973                 rcu_assign_pointer(ips_mchdev, gt->i915);
1974                 ips_ping_for_i915_load();
1975         }
1976 }
1977
1978 void intel_rps_driver_unregister(struct intel_rps *rps)
1979 {
1980         if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps))
1981                 rcu_assign_pointer(ips_mchdev, NULL);
1982 }
1983
1984 static struct drm_i915_private *mchdev_get(void)
1985 {
1986         struct drm_i915_private *i915;
1987
1988         rcu_read_lock();
1989         i915 = rcu_dereference(ips_mchdev);
1990         if (i915 && !kref_get_unless_zero(&i915->drm.ref))
1991                 i915 = NULL;
1992         rcu_read_unlock();
1993
1994         return i915;
1995 }
1996
1997 /**
1998  * i915_read_mch_val - return value for IPS use
1999  *
2000  * Calculate and return a value for the IPS driver to use when deciding whether
2001  * we have thermal and power headroom to increase CPU or GPU power budget.
2002  */
2003 unsigned long i915_read_mch_val(void)
2004 {
2005         struct drm_i915_private *i915;
2006         unsigned long chipset_val = 0;
2007         unsigned long graphics_val = 0;
2008         intel_wakeref_t wakeref;
2009
2010         i915 = mchdev_get();
2011         if (!i915)
2012                 return 0;
2013
2014         with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
2015                 struct intel_ips *ips = &i915->gt.rps.ips;
2016
2017                 spin_lock_irq(&mchdev_lock);
2018                 chipset_val = __ips_chipset_val(ips);
2019                 graphics_val = __ips_gfx_val(ips);
2020                 spin_unlock_irq(&mchdev_lock);
2021         }
2022
2023         drm_dev_put(&i915->drm);
2024         return chipset_val + graphics_val;
2025 }
2026 EXPORT_SYMBOL_GPL(i915_read_mch_val);
2027
2028 /**
2029  * i915_gpu_raise - raise GPU frequency limit
2030  *
2031  * Raise the limit; IPS indicates we have thermal headroom.
2032  */
2033 bool i915_gpu_raise(void)
2034 {
2035         struct drm_i915_private *i915;
2036         struct intel_rps *rps;
2037
2038         i915 = mchdev_get();
2039         if (!i915)
2040                 return false;
2041
2042         rps = &i915->gt.rps;
2043
2044         spin_lock_irq(&mchdev_lock);
2045         if (rps->max_freq_softlimit < rps->max_freq)
2046                 rps->max_freq_softlimit++;
2047         spin_unlock_irq(&mchdev_lock);
2048
2049         drm_dev_put(&i915->drm);
2050         return true;
2051 }
2052 EXPORT_SYMBOL_GPL(i915_gpu_raise);
2053
2054 /**
2055  * i915_gpu_lower - lower GPU frequency limit
2056  *
2057  * IPS indicates we're close to a thermal limit, so throttle back the GPU
2058  * frequency maximum.
2059  */
2060 bool i915_gpu_lower(void)
2061 {
2062         struct drm_i915_private *i915;
2063         struct intel_rps *rps;
2064
2065         i915 = mchdev_get();
2066         if (!i915)
2067                 return false;
2068
2069         rps = &i915->gt.rps;
2070
2071         spin_lock_irq(&mchdev_lock);
2072         if (rps->max_freq_softlimit > rps->min_freq)
2073                 rps->max_freq_softlimit--;
2074         spin_unlock_irq(&mchdev_lock);
2075
2076         drm_dev_put(&i915->drm);
2077         return true;
2078 }
2079 EXPORT_SYMBOL_GPL(i915_gpu_lower);
2080
2081 /**
2082  * i915_gpu_busy - indicate GPU business to IPS
2083  *
2084  * Tell the IPS driver whether or not the GPU is busy.
2085  */
2086 bool i915_gpu_busy(void)
2087 {
2088         struct drm_i915_private *i915;
2089         bool ret;
2090
2091         i915 = mchdev_get();
2092         if (!i915)
2093                 return false;
2094
2095         ret = i915->gt.awake;
2096
2097         drm_dev_put(&i915->drm);
2098         return ret;
2099 }
2100 EXPORT_SYMBOL_GPL(i915_gpu_busy);
2101
2102 /**
2103  * i915_gpu_turbo_disable - disable graphics turbo
2104  *
2105  * Disable graphics turbo by resetting the max frequency and setting the
2106  * current frequency to the default.
2107  */
2108 bool i915_gpu_turbo_disable(void)
2109 {
2110         struct drm_i915_private *i915;
2111         struct intel_rps *rps;
2112         bool ret;
2113
2114         i915 = mchdev_get();
2115         if (!i915)
2116                 return false;
2117
2118         rps = &i915->gt.rps;
2119
2120         spin_lock_irq(&mchdev_lock);
2121         rps->max_freq_softlimit = rps->min_freq;
2122         ret = !__gen5_rps_set(&i915->gt.rps, rps->min_freq);
2123         spin_unlock_irq(&mchdev_lock);
2124
2125         drm_dev_put(&i915->drm);
2126         return ret;
2127 }
2128 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
2129
2130 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2131 #include "selftest_rps.c"
2132 #endif