drivers/gpu/drm/i915/gt/intel_gt_pm.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include <linux/suspend.h>
   7
   8 #include "i915_drv.h"
   9 #include "i915_globals.h"
  10 #include "i915_params.h"
  11 #include "intel_context.h"
  12 #include "intel_engine_pm.h"
  13 #include "intel_gt.h"
  14 #include "intel_gt_clock_utils.h"
  15 #include "intel_gt_pm.h"
  16 #include "intel_gt_requests.h"
  17 #include "intel_llc.h"
  18 #include "intel_pm.h"
  19 #include "intel_rc6.h"
  20 #include "intel_rps.h"
  21 #include "intel_wakeref.h"
  22
  23 static void user_forcewake(struct intel_gt *gt, bool suspend)
  24 {
  25         int count = atomic_read(&gt->user_wakeref);
  26
  27         /* Inside suspend/resume so single threaded, no races to worry about. */
  28         if (likely(!count))
  29                 return;
  30
  31         intel_gt_pm_get(gt);
  32         if (suspend) {
  33                 GEM_BUG_ON(count > atomic_read(&gt->wakeref.count));
  34                 atomic_sub(count, &gt->wakeref.count);
  35         } else {
  36                 atomic_add(count, &gt->wakeref.count);
  37         }
  38         intel_gt_pm_put(gt);
  39 }
  40
  41 static void runtime_begin(struct intel_gt *gt)
  42 {
  43         local_irq_disable();
  44         write_seqcount_begin(&gt->stats.lock);
  45         gt->stats.start = ktime_get();
  46         gt->stats.active = true;
  47         write_seqcount_end(&gt->stats.lock);
  48         local_irq_enable();
  49 }
  50
  51 static void runtime_end(struct intel_gt *gt)
  52 {
  53         local_irq_disable();
  54         write_seqcount_begin(&gt->stats.lock);
  55         gt->stats.active = false;
  56         gt->stats.total =
  57                 ktime_add(gt->stats.total,
  58                           ktime_sub(ktime_get(), gt->stats.start));
  59         write_seqcount_end(&gt->stats.lock);
  60         local_irq_enable();
  61 }
  62
  63 static int __gt_unpark(struct intel_wakeref *wf)
  64 {
  65         struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
  66         struct drm_i915_private *i915 = gt->i915;
  67
  68         GT_TRACE(gt, "\n");
  69
  70         i915_globals_unpark();
  71
  72         /*
  73          * It seems that the DMC likes to transition between the DC states a lot
  74          * when there are no connected displays (no active power domains) during
  75          * command submission.
  76          *
  77          * This activity has negative impact on the performance of the chip with
  78          * huge latencies observed in the interrupt handler and elsewhere.
  79          *
  80          * Work around it by grabbing a GT IRQ power domain whilst there is any
  81          * GT activity, preventing any DC state transitions.
  82          */
  83         gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
  84         GEM_BUG_ON(!gt->awake);
  85
  86         intel_rc6_unpark(&gt->rc6);
  87         intel_rps_unpark(&gt->rps);
  88         i915_pmu_gt_unparked(i915);
  89
  90         intel_gt_unpark_requests(gt);
  91         runtime_begin(gt);
  92
  93         return 0;
  94 }
  95
  96 static int __gt_park(struct intel_wakeref *wf)
  97 {
  98         struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
  99         intel_wakeref_t wakeref = fetch_and_zero(&gt->awake);
 100         struct drm_i915_private *i915 = gt->i915;
 101
 102         GT_TRACE(gt, "\n");
 103
 104         runtime_end(gt);
 105         intel_gt_park_requests(gt);
 106
 107         i915_vma_parked(gt);
 108         i915_pmu_gt_parked(i915);
 109         intel_rps_park(&gt->rps);
 110         intel_rc6_park(&gt->rc6);
 111
 112         /* Everything switched off, flush any residual interrupt just in case */
 113         intel_synchronize_irq(i915);
 114
 115         /* Defer dropping the display power well for 100ms, it's slow! */
 116         GEM_BUG_ON(!wakeref);
 117         intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref);
 118
 119         i915_globals_park();
 120
 121         return 0;
 122 }
 123
 124 static const struct intel_wakeref_ops wf_ops = {
 125         .get = __gt_unpark,
 126         .put = __gt_park,
 127 };
 128
 129 void intel_gt_pm_init_early(struct intel_gt *gt)
 130 {
 131         intel_wakeref_init(&gt->wakeref, gt->uncore->rpm, &wf_ops);
 132         seqcount_mutex_init(&gt->stats.lock, &gt->wakeref.mutex);
 133 }
 134
 135 void intel_gt_pm_init(struct intel_gt *gt)
 136 {
 137         /*
 138          * Enabling power-management should be "self-healing". If we cannot
 139          * enable a feature, simply leave it disabled with a notice to the
 140          * user.
 141          */
 142         intel_rc6_init(&gt->rc6);
 143         intel_rps_init(&gt->rps);
 144 }
 145
 146 static bool reset_engines(struct intel_gt *gt)
 147 {
 148         if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
 149                 return false;
 150
 151         return __intel_gt_reset(gt, ALL_ENGINES) == 0;
 152 }
 153
 154 static void gt_sanitize(struct intel_gt *gt, bool force)
 155 {
 156         struct intel_engine_cs *engine;
 157         enum intel_engine_id id;
 158         intel_wakeref_t wakeref;
 159
 160         GT_TRACE(gt, "force:%s", yesno(force));
 161
 162         /* Use a raw wakeref to avoid calling intel_display_power_get early */
 163         wakeref = intel_runtime_pm_get(gt->uncore->rpm);
 164         intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
 165
 166         intel_gt_check_clock_frequency(gt);
 167
 168         /*
 169          * As we have just resumed the machine and woken the device up from
 170          * deep PCI sleep (presumably D3_cold), assume the HW has been reset
 171          * back to defaults, recovering from whatever wedged state we left it
 172          * in and so worth trying to use the device once more.
 173          */
 174         if (intel_gt_is_wedged(gt))
 175                 intel_gt_unset_wedged(gt);
 176
 177         intel_uc_sanitize(&gt->uc);
 178
 179         for_each_engine(engine, gt, id)
 180                 if (engine->reset.prepare)
 181                         engine->reset.prepare(engine);
 182
 183         intel_uc_reset_prepare(&gt->uc);
 184
 185         for_each_engine(engine, gt, id)
 186                 if (engine->sanitize)
 187                         engine->sanitize(engine);
 188
 189         if (reset_engines(gt) || force) {
 190                 for_each_engine(engine, gt, id)
 191                         __intel_engine_reset(engine, false);
 192         }
 193
 194         for_each_engine(engine, gt, id)
 195                 if (engine->reset.finish)
 196                         engine->reset.finish(engine);
 197
 198         intel_rps_sanitize(&gt->rps);
 199
 200         intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 201         intel_runtime_pm_put(gt->uncore->rpm, wakeref);
 202 }
 203
 204 void intel_gt_pm_fini(struct intel_gt *gt)
 205 {
 206         intel_rc6_fini(&gt->rc6);
 207 }
 208
 209 int intel_gt_resume(struct intel_gt *gt)
 210 {
 211         struct intel_engine_cs *engine;
 212         enum intel_engine_id id;
 213         int err;
 214
 215         err = intel_gt_has_unrecoverable_error(gt);
 216         if (err)
 217                 return err;
 218
 219         GT_TRACE(gt, "\n");
 220
 221         /*
 222          * After resume, we may need to poke into the pinned kernel
 223          * contexts to paper over any damage caused by the sudden suspend.
 224          * Only the kernel contexts should remain pinned over suspend,
 225          * allowing us to fixup the user contexts on their first pin.
 226          */
 227         gt_sanitize(gt, true);
 228
 229         intel_gt_pm_get(gt);
 230
 231         intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
 232         intel_rc6_sanitize(&gt->rc6);
 233         if (intel_gt_is_wedged(gt)) {
 234                 err = -EIO;
 235                 goto out_fw;
 236         }
 237
 238         /* Only when the HW is re-initialised, can we replay the requests */
 239         err = intel_gt_init_hw(gt);
 240         if (err) {
 241                 i915_probe_error(gt->i915,
 242                                  "Failed to initialize GPU, declaring it wedged!\n");
 243                 goto err_wedged;
 244         }
 245
 246         intel_rps_enable(&gt->rps);
 247         intel_llc_enable(&gt->llc);
 248
 249         for_each_engine(engine, gt, id) {
 250                 intel_engine_pm_get(engine);
 251
 252                 engine->serial++; /* kernel context lost */
 253                 err = intel_engine_resume(engine);
 254
 255                 intel_engine_pm_put(engine);
 256                 if (err) {
 257                         drm_err(&gt->i915->drm,
 258                                 "Failed to restart %s (%d)\n",
 259                                 engine->name, err);
 260                         goto err_wedged;
 261                 }
 262         }
 263
 264         intel_rc6_enable(&gt->rc6);
 265
 266         intel_uc_resume(&gt->uc);
 267
 268         user_forcewake(gt, false);
 269
 270 out_fw:
 271         intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 272         intel_gt_pm_put(gt);
 273         return err;
 274
 275 err_wedged:
 276         intel_gt_set_wedged(gt);
 277         goto out_fw;
 278 }
 279
 280 static void wait_for_suspend(struct intel_gt *gt)
 281 {
 282         if (!intel_gt_pm_is_awake(gt))
 283                 return;
 284
 285         if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) {
 286                 /*
 287                  * Forcibly cancel outstanding work and leave
 288                  * the gpu quiet.
 289                  */
 290                 intel_gt_set_wedged(gt);
 291                 intel_gt_retire_requests(gt);
 292         }
 293
 294         intel_gt_pm_wait_for_idle(gt);
 295 }
 296
 297 void intel_gt_suspend_prepare(struct intel_gt *gt)
 298 {
 299         user_forcewake(gt, true);
 300         wait_for_suspend(gt);
 301
 302         intel_uc_suspend(&gt->uc);
 303 }
 304
 305 static suspend_state_t pm_suspend_target(void)
 306 {
 307 #if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP)
 308         return pm_suspend_target_state;
 309 #else
 310         return PM_SUSPEND_TO_IDLE;
 311 #endif
 312 }
 313
 314 void intel_gt_suspend_late(struct intel_gt *gt)
 315 {
 316         intel_wakeref_t wakeref;
 317
 318         /* We expect to be idle already; but also want to be independent */
 319         wait_for_suspend(gt);
 320
 321         if (is_mock_gt(gt))
 322                 return;
 323
 324         GEM_BUG_ON(gt->awake);
 325
 326         /*
 327          * On disabling the device, we want to turn off HW access to memory
 328          * that we no longer own.
 329          *
 330          * However, not all suspend-states disable the device. S0 (s2idle)
 331          * is effectively runtime-suspend, the device is left powered on
 332          * but needs to be put into a low power state. We need to keep
 333          * powermanagement enabled, but we also retain system state and so
 334          * it remains safe to keep on using our allocated memory.
 335          */
 336         if (pm_suspend_target() == PM_SUSPEND_TO_IDLE)
 337                 return;
 338
 339         with_intel_runtime_pm(gt->uncore->rpm, wakeref) {
 340                 intel_rps_disable(&gt->rps);
 341                 intel_rc6_disable(&gt->rc6);
 342                 intel_llc_disable(&gt->llc);
 343         }
 344
 345         gt_sanitize(gt, false);
 346
 347         GT_TRACE(gt, "\n");
 348 }
 349
 350 void intel_gt_runtime_suspend(struct intel_gt *gt)
 351 {
 352         intel_uc_runtime_suspend(&gt->uc);
 353
 354         GT_TRACE(gt, "\n");
 355 }
 356
 357 int intel_gt_runtime_resume(struct intel_gt *gt)
 358 {
 359         GT_TRACE(gt, "\n");
 360         intel_gt_init_swizzling(gt);
 361         intel_ggtt_restore_fences(gt->ggtt);
 362
 363         return intel_uc_runtime_resume(&gt->uc);
 364 }
 365
 366 static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt)
 367 {
 368         ktime_t total = gt->stats.total;
 369
 370         if (gt->stats.active)
 371                 total = ktime_add(total,
 372                                   ktime_sub(ktime_get(), gt->stats.start));
 373
 374         return total;
 375 }
 376
 377 ktime_t intel_gt_get_awake_time(const struct intel_gt *gt)
 378 {
 379         unsigned int seq;
 380         ktime_t total;
 381
 382         do {
 383                 seq = read_seqcount_begin(&gt->stats.lock);
 384                 total = __intel_gt_get_awake_time(gt);
 385         } while (read_seqcount_retry(&gt->stats.lock, seq));
 386
 387         return total;
 388 }
 389
 390 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 391 #include "selftest_gt_pm.c"
 392 #endif