drivers/gpu/drm/i915/gt/intel_gt_requests.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include <linux/workqueue.h>
   7
   8 #include "i915_drv.h" /* for_each_engine() */
   9 #include "i915_request.h"
  10 #include "intel_engine_heartbeat.h"
  11 #include "intel_execlists_submission.h"
  12 #include "intel_gt.h"
  13 #include "intel_gt_pm.h"
  14 #include "intel_gt_requests.h"
  15 #include "intel_timeline.h"
  16
  17 static bool retire_requests(struct intel_timeline *tl)
  18 {
  19         struct i915_request *rq, *rn;
  20
  21         list_for_each_entry_safe(rq, rn, &tl->requests, link)
  22                 if (!i915_request_retire(rq))
  23                         return false;
  24
  25         /* And check nothing new was submitted */
  26         return !i915_active_fence_isset(&tl->last_request);
  27 }
  28
  29 static bool engine_active(const struct intel_engine_cs *engine)
  30 {
  31         return !list_empty(&engine->kernel_context->timeline->requests);
  32 }
  33
  34 static bool flush_submission(struct intel_gt *gt, long timeout)
  35 {
  36         struct intel_engine_cs *engine;
  37         enum intel_engine_id id;
  38         bool active = false;
  39
  40         if (!timeout)
  41                 return false;
  42
  43         if (!intel_gt_pm_is_awake(gt))
  44                 return false;
  45
  46         for_each_engine(engine, gt, id) {
  47                 intel_engine_flush_submission(engine);
  48
  49                 /* Flush the background retirement and idle barriers */
  50                 flush_work(&engine->retire_work);
  51                 flush_delayed_work(&engine->wakeref.work);
  52
  53                 /* Is the idle barrier still outstanding? */
  54                 active |= engine_active(engine);
  55         }
  56
  57         return active;
  58 }
  59
  60 static void engine_retire(struct work_struct *work)
  61 {
  62         struct intel_engine_cs *engine =
  63                 container_of(work, typeof(*engine), retire_work);
  64         struct intel_timeline *tl = xchg(&engine->retire, NULL);
  65
  66         do {
  67                 struct intel_timeline *next = xchg(&tl->retire, NULL);
  68
  69                 /*
  70                  * Our goal here is to retire _idle_ timelines as soon as
  71                  * possible (as they are idle, we do not expect userspace
  72                  * to be cleaning up anytime soon).
  73                  *
  74                  * If the timeline is currently locked, either it is being
  75                  * retired elsewhere or about to be!
  76                  */
  77                 if (mutex_trylock(&tl->mutex)) {
  78                         retire_requests(tl);
  79                         mutex_unlock(&tl->mutex);
  80                 }
  81                 intel_timeline_put(tl);
  82
  83                 GEM_BUG_ON(!next);
  84                 tl = ptr_mask_bits(next, 1);
  85         } while (tl);
  86 }
  87
  88 static bool add_retire(struct intel_engine_cs *engine,
  89                        struct intel_timeline *tl)
  90 {
  91 #define STUB ((struct intel_timeline *)1)
  92         struct intel_timeline *first;
  93
  94         /*
  95          * We open-code a llist here to include the additional tag [BIT(0)]
  96          * so that we know when the timeline is already on a
  97          * retirement queue: either this engine or another.
  98          */
  99
 100         if (cmpxchg(&tl->retire, NULL, STUB)) /* already queued */
 101                 return false;
 102
 103         intel_timeline_get(tl);
 104         first = READ_ONCE(engine->retire);
 105         do
 106                 tl->retire = ptr_pack_bits(first, 1, 1);
 107         while (!try_cmpxchg(&engine->retire, &first, tl));
 108
 109         return !first;
 110 }
 111
 112 void intel_engine_add_retire(struct intel_engine_cs *engine,
 113                              struct intel_timeline *tl)
 114 {
 115         /* We don't deal well with the engine disappearing beneath us */
 116         GEM_BUG_ON(intel_engine_is_virtual(engine));
 117
 118         if (add_retire(engine, tl))
 119                 schedule_work(&engine->retire_work);
 120 }
 121
 122 void intel_engine_init_retire(struct intel_engine_cs *engine)
 123 {
 124         INIT_WORK(&engine->retire_work, engine_retire);
 125 }
 126
 127 void intel_engine_fini_retire(struct intel_engine_cs *engine)
 128 {
 129         flush_work(&engine->retire_work);
 130         GEM_BUG_ON(engine->retire);
 131 }
 132
 133 long intel_gt_retire_requests_timeout(struct intel_gt *gt, long timeout)
 134 {
 135         struct intel_gt_timelines *timelines = &gt->timelines;
 136         struct intel_timeline *tl, *tn;
 137         unsigned long active_count = 0;
 138         LIST_HEAD(free);
 139
 140         flush_submission(gt, timeout); /* kick the ksoftirqd tasklets */
 141         spin_lock(&timelines->lock);
 142         list_for_each_entry_safe(tl, tn, &timelines->active_list, link) {
 143                 if (!mutex_trylock(&tl->mutex)) {
 144                         active_count++; /* report busy to caller, try again? */
 145                         continue;
 146                 }
 147
 148                 intel_timeline_get(tl);
 149                 GEM_BUG_ON(!atomic_read(&tl->active_count));
 150                 atomic_inc(&tl->active_count); /* pin the list element */
 151                 spin_unlock(&timelines->lock);
 152
 153                 if (timeout > 0) {
 154                         struct dma_fence *fence;
 155
 156                         fence = i915_active_fence_get(&tl->last_request);
 157                         if (fence) {
 158                                 mutex_unlock(&tl->mutex);
 159
 160                                 timeout = dma_fence_wait_timeout(fence,
 161                                                                  true,
 162                                                                  timeout);
 163                                 dma_fence_put(fence);
 164
 165                                 /* Retirement is best effort */
 166                                 if (!mutex_trylock(&tl->mutex)) {
 167                                         active_count++;
 168                                         goto out_active;
 169                                 }
 170                         }
 171                 }
 172
 173                 if (!retire_requests(tl))
 174                         active_count++;
 175                 mutex_unlock(&tl->mutex);
 176
 177 out_active:     spin_lock(&timelines->lock);
 178
 179                 /* Resume list iteration after reacquiring spinlock */
 180                 list_safe_reset_next(tl, tn, link);
 181                 if (atomic_dec_and_test(&tl->active_count))
 182                         list_del(&tl->link);
 183
 184                 /* Defer the final release to after the spinlock */
 185                 if (refcount_dec_and_test(&tl->kref.refcount)) {
 186                         GEM_BUG_ON(atomic_read(&tl->active_count));
 187                         list_add(&tl->link, &free);
 188                 }
 189         }
 190         spin_unlock(&timelines->lock);
 191
 192         list_for_each_entry_safe(tl, tn, &free, link)
 193                 __intel_timeline_free(&tl->kref);
 194
 195         if (flush_submission(gt, timeout)) /* Wait, there's more! */
 196                 active_count++;
 197
 198         return active_count ? timeout : 0;
 199 }
 200
 201 int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout)
 202 {
 203         /* If the device is asleep, we have no requests outstanding */
 204         if (!intel_gt_pm_is_awake(gt))
 205                 return 0;
 206
 207         while ((timeout = intel_gt_retire_requests_timeout(gt, timeout)) > 0) {
 208                 cond_resched();
 209                 if (signal_pending(current))
 210                         return -EINTR;
 211         }
 212
 213         return timeout;
 214 }
 215
 216 static void retire_work_handler(struct work_struct *work)
 217 {
 218         struct intel_gt *gt =
 219                 container_of(work, typeof(*gt), requests.retire_work.work);
 220
 221         schedule_delayed_work(&gt->requests.retire_work,
 222                               round_jiffies_up_relative(HZ));
 223         intel_gt_retire_requests(gt);
 224 }
 225
 226 void intel_gt_init_requests(struct intel_gt *gt)
 227 {
 228         INIT_DELAYED_WORK(&gt->requests.retire_work, retire_work_handler);
 229 }
 230
 231 void intel_gt_park_requests(struct intel_gt *gt)
 232 {
 233         cancel_delayed_work(&gt->requests.retire_work);
 234 }
 235
 236 void intel_gt_unpark_requests(struct intel_gt *gt)
 237 {
 238         schedule_delayed_work(&gt->requests.retire_work,
 239                               round_jiffies_up_relative(HZ));
 240 }
 241
 242 void intel_gt_fini_requests(struct intel_gt *gt)
 243 {
 244         /* Wait until the work is marked as finished before unloading! */
 245         cancel_delayed_work_sync(&gt->requests.retire_work);
 246
 247         flush_work(&gt->watchdog.work);
 248 }
 249
 250 void intel_gt_watchdog_work(struct work_struct *work)
 251 {
 252         struct intel_gt *gt =
 253                 container_of(work, typeof(*gt), watchdog.work);
 254         struct i915_request *rq, *rn;
 255         struct llist_node *first;
 256
 257         first = llist_del_all(&gt->watchdog.list);
 258         if (!first)
 259                 return;
 260
 261         llist_for_each_entry_safe(rq, rn, first, watchdog.link) {
 262                 if (!i915_request_completed(rq)) {
 263                         struct dma_fence *f = &rq->fence;
 264
 265                         pr_notice("Fence expiration time out i915-%s:%s:%llx!\n",
 266                                   f->ops->get_driver_name(f),
 267                                   f->ops->get_timeline_name(f),
 268                                   f->seqno);
 269                         i915_request_cancel(rq, -EINTR);
 270                 }
 271                 i915_request_put(rq);
 272         }
 273 }