drivers/gpu/drm/i915/gt/intel_breadcrumbs.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/kthread.h>
  26 #include <trace/events/dma_fence.h>
  27 #include <uapi/linux/sched/types.h>
  28
  29 #include "i915_drv.h"
  30 #include "i915_trace.h"
  31 #include "intel_breadcrumbs.h"
  32 #include "intel_context.h"
  33 #include "intel_engine_pm.h"
  34 #include "intel_gt_pm.h"
  35 #include "intel_gt_requests.h"
  36
  37 static bool irq_enable(struct intel_engine_cs *engine)
  38 {
  39         if (!engine->irq_enable)
  40                 return false;
  41
  42         /* Caller disables interrupts */
  43         spin_lock(&engine->gt->irq_lock);
  44         engine->irq_enable(engine);
  45         spin_unlock(&engine->gt->irq_lock);
  46
  47         return true;
  48 }
  49
  50 static void irq_disable(struct intel_engine_cs *engine)
  51 {
  52         if (!engine->irq_disable)
  53                 return;
  54
  55         /* Caller disables interrupts */
  56         spin_lock(&engine->gt->irq_lock);
  57         engine->irq_disable(engine);
  58         spin_unlock(&engine->gt->irq_lock);
  59 }
  60
  61 static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)
  62 {
  63         /*
  64          * Since we are waiting on a request, the GPU should be busy
  65          * and should have its own rpm reference.
  66          */
  67         if (GEM_WARN_ON(!intel_gt_pm_get_if_awake(b->irq_engine->gt)))
  68                 return;
  69
  70         /*
  71          * The breadcrumb irq will be disarmed on the interrupt after the
  72          * waiters are signaled. This gives us a single interrupt window in
  73          * which we can add a new waiter and avoid the cost of re-enabling
  74          * the irq.
  75          */
  76         WRITE_ONCE(b->irq_armed, true);
  77
  78         /* Requests may have completed before we could enable the interrupt. */
  79         if (!b->irq_enabled++ && irq_enable(b->irq_engine))
  80                 irq_work_queue(&b->irq_work);
  81 }
  82
  83 static void intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)
  84 {
  85         if (!b->irq_engine)
  86                 return;
  87
  88         spin_lock(&b->irq_lock);
  89         if (!b->irq_armed)
  90                 __intel_breadcrumbs_arm_irq(b);
  91         spin_unlock(&b->irq_lock);
  92 }
  93
  94 static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)
  95 {
  96         GEM_BUG_ON(!b->irq_enabled);
  97         if (!--b->irq_enabled)
  98                 irq_disable(b->irq_engine);
  99
 100         WRITE_ONCE(b->irq_armed, false);
 101         intel_gt_pm_put_async(b->irq_engine->gt);
 102 }
 103
 104 static void intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)
 105 {
 106         spin_lock(&b->irq_lock);
 107         if (b->irq_armed)
 108                 __intel_breadcrumbs_disarm_irq(b);
 109         spin_unlock(&b->irq_lock);
 110 }
 111
 112 static void add_signaling_context(struct intel_breadcrumbs *b,
 113                                   struct intel_context *ce)
 114 {
 115         lockdep_assert_held(&ce->signal_lock);
 116
 117         spin_lock(&b->signalers_lock);
 118         list_add_rcu(&ce->signal_link, &b->signalers);
 119         spin_unlock(&b->signalers_lock);
 120 }
 121
 122 static bool remove_signaling_context(struct intel_breadcrumbs *b,
 123                                      struct intel_context *ce)
 124 {
 125         lockdep_assert_held(&ce->signal_lock);
 126
 127         if (!list_empty(&ce->signals))
 128                 return false;
 129
 130         spin_lock(&b->signalers_lock);
 131         list_del_rcu(&ce->signal_link);
 132         spin_unlock(&b->signalers_lock);
 133
 134         return true;
 135 }
 136
 137 static inline bool __request_completed(const struct i915_request *rq)
 138 {
 139         return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno);
 140 }
 141
 142 __maybe_unused static bool
 143 check_signal_order(struct intel_context *ce, struct i915_request *rq)
 144 {
 145         if (rq->context != ce)
 146                 return false;
 147
 148         if (!list_is_last(&rq->signal_link, &ce->signals) &&
 149             i915_seqno_passed(rq->fence.seqno,
 150                               list_next_entry(rq, signal_link)->fence.seqno))
 151                 return false;
 152
 153         if (!list_is_first(&rq->signal_link, &ce->signals) &&
 154             i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno,
 155                               rq->fence.seqno))
 156                 return false;
 157
 158         return true;
 159 }
 160
 161 static bool
 162 __dma_fence_signal(struct dma_fence *fence)
 163 {
 164         return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags);
 165 }
 166
 167 static void
 168 __dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp)
 169 {
 170         fence->timestamp = timestamp;
 171         set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
 172         trace_dma_fence_signaled(fence);
 173 }
 174
 175 static void
 176 __dma_fence_signal__notify(struct dma_fence *fence,
 177                            const struct list_head *list)
 178 {
 179         struct dma_fence_cb *cur, *tmp;
 180
 181         lockdep_assert_held(fence->lock);
 182
 183         list_for_each_entry_safe(cur, tmp, list, node) {
 184                 INIT_LIST_HEAD(&cur->node);
 185                 cur->func(fence, cur);
 186         }
 187 }
 188
 189 static void add_retire(struct intel_breadcrumbs *b, struct intel_timeline *tl)
 190 {
 191         if (b->irq_engine)
 192                 intel_engine_add_retire(b->irq_engine, tl);
 193 }
 194
 195 static bool __signal_request(struct i915_request *rq)
 196 {
 197         GEM_BUG_ON(test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags));
 198
 199         if (!__dma_fence_signal(&rq->fence)) {
 200                 i915_request_put(rq);
 201                 return false;
 202         }
 203
 204         return true;
 205 }
 206
 207 static struct llist_node *
 208 slist_add(struct llist_node *node, struct llist_node *head)
 209 {
 210         node->next = head;
 211         return node;
 212 }
 213
 214 static void signal_irq_work(struct irq_work *work)
 215 {
 216         struct intel_breadcrumbs *b = container_of(work, typeof(*b), irq_work);
 217         const ktime_t timestamp = ktime_get();
 218         struct llist_node *signal, *sn;
 219         struct intel_context *ce;
 220
 221         signal = NULL;
 222         if (unlikely(!llist_empty(&b->signaled_requests)))
 223                 signal = llist_del_all(&b->signaled_requests);
 224
 225         /*
 226          * Keep the irq armed until the interrupt after all listeners are gone.
 227          *
 228          * Enabling/disabling the interrupt is rather costly, roughly a couple
 229          * of hundred microseconds. If we are proactive and enable/disable
 230          * the interrupt around every request that wants a breadcrumb, we
 231          * quickly drown in the extra orders of magnitude of latency imposed
 232          * on request submission.
 233          *
 234          * So we try to be lazy, and keep the interrupts enabled until no
 235          * more listeners appear within a breadcrumb interrupt interval (that
 236          * is until a request completes that no one cares about). The
 237          * observation is that listeners come in batches, and will often
 238          * listen to a bunch of requests in succession. Though note on icl+,
 239          * interrupts are always enabled due to concerns with rc6 being
 240          * dysfunctional with per-engine interrupt masking.
 241          *
 242          * We also try to avoid raising too many interrupts, as they may
 243          * be generated by userspace batches and it is unfortunately rather
 244          * too easy to drown the CPU under a flood of GPU interrupts. Thus
 245          * whenever no one appears to be listening, we turn off the interrupts.
 246          * Fewer interrupts should conserve power -- at the very least, fewer
 247          * interrupt draw less ire from other users of the system and tools
 248          * like powertop.
 249          */
 250         if (!signal && READ_ONCE(b->irq_armed) && list_empty(&b->signalers))
 251                 intel_breadcrumbs_disarm_irq(b);
 252
 253         rcu_read_lock();
 254         list_for_each_entry_rcu(ce, &b->signalers, signal_link) {
 255                 struct i915_request *rq;
 256
 257                 list_for_each_entry_rcu(rq, &ce->signals, signal_link) {
 258                         bool release;
 259
 260                         if (!__request_completed(rq))
 261                                 break;
 262
 263                         if (!test_and_clear_bit(I915_FENCE_FLAG_SIGNAL,
 264                                                 &rq->fence.flags))
 265                                 break;
 266
 267                         /*
 268                          * Queue for execution after dropping the signaling
 269                          * spinlock as the callback chain may end up adding
 270                          * more signalers to the same context or engine.
 271                          */
 272                         spin_lock(&ce->signal_lock);
 273                         list_del_rcu(&rq->signal_link);
 274                         release = remove_signaling_context(b, ce);
 275                         spin_unlock(&ce->signal_lock);
 276
 277                         if (__signal_request(rq))
 278                                 /* We own signal_node now, xfer to local list */
 279                                 signal = slist_add(&rq->signal_node, signal);
 280
 281                         if (release) {
 282                                 add_retire(b, ce->timeline);
 283                                 intel_context_put(ce);
 284                         }
 285                 }
 286         }
 287         rcu_read_unlock();
 288
 289         llist_for_each_safe(signal, sn, signal) {
 290                 struct i915_request *rq =
 291                         llist_entry(signal, typeof(*rq), signal_node);
 292                 struct list_head cb_list;
 293
 294                 spin_lock(&rq->lock);
 295                 list_replace(&rq->fence.cb_list, &cb_list);
 296                 __dma_fence_signal__timestamp(&rq->fence, timestamp);
 297                 __dma_fence_signal__notify(&rq->fence, &cb_list);
 298                 spin_unlock(&rq->lock);
 299
 300                 i915_request_put(rq);
 301         }
 302
 303         if (!READ_ONCE(b->irq_armed) && !list_empty(&b->signalers))
 304                 intel_breadcrumbs_arm_irq(b);
 305 }
 306
 307 struct intel_breadcrumbs *
 308 intel_breadcrumbs_create(struct intel_engine_cs *irq_engine)
 309 {
 310         struct intel_breadcrumbs *b;
 311
 312         b = kzalloc(sizeof(*b), GFP_KERNEL);
 313         if (!b)
 314                 return NULL;
 315
 316         b->irq_engine = irq_engine;
 317
 318         spin_lock_init(&b->signalers_lock);
 319         INIT_LIST_HEAD(&b->signalers);
 320         init_llist_head(&b->signaled_requests);
 321
 322         spin_lock_init(&b->irq_lock);
 323         init_irq_work(&b->irq_work, signal_irq_work);
 324
 325         return b;
 326 }
 327
 328 void intel_breadcrumbs_reset(struct intel_breadcrumbs *b)
 329 {
 330         unsigned long flags;
 331
 332         if (!b->irq_engine)
 333                 return;
 334
 335         spin_lock_irqsave(&b->irq_lock, flags);
 336
 337         if (b->irq_enabled)
 338                 irq_enable(b->irq_engine);
 339         else
 340                 irq_disable(b->irq_engine);
 341
 342         spin_unlock_irqrestore(&b->irq_lock, flags);
 343 }
 344
 345 void intel_breadcrumbs_park(struct intel_breadcrumbs *b)
 346 {
 347         /* Kick the work once more to drain the signalers */
 348         irq_work_sync(&b->irq_work);
 349         while (unlikely(READ_ONCE(b->irq_armed))) {
 350                 local_irq_disable();
 351                 signal_irq_work(&b->irq_work);
 352                 local_irq_enable();
 353                 cond_resched();
 354         }
 355         GEM_BUG_ON(!list_empty(&b->signalers));
 356 }
 357
 358 void intel_breadcrumbs_free(struct intel_breadcrumbs *b)
 359 {
 360         irq_work_sync(&b->irq_work);
 361         GEM_BUG_ON(!list_empty(&b->signalers));
 362         GEM_BUG_ON(b->irq_armed);
 363         kfree(b);
 364 }
 365
 366 static void insert_breadcrumb(struct i915_request *rq)
 367 {
 368         struct intel_breadcrumbs *b = READ_ONCE(rq->engine)->breadcrumbs;
 369         struct intel_context *ce = rq->context;
 370         struct list_head *pos;
 371
 372         if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags))
 373                 return;
 374
 375         i915_request_get(rq);
 376
 377         /*
 378          * If the request is already completed, we can transfer it
 379          * straight onto a signaled list, and queue the irq worker for
 380          * its signal completion.
 381          */
 382         if (__request_completed(rq)) {
 383                 if (__signal_request(rq) &&
 384                     llist_add(&rq->signal_node, &b->signaled_requests))
 385                         irq_work_queue(&b->irq_work);
 386                 return;
 387         }
 388
 389         if (list_empty(&ce->signals)) {
 390                 intel_context_get(ce);
 391                 add_signaling_context(b, ce);
 392                 pos = &ce->signals;
 393         } else {
 394                 /*
 395                  * We keep the seqno in retirement order, so we can break
 396                  * inside intel_engine_signal_breadcrumbs as soon as we've
 397                  * passed the last completed request (or seen a request that
 398                  * hasn't event started). We could walk the timeline->requests,
 399                  * but keeping a separate signalers_list has the advantage of
 400                  * hopefully being much smaller than the full list and so
 401                  * provides faster iteration and detection when there are no
 402                  * more interrupts required for this context.
 403                  *
 404                  * We typically expect to add new signalers in order, so we
 405                  * start looking for our insertion point from the tail of
 406                  * the list.
 407                  */
 408                 list_for_each_prev(pos, &ce->signals) {
 409                         struct i915_request *it =
 410                                 list_entry(pos, typeof(*it), signal_link);
 411
 412                         if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno))
 413                                 break;
 414                 }
 415         }
 416         list_add_rcu(&rq->signal_link, pos);
 417         GEM_BUG_ON(!check_signal_order(ce, rq));
 418         GEM_BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags));
 419         set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
 420
 421         /*
 422          * Defer enabling the interrupt to after HW submission and recheck
 423          * the request as it may have completed and raised the interrupt as
 424          * we were attaching it into the lists.
 425          */
 426         irq_work_queue(&b->irq_work);
 427 }
 428
 429 bool i915_request_enable_breadcrumb(struct i915_request *rq)
 430 {
 431         struct intel_context *ce = rq->context;
 432
 433         /* Serialises with i915_request_retire() using rq->lock */
 434         if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags))
 435                 return true;
 436
 437         /*
 438          * Peek at i915_request_submit()/i915_request_unsubmit() status.
 439          *
 440          * If the request is not yet active (and not signaled), we will
 441          * attach the breadcrumb later.
 442          */
 443         if (!test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags))
 444                 return true;
 445
 446         spin_lock(&ce->signal_lock);
 447         if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags))
 448                 insert_breadcrumb(rq);
 449         spin_unlock(&ce->signal_lock);
 450
 451         return true;
 452 }
 453
 454 void i915_request_cancel_breadcrumb(struct i915_request *rq)
 455 {
 456         struct intel_context *ce = rq->context;
 457         bool release;
 458
 459         if (!test_and_clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags))
 460                 return;
 461
 462         spin_lock(&ce->signal_lock);
 463         list_del_rcu(&rq->signal_link);
 464         release = remove_signaling_context(rq->engine->breadcrumbs, ce);
 465         spin_unlock(&ce->signal_lock);
 466         if (release)
 467                 intel_context_put(ce);
 468
 469         i915_request_put(rq);
 470 }
 471
 472 static void print_signals(struct intel_breadcrumbs *b, struct drm_printer *p)
 473 {
 474         struct intel_context *ce;
 475         struct i915_request *rq;
 476
 477         drm_printf(p, "Signals:\n");
 478
 479         rcu_read_lock();
 480         list_for_each_entry_rcu(ce, &b->signalers, signal_link) {
 481                 list_for_each_entry_rcu(rq, &ce->signals, signal_link)
 482                         drm_printf(p, "\t[%llx:%llx%s] @ %dms\n",
 483                                    rq->fence.context, rq->fence.seqno,
 484                                    i915_request_completed(rq) ? "!" :
 485                                    i915_request_started(rq) ? "*" :
 486                                    "",
 487                                    jiffies_to_msecs(jiffies - rq->emitted_jiffies));
 488         }
 489         rcu_read_unlock();
 490 }
 491
 492 void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine,
 493                                     struct drm_printer *p)
 494 {
 495         struct intel_breadcrumbs *b;
 496
 497         b = engine->breadcrumbs;
 498         if (!b)
 499                 return;
 500
 501         drm_printf(p, "IRQ: %s\n", enableddisabled(b->irq_armed));
 502         if (!list_empty(&b->signalers))
 503                 print_signals(b, p);
 504 }