drivers/gpu/drm/i915/i915_sw_fence.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * (C) Copyright 2016 Intel Corporation
   5  */
   6
   7 #include <linux/slab.h>
   8 #include <linux/dma-fence.h>
   9 #include <linux/irq_work.h>
  10 #include <linux/reservation.h>
  11
  12 #include "i915_sw_fence.h"
  13 #include "i915_selftest.h"
  14
  15 #define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */
  16
  17 static DEFINE_SPINLOCK(i915_sw_fence_lock);
  18
  19 enum {
  20         DEBUG_FENCE_IDLE = 0,
  21         DEBUG_FENCE_NOTIFY,
  22 };
  23
  24 static void *i915_sw_fence_debug_hint(void *addr)
  25 {
  26         return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK);
  27 }
  28
  29 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
  30
  31 static struct debug_obj_descr i915_sw_fence_debug_descr = {
  32         .name = "i915_sw_fence",
  33         .debug_hint = i915_sw_fence_debug_hint,
  34 };
  35
  36 static inline void debug_fence_init(struct i915_sw_fence *fence)
  37 {
  38         debug_object_init(fence, &i915_sw_fence_debug_descr);
  39 }
  40
  41 static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
  42 {
  43         debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr);
  44 }
  45
  46 static inline void debug_fence_activate(struct i915_sw_fence *fence)
  47 {
  48         debug_object_activate(fence, &i915_sw_fence_debug_descr);
  49 }
  50
  51 static inline void debug_fence_set_state(struct i915_sw_fence *fence,
  52                                          int old, int new)
  53 {
  54         debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
  55 }
  56
  57 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
  58 {
  59         debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
  60 }
  61
  62 static inline void debug_fence_destroy(struct i915_sw_fence *fence)
  63 {
  64         debug_object_destroy(fence, &i915_sw_fence_debug_descr);
  65 }
  66
  67 static inline void debug_fence_free(struct i915_sw_fence *fence)
  68 {
  69         debug_object_free(fence, &i915_sw_fence_debug_descr);
  70         smp_wmb(); /* flush the change in state before reallocation */
  71 }
  72
  73 static inline void debug_fence_assert(struct i915_sw_fence *fence)
  74 {
  75         debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
  76 }
  77
  78 #else
  79
  80 static inline void debug_fence_init(struct i915_sw_fence *fence)
  81 {
  82 }
  83
  84 static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
  85 {
  86 }
  87
  88 static inline void debug_fence_activate(struct i915_sw_fence *fence)
  89 {
  90 }
  91
  92 static inline void debug_fence_set_state(struct i915_sw_fence *fence,
  93                                          int old, int new)
  94 {
  95 }
  96
  97 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
  98 {
  99 }
 100
 101 static inline void debug_fence_destroy(struct i915_sw_fence *fence)
 102 {
 103 }
 104
 105 static inline void debug_fence_free(struct i915_sw_fence *fence)
 106 {
 107 }
 108
 109 static inline void debug_fence_assert(struct i915_sw_fence *fence)
 110 {
 111 }
 112
 113 #endif
 114
 115 static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
 116                                   enum i915_sw_fence_notify state)
 117 {
 118         i915_sw_fence_notify_t fn;
 119
 120         fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
 121         return fn(fence, state);
 122 }
 123
 124 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
 125 void i915_sw_fence_fini(struct i915_sw_fence *fence)
 126 {
 127         debug_fence_free(fence);
 128 }
 129 #endif
 130
 131 static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
 132                                         struct list_head *continuation)
 133 {
 134         wait_queue_head_t *x = &fence->wait;
 135         wait_queue_entry_t *pos, *next;
 136         unsigned long flags;
 137
 138         debug_fence_deactivate(fence);
 139         atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */
 140
 141         /*
 142          * To prevent unbounded recursion as we traverse the graph of
 143          * i915_sw_fences, we move the entry list from this, the next ready
 144          * fence, to the tail of the original fence's entry list
 145          * (and so added to the list to be woken).
 146          */
 147
 148         spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
 149         if (continuation) {
 150                 list_for_each_entry_safe(pos, next, &x->head, entry) {
 151                         if (pos->func == autoremove_wake_function)
 152                                 pos->func(pos, TASK_NORMAL, 0, continuation);
 153                         else
 154                                 list_move_tail(&pos->entry, continuation);
 155                 }
 156         } else {
 157                 LIST_HEAD(extra);
 158
 159                 do {
 160                         list_for_each_entry_safe(pos, next, &x->head, entry)
 161                                 pos->func(pos, TASK_NORMAL, 0, &extra);
 162
 163                         if (list_empty(&extra))
 164                                 break;
 165
 166                         list_splice_tail_init(&extra, &x->head);
 167                 } while (1);
 168         }
 169         spin_unlock_irqrestore(&x->lock, flags);
 170
 171         debug_fence_assert(fence);
 172 }
 173
 174 static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
 175                                      struct list_head *continuation)
 176 {
 177         debug_fence_assert(fence);
 178
 179         if (!atomic_dec_and_test(&fence->pending))
 180                 return;
 181
 182         debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);
 183
 184         if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
 185                 return;
 186
 187         debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);
 188
 189         __i915_sw_fence_wake_up_all(fence, continuation);
 190
 191         debug_fence_destroy(fence);
 192         __i915_sw_fence_notify(fence, FENCE_FREE);
 193 }
 194
 195 void i915_sw_fence_complete(struct i915_sw_fence *fence)
 196 {
 197         debug_fence_assert(fence);
 198
 199         if (WARN_ON(i915_sw_fence_done(fence)))
 200                 return;
 201
 202         __i915_sw_fence_complete(fence, NULL);
 203 }
 204
 205 void i915_sw_fence_await(struct i915_sw_fence *fence)
 206 {
 207         debug_fence_assert(fence);
 208         WARN_ON(atomic_inc_return(&fence->pending) <= 1);
 209 }
 210
 211 void __i915_sw_fence_init(struct i915_sw_fence *fence,
 212                           i915_sw_fence_notify_t fn,
 213                           const char *name,
 214                           struct lock_class_key *key)
 215 {
 216         BUG_ON(!fn || (unsigned long)fn & ~I915_SW_FENCE_MASK);
 217
 218         debug_fence_init(fence);
 219
 220         __init_waitqueue_head(&fence->wait, name, key);
 221         atomic_set(&fence->pending, 1);
 222         fence->flags = (unsigned long)fn;
 223 }
 224
 225 void i915_sw_fence_commit(struct i915_sw_fence *fence)
 226 {
 227         debug_fence_activate(fence);
 228         i915_sw_fence_complete(fence);
 229 }
 230
 231 static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key)
 232 {
 233         list_del(&wq->entry);
 234         __i915_sw_fence_complete(wq->private, key);
 235
 236         if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
 237                 kfree(wq);
 238         return 0;
 239 }
 240
 241 static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
 242                                     const struct i915_sw_fence * const signaler)
 243 {
 244         wait_queue_entry_t *wq;
 245
 246         if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
 247                 return false;
 248
 249         if (fence == signaler)
 250                 return true;
 251
 252         list_for_each_entry(wq, &fence->wait.head, entry) {
 253                 if (wq->func != i915_sw_fence_wake)
 254                         continue;
 255
 256                 if (__i915_sw_fence_check_if_after(wq->private, signaler))
 257                         return true;
 258         }
 259
 260         return false;
 261 }
 262
 263 static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
 264 {
 265         wait_queue_entry_t *wq;
 266
 267         if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
 268                 return;
 269
 270         list_for_each_entry(wq, &fence->wait.head, entry) {
 271                 if (wq->func != i915_sw_fence_wake)
 272                         continue;
 273
 274                 __i915_sw_fence_clear_checked_bit(wq->private);
 275         }
 276 }
 277
 278 static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
 279                                   const struct i915_sw_fence * const signaler)
 280 {
 281         unsigned long flags;
 282         bool err;
 283
 284         if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG))
 285                 return false;
 286
 287         spin_lock_irqsave(&i915_sw_fence_lock, flags);
 288         err = __i915_sw_fence_check_if_after(fence, signaler);
 289         __i915_sw_fence_clear_checked_bit(fence);
 290         spin_unlock_irqrestore(&i915_sw_fence_lock, flags);
 291
 292         return err;
 293 }
 294
 295 static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 296                                           struct i915_sw_fence *signaler,
 297                                           wait_queue_entry_t *wq, gfp_t gfp)
 298 {
 299         unsigned long flags;
 300         int pending;
 301
 302         debug_fence_assert(fence);
 303         might_sleep_if(gfpflags_allow_blocking(gfp));
 304
 305         if (i915_sw_fence_done(signaler))
 306                 return 0;
 307
 308         debug_fence_assert(signaler);
 309
 310         /* The dependency graph must be acyclic. */
 311         if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
 312                 return -EINVAL;
 313
 314         pending = 0;
 315         if (!wq) {
 316                 wq = kmalloc(sizeof(*wq), gfp);
 317                 if (!wq) {
 318                         if (!gfpflags_allow_blocking(gfp))
 319                                 return -ENOMEM;
 320
 321                         i915_sw_fence_wait(signaler);
 322                         return 0;
 323                 }
 324
 325                 pending |= I915_SW_FENCE_FLAG_ALLOC;
 326         }
 327
 328         INIT_LIST_HEAD(&wq->entry);
 329         wq->flags = pending;
 330         wq->func = i915_sw_fence_wake;
 331         wq->private = fence;
 332
 333         i915_sw_fence_await(fence);
 334
 335         spin_lock_irqsave(&signaler->wait.lock, flags);
 336         if (likely(!i915_sw_fence_done(signaler))) {
 337                 __add_wait_queue_entry_tail(&signaler->wait, wq);
 338                 pending = 1;
 339         } else {
 340                 i915_sw_fence_wake(wq, 0, 0, NULL);
 341                 pending = 0;
 342         }
 343         spin_unlock_irqrestore(&signaler->wait.lock, flags);
 344
 345         return pending;
 346 }
 347
 348 int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 349                                  struct i915_sw_fence *signaler,
 350                                  wait_queue_entry_t *wq)
 351 {
 352         return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
 353 }
 354
 355 int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
 356                                      struct i915_sw_fence *signaler,
 357                                      gfp_t gfp)
 358 {
 359         return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
 360 }
 361
 362 struct i915_sw_dma_fence_cb_timer {
 363         struct i915_sw_dma_fence_cb base;
 364         struct dma_fence *dma;
 365         struct timer_list timer;
 366         struct irq_work work;
 367         struct rcu_head rcu;
 368 };
 369
 370 static void dma_i915_sw_fence_wake(struct dma_fence *dma,
 371                                    struct dma_fence_cb *data)
 372 {
 373         struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);
 374
 375         i915_sw_fence_complete(cb->fence);
 376         kfree(cb);
 377 }
 378
 379 static void timer_i915_sw_fence_wake(struct timer_list *t)
 380 {
 381         struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer);
 382         struct i915_sw_fence *fence;
 383
 384         fence = xchg(&cb->base.fence, NULL);
 385         if (!fence)
 386                 return;
 387
 388         pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%pS)\n",
 389                   cb->dma->ops->get_driver_name(cb->dma),
 390                   cb->dma->ops->get_timeline_name(cb->dma),
 391                   cb->dma->seqno,
 392                   i915_sw_fence_debug_hint(fence));
 393
 394         i915_sw_fence_complete(fence);
 395 }
 396
 397 static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma,
 398                                          struct dma_fence_cb *data)
 399 {
 400         struct i915_sw_dma_fence_cb_timer *cb =
 401                 container_of(data, typeof(*cb), base.base);
 402         struct i915_sw_fence *fence;
 403
 404         fence = xchg(&cb->base.fence, NULL);
 405         if (fence)
 406                 i915_sw_fence_complete(fence);
 407
 408         irq_work_queue(&cb->work);
 409 }
 410
 411 static void irq_i915_sw_fence_work(struct irq_work *wrk)
 412 {
 413         struct i915_sw_dma_fence_cb_timer *cb =
 414                 container_of(wrk, typeof(*cb), work);
 415
 416         del_timer_sync(&cb->timer);
 417         dma_fence_put(cb->dma);
 418
 419         kfree_rcu(cb, rcu);
 420 }
 421
 422 int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
 423                                   struct dma_fence *dma,
 424                                   unsigned long timeout,
 425                                   gfp_t gfp)
 426 {
 427         struct i915_sw_dma_fence_cb *cb;
 428         dma_fence_func_t func;
 429         int ret;
 430
 431         debug_fence_assert(fence);
 432         might_sleep_if(gfpflags_allow_blocking(gfp));
 433
 434         if (dma_fence_is_signaled(dma))
 435                 return 0;
 436
 437         cb = kmalloc(timeout ?
 438                      sizeof(struct i915_sw_dma_fence_cb_timer) :
 439                      sizeof(struct i915_sw_dma_fence_cb),
 440                      gfp);
 441         if (!cb) {
 442                 if (!gfpflags_allow_blocking(gfp))
 443                         return -ENOMEM;
 444
 445                 return dma_fence_wait(dma, false);
 446         }
 447
 448         cb->fence = fence;
 449         i915_sw_fence_await(fence);
 450
 451         func = dma_i915_sw_fence_wake;
 452         if (timeout) {
 453                 struct i915_sw_dma_fence_cb_timer *timer =
 454                         container_of(cb, typeof(*timer), base);
 455
 456                 timer->dma = dma_fence_get(dma);
 457                 init_irq_work(&timer->work, irq_i915_sw_fence_work);
 458
 459                 timer_setup(&timer->timer,
 460                             timer_i915_sw_fence_wake, TIMER_IRQSAFE);
 461                 mod_timer(&timer->timer, round_jiffies_up(jiffies + timeout));
 462
 463                 func = dma_i915_sw_fence_wake_timer;
 464         }
 465
 466         ret = dma_fence_add_callback(dma, &cb->base, func);
 467         if (ret == 0) {
 468                 ret = 1;
 469         } else {
 470                 func(dma, &cb->base);
 471                 if (ret == -ENOENT) /* fence already signaled */
 472                         ret = 0;
 473         }
 474
 475         return ret;
 476 }
 477
 478 static void __dma_i915_sw_fence_wake(struct dma_fence *dma,
 479                                      struct dma_fence_cb *data)
 480 {
 481         struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);
 482
 483         i915_sw_fence_complete(cb->fence);
 484 }
 485
 486 int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
 487                                     struct dma_fence *dma,
 488                                     struct i915_sw_dma_fence_cb *cb)
 489 {
 490         int ret;
 491
 492         debug_fence_assert(fence);
 493
 494         if (dma_fence_is_signaled(dma))
 495                 return 0;
 496
 497         cb->fence = fence;
 498         i915_sw_fence_await(fence);
 499
 500         ret = dma_fence_add_callback(dma, &cb->base, __dma_i915_sw_fence_wake);
 501         if (ret == 0) {
 502                 ret = 1;
 503         } else {
 504                 i915_sw_fence_complete(fence);
 505                 if (ret == -ENOENT) /* fence already signaled */
 506                         ret = 0;
 507         }
 508
 509         return ret;
 510 }
 511
 512 int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
 513                                     struct reservation_object *resv,
 514                                     const struct dma_fence_ops *exclude,
 515                                     bool write,
 516                                     unsigned long timeout,
 517                                     gfp_t gfp)
 518 {
 519         struct dma_fence *excl;
 520         int ret = 0, pending;
 521
 522         debug_fence_assert(fence);
 523         might_sleep_if(gfpflags_allow_blocking(gfp));
 524
 525         if (write) {
 526                 struct dma_fence **shared;
 527                 unsigned int count, i;
 528
 529                 ret = reservation_object_get_fences_rcu(resv,
 530                                                         &excl, &count, &shared);
 531                 if (ret)
 532                         return ret;
 533
 534                 for (i = 0; i < count; i++) {
 535                         if (shared[i]->ops == exclude)
 536                                 continue;
 537
 538                         pending = i915_sw_fence_await_dma_fence(fence,
 539                                                                 shared[i],
 540                                                                 timeout,
 541                                                                 gfp);
 542                         if (pending < 0) {
 543                                 ret = pending;
 544                                 break;
 545                         }
 546
 547                         ret |= pending;
 548                 }
 549
 550                 for (i = 0; i < count; i++)
 551                         dma_fence_put(shared[i]);
 552                 kfree(shared);
 553         } else {
 554                 excl = reservation_object_get_excl_rcu(resv);
 555         }
 556
 557         if (ret >= 0 && excl && excl->ops != exclude) {
 558                 pending = i915_sw_fence_await_dma_fence(fence,
 559                                                         excl,
 560                                                         timeout,
 561                                                         gfp);
 562                 if (pending < 0)
 563                         ret = pending;
 564                 else
 565                         ret |= pending;
 566         }
 567
 568         dma_fence_put(excl);
 569
 570         return ret;
 571 }
 572
 573 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 574 #include "selftests/lib_sw_fence.c"
 575 #include "selftests/i915_sw_fence.c"
 576 #endif