drivers/gpu/drm/i915/i915_active.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2019 Intel Corporation
   5  */
   6
   7 #include <linux/debugobjects.h>
   8
   9 #include "gt/intel_engine_pm.h"
  10
  11 #include "i915_drv.h"
  12 #include "i915_active.h"
  13 #include "i915_globals.h"
  14
  15 #define BKL(ref) (&(ref)->i915->drm.struct_mutex)
  16
  17 /*
  18  * Active refs memory management
  19  *
  20  * To be more economical with memory, we reap all the i915_active trees as
  21  * they idle (when we know the active requests are inactive) and allocate the
  22  * nodes from a local slab cache to hopefully reduce the fragmentation.
  23  */
  24 static struct i915_global_active {
  25         struct i915_global base;
  26         struct kmem_cache *slab_cache;
  27 } global;
  28
  29 struct active_node {
  30         struct i915_active_request base;
  31         struct i915_active *ref;
  32         struct rb_node node;
  33         u64 timeline;
  34 };
  35
  36 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) && IS_ENABLED(CONFIG_DEBUG_OBJECTS)
  37
  38 static void *active_debug_hint(void *addr)
  39 {
  40         struct i915_active *ref = addr;
  41
  42         return (void *)ref->active ?: (void *)ref->retire ?: (void *)ref;
  43 }
  44
  45 static struct debug_obj_descr active_debug_desc = {
  46         .name = "i915_active",
  47         .debug_hint = active_debug_hint,
  48 };
  49
  50 static void debug_active_init(struct i915_active *ref)
  51 {
  52         debug_object_init(ref, &active_debug_desc);
  53 }
  54
  55 static void debug_active_activate(struct i915_active *ref)
  56 {
  57         debug_object_activate(ref, &active_debug_desc);
  58 }
  59
  60 static void debug_active_deactivate(struct i915_active *ref)
  61 {
  62         debug_object_deactivate(ref, &active_debug_desc);
  63 }
  64
  65 static void debug_active_fini(struct i915_active *ref)
  66 {
  67         debug_object_free(ref, &active_debug_desc);
  68 }
  69
  70 static void debug_active_assert(struct i915_active *ref)
  71 {
  72         debug_object_assert_init(ref, &active_debug_desc);
  73 }
  74
  75 #else
  76
  77 static inline void debug_active_init(struct i915_active *ref) { }
  78 static inline void debug_active_activate(struct i915_active *ref) { }
  79 static inline void debug_active_deactivate(struct i915_active *ref) { }
  80 static inline void debug_active_fini(struct i915_active *ref) { }
  81 static inline void debug_active_assert(struct i915_active *ref) { }
  82
  83 #endif
  84
  85 static void
  86 __active_retire(struct i915_active *ref)
  87 {
  88         struct active_node *it, *n;
  89         struct rb_root root;
  90         bool retire = false;
  91
  92         lockdep_assert_held(&ref->mutex);
  93
  94         /* return the unused nodes to our slabcache -- flushing the allocator */
  95         if (atomic_dec_and_test(&ref->count)) {
  96                 debug_active_deactivate(ref);
  97                 root = ref->tree;
  98                 ref->tree = RB_ROOT;
  99                 ref->cache = NULL;
 100                 retire = true;
 101         }
 102
 103         mutex_unlock(&ref->mutex);
 104         if (!retire)
 105                 return;
 106
 107         ref->retire(ref);
 108
 109         rbtree_postorder_for_each_entry_safe(it, n, &root, node) {
 110                 GEM_BUG_ON(i915_active_request_isset(&it->base));
 111                 kmem_cache_free(global.slab_cache, it);
 112         }
 113 }
 114
 115 static void
 116 active_retire(struct i915_active *ref)
 117 {
 118         GEM_BUG_ON(!atomic_read(&ref->count));
 119         if (atomic_add_unless(&ref->count, -1, 1))
 120                 return;
 121
 122         /* One active may be flushed from inside the acquire of another */
 123         mutex_lock_nested(&ref->mutex, SINGLE_DEPTH_NESTING);
 124         __active_retire(ref);
 125 }
 126
 127 static void
 128 node_retire(struct i915_active_request *base, struct i915_request *rq)
 129 {
 130         active_retire(container_of(base, struct active_node, base)->ref);
 131 }
 132
 133 static struct i915_active_request *
 134 active_instance(struct i915_active *ref, u64 idx)
 135 {
 136         struct active_node *node, *prealloc;
 137         struct rb_node **p, *parent;
 138
 139         /*
 140          * We track the most recently used timeline to skip a rbtree search
 141          * for the common case, under typical loads we never need the rbtree
 142          * at all. We can reuse the last slot if it is empty, that is
 143          * after the previous activity has been retired, or if it matches the
 144          * current timeline.
 145          */
 146         node = READ_ONCE(ref->cache);
 147         if (node && node->timeline == idx)
 148                 return &node->base;
 149
 150         /* Preallocate a replacement, just in case */
 151         prealloc = kmem_cache_alloc(global.slab_cache, GFP_KERNEL);
 152         if (!prealloc)
 153                 return NULL;
 154
 155         mutex_lock(&ref->mutex);
 156         GEM_BUG_ON(i915_active_is_idle(ref));
 157
 158         parent = NULL;
 159         p = &ref->tree.rb_node;
 160         while (*p) {
 161                 parent = *p;
 162
 163                 node = rb_entry(parent, struct active_node, node);
 164                 if (node->timeline == idx) {
 165                         kmem_cache_free(global.slab_cache, prealloc);
 166                         goto out;
 167                 }
 168
 169                 if (node->timeline < idx)
 170                         p = &parent->rb_right;
 171                 else
 172                         p = &parent->rb_left;
 173         }
 174
 175         node = prealloc;
 176         i915_active_request_init(&node->base, NULL, node_retire);
 177         node->ref = ref;
 178         node->timeline = idx;
 179
 180         rb_link_node(&node->node, parent, p);
 181         rb_insert_color(&node->node, &ref->tree);
 182
 183 out:
 184         ref->cache = node;
 185         mutex_unlock(&ref->mutex);
 186
 187         return &node->base;
 188 }
 189
 190 void __i915_active_init(struct drm_i915_private *i915,
 191                         struct i915_active *ref,
 192                         int (*active)(struct i915_active *ref),
 193                         void (*retire)(struct i915_active *ref),
 194                         struct lock_class_key *key)
 195 {
 196         debug_active_init(ref);
 197
 198         ref->i915 = i915;
 199         ref->active = active;
 200         ref->retire = retire;
 201         ref->tree = RB_ROOT;
 202         ref->cache = NULL;
 203         init_llist_head(&ref->barriers);
 204         atomic_set(&ref->count, 0);
 205         __mutex_init(&ref->mutex, "i915_active", key);
 206 }
 207
 208 int i915_active_ref(struct i915_active *ref,
 209                     u64 timeline,
 210                     struct i915_request *rq)
 211 {
 212         struct i915_active_request *active;
 213         int err;
 214
 215         /* Prevent reaping in case we malloc/wait while building the tree */
 216         err = i915_active_acquire(ref);
 217         if (err)
 218                 return err;
 219
 220         active = active_instance(ref, timeline);
 221         if (!active) {
 222                 err = -ENOMEM;
 223                 goto out;
 224         }
 225
 226         if (!i915_active_request_isset(active))
 227                 atomic_inc(&ref->count);
 228         __i915_active_request_set(active, rq);
 229
 230 out:
 231         i915_active_release(ref);
 232         return err;
 233 }
 234
 235 int i915_active_acquire(struct i915_active *ref)
 236 {
 237         int err;
 238
 239         debug_active_assert(ref);
 240         if (atomic_add_unless(&ref->count, 1, 0))
 241                 return 0;
 242
 243         err = mutex_lock_interruptible(&ref->mutex);
 244         if (err)
 245                 return err;
 246
 247         if (!atomic_read(&ref->count) && ref->active)
 248                 err = ref->active(ref);
 249         if (!err) {
 250                 debug_active_activate(ref);
 251                 atomic_inc(&ref->count);
 252         }
 253
 254         mutex_unlock(&ref->mutex);
 255
 256         return err;
 257 }
 258
 259 void i915_active_release(struct i915_active *ref)
 260 {
 261         debug_active_assert(ref);
 262         active_retire(ref);
 263 }
 264
 265 int i915_active_wait(struct i915_active *ref)
 266 {
 267         struct active_node *it, *n;
 268         int err;
 269
 270         might_sleep();
 271         if (RB_EMPTY_ROOT(&ref->tree))
 272                 return 0;
 273
 274         err = mutex_lock_interruptible(&ref->mutex);
 275         if (err)
 276                 return err;
 277
 278         if (!atomic_add_unless(&ref->count, 1, 0)) {
 279                 mutex_unlock(&ref->mutex);
 280                 return 0;
 281         }
 282
 283         rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
 284                 err = i915_active_request_retire(&it->base, BKL(ref));
 285                 if (err)
 286                         break;
 287         }
 288
 289         __active_retire(ref);
 290         if (err)
 291                 return err;
 292
 293         if (!i915_active_is_idle(ref))
 294                 return -EBUSY;
 295
 296         return 0;
 297 }
 298
 299 int i915_request_await_active_request(struct i915_request *rq,
 300                                       struct i915_active_request *active)
 301 {
 302         struct i915_request *barrier =
 303                 i915_active_request_raw(active, &rq->i915->drm.struct_mutex);
 304
 305         return barrier ? i915_request_await_dma_fence(rq, &barrier->fence) : 0;
 306 }
 307
 308 int i915_request_await_active(struct i915_request *rq, struct i915_active *ref)
 309 {
 310         struct active_node *it, *n;
 311         int err;
 312
 313         if (RB_EMPTY_ROOT(&ref->tree))
 314                 return 0;
 315
 316         /* await allocates and so we need to avoid hitting the shrinker */
 317         err = i915_active_acquire(ref);
 318         if (err)
 319                 return err;
 320
 321         mutex_lock(&ref->mutex);
 322         rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
 323                 err = i915_request_await_active_request(rq, &it->base);
 324                 if (err)
 325                         break;
 326         }
 327         mutex_unlock(&ref->mutex);
 328
 329         i915_active_release(ref);
 330         return err;
 331 }
 332
 333 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
 334 void i915_active_fini(struct i915_active *ref)
 335 {
 336         debug_active_fini(ref);
 337         GEM_BUG_ON(!RB_EMPTY_ROOT(&ref->tree));
 338         GEM_BUG_ON(atomic_read(&ref->count));
 339         mutex_destroy(&ref->mutex);
 340 }
 341 #endif
 342
 343 int i915_active_acquire_preallocate_barrier(struct i915_active *ref,
 344                                             struct intel_engine_cs *engine)
 345 {
 346         struct drm_i915_private *i915 = engine->i915;
 347         struct llist_node *pos, *next;
 348         unsigned long tmp;
 349         int err;
 350
 351         GEM_BUG_ON(!engine->mask);
 352         for_each_engine_masked(engine, i915, engine->mask, tmp) {
 353                 struct intel_context *kctx = engine->kernel_context;
 354                 struct active_node *node;
 355
 356                 node = kmem_cache_alloc(global.slab_cache, GFP_KERNEL);
 357                 if (unlikely(!node)) {
 358                         err = -ENOMEM;
 359                         goto unwind;
 360                 }
 361
 362                 i915_active_request_init(&node->base,
 363                                          (void *)engine, node_retire);
 364                 node->timeline = kctx->ring->timeline->fence_context;
 365                 node->ref = ref;
 366                 atomic_inc(&ref->count);
 367
 368                 intel_engine_pm_get(engine);
 369                 llist_add((struct llist_node *)&node->base.link,
 370                           &ref->barriers);
 371         }
 372
 373         return 0;
 374
 375 unwind:
 376         llist_for_each_safe(pos, next, llist_del_all(&ref->barriers)) {
 377                 struct active_node *node;
 378
 379                 node = container_of((struct list_head *)pos,
 380                                     typeof(*node), base.link);
 381                 engine = (void *)rcu_access_pointer(node->base.request);
 382
 383                 intel_engine_pm_put(engine);
 384                 kmem_cache_free(global.slab_cache, node);
 385         }
 386         return err;
 387 }
 388
 389 void i915_active_acquire_barrier(struct i915_active *ref)
 390 {
 391         struct llist_node *pos, *next;
 392
 393         GEM_BUG_ON(i915_active_is_idle(ref));
 394
 395         mutex_lock_nested(&ref->mutex, SINGLE_DEPTH_NESTING);
 396         llist_for_each_safe(pos, next, llist_del_all(&ref->barriers)) {
 397                 struct intel_engine_cs *engine;
 398                 struct active_node *node;
 399                 struct rb_node **p, *parent;
 400
 401                 node = container_of((struct list_head *)pos,
 402                                     typeof(*node), base.link);
 403
 404                 engine = (void *)rcu_access_pointer(node->base.request);
 405                 RCU_INIT_POINTER(node->base.request, ERR_PTR(-EAGAIN));
 406
 407                 parent = NULL;
 408                 p = &ref->tree.rb_node;
 409                 while (*p) {
 410                         parent = *p;
 411                         if (rb_entry(parent,
 412                                      struct active_node,
 413                                      node)->timeline < node->timeline)
 414                                 p = &parent->rb_right;
 415                         else
 416                                 p = &parent->rb_left;
 417                 }
 418                 rb_link_node(&node->node, parent, p);
 419                 rb_insert_color(&node->node, &ref->tree);
 420
 421                 llist_add((struct llist_node *)&node->base.link,
 422                           &engine->barrier_tasks);
 423                 intel_engine_pm_put(engine);
 424         }
 425         mutex_unlock(&ref->mutex);
 426 }
 427
 428 void i915_request_add_barriers(struct i915_request *rq)
 429 {
 430         struct intel_engine_cs *engine = rq->engine;
 431         struct llist_node *node, *next;
 432
 433         llist_for_each_safe(node, next, llist_del_all(&engine->barrier_tasks))
 434                 list_add_tail((struct list_head *)node, &rq->active_list);
 435 }
 436
 437 int i915_active_request_set(struct i915_active_request *active,
 438                             struct i915_request *rq)
 439 {
 440         int err;
 441
 442         /* Must maintain ordering wrt previous active requests */
 443         err = i915_request_await_active_request(rq, active);
 444         if (err)
 445                 return err;
 446
 447         __i915_active_request_set(active, rq);
 448         return 0;
 449 }
 450
 451 void i915_active_retire_noop(struct i915_active_request *active,
 452                              struct i915_request *request)
 453 {
 454         /* Space left intentionally blank */
 455 }
 456
 457 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 458 #include "selftests/i915_active.c"
 459 #endif
 460
 461 static void i915_global_active_shrink(void)
 462 {
 463         kmem_cache_shrink(global.slab_cache);
 464 }
 465
 466 static void i915_global_active_exit(void)
 467 {
 468         kmem_cache_destroy(global.slab_cache);
 469 }
 470
 471 static struct i915_global_active global = { {
 472         .shrink = i915_global_active_shrink,
 473         .exit = i915_global_active_exit,
 474 } };
 475
 476 int __init i915_global_active_init(void)
 477 {
 478         global.slab_cache = KMEM_CACHE(active_node, SLAB_HWCACHE_ALIGN);
 479         if (!global.slab_cache)
 480                 return -ENOMEM;
 481
 482         i915_global_register(&global.base);
 483         return 0;
 484 }