drivers/gpu/drm/i915/gt/intel_timeline.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2016-2018 Intel Corporation
   5  */
   6
   7 #include "i915_drv.h"
   8
   9 #include "i915_active.h"
  10 #include "i915_syncmap.h"
  11 #include "intel_gt.h"
  12 #include "intel_ring.h"
  13 #include "intel_timeline.h"
  14
  15 #define ptr_set_bit(ptr, bit) ((typeof(ptr))((unsigned long)(ptr) | BIT(bit)))
  16 #define ptr_test_bit(ptr, bit) ((unsigned long)(ptr) & BIT(bit))
  17
  18 #define CACHELINE_BITS 6
  19 #define CACHELINE_FREE CACHELINE_BITS
  20
  21 struct intel_timeline_hwsp {
  22         struct intel_gt *gt;
  23         struct intel_gt_timelines *gt_timelines;
  24         struct list_head free_link;
  25         struct i915_vma *vma;
  26         u64 free_bitmap;
  27 };
  28
  29 static struct i915_vma *__hwsp_alloc(struct intel_gt *gt)
  30 {
  31         struct drm_i915_private *i915 = gt->i915;
  32         struct drm_i915_gem_object *obj;
  33         struct i915_vma *vma;
  34
  35         obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
  36         if (IS_ERR(obj))
  37                 return ERR_CAST(obj);
  38
  39         i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
  40
  41         vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
  42         if (IS_ERR(vma))
  43                 i915_gem_object_put(obj);
  44
  45         return vma;
  46 }
  47
  48 static struct i915_vma *
  49 hwsp_alloc(struct intel_timeline *timeline, unsigned int *cacheline)
  50 {
  51         struct intel_gt_timelines *gt = &timeline->gt->timelines;
  52         struct intel_timeline_hwsp *hwsp;
  53
  54         BUILD_BUG_ON(BITS_PER_TYPE(u64) * CACHELINE_BYTES > PAGE_SIZE);
  55
  56         spin_lock_irq(&gt->hwsp_lock);
  57
  58         /* hwsp_free_list only contains HWSP that have available cachelines */
  59         hwsp = list_first_entry_or_null(&gt->hwsp_free_list,
  60                                         typeof(*hwsp), free_link);
  61         if (!hwsp) {
  62                 struct i915_vma *vma;
  63
  64                 spin_unlock_irq(&gt->hwsp_lock);
  65
  66                 hwsp = kmalloc(sizeof(*hwsp), GFP_KERNEL);
  67                 if (!hwsp)
  68                         return ERR_PTR(-ENOMEM);
  69
  70                 vma = __hwsp_alloc(timeline->gt);
  71                 if (IS_ERR(vma)) {
  72                         kfree(hwsp);
  73                         return vma;
  74                 }
  75
  76                 GT_TRACE(timeline->gt, "new HWSP allocated\n");
  77
  78                 vma->private = hwsp;
  79                 hwsp->gt = timeline->gt;
  80                 hwsp->vma = vma;
  81                 hwsp->free_bitmap = ~0ull;
  82                 hwsp->gt_timelines = gt;
  83
  84                 spin_lock_irq(&gt->hwsp_lock);
  85                 list_add(&hwsp->free_link, &gt->hwsp_free_list);
  86         }
  87
  88         GEM_BUG_ON(!hwsp->free_bitmap);
  89         *cacheline = __ffs64(hwsp->free_bitmap);
  90         hwsp->free_bitmap &= ~BIT_ULL(*cacheline);
  91         if (!hwsp->free_bitmap)
  92                 list_del(&hwsp->free_link);
  93
  94         spin_unlock_irq(&gt->hwsp_lock);
  95
  96         GEM_BUG_ON(hwsp->vma->private != hwsp);
  97         return hwsp->vma;
  98 }
  99
 100 static void __idle_hwsp_free(struct intel_timeline_hwsp *hwsp, int cacheline)
 101 {
 102         struct intel_gt_timelines *gt = hwsp->gt_timelines;
 103         unsigned long flags;
 104
 105         spin_lock_irqsave(&gt->hwsp_lock, flags);
 106
 107         /* As a cacheline becomes available, publish the HWSP on the freelist */
 108         if (!hwsp->free_bitmap)
 109                 list_add_tail(&hwsp->free_link, &gt->hwsp_free_list);
 110
 111         GEM_BUG_ON(cacheline >= BITS_PER_TYPE(hwsp->free_bitmap));
 112         hwsp->free_bitmap |= BIT_ULL(cacheline);
 113
 114         /* And if no one is left using it, give the page back to the system */
 115         if (hwsp->free_bitmap == ~0ull) {
 116                 i915_vma_put(hwsp->vma);
 117                 list_del(&hwsp->free_link);
 118                 kfree(hwsp);
 119         }
 120
 121         spin_unlock_irqrestore(&gt->hwsp_lock, flags);
 122 }
 123
 124 static void __rcu_cacheline_free(struct rcu_head *rcu)
 125 {
 126         struct intel_timeline_cacheline *cl =
 127                 container_of(rcu, typeof(*cl), rcu);
 128
 129         i915_active_fini(&cl->active);
 130         kfree(cl);
 131 }
 132
 133 static void __idle_cacheline_free(struct intel_timeline_cacheline *cl)
 134 {
 135         GEM_BUG_ON(!i915_active_is_idle(&cl->active));
 136
 137         i915_gem_object_unpin_map(cl->hwsp->vma->obj);
 138         i915_vma_put(cl->hwsp->vma);
 139         __idle_hwsp_free(cl->hwsp, ptr_unmask_bits(cl->vaddr, CACHELINE_BITS));
 140
 141         call_rcu(&cl->rcu, __rcu_cacheline_free);
 142 }
 143
 144 __i915_active_call
 145 static void __cacheline_retire(struct i915_active *active)
 146 {
 147         struct intel_timeline_cacheline *cl =
 148                 container_of(active, typeof(*cl), active);
 149
 150         i915_vma_unpin(cl->hwsp->vma);
 151         if (ptr_test_bit(cl->vaddr, CACHELINE_FREE))
 152                 __idle_cacheline_free(cl);
 153 }
 154
 155 static int __cacheline_active(struct i915_active *active)
 156 {
 157         struct intel_timeline_cacheline *cl =
 158                 container_of(active, typeof(*cl), active);
 159
 160         __i915_vma_pin(cl->hwsp->vma);
 161         return 0;
 162 }
 163
 164 static struct intel_timeline_cacheline *
 165 cacheline_alloc(struct intel_timeline_hwsp *hwsp, unsigned int cacheline)
 166 {
 167         struct intel_timeline_cacheline *cl;
 168         void *vaddr;
 169
 170         GEM_BUG_ON(cacheline >= BIT(CACHELINE_BITS));
 171
 172         cl = kmalloc(sizeof(*cl), GFP_KERNEL);
 173         if (!cl)
 174                 return ERR_PTR(-ENOMEM);
 175
 176         vaddr = i915_gem_object_pin_map(hwsp->vma->obj, I915_MAP_WB);
 177         if (IS_ERR(vaddr)) {
 178                 kfree(cl);
 179                 return ERR_CAST(vaddr);
 180         }
 181
 182         i915_vma_get(hwsp->vma);
 183         cl->hwsp = hwsp;
 184         cl->vaddr = page_pack_bits(vaddr, cacheline);
 185
 186         i915_active_init(&cl->active, __cacheline_active, __cacheline_retire);
 187
 188         return cl;
 189 }
 190
 191 static void cacheline_acquire(struct intel_timeline_cacheline *cl,
 192                               u32 ggtt_offset)
 193 {
 194         if (!cl)
 195                 return;
 196
 197         cl->ggtt_offset = ggtt_offset;
 198         i915_active_acquire(&cl->active);
 199 }
 200
 201 static void cacheline_release(struct intel_timeline_cacheline *cl)
 202 {
 203         if (cl)
 204                 i915_active_release(&cl->active);
 205 }
 206
 207 static void cacheline_free(struct intel_timeline_cacheline *cl)
 208 {
 209         if (!i915_active_acquire_if_busy(&cl->active)) {
 210                 __idle_cacheline_free(cl);
 211                 return;
 212         }
 213
 214         GEM_BUG_ON(ptr_test_bit(cl->vaddr, CACHELINE_FREE));
 215         cl->vaddr = ptr_set_bit(cl->vaddr, CACHELINE_FREE);
 216
 217         i915_active_release(&cl->active);
 218 }
 219
 220 static int intel_timeline_init(struct intel_timeline *timeline,
 221                                struct intel_gt *gt,
 222                                struct i915_vma *hwsp,
 223                                unsigned int offset)
 224 {
 225         void *vaddr;
 226
 227         kref_init(&timeline->kref);
 228         atomic_set(&timeline->pin_count, 0);
 229
 230         timeline->gt = gt;
 231
 232         timeline->has_initial_breadcrumb = !hwsp;
 233         timeline->hwsp_cacheline = NULL;
 234
 235         if (!hwsp) {
 236                 struct intel_timeline_cacheline *cl;
 237                 unsigned int cacheline;
 238
 239                 hwsp = hwsp_alloc(timeline, &cacheline);
 240                 if (IS_ERR(hwsp))
 241                         return PTR_ERR(hwsp);
 242
 243                 cl = cacheline_alloc(hwsp->private, cacheline);
 244                 if (IS_ERR(cl)) {
 245                         __idle_hwsp_free(hwsp->private, cacheline);
 246                         return PTR_ERR(cl);
 247                 }
 248
 249                 timeline->hwsp_cacheline = cl;
 250                 timeline->hwsp_offset = cacheline * CACHELINE_BYTES;
 251
 252                 vaddr = page_mask_bits(cl->vaddr);
 253         } else {
 254                 timeline->hwsp_offset = offset;
 255                 vaddr = i915_gem_object_pin_map(hwsp->obj, I915_MAP_WB);
 256                 if (IS_ERR(vaddr))
 257                         return PTR_ERR(vaddr);
 258         }
 259
 260         timeline->hwsp_seqno =
 261                 memset(vaddr + timeline->hwsp_offset, 0, CACHELINE_BYTES);
 262
 263         timeline->hwsp_ggtt = i915_vma_get(hwsp);
 264         GEM_BUG_ON(timeline->hwsp_offset >= hwsp->size);
 265
 266         timeline->fence_context = dma_fence_context_alloc(1);
 267
 268         mutex_init(&timeline->mutex);
 269
 270         INIT_ACTIVE_FENCE(&timeline->last_request);
 271         INIT_LIST_HEAD(&timeline->requests);
 272
 273         i915_syncmap_init(&timeline->sync);
 274
 275         return 0;
 276 }
 277
 278 void intel_gt_init_timelines(struct intel_gt *gt)
 279 {
 280         struct intel_gt_timelines *timelines = &gt->timelines;
 281
 282         spin_lock_init(&timelines->lock);
 283         INIT_LIST_HEAD(&timelines->active_list);
 284
 285         spin_lock_init(&timelines->hwsp_lock);
 286         INIT_LIST_HEAD(&timelines->hwsp_free_list);
 287 }
 288
 289 static void intel_timeline_fini(struct intel_timeline *timeline)
 290 {
 291         GEM_BUG_ON(atomic_read(&timeline->pin_count));
 292         GEM_BUG_ON(!list_empty(&timeline->requests));
 293         GEM_BUG_ON(timeline->retire);
 294
 295         if (timeline->hwsp_cacheline)
 296                 cacheline_free(timeline->hwsp_cacheline);
 297         else
 298                 i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);
 299
 300         i915_vma_put(timeline->hwsp_ggtt);
 301 }
 302
 303 struct intel_timeline *
 304 __intel_timeline_create(struct intel_gt *gt,
 305                         struct i915_vma *global_hwsp,
 306                         unsigned int offset)
 307 {
 308         struct intel_timeline *timeline;
 309         int err;
 310
 311         timeline = kzalloc(sizeof(*timeline), GFP_KERNEL);
 312         if (!timeline)
 313                 return ERR_PTR(-ENOMEM);
 314
 315         err = intel_timeline_init(timeline, gt, global_hwsp, offset);
 316         if (err) {
 317                 kfree(timeline);
 318                 return ERR_PTR(err);
 319         }
 320
 321         return timeline;
 322 }
 323
 324 void __intel_timeline_pin(struct intel_timeline *tl)
 325 {
 326         GEM_BUG_ON(!atomic_read(&tl->pin_count));
 327         atomic_inc(&tl->pin_count);
 328 }
 329
 330 int intel_timeline_pin(struct intel_timeline *tl, struct i915_gem_ww_ctx *ww)
 331 {
 332         int err;
 333
 334         if (atomic_add_unless(&tl->pin_count, 1, 0))
 335                 return 0;
 336
 337         err = i915_ggtt_pin(tl->hwsp_ggtt, ww, 0, PIN_HIGH);
 338         if (err)
 339                 return err;
 340
 341         tl->hwsp_offset =
 342                 i915_ggtt_offset(tl->hwsp_ggtt) +
 343                 offset_in_page(tl->hwsp_offset);
 344         GT_TRACE(tl->gt, "timeline:%llx using HWSP offset:%x\n",
 345                  tl->fence_context, tl->hwsp_offset);
 346
 347         cacheline_acquire(tl->hwsp_cacheline, tl->hwsp_offset);
 348         if (atomic_fetch_inc(&tl->pin_count)) {
 349                 cacheline_release(tl->hwsp_cacheline);
 350                 __i915_vma_unpin(tl->hwsp_ggtt);
 351         }
 352
 353         return 0;
 354 }
 355
 356 void intel_timeline_reset_seqno(const struct intel_timeline *tl)
 357 {
 358         /* Must be pinned to be writable, and no requests in flight. */
 359         GEM_BUG_ON(!atomic_read(&tl->pin_count));
 360         WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno);
 361 }
 362
 363 void intel_timeline_enter(struct intel_timeline *tl)
 364 {
 365         struct intel_gt_timelines *timelines = &tl->gt->timelines;
 366
 367         /*
 368          * Pretend we are serialised by the timeline->mutex.
 369          *
 370          * While generally true, there are a few exceptions to the rule
 371          * for the engine->kernel_context being used to manage power
 372          * transitions. As the engine_park may be called from under any
 373          * timeline, it uses the power mutex as a global serialisation
 374          * lock to prevent any other request entering its timeline.
 375          *
 376          * The rule is generally tl->mutex, otherwise engine->wakeref.mutex.
 377          *
 378          * However, intel_gt_retire_request() does not know which engine
 379          * it is retiring along and so cannot partake in the engine-pm
 380          * barrier, and there we use the tl->active_count as a means to
 381          * pin the timeline in the active_list while the locks are dropped.
 382          * Ergo, as that is outside of the engine-pm barrier, we need to
 383          * use atomic to manipulate tl->active_count.
 384          */
 385         lockdep_assert_held(&tl->mutex);
 386
 387         if (atomic_add_unless(&tl->active_count, 1, 0))
 388                 return;
 389
 390         spin_lock(&timelines->lock);
 391         if (!atomic_fetch_inc(&tl->active_count)) {
 392                 /*
 393                  * The HWSP is volatile, and may have been lost while inactive,
 394                  * e.g. across suspend/resume. Be paranoid, and ensure that
 395                  * the HWSP value matches our seqno so we don't proclaim
 396                  * the next request as already complete.
 397                  */
 398                 intel_timeline_reset_seqno(tl);
 399                 list_add_tail(&tl->link, &timelines->active_list);
 400         }
 401         spin_unlock(&timelines->lock);
 402 }
 403
 404 void intel_timeline_exit(struct intel_timeline *tl)
 405 {
 406         struct intel_gt_timelines *timelines = &tl->gt->timelines;
 407
 408         /* See intel_timeline_enter() */
 409         lockdep_assert_held(&tl->mutex);
 410
 411         GEM_BUG_ON(!atomic_read(&tl->active_count));
 412         if (atomic_add_unless(&tl->active_count, -1, 1))
 413                 return;
 414
 415         spin_lock(&timelines->lock);
 416         if (atomic_dec_and_test(&tl->active_count))
 417                 list_del(&tl->link);
 418         spin_unlock(&timelines->lock);
 419
 420         /*
 421          * Since this timeline is idle, all bariers upon which we were waiting
 422          * must also be complete and so we can discard the last used barriers
 423          * without loss of information.
 424          */
 425         i915_syncmap_free(&tl->sync);
 426 }
 427
 428 static u32 timeline_advance(struct intel_timeline *tl)
 429 {
 430         GEM_BUG_ON(!atomic_read(&tl->pin_count));
 431         GEM_BUG_ON(tl->seqno & tl->has_initial_breadcrumb);
 432
 433         return tl->seqno += 1 + tl->has_initial_breadcrumb;
 434 }
 435
 436 static void timeline_rollback(struct intel_timeline *tl)
 437 {
 438         tl->seqno -= 1 + tl->has_initial_breadcrumb;
 439 }
 440
 441 static noinline int
 442 __intel_timeline_get_seqno(struct intel_timeline *tl,
 443                            struct i915_request *rq,
 444                            u32 *seqno)
 445 {
 446         struct intel_timeline_cacheline *cl;
 447         unsigned int cacheline;
 448         struct i915_vma *vma;
 449         void *vaddr;
 450         int err;
 451
 452         might_lock(&tl->gt->ggtt->vm.mutex);
 453         GT_TRACE(tl->gt, "timeline:%llx wrapped\n", tl->fence_context);
 454
 455         /*
 456          * If there is an outstanding GPU reference to this cacheline,
 457          * such as it being sampled by a HW semaphore on another timeline,
 458          * we cannot wraparound our seqno value (the HW semaphore does
 459          * a strict greater-than-or-equals compare, not i915_seqno_passed).
 460          * So if the cacheline is still busy, we must detach ourselves
 461          * from it and leave it inflight alongside its users.
 462          *
 463          * However, if nobody is watching and we can guarantee that nobody
 464          * will, we could simply reuse the same cacheline.
 465          *
 466          * if (i915_active_request_is_signaled(&tl->last_request) &&
 467          *     i915_active_is_signaled(&tl->hwsp_cacheline->active))
 468          *      return 0;
 469          *
 470          * That seems unlikely for a busy timeline that needed to wrap in
 471          * the first place, so just replace the cacheline.
 472          */
 473
 474         vma = hwsp_alloc(tl, &cacheline);
 475         if (IS_ERR(vma)) {
 476                 err = PTR_ERR(vma);
 477                 goto err_rollback;
 478         }
 479
 480         err = i915_ggtt_pin(vma, NULL, 0, PIN_HIGH);
 481         if (err) {
 482                 __idle_hwsp_free(vma->private, cacheline);
 483                 goto err_rollback;
 484         }
 485
 486         cl = cacheline_alloc(vma->private, cacheline);
 487         if (IS_ERR(cl)) {
 488                 err = PTR_ERR(cl);
 489                 __idle_hwsp_free(vma->private, cacheline);
 490                 goto err_unpin;
 491         }
 492         GEM_BUG_ON(cl->hwsp->vma != vma);
 493
 494         /*
 495          * Attach the old cacheline to the current request, so that we only
 496          * free it after the current request is retired, which ensures that
 497          * all writes into the cacheline from previous requests are complete.
 498          */
 499         err = i915_active_ref(&tl->hwsp_cacheline->active,
 500                               tl->fence_context,
 501                               &rq->fence);
 502         if (err)
 503                 goto err_cacheline;
 504
 505         cacheline_release(tl->hwsp_cacheline); /* ownership now xfered to rq */
 506         cacheline_free(tl->hwsp_cacheline);
 507
 508         i915_vma_unpin(tl->hwsp_ggtt); /* binding kept alive by old cacheline */
 509         i915_vma_put(tl->hwsp_ggtt);
 510
 511         tl->hwsp_ggtt = i915_vma_get(vma);
 512
 513         vaddr = page_mask_bits(cl->vaddr);
 514         tl->hwsp_offset = cacheline * CACHELINE_BYTES;
 515         tl->hwsp_seqno =
 516                 memset(vaddr + tl->hwsp_offset, 0, CACHELINE_BYTES);
 517
 518         tl->hwsp_offset += i915_ggtt_offset(vma);
 519         GT_TRACE(tl->gt, "timeline:%llx using HWSP offset:%x\n",
 520                  tl->fence_context, tl->hwsp_offset);
 521
 522         cacheline_acquire(cl, tl->hwsp_offset);
 523         tl->hwsp_cacheline = cl;
 524
 525         *seqno = timeline_advance(tl);
 526         GEM_BUG_ON(i915_seqno_passed(*tl->hwsp_seqno, *seqno));
 527         return 0;
 528
 529 err_cacheline:
 530         cacheline_free(cl);
 531 err_unpin:
 532         i915_vma_unpin(vma);
 533 err_rollback:
 534         timeline_rollback(tl);
 535         return err;
 536 }
 537
 538 int intel_timeline_get_seqno(struct intel_timeline *tl,
 539                              struct i915_request *rq,
 540                              u32 *seqno)
 541 {
 542         *seqno = timeline_advance(tl);
 543
 544         /* Replace the HWSP on wraparound for HW semaphores */
 545         if (unlikely(!*seqno && tl->hwsp_cacheline))
 546                 return __intel_timeline_get_seqno(tl, rq, seqno);
 547
 548         return 0;
 549 }
 550
 551 static int cacheline_ref(struct intel_timeline_cacheline *cl,
 552                          struct i915_request *rq)
 553 {
 554         return i915_active_add_request(&cl->active, rq);
 555 }
 556
 557 int intel_timeline_read_hwsp(struct i915_request *from,
 558                              struct i915_request *to,
 559                              u32 *hwsp)
 560 {
 561         struct intel_timeline_cacheline *cl;
 562         int err;
 563
 564         GEM_BUG_ON(!rcu_access_pointer(from->hwsp_cacheline));
 565
 566         rcu_read_lock();
 567         cl = rcu_dereference(from->hwsp_cacheline);
 568         if (i915_request_completed(from)) /* confirm cacheline is valid */
 569                 goto unlock;
 570         if (unlikely(!i915_active_acquire_if_busy(&cl->active)))
 571                 goto unlock; /* seqno wrapped and completed! */
 572         if (unlikely(i915_request_completed(from)))
 573                 goto release;
 574         rcu_read_unlock();
 575
 576         err = cacheline_ref(cl, to);
 577         if (err)
 578                 goto out;
 579
 580         *hwsp = cl->ggtt_offset;
 581 out:
 582         i915_active_release(&cl->active);
 583         return err;
 584
 585 release:
 586         i915_active_release(&cl->active);
 587 unlock:
 588         rcu_read_unlock();
 589         return 1;
 590 }
 591
 592 void intel_timeline_unpin(struct intel_timeline *tl)
 593 {
 594         GEM_BUG_ON(!atomic_read(&tl->pin_count));
 595         if (!atomic_dec_and_test(&tl->pin_count))
 596                 return;
 597
 598         cacheline_release(tl->hwsp_cacheline);
 599
 600         __i915_vma_unpin(tl->hwsp_ggtt);
 601 }
 602
 603 void __intel_timeline_free(struct kref *kref)
 604 {
 605         struct intel_timeline *timeline =
 606                 container_of(kref, typeof(*timeline), kref);
 607
 608         intel_timeline_fini(timeline);
 609         kfree_rcu(timeline, rcu);
 610 }
 611
 612 void intel_gt_fini_timelines(struct intel_gt *gt)
 613 {
 614         struct intel_gt_timelines *timelines = &gt->timelines;
 615
 616         GEM_BUG_ON(!list_empty(&timelines->active_list));
 617         GEM_BUG_ON(!list_empty(&timelines->hwsp_free_list));
 618 }
 619
 620 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 621 #include "gt/selftests/mock_timeline.c"
 622 #include "gt/selftest_timeline.c"
 623 #endif