drivers/gpu/drm/i915/i915_timeline.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2016-2018 Intel Corporation
   5  */
   6
   7 #include "i915_drv.h"
   8
   9 #include "i915_timeline.h"
  10 #include "i915_syncmap.h"
  11
  12 struct i915_timeline_hwsp {
  13         struct i915_vma *vma;
  14         struct list_head free_link;
  15         u64 free_bitmap;
  16 };
  17
  18 static inline struct i915_timeline_hwsp *
  19 i915_timeline_hwsp(const struct i915_timeline *tl)
  20 {
  21         return tl->hwsp_ggtt->private;
  22 }
  23
  24 static struct i915_vma *__hwsp_alloc(struct drm_i915_private *i915)
  25 {
  26         struct drm_i915_gem_object *obj;
  27         struct i915_vma *vma;
  28
  29         obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
  30         if (IS_ERR(obj))
  31                 return ERR_CAST(obj);
  32
  33         i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
  34
  35         vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
  36         if (IS_ERR(vma))
  37                 i915_gem_object_put(obj);
  38
  39         return vma;
  40 }
  41
  42 static struct i915_vma *
  43 hwsp_alloc(struct i915_timeline *timeline, unsigned int *cacheline)
  44 {
  45         struct drm_i915_private *i915 = timeline->i915;
  46         struct i915_gt_timelines *gt = &i915->gt.timelines;
  47         struct i915_timeline_hwsp *hwsp;
  48
  49         BUILD_BUG_ON(BITS_PER_TYPE(u64) * CACHELINE_BYTES > PAGE_SIZE);
  50
  51         spin_lock(&gt->hwsp_lock);
  52
  53         /* hwsp_free_list only contains HWSP that have available cachelines */
  54         hwsp = list_first_entry_or_null(&gt->hwsp_free_list,
  55                                         typeof(*hwsp), free_link);
  56         if (!hwsp) {
  57                 struct i915_vma *vma;
  58
  59                 spin_unlock(&gt->hwsp_lock);
  60
  61                 hwsp = kmalloc(sizeof(*hwsp), GFP_KERNEL);
  62                 if (!hwsp)
  63                         return ERR_PTR(-ENOMEM);
  64
  65                 vma = __hwsp_alloc(i915);
  66                 if (IS_ERR(vma)) {
  67                         kfree(hwsp);
  68                         return vma;
  69                 }
  70
  71                 vma->private = hwsp;
  72                 hwsp->vma = vma;
  73                 hwsp->free_bitmap = ~0ull;
  74
  75                 spin_lock(&gt->hwsp_lock);
  76                 list_add(&hwsp->free_link, &gt->hwsp_free_list);
  77         }
  78
  79         GEM_BUG_ON(!hwsp->free_bitmap);
  80         *cacheline = __ffs64(hwsp->free_bitmap);
  81         hwsp->free_bitmap &= ~BIT_ULL(*cacheline);
  82         if (!hwsp->free_bitmap)
  83                 list_del(&hwsp->free_link);
  84
  85         spin_unlock(&gt->hwsp_lock);
  86
  87         GEM_BUG_ON(hwsp->vma->private != hwsp);
  88         return hwsp->vma;
  89 }
  90
  91 static void hwsp_free(struct i915_timeline *timeline)
  92 {
  93         struct i915_gt_timelines *gt = &timeline->i915->gt.timelines;
  94         struct i915_timeline_hwsp *hwsp;
  95
  96         hwsp = i915_timeline_hwsp(timeline);
  97         if (!hwsp) /* leave global HWSP alone! */
  98                 return;
  99
 100         spin_lock(&gt->hwsp_lock);
 101
 102         /* As a cacheline becomes available, publish the HWSP on the freelist */
 103         if (!hwsp->free_bitmap)
 104                 list_add_tail(&hwsp->free_link, &gt->hwsp_free_list);
 105
 106         hwsp->free_bitmap |= BIT_ULL(timeline->hwsp_offset / CACHELINE_BYTES);
 107
 108         /* And if no one is left using it, give the page back to the system */
 109         if (hwsp->free_bitmap == ~0ull) {
 110                 i915_vma_put(hwsp->vma);
 111                 list_del(&hwsp->free_link);
 112                 kfree(hwsp);
 113         }
 114
 115         spin_unlock(&gt->hwsp_lock);
 116 }
 117
 118 int i915_timeline_init(struct drm_i915_private *i915,
 119                        struct i915_timeline *timeline,
 120                        const char *name,
 121                        struct i915_vma *hwsp)
 122 {
 123         void *vaddr;
 124
 125         /*
 126          * Ideally we want a set of engines on a single leaf as we expect
 127          * to mostly be tracking synchronisation between engines. It is not
 128          * a huge issue if this is not the case, but we may want to mitigate
 129          * any page crossing penalties if they become an issue.
 130          *
 131          * Called during early_init before we know how many engines there are.
 132          */
 133         BUILD_BUG_ON(KSYNCMAP < I915_NUM_ENGINES);
 134
 135         timeline->i915 = i915;
 136         timeline->name = name;
 137         timeline->pin_count = 0;
 138         timeline->has_initial_breadcrumb = !hwsp;
 139
 140         timeline->hwsp_offset = I915_GEM_HWS_SEQNO_ADDR;
 141         if (!hwsp) {
 142                 unsigned int cacheline;
 143
 144                 hwsp = hwsp_alloc(timeline, &cacheline);
 145                 if (IS_ERR(hwsp))
 146                         return PTR_ERR(hwsp);
 147
 148                 timeline->hwsp_offset = cacheline * CACHELINE_BYTES;
 149         }
 150         timeline->hwsp_ggtt = i915_vma_get(hwsp);
 151
 152         vaddr = i915_gem_object_pin_map(hwsp->obj, I915_MAP_WB);
 153         if (IS_ERR(vaddr)) {
 154                 hwsp_free(timeline);
 155                 i915_vma_put(hwsp);
 156                 return PTR_ERR(vaddr);
 157         }
 158
 159         timeline->hwsp_seqno =
 160                 memset(vaddr + timeline->hwsp_offset, 0, CACHELINE_BYTES);
 161
 162         timeline->fence_context = dma_fence_context_alloc(1);
 163
 164         spin_lock_init(&timeline->lock);
 165
 166         INIT_ACTIVE_REQUEST(&timeline->barrier);
 167         INIT_ACTIVE_REQUEST(&timeline->last_request);
 168         INIT_LIST_HEAD(&timeline->requests);
 169
 170         i915_syncmap_init(&timeline->sync);
 171
 172         return 0;
 173 }
 174
 175 void i915_timelines_init(struct drm_i915_private *i915)
 176 {
 177         struct i915_gt_timelines *gt = &i915->gt.timelines;
 178
 179         mutex_init(&gt->mutex);
 180         INIT_LIST_HEAD(&gt->active_list);
 181
 182         spin_lock_init(&gt->hwsp_lock);
 183         INIT_LIST_HEAD(&gt->hwsp_free_list);
 184
 185         /* via i915_gem_wait_for_idle() */
 186         i915_gem_shrinker_taints_mutex(i915, &gt->mutex);
 187 }
 188
 189 static void timeline_add_to_active(struct i915_timeline *tl)
 190 {
 191         struct i915_gt_timelines *gt = &tl->i915->gt.timelines;
 192
 193         mutex_lock(&gt->mutex);
 194         list_add(&tl->link, &gt->active_list);
 195         mutex_unlock(&gt->mutex);
 196 }
 197
 198 static void timeline_remove_from_active(struct i915_timeline *tl)
 199 {
 200         struct i915_gt_timelines *gt = &tl->i915->gt.timelines;
 201
 202         mutex_lock(&gt->mutex);
 203         list_del(&tl->link);
 204         mutex_unlock(&gt->mutex);
 205 }
 206
 207 /**
 208  * i915_timelines_park - called when the driver idles
 209  * @i915: the drm_i915_private device
 210  *
 211  * When the driver is completely idle, we know that all of our sync points
 212  * have been signaled and our tracking is then entirely redundant. Any request
 213  * to wait upon an older sync point will be completed instantly as we know
 214  * the fence is signaled and therefore we will not even look them up in the
 215  * sync point map.
 216  */
 217 void i915_timelines_park(struct drm_i915_private *i915)
 218 {
 219         struct i915_gt_timelines *gt = &i915->gt.timelines;
 220         struct i915_timeline *timeline;
 221
 222         mutex_lock(&gt->mutex);
 223         list_for_each_entry(timeline, &gt->active_list, link) {
 224                 /*
 225                  * All known fences are completed so we can scrap
 226                  * the current sync point tracking and start afresh,
 227                  * any attempt to wait upon a previous sync point
 228                  * will be skipped as the fence was signaled.
 229                  */
 230                 i915_syncmap_free(&timeline->sync);
 231         }
 232         mutex_unlock(&gt->mutex);
 233 }
 234
 235 void i915_timeline_fini(struct i915_timeline *timeline)
 236 {
 237         GEM_BUG_ON(timeline->pin_count);
 238         GEM_BUG_ON(!list_empty(&timeline->requests));
 239         GEM_BUG_ON(i915_active_request_isset(&timeline->barrier));
 240
 241         i915_syncmap_free(&timeline->sync);
 242         hwsp_free(timeline);
 243
 244         i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);
 245         i915_vma_put(timeline->hwsp_ggtt);
 246 }
 247
 248 struct i915_timeline *
 249 i915_timeline_create(struct drm_i915_private *i915,
 250                      const char *name,
 251                      struct i915_vma *global_hwsp)
 252 {
 253         struct i915_timeline *timeline;
 254         int err;
 255
 256         timeline = kzalloc(sizeof(*timeline), GFP_KERNEL);
 257         if (!timeline)
 258                 return ERR_PTR(-ENOMEM);
 259
 260         err = i915_timeline_init(i915, timeline, name, global_hwsp);
 261         if (err) {
 262                 kfree(timeline);
 263                 return ERR_PTR(err);
 264         }
 265
 266         kref_init(&timeline->kref);
 267
 268         return timeline;
 269 }
 270
 271 int i915_timeline_pin(struct i915_timeline *tl)
 272 {
 273         int err;
 274
 275         if (tl->pin_count++)
 276                 return 0;
 277         GEM_BUG_ON(!tl->pin_count);
 278
 279         err = i915_vma_pin(tl->hwsp_ggtt, 0, 0, PIN_GLOBAL | PIN_HIGH);
 280         if (err)
 281                 goto unpin;
 282
 283         tl->hwsp_offset =
 284                 i915_ggtt_offset(tl->hwsp_ggtt) +
 285                 offset_in_page(tl->hwsp_offset);
 286
 287         timeline_add_to_active(tl);
 288
 289         return 0;
 290
 291 unpin:
 292         tl->pin_count = 0;
 293         return err;
 294 }
 295
 296 void i915_timeline_unpin(struct i915_timeline *tl)
 297 {
 298         GEM_BUG_ON(!tl->pin_count);
 299         if (--tl->pin_count)
 300                 return;
 301
 302         timeline_remove_from_active(tl);
 303
 304         /*
 305          * Since this timeline is idle, all bariers upon which we were waiting
 306          * must also be complete and so we can discard the last used barriers
 307          * without loss of information.
 308          */
 309         i915_syncmap_free(&tl->sync);
 310
 311         __i915_vma_unpin(tl->hwsp_ggtt);
 312 }
 313
 314 void __i915_timeline_free(struct kref *kref)
 315 {
 316         struct i915_timeline *timeline =
 317                 container_of(kref, typeof(*timeline), kref);
 318
 319         i915_timeline_fini(timeline);
 320         kfree(timeline);
 321 }
 322
 323 void i915_timelines_fini(struct drm_i915_private *i915)
 324 {
 325         struct i915_gt_timelines *gt = &i915->gt.timelines;
 326
 327         GEM_BUG_ON(!list_empty(&gt->active_list));
 328         GEM_BUG_ON(!list_empty(&gt->hwsp_free_list));
 329
 330         mutex_destroy(&gt->mutex);
 331 }
 332
 333 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 334 #include "selftests/mock_timeline.c"
 335 #include "selftests/i915_timeline.c"
 336 #endif