drivers/gpu/drm/i915/gt/intel_ring.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include "gem/i915_gem_lmem.h"
   7 #include "gem/i915_gem_object.h"
   8
   9 #include "i915_drv.h"
  10 #include "i915_vma.h"
  11 #include "intel_engine.h"
  12 #include "intel_gpu_commands.h"
  13 #include "intel_ring.h"
  14 #include "intel_timeline.h"
  15
  16 unsigned int intel_ring_update_space(struct intel_ring *ring)
  17 {
  18         unsigned int space;
  19
  20         space = __intel_ring_space(ring->head, ring->emit, ring->size);
  21
  22         ring->space = space;
  23         return space;
  24 }
  25
  26 void __intel_ring_pin(struct intel_ring *ring)
  27 {
  28         GEM_BUG_ON(!atomic_read(&ring->pin_count));
  29         atomic_inc(&ring->pin_count);
  30 }
  31
  32 int intel_ring_pin(struct intel_ring *ring, struct i915_gem_ww_ctx *ww)
  33 {
  34         struct i915_vma *vma = ring->vma;
  35         unsigned int flags;
  36         void *addr;
  37         int ret;
  38
  39         if (atomic_fetch_inc(&ring->pin_count))
  40                 return 0;
  41
  42         /* Ring wraparound at offset 0 sometimes hangs. No idea why. */
  43         flags = PIN_OFFSET_BIAS | i915_ggtt_pin_bias(vma);
  44
  45         if (i915_gem_object_is_stolen(vma->obj))
  46                 flags |= PIN_MAPPABLE;
  47         else
  48                 flags |= PIN_HIGH;
  49
  50         ret = i915_ggtt_pin(vma, ww, 0, flags);
  51         if (unlikely(ret))
  52                 goto err_unpin;
  53
  54         if (i915_vma_is_map_and_fenceable(vma)) {
  55                 addr = (void __force *)i915_vma_pin_iomap(vma);
  56         } else {
  57                 int type = i915_coherent_map_type(vma->vm->i915, vma->obj, false);
  58
  59                 addr = i915_gem_object_pin_map(vma->obj, type);
  60         }
  61
  62         if (IS_ERR(addr)) {
  63                 ret = PTR_ERR(addr);
  64                 goto err_ring;
  65         }
  66
  67         i915_vma_make_unshrinkable(vma);
  68
  69         /* Discard any unused bytes beyond that submitted to hw. */
  70         intel_ring_reset(ring, ring->emit);
  71
  72         ring->vaddr = addr;
  73         return 0;
  74
  75 err_ring:
  76         i915_vma_unpin(vma);
  77 err_unpin:
  78         atomic_dec(&ring->pin_count);
  79         return ret;
  80 }
  81
  82 void intel_ring_reset(struct intel_ring *ring, u32 tail)
  83 {
  84         tail = intel_ring_wrap(ring, tail);
  85         ring->tail = tail;
  86         ring->head = tail;
  87         ring->emit = tail;
  88         intel_ring_update_space(ring);
  89 }
  90
  91 void intel_ring_unpin(struct intel_ring *ring)
  92 {
  93         struct i915_vma *vma = ring->vma;
  94
  95         if (!atomic_dec_and_test(&ring->pin_count))
  96                 return;
  97
  98         i915_vma_unset_ggtt_write(vma);
  99         if (i915_vma_is_map_and_fenceable(vma))
 100                 i915_vma_unpin_iomap(vma);
 101         else
 102                 i915_gem_object_unpin_map(vma->obj);
 103
 104         i915_vma_make_purgeable(vma);
 105         i915_vma_unpin(vma);
 106 }
 107
 108 static struct i915_vma *create_ring_vma(struct i915_ggtt *ggtt, int size)
 109 {
 110         struct i915_address_space *vm = &ggtt->vm;
 111         struct drm_i915_private *i915 = vm->i915;
 112         struct drm_i915_gem_object *obj;
 113         struct i915_vma *vma;
 114
 115         obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_VOLATILE);
 116         if (IS_ERR(obj) && i915_ggtt_has_aperture(ggtt))
 117                 obj = i915_gem_object_create_stolen(i915, size);
 118         if (IS_ERR(obj))
 119                 obj = i915_gem_object_create_internal(i915, size);
 120         if (IS_ERR(obj))
 121                 return ERR_CAST(obj);
 122
 123         /*
 124          * Mark ring buffers as read-only from GPU side (so no stray overwrites)
 125          * if supported by the platform's GGTT.
 126          */
 127         if (vm->has_read_only)
 128                 i915_gem_object_set_readonly(obj);
 129
 130         vma = i915_vma_instance(obj, vm, NULL);
 131         if (IS_ERR(vma))
 132                 goto err;
 133
 134         return vma;
 135
 136 err:
 137         i915_gem_object_put(obj);
 138         return vma;
 139 }
 140
 141 struct intel_ring *
 142 intel_engine_create_ring(struct intel_engine_cs *engine, int size)
 143 {
 144         struct drm_i915_private *i915 = engine->i915;
 145         struct intel_ring *ring;
 146         struct i915_vma *vma;
 147
 148         GEM_BUG_ON(!is_power_of_2(size));
 149         GEM_BUG_ON(RING_CTL_SIZE(size) & ~RING_NR_PAGES);
 150
 151         ring = kzalloc(sizeof(*ring), GFP_KERNEL);
 152         if (!ring)
 153                 return ERR_PTR(-ENOMEM);
 154
 155         kref_init(&ring->ref);
 156         ring->size = size;
 157         ring->wrap = BITS_PER_TYPE(ring->size) - ilog2(size);
 158
 159         /*
 160          * Workaround an erratum on the i830 which causes a hang if
 161          * the TAIL pointer points to within the last 2 cachelines
 162          * of the buffer.
 163          */
 164         ring->effective_size = size;
 165         if (IS_I830(i915) || IS_I845G(i915))
 166                 ring->effective_size -= 2 * CACHELINE_BYTES;
 167
 168         intel_ring_update_space(ring);
 169
 170         vma = create_ring_vma(engine->gt->ggtt, size);
 171         if (IS_ERR(vma)) {
 172                 kfree(ring);
 173                 return ERR_CAST(vma);
 174         }
 175         ring->vma = vma;
 176
 177         return ring;
 178 }
 179
 180 void intel_ring_free(struct kref *ref)
 181 {
 182         struct intel_ring *ring = container_of(ref, typeof(*ring), ref);
 183
 184         i915_vma_put(ring->vma);
 185         kfree(ring);
 186 }
 187
 188 static noinline int
 189 wait_for_space(struct intel_ring *ring,
 190                struct intel_timeline *tl,
 191                unsigned int bytes)
 192 {
 193         struct i915_request *target;
 194         long timeout;
 195
 196         if (intel_ring_update_space(ring) >= bytes)
 197                 return 0;
 198
 199         GEM_BUG_ON(list_empty(&tl->requests));
 200         list_for_each_entry(target, &tl->requests, link) {
 201                 if (target->ring != ring)
 202                         continue;
 203
 204                 /* Would completion of this request free enough space? */
 205                 if (bytes <= __intel_ring_space(target->postfix,
 206                                                 ring->emit, ring->size))
 207                         break;
 208         }
 209
 210         if (GEM_WARN_ON(&target->link == &tl->requests))
 211                 return -ENOSPC;
 212
 213         timeout = i915_request_wait(target,
 214                                     I915_WAIT_INTERRUPTIBLE,
 215                                     MAX_SCHEDULE_TIMEOUT);
 216         if (timeout < 0)
 217                 return timeout;
 218
 219         i915_request_retire_upto(target);
 220
 221         intel_ring_update_space(ring);
 222         GEM_BUG_ON(ring->space < bytes);
 223         return 0;
 224 }
 225
 226 u32 *intel_ring_begin(struct i915_request *rq, unsigned int num_dwords)
 227 {
 228         struct intel_ring *ring = rq->ring;
 229         const unsigned int remain_usable = ring->effective_size - ring->emit;
 230         const unsigned int bytes = num_dwords * sizeof(u32);
 231         unsigned int need_wrap = 0;
 232         unsigned int total_bytes;
 233         u32 *cs;
 234
 235         /* Packets must be qword aligned. */
 236         GEM_BUG_ON(num_dwords & 1);
 237
 238         total_bytes = bytes + rq->reserved_space;
 239         GEM_BUG_ON(total_bytes > ring->effective_size);
 240
 241         if (unlikely(total_bytes > remain_usable)) {
 242                 const int remain_actual = ring->size - ring->emit;
 243
 244                 if (bytes > remain_usable) {
 245                         /*
 246                          * Not enough space for the basic request. So need to
 247                          * flush out the remainder and then wait for
 248                          * base + reserved.
 249                          */
 250                         total_bytes += remain_actual;
 251                         need_wrap = remain_actual | 1;
 252                 } else  {
 253                         /*
 254                          * The base request will fit but the reserved space
 255                          * falls off the end. So we don't need an immediate
 256                          * wrap and only need to effectively wait for the
 257                          * reserved size from the start of ringbuffer.
 258                          */
 259                         total_bytes = rq->reserved_space + remain_actual;
 260                 }
 261         }
 262
 263         if (unlikely(total_bytes > ring->space)) {
 264                 int ret;
 265
 266                 /*
 267                  * Space is reserved in the ringbuffer for finalising the
 268                  * request, as that cannot be allowed to fail. During request
 269                  * finalisation, reserved_space is set to 0 to stop the
 270                  * overallocation and the assumption is that then we never need
 271                  * to wait (which has the risk of failing with EINTR).
 272                  *
 273                  * See also i915_request_alloc() and i915_request_add().
 274                  */
 275                 GEM_BUG_ON(!rq->reserved_space);
 276
 277                 ret = wait_for_space(ring,
 278                                      i915_request_timeline(rq),
 279                                      total_bytes);
 280                 if (unlikely(ret))
 281                         return ERR_PTR(ret);
 282         }
 283
 284         if (unlikely(need_wrap)) {
 285                 need_wrap &= ~1;
 286                 GEM_BUG_ON(need_wrap > ring->space);
 287                 GEM_BUG_ON(ring->emit + need_wrap > ring->size);
 288                 GEM_BUG_ON(!IS_ALIGNED(need_wrap, sizeof(u64)));
 289
 290                 /* Fill the tail with MI_NOOP */
 291                 memset64(ring->vaddr + ring->emit, 0, need_wrap / sizeof(u64));
 292                 ring->space -= need_wrap;
 293                 ring->emit = 0;
 294         }
 295
 296         GEM_BUG_ON(ring->emit > ring->size - bytes);
 297         GEM_BUG_ON(ring->space < bytes);
 298         cs = ring->vaddr + ring->emit;
 299         GEM_DEBUG_EXEC(memset32(cs, POISON_INUSE, bytes / sizeof(*cs)));
 300         ring->emit += bytes;
 301         ring->space -= bytes;
 302
 303         return cs;
 304 }
 305
 306 /* Align the ring tail to a cacheline boundary */
 307 int intel_ring_cacheline_align(struct i915_request *rq)
 308 {
 309         int num_dwords;
 310         void *cs;
 311
 312         num_dwords = (rq->ring->emit & (CACHELINE_BYTES - 1)) / sizeof(u32);
 313         if (num_dwords == 0)
 314                 return 0;
 315
 316         num_dwords = CACHELINE_DWORDS - num_dwords;
 317         GEM_BUG_ON(num_dwords & 1);
 318
 319         cs = intel_ring_begin(rq, num_dwords);
 320         if (IS_ERR(cs))
 321                 return PTR_ERR(cs);
 322
 323         memset64(cs, (u64)MI_NOOP << 32 | MI_NOOP, num_dwords / 2);
 324         intel_ring_advance(rq, cs + num_dwords);
 325
 326         GEM_BUG_ON(rq->ring->emit & (CACHELINE_BYTES - 1));
 327         return 0;
 328 }
 329
 330 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 331 #include "selftest_ring.c"
 332 #endif