drivers/gpu/drm/i915/gem/i915_gem_wait.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2016 Intel Corporation
   5  */
   6
   7 #include <linux/dma-fence-array.h>
   8 #include <linux/dma-fence-chain.h>
   9 #include <linux/jiffies.h>
  10
  11 #include "gt/intel_engine.h"
  12
  13 #include "i915_gem_ioctls.h"
  14 #include "i915_gem_object.h"
  15
  16 static long
  17 i915_gem_object_wait_fence(struct dma_fence *fence,
  18                            unsigned int flags,
  19                            long timeout)
  20 {
  21         BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
  22
  23         if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
  24                 return timeout;
  25
  26         if (dma_fence_is_i915(fence))
  27                 return i915_request_wait_timeout(to_request(fence), flags, timeout);
  28
  29         return dma_fence_wait_timeout(fence,
  30                                       flags & I915_WAIT_INTERRUPTIBLE,
  31                                       timeout);
  32 }
  33
  34 static long
  35 i915_gem_object_wait_reservation(struct dma_resv *resv,
  36                                  unsigned int flags,
  37                                  long timeout)
  38 {
  39         struct dma_resv_iter cursor;
  40         struct dma_fence *fence;
  41         long ret = timeout ?: 1;
  42
  43         dma_resv_iter_begin(&cursor, resv,
  44                             dma_resv_usage_rw(flags & I915_WAIT_ALL));
  45         dma_resv_for_each_fence_unlocked(&cursor, fence) {
  46                 ret = i915_gem_object_wait_fence(fence, flags, timeout);
  47                 if (ret <= 0)
  48                         break;
  49
  50                 if (timeout)
  51                         timeout = ret;
  52         }
  53         dma_resv_iter_end(&cursor);
  54
  55         return ret;
  56 }
  57
  58 static void fence_set_priority(struct dma_fence *fence,
  59                                const struct i915_sched_attr *attr)
  60 {
  61         struct i915_request *rq;
  62         struct intel_engine_cs *engine;
  63
  64         if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence))
  65                 return;
  66
  67         rq = to_request(fence);
  68         engine = rq->engine;
  69
  70         rcu_read_lock(); /* RCU serialisation for set-wedged protection */
  71         if (engine->sched_engine->schedule)
  72                 engine->sched_engine->schedule(rq, attr);
  73         rcu_read_unlock();
  74 }
  75
  76 static inline bool __dma_fence_is_chain(const struct dma_fence *fence)
  77 {
  78         return fence->ops == &dma_fence_chain_ops;
  79 }
  80
  81 void i915_gem_fence_wait_priority(struct dma_fence *fence,
  82                                   const struct i915_sched_attr *attr)
  83 {
  84         if (dma_fence_is_signaled(fence))
  85                 return;
  86
  87         local_bh_disable();
  88
  89         /* Recurse once into a fence-array */
  90         if (dma_fence_is_array(fence)) {
  91                 struct dma_fence_array *array = to_dma_fence_array(fence);
  92                 int i;
  93
  94                 for (i = 0; i < array->num_fences; i++)
  95                         fence_set_priority(array->fences[i], attr);
  96         } else if (__dma_fence_is_chain(fence)) {
  97                 struct dma_fence *iter;
  98
  99                 /* The chain is ordered; if we boost the last, we boost all */
 100                 dma_fence_chain_for_each(iter, fence) {
 101                         fence_set_priority(to_dma_fence_chain(iter)->fence,
 102                                            attr);
 103                         break;
 104                 }
 105                 dma_fence_put(iter);
 106         } else {
 107                 fence_set_priority(fence, attr);
 108         }
 109
 110         local_bh_enable(); /* kick the tasklets if queues were reprioritised */
 111 }
 112
 113 int
 114 i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
 115                               unsigned int flags,
 116                               const struct i915_sched_attr *attr)
 117 {
 118         struct dma_resv_iter cursor;
 119         struct dma_fence *fence;
 120
 121         dma_resv_iter_begin(&cursor, obj->base.resv,
 122                             dma_resv_usage_rw(flags & I915_WAIT_ALL));
 123         dma_resv_for_each_fence_unlocked(&cursor, fence)
 124                 i915_gem_fence_wait_priority(fence, attr);
 125         dma_resv_iter_end(&cursor);
 126         return 0;
 127 }
 128
 129 /**
 130  * Waits for rendering to the object to be completed
 131  * @obj: i915 gem object
 132  * @flags: how to wait (under a lock, for all rendering or just for writes etc)
 133  * @timeout: how long to wait
 134  */
 135 int
 136 i915_gem_object_wait(struct drm_i915_gem_object *obj,
 137                      unsigned int flags,
 138                      long timeout)
 139 {
 140         might_sleep();
 141         GEM_BUG_ON(timeout < 0);
 142
 143         timeout = i915_gem_object_wait_reservation(obj->base.resv,
 144                                                    flags, timeout);
 145
 146         if (timeout < 0)
 147                 return timeout;
 148
 149         return !timeout ? -ETIME : 0;
 150 }
 151
 152 static inline unsigned long nsecs_to_jiffies_timeout(const u64 n)
 153 {
 154         /* nsecs_to_jiffies64() does not guard against overflow */
 155         if (NSEC_PER_SEC % HZ &&
 156             div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ)
 157                 return MAX_JIFFY_OFFSET;
 158
 159         return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1);
 160 }
 161
 162 static unsigned long to_wait_timeout(s64 timeout_ns)
 163 {
 164         if (timeout_ns < 0)
 165                 return MAX_SCHEDULE_TIMEOUT;
 166
 167         if (timeout_ns == 0)
 168                 return 0;
 169
 170         return nsecs_to_jiffies_timeout(timeout_ns);
 171 }
 172
 173 /**
 174  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
 175  * @dev: drm device pointer
 176  * @data: ioctl data blob
 177  * @file: drm file pointer
 178  *
 179  * Returns 0 if successful, else an error is returned with the remaining time in
 180  * the timeout parameter.
 181  *  -ETIME: object is still busy after timeout
 182  *  -ERESTARTSYS: signal interrupted the wait
 183  *  -ENONENT: object doesn't exist
 184  * Also possible, but rare:
 185  *  -EAGAIN: incomplete, restart syscall
 186  *  -ENOMEM: damn
 187  *  -ENODEV: Internal IRQ fail
 188  *  -E?: The add request failed
 189  *
 190  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
 191  * non-zero timeout parameter the wait ioctl will wait for the given number of
 192  * nanoseconds on an object becoming unbusy. Since the wait itself does so
 193  * without holding struct_mutex the object may become re-busied before this
 194  * function completes. A similar but shorter * race condition exists in the busy
 195  * ioctl
 196  */
 197 int
 198 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 199 {
 200         struct drm_i915_gem_wait *args = data;
 201         struct drm_i915_gem_object *obj;
 202         ktime_t start;
 203         long ret;
 204
 205         if (args->flags != 0)
 206                 return -EINVAL;
 207
 208         obj = i915_gem_object_lookup(file, args->bo_handle);
 209         if (!obj)
 210                 return -ENOENT;
 211
 212         start = ktime_get();
 213
 214         ret = i915_gem_object_wait(obj,
 215                                    I915_WAIT_INTERRUPTIBLE |
 216                                    I915_WAIT_PRIORITY |
 217                                    I915_WAIT_ALL,
 218                                    to_wait_timeout(args->timeout_ns));
 219
 220         if (args->timeout_ns > 0) {
 221                 args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));
 222                 if (args->timeout_ns < 0)
 223                         args->timeout_ns = 0;
 224
 225                 /*
 226                  * Apparently ktime isn't accurate enough and occasionally has a
 227                  * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
 228                  * things up to make the test happy. We allow up to 1 jiffy.
 229                  *
 230                  * This is a regression from the timespec->ktime conversion.
 231                  */
 232                 if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns))
 233                         args->timeout_ns = 0;
 234
 235                 /* Asked to wait beyond the jiffie/scheduler precision? */
 236                 if (ret == -ETIME && args->timeout_ns)
 237                         ret = -EAGAIN;
 238         }
 239
 240         i915_gem_object_put(obj);
 241         return ret;
 242 }
 243
 244 /**
 245  * i915_gem_object_wait_migration - Sync an accelerated migration operation
 246  * @obj: The migrating object.
 247  * @flags: waiting flags. Currently supports only I915_WAIT_INTERRUPTIBLE.
 248  *
 249  * Wait for any pending async migration operation on the object,
 250  * whether it's explicitly (i915_gem_object_migrate()) or implicitly
 251  * (swapin, initial clearing) initiated.
 252  *
 253  * Return: 0 if successful, -ERESTARTSYS if a signal was hit during waiting.
 254  */
 255 int i915_gem_object_wait_migration(struct drm_i915_gem_object *obj,
 256                                    unsigned int flags)
 257 {
 258         might_sleep();
 259
 260         return i915_gem_object_wait_moving_fence(obj, !!(flags & I915_WAIT_INTERRUPTIBLE));
 261 }