2 * SPDX-License-Identifier: MIT
4 * Copyright © 2016 Intel Corporation
7 #include <linux/dma-fence-array.h>
8 #include <linux/dma-fence-chain.h>
9 #include <linux/jiffies.h>
11 #include "gt/intel_engine.h"
13 #include "i915_gem_ioctls.h"
14 #include "i915_gem_object.h"
17 i915_gem_object_wait_fence(struct dma_fence *fence,
21 BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
23 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
26 if (dma_fence_is_i915(fence))
27 return i915_request_wait_timeout(to_request(fence), flags, timeout);
29 return dma_fence_wait_timeout(fence,
30 flags & I915_WAIT_INTERRUPTIBLE,
35 i915_gem_object_wait_reservation(struct dma_resv *resv,
39 struct dma_resv_iter cursor;
40 struct dma_fence *fence;
41 long ret = timeout ?: 1;
43 dma_resv_iter_begin(&cursor, resv,
44 dma_resv_usage_rw(flags & I915_WAIT_ALL));
45 dma_resv_for_each_fence_unlocked(&cursor, fence) {
46 ret = i915_gem_object_wait_fence(fence, flags, timeout);
53 dma_resv_iter_end(&cursor);
58 static void fence_set_priority(struct dma_fence *fence,
59 const struct i915_sched_attr *attr)
61 struct i915_request *rq;
62 struct intel_engine_cs *engine;
64 if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence))
67 rq = to_request(fence);
70 rcu_read_lock(); /* RCU serialisation for set-wedged protection */
71 if (engine->sched_engine->schedule)
72 engine->sched_engine->schedule(rq, attr);
76 static inline bool __dma_fence_is_chain(const struct dma_fence *fence)
78 return fence->ops == &dma_fence_chain_ops;
81 void i915_gem_fence_wait_priority(struct dma_fence *fence,
82 const struct i915_sched_attr *attr)
84 if (dma_fence_is_signaled(fence))
89 /* Recurse once into a fence-array */
90 if (dma_fence_is_array(fence)) {
91 struct dma_fence_array *array = to_dma_fence_array(fence);
94 for (i = 0; i < array->num_fences; i++)
95 fence_set_priority(array->fences[i], attr);
96 } else if (__dma_fence_is_chain(fence)) {
97 struct dma_fence *iter;
99 /* The chain is ordered; if we boost the last, we boost all */
100 dma_fence_chain_for_each(iter, fence) {
101 fence_set_priority(to_dma_fence_chain(iter)->fence,
107 fence_set_priority(fence, attr);
110 local_bh_enable(); /* kick the tasklets if queues were reprioritised */
114 i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
116 const struct i915_sched_attr *attr)
118 struct dma_resv_iter cursor;
119 struct dma_fence *fence;
121 dma_resv_iter_begin(&cursor, obj->base.resv,
122 dma_resv_usage_rw(flags & I915_WAIT_ALL));
123 dma_resv_for_each_fence_unlocked(&cursor, fence)
124 i915_gem_fence_wait_priority(fence, attr);
125 dma_resv_iter_end(&cursor);
130 * Waits for rendering to the object to be completed
131 * @obj: i915 gem object
132 * @flags: how to wait (under a lock, for all rendering or just for writes etc)
133 * @timeout: how long to wait
136 i915_gem_object_wait(struct drm_i915_gem_object *obj,
141 GEM_BUG_ON(timeout < 0);
143 timeout = i915_gem_object_wait_reservation(obj->base.resv,
149 return !timeout ? -ETIME : 0;
152 static inline unsigned long nsecs_to_jiffies_timeout(const u64 n)
154 /* nsecs_to_jiffies64() does not guard against overflow */
155 if (NSEC_PER_SEC % HZ &&
156 div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ)
157 return MAX_JIFFY_OFFSET;
159 return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1);
162 static unsigned long to_wait_timeout(s64 timeout_ns)
165 return MAX_SCHEDULE_TIMEOUT;
170 return nsecs_to_jiffies_timeout(timeout_ns);
174 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
175 * @dev: drm device pointer
176 * @data: ioctl data blob
177 * @file: drm file pointer
179 * Returns 0 if successful, else an error is returned with the remaining time in
180 * the timeout parameter.
181 * -ETIME: object is still busy after timeout
182 * -ERESTARTSYS: signal interrupted the wait
183 * -ENONENT: object doesn't exist
184 * Also possible, but rare:
185 * -EAGAIN: incomplete, restart syscall
187 * -ENODEV: Internal IRQ fail
188 * -E?: The add request failed
190 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
191 * non-zero timeout parameter the wait ioctl will wait for the given number of
192 * nanoseconds on an object becoming unbusy. Since the wait itself does so
193 * without holding struct_mutex the object may become re-busied before this
194 * function completes. A similar but shorter * race condition exists in the busy
198 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
200 struct drm_i915_gem_wait *args = data;
201 struct drm_i915_gem_object *obj;
205 if (args->flags != 0)
208 obj = i915_gem_object_lookup(file, args->bo_handle);
214 ret = i915_gem_object_wait(obj,
215 I915_WAIT_INTERRUPTIBLE |
218 to_wait_timeout(args->timeout_ns));
220 if (args->timeout_ns > 0) {
221 args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));
222 if (args->timeout_ns < 0)
223 args->timeout_ns = 0;
226 * Apparently ktime isn't accurate enough and occasionally has a
227 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
228 * things up to make the test happy. We allow up to 1 jiffy.
230 * This is a regression from the timespec->ktime conversion.
232 if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns))
233 args->timeout_ns = 0;
235 /* Asked to wait beyond the jiffie/scheduler precision? */
236 if (ret == -ETIME && args->timeout_ns)
240 i915_gem_object_put(obj);
245 * i915_gem_object_wait_migration - Sync an accelerated migration operation
246 * @obj: The migrating object.
247 * @flags: waiting flags. Currently supports only I915_WAIT_INTERRUPTIBLE.
249 * Wait for any pending async migration operation on the object,
250 * whether it's explicitly (i915_gem_object_migrate()) or implicitly
251 * (swapin, initial clearing) initiated.
253 * Return: 0 if successful, -ERESTARTSYS if a signal was hit during waiting.
255 int i915_gem_object_wait_migration(struct drm_i915_gem_object *obj,
260 return i915_gem_object_wait_moving_fence(obj, !!(flags & I915_WAIT_INTERRUPTIBLE));