2 * Copyright 2017 Red Hat
3 * Parts ported from amdgpu (fence wait code).
4 * Copyright 2016 Advanced Micro Devices, Inc.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
32 * DRM synchronisation objects (syncobj, see struct &drm_syncobj) are
33 * persistent objects that contain an optional fence. The fence can be updated
34 * with a new fence, or be NULL.
36 * syncobj's can be waited upon, where it will wait for the underlying
39 * syncobj's can be export to fd's and back, these fd's are opaque and
40 * have no other use case, except passing the syncobj between processes.
42 * Their primary use-case is to implement Vulkan fences and semaphores.
44 * syncobj have a kref reference count, but also have an optional file.
45 * The file is only created once the syncobj is exported.
46 * The file takes a reference on the kref.
50 #include <linux/file.h>
52 #include <linux/anon_inodes.h>
53 #include <linux/sync_file.h>
54 #include <linux/sched/signal.h>
56 #include "drm_internal.h"
57 #include <drm/drm_syncobj.h>
59 struct syncobj_wait_entry {
60 struct list_head node;
61 struct task_struct *task;
62 struct dma_fence *fence;
63 struct dma_fence_cb fence_cb;
66 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
67 struct syncobj_wait_entry *wait);
70 * drm_syncobj_find - lookup and reference a sync object.
71 * @file_private: drm file private pointer
72 * @handle: sync object handle to lookup.
74 * Returns a reference to the syncobj pointed to by handle or NULL. The
75 * reference must be released by calling drm_syncobj_put().
77 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
80 struct drm_syncobj *syncobj;
82 spin_lock(&file_private->syncobj_table_lock);
84 /* Check if we currently have a reference on the object */
85 syncobj = idr_find(&file_private->syncobj_idr, handle);
87 drm_syncobj_get(syncobj);
89 spin_unlock(&file_private->syncobj_table_lock);
93 EXPORT_SYMBOL(drm_syncobj_find);
95 static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,
96 struct syncobj_wait_entry *wait)
101 spin_lock(&syncobj->lock);
102 /* We've already tried once to get a fence and failed. Now that we
103 * have the lock, try one more time just to be sure we don't add a
104 * callback when a fence has already been set.
107 wait->fence = dma_fence_get(
108 rcu_dereference_protected(syncobj->fence, 1));
110 list_add_tail(&wait->node, &syncobj->cb_list);
111 spin_unlock(&syncobj->lock);
114 static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj,
115 struct syncobj_wait_entry *wait)
117 if (!wait->node.next)
120 spin_lock(&syncobj->lock);
121 list_del_init(&wait->node);
122 spin_unlock(&syncobj->lock);
126 * drm_syncobj_replace_fence - replace fence in a sync object.
127 * @syncobj: Sync object to replace fence in
128 * @fence: fence to install in sync file.
130 * This replaces the fence on a sync object.
132 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
133 struct dma_fence *fence)
135 struct dma_fence *old_fence;
136 struct syncobj_wait_entry *cur, *tmp;
139 dma_fence_get(fence);
141 spin_lock(&syncobj->lock);
143 old_fence = rcu_dereference_protected(syncobj->fence,
144 lockdep_is_held(&syncobj->lock));
145 rcu_assign_pointer(syncobj->fence, fence);
147 if (fence != old_fence) {
148 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node) {
149 list_del_init(&cur->node);
150 syncobj_wait_syncobj_func(syncobj, cur);
154 spin_unlock(&syncobj->lock);
156 dma_fence_put(old_fence);
158 EXPORT_SYMBOL(drm_syncobj_replace_fence);
161 * drm_syncobj_assign_null_handle - assign a stub fence to the sync object
162 * @syncobj: sync object to assign the fence on
164 * Assign a already signaled stub fence to the sync object.
166 static void drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
168 struct dma_fence *fence = dma_fence_get_stub();
170 drm_syncobj_replace_fence(syncobj, fence);
171 dma_fence_put(fence);
175 * drm_syncobj_find_fence - lookup and reference the fence in a sync object
176 * @file_private: drm file private pointer
177 * @handle: sync object handle to lookup.
178 * @point: timeline point
179 * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not
180 * @fence: out parameter for the fence
182 * This is just a convenience function that combines drm_syncobj_find() and
183 * drm_syncobj_fence_get().
185 * Returns 0 on success or a negative error value on failure. On success @fence
186 * contains a reference to the fence, which must be released by calling
189 int drm_syncobj_find_fence(struct drm_file *file_private,
190 u32 handle, u64 point, u64 flags,
191 struct dma_fence **fence)
193 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
199 *fence = drm_syncobj_fence_get(syncobj);
203 drm_syncobj_put(syncobj);
206 EXPORT_SYMBOL(drm_syncobj_find_fence);
209 * drm_syncobj_free - free a sync object.
210 * @kref: kref to free.
212 * Only to be called from kref_put in drm_syncobj_put.
214 void drm_syncobj_free(struct kref *kref)
216 struct drm_syncobj *syncobj = container_of(kref,
219 drm_syncobj_replace_fence(syncobj, NULL);
222 EXPORT_SYMBOL(drm_syncobj_free);
225 * drm_syncobj_create - create a new syncobj
226 * @out_syncobj: returned syncobj
227 * @flags: DRM_SYNCOBJ_* flags
228 * @fence: if non-NULL, the syncobj will represent this fence
230 * This is the first function to create a sync object. After creating, drivers
231 * probably want to make it available to userspace, either through
232 * drm_syncobj_get_handle() or drm_syncobj_get_fd().
234 * Returns 0 on success or a negative error value on failure.
236 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
237 struct dma_fence *fence)
239 struct drm_syncobj *syncobj;
241 syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
245 kref_init(&syncobj->refcount);
246 INIT_LIST_HEAD(&syncobj->cb_list);
247 spin_lock_init(&syncobj->lock);
249 if (flags & DRM_SYNCOBJ_CREATE_SIGNALED)
250 drm_syncobj_assign_null_handle(syncobj);
253 drm_syncobj_replace_fence(syncobj, fence);
255 *out_syncobj = syncobj;
258 EXPORT_SYMBOL(drm_syncobj_create);
261 * drm_syncobj_get_handle - get a handle from a syncobj
262 * @file_private: drm file private pointer
263 * @syncobj: Sync object to export
264 * @handle: out parameter with the new handle
266 * Exports a sync object created with drm_syncobj_create() as a handle on
267 * @file_private to userspace.
269 * Returns 0 on success or a negative error value on failure.
271 int drm_syncobj_get_handle(struct drm_file *file_private,
272 struct drm_syncobj *syncobj, u32 *handle)
276 /* take a reference to put in the idr */
277 drm_syncobj_get(syncobj);
279 idr_preload(GFP_KERNEL);
280 spin_lock(&file_private->syncobj_table_lock);
281 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
282 spin_unlock(&file_private->syncobj_table_lock);
287 drm_syncobj_put(syncobj);
294 EXPORT_SYMBOL(drm_syncobj_get_handle);
296 static int drm_syncobj_create_as_handle(struct drm_file *file_private,
297 u32 *handle, uint32_t flags)
300 struct drm_syncobj *syncobj;
302 ret = drm_syncobj_create(&syncobj, flags, NULL);
306 ret = drm_syncobj_get_handle(file_private, syncobj, handle);
307 drm_syncobj_put(syncobj);
311 static int drm_syncobj_destroy(struct drm_file *file_private,
314 struct drm_syncobj *syncobj;
316 spin_lock(&file_private->syncobj_table_lock);
317 syncobj = idr_remove(&file_private->syncobj_idr, handle);
318 spin_unlock(&file_private->syncobj_table_lock);
323 drm_syncobj_put(syncobj);
327 static int drm_syncobj_file_release(struct inode *inode, struct file *file)
329 struct drm_syncobj *syncobj = file->private_data;
331 drm_syncobj_put(syncobj);
335 static const struct file_operations drm_syncobj_file_fops = {
336 .release = drm_syncobj_file_release,
340 * drm_syncobj_get_fd - get a file descriptor from a syncobj
341 * @syncobj: Sync object to export
342 * @p_fd: out parameter with the new file descriptor
344 * Exports a sync object created with drm_syncobj_create() as a file descriptor.
346 * Returns 0 on success or a negative error value on failure.
348 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
353 fd = get_unused_fd_flags(O_CLOEXEC);
357 file = anon_inode_getfile("syncobj_file",
358 &drm_syncobj_file_fops,
362 return PTR_ERR(file);
365 drm_syncobj_get(syncobj);
366 fd_install(fd, file);
371 EXPORT_SYMBOL(drm_syncobj_get_fd);
373 static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
374 u32 handle, int *p_fd)
376 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
382 ret = drm_syncobj_get_fd(syncobj, p_fd);
383 drm_syncobj_put(syncobj);
387 static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
390 struct drm_syncobj *syncobj;
398 if (file->f_op != &drm_syncobj_file_fops) {
403 /* take a reference to put in the idr */
404 syncobj = file->private_data;
405 drm_syncobj_get(syncobj);
407 idr_preload(GFP_KERNEL);
408 spin_lock(&file_private->syncobj_table_lock);
409 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
410 spin_unlock(&file_private->syncobj_table_lock);
417 drm_syncobj_put(syncobj);
423 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
426 struct dma_fence *fence = sync_file_get_fence(fd);
427 struct drm_syncobj *syncobj;
432 syncobj = drm_syncobj_find(file_private, handle);
434 dma_fence_put(fence);
438 drm_syncobj_replace_fence(syncobj, fence);
439 dma_fence_put(fence);
440 drm_syncobj_put(syncobj);
444 static int drm_syncobj_export_sync_file(struct drm_file *file_private,
445 int handle, int *p_fd)
448 struct dma_fence *fence;
449 struct sync_file *sync_file;
450 int fd = get_unused_fd_flags(O_CLOEXEC);
455 ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence);
459 sync_file = sync_file_create(fence);
461 dma_fence_put(fence);
468 fd_install(fd, sync_file->file);
477 * drm_syncobj_open - initalizes syncobj file-private structures at devnode open time
478 * @file_private: drm file-private structure to set up
480 * Called at device open time, sets up the structure for handling refcounting
484 drm_syncobj_open(struct drm_file *file_private)
486 idr_init_base(&file_private->syncobj_idr, 1);
487 spin_lock_init(&file_private->syncobj_table_lock);
491 drm_syncobj_release_handle(int id, void *ptr, void *data)
493 struct drm_syncobj *syncobj = ptr;
495 drm_syncobj_put(syncobj);
500 * drm_syncobj_release - release file-private sync object resources
501 * @file_private: drm file-private structure to clean up
503 * Called at close time when the filp is going away.
505 * Releases any remaining references on objects by this filp.
508 drm_syncobj_release(struct drm_file *file_private)
510 idr_for_each(&file_private->syncobj_idr,
511 &drm_syncobj_release_handle, file_private);
512 idr_destroy(&file_private->syncobj_idr);
516 drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
517 struct drm_file *file_private)
519 struct drm_syncobj_create *args = data;
521 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
524 /* no valid flags yet */
525 if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
528 return drm_syncobj_create_as_handle(file_private,
529 &args->handle, args->flags);
533 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
534 struct drm_file *file_private)
536 struct drm_syncobj_destroy *args = data;
538 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
541 /* make sure padding is empty */
544 return drm_syncobj_destroy(file_private, args->handle);
548 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
549 struct drm_file *file_private)
551 struct drm_syncobj_handle *args = data;
553 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
559 if (args->flags != 0 &&
560 args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
563 if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
564 return drm_syncobj_export_sync_file(file_private, args->handle,
567 return drm_syncobj_handle_to_fd(file_private, args->handle,
572 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
573 struct drm_file *file_private)
575 struct drm_syncobj_handle *args = data;
577 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
583 if (args->flags != 0 &&
584 args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
587 if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
588 return drm_syncobj_import_sync_file_fence(file_private,
592 return drm_syncobj_fd_to_handle(file_private, args->fd,
596 static void syncobj_wait_fence_func(struct dma_fence *fence,
597 struct dma_fence_cb *cb)
599 struct syncobj_wait_entry *wait =
600 container_of(cb, struct syncobj_wait_entry, fence_cb);
602 wake_up_process(wait->task);
605 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
606 struct syncobj_wait_entry *wait)
608 /* This happens inside the syncobj lock */
609 wait->fence = dma_fence_get(rcu_dereference_protected(syncobj->fence,
610 lockdep_is_held(&syncobj->lock)));
611 wake_up_process(wait->task);
614 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
620 struct syncobj_wait_entry *entries;
621 struct dma_fence *fence;
622 uint32_t signaled_count, i;
624 entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
628 /* Walk the list of sync objects and initialize entries. We do
629 * this up-front so that we can properly return -EINVAL if there is
630 * a syncobj with a missing fence and then never have the chance of
631 * returning -EINVAL again.
634 for (i = 0; i < count; ++i) {
635 entries[i].task = current;
636 entries[i].fence = drm_syncobj_fence_get(syncobjs[i]);
637 if (!entries[i].fence) {
638 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
642 goto cleanup_entries;
646 if (dma_fence_is_signaled(entries[i].fence)) {
647 if (signaled_count == 0 && idx)
653 if (signaled_count == count ||
654 (signaled_count > 0 &&
655 !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
656 goto cleanup_entries;
658 /* There's a very annoying laxness in the dma_fence API here, in
659 * that backends are not required to automatically report when a
660 * fence is signaled prior to fence->ops->enable_signaling() being
661 * called. So here if we fail to match signaled_count, we need to
662 * fallthough and try a 0 timeout wait!
665 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
666 for (i = 0; i < count; ++i)
667 drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]);
671 set_current_state(TASK_INTERRUPTIBLE);
674 for (i = 0; i < count; ++i) {
675 fence = entries[i].fence;
679 if (dma_fence_is_signaled(fence) ||
680 (!entries[i].fence_cb.func &&
681 dma_fence_add_callback(fence,
682 &entries[i].fence_cb,
683 syncobj_wait_fence_func))) {
684 /* The fence has been signaled */
685 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
695 if (signaled_count == count)
703 if (signal_pending(current)) {
704 timeout = -ERESTARTSYS;
708 timeout = schedule_timeout(timeout);
712 __set_current_state(TASK_RUNNING);
715 for (i = 0; i < count; ++i) {
716 drm_syncobj_remove_wait(syncobjs[i], &entries[i]);
717 if (entries[i].fence_cb.func)
718 dma_fence_remove_callback(entries[i].fence,
719 &entries[i].fence_cb);
720 dma_fence_put(entries[i].fence);
728 * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
730 * @timeout_nsec: timeout nsec component in ns, 0 for poll
732 * Calculate the timeout in jiffies from an absolute time in sec/nsec.
734 static signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
736 ktime_t abs_timeout, now;
737 u64 timeout_ns, timeout_jiffies64;
739 /* make 0 timeout means poll - absolute 0 doesn't seem valid */
740 if (timeout_nsec == 0)
743 abs_timeout = ns_to_ktime(timeout_nsec);
746 if (!ktime_after(abs_timeout, now))
749 timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
751 timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
752 /* clamp timeout to avoid infinite timeout */
753 if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
754 return MAX_SCHEDULE_TIMEOUT - 1;
756 return timeout_jiffies64 + 1;
759 static int drm_syncobj_array_wait(struct drm_device *dev,
760 struct drm_file *file_private,
761 struct drm_syncobj_wait *wait,
762 struct drm_syncobj **syncobjs)
764 signed long timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
767 timeout = drm_syncobj_array_wait_timeout(syncobjs,
774 wait->first_signaled = first;
778 static int drm_syncobj_array_find(struct drm_file *file_private,
779 void __user *user_handles,
780 uint32_t count_handles,
781 struct drm_syncobj ***syncobjs_out)
783 uint32_t i, *handles;
784 struct drm_syncobj **syncobjs;
787 handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
791 if (copy_from_user(handles, user_handles,
792 sizeof(uint32_t) * count_handles)) {
794 goto err_free_handles;
797 syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
798 if (syncobjs == NULL) {
800 goto err_free_handles;
803 for (i = 0; i < count_handles; i++) {
804 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
807 goto err_put_syncobjs;
812 *syncobjs_out = syncobjs;
817 drm_syncobj_put(syncobjs[i]);
825 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
829 for (i = 0; i < count; i++)
830 drm_syncobj_put(syncobjs[i]);
835 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
836 struct drm_file *file_private)
838 struct drm_syncobj_wait *args = data;
839 struct drm_syncobj **syncobjs;
842 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
845 if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
846 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
849 if (args->count_handles == 0)
852 ret = drm_syncobj_array_find(file_private,
853 u64_to_user_ptr(args->handles),
859 ret = drm_syncobj_array_wait(dev, file_private,
862 drm_syncobj_array_free(syncobjs, args->count_handles);
868 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
869 struct drm_file *file_private)
871 struct drm_syncobj_array *args = data;
872 struct drm_syncobj **syncobjs;
876 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
882 if (args->count_handles == 0)
885 ret = drm_syncobj_array_find(file_private,
886 u64_to_user_ptr(args->handles),
892 for (i = 0; i < args->count_handles; i++)
893 drm_syncobj_replace_fence(syncobjs[i], NULL);
895 drm_syncobj_array_free(syncobjs, args->count_handles);
901 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
902 struct drm_file *file_private)
904 struct drm_syncobj_array *args = data;
905 struct drm_syncobj **syncobjs;
909 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
915 if (args->count_handles == 0)
918 ret = drm_syncobj_array_find(file_private,
919 u64_to_user_ptr(args->handles),
925 for (i = 0; i < args->count_handles; i++)
926 drm_syncobj_assign_null_handle(syncobjs[i]);
928 drm_syncobj_array_free(syncobjs, args->count_handles);