drivers/gpu/drm/drm_syncobj.c

   1 /*
   2  * Copyright 2017 Red Hat
   3  * Parts ported from amdgpu (fence wait code).
   4  * Copyright 2016 Advanced Micro Devices, Inc.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  23  * IN THE SOFTWARE.
  24  *
  25  * Authors:
  26  *
  27  */
  28
  29 /**
  30  * DOC: Overview
  31  *
  32  * DRM synchronisation objects (syncobj, see struct &drm_syncobj) provide a
  33  * container for a synchronization primitive which can be used by userspace
  34  * to explicitly synchronize GPU commands, can be shared between userspace
  35  * processes, and can be shared between different DRM drivers.
  36  * Their primary use-case is to implement Vulkan fences and semaphores.
  37  * The syncobj userspace API provides ioctls for several operations:
  38  *
  39  *  - Creation and destruction of syncobjs
  40  *  - Import and export of syncobjs to/from a syncobj file descriptor
  41  *  - Import and export a syncobj's underlying fence to/from a sync file
  42  *  - Reset a syncobj (set its fence to NULL)
  43  *  - Signal a syncobj (set a trivially signaled fence)
  44  *  - Wait for a syncobj's fence to appear and be signaled
  45  *
  46  * The syncobj userspace API also provides operations to manipulate a syncobj
  47  * in terms of a timeline of struct &dma_fence_chain rather than a single
  48  * struct &dma_fence, through the following operations:
  49  *
  50  *   - Signal a given point on the timeline
  51  *   - Wait for a given point to appear and/or be signaled
  52  *   - Import and export from/to a given point of a timeline
  53  *
  54  * At it's core, a syncobj is simply a wrapper around a pointer to a struct
  55  * &dma_fence which may be NULL.
  56  * When a syncobj is first created, its pointer is either NULL or a pointer
  57  * to an already signaled fence depending on whether the
  58  * &DRM_SYNCOBJ_CREATE_SIGNALED flag is passed to
  59  * &DRM_IOCTL_SYNCOBJ_CREATE.
  60  *
  61  * If the syncobj is considered as a binary (its state is either signaled or
  62  * unsignaled) primitive, when GPU work is enqueued in a DRM driver to signal
  63  * the syncobj, the syncobj's fence is replaced with a fence which will be
  64  * signaled by the completion of that work.
  65  * If the syncobj is considered as a timeline primitive, when GPU work is
  66  * enqueued in a DRM driver to signal the a given point of the syncobj, a new
  67  * struct &dma_fence_chain pointing to the DRM driver's fence and also
  68  * pointing to the previous fence that was in the syncobj. The new struct
  69  * &dma_fence_chain fence replace the syncobj's fence and will be signaled by
  70  * completion of the DRM driver's work and also any work associated with the
  71  * fence previously in the syncobj.
  72  *
  73  * When GPU work which waits on a syncobj is enqueued in a DRM driver, at the
  74  * time the work is enqueued, it waits on the syncobj's fence before
  75  * submitting the work to hardware. That fence is either :
  76  *
  77  *    - The syncobj's current fence if the syncobj is considered as a binary
  78  *      primitive.
  79  *    - The struct &dma_fence associated with a given point if the syncobj is
  80  *      considered as a timeline primitive.
  81  *
  82  * If the syncobj's fence is NULL or not present in the syncobj's timeline,
  83  * the enqueue operation is expected to fail.
  84  *
  85  * With binary syncobj, all manipulation of the syncobjs's fence happens in
  86  * terms of the current fence at the time the ioctl is called by userspace
  87  * regardless of whether that operation is an immediate host-side operation
  88  * (signal or reset) or or an operation which is enqueued in some driver
  89  * queue. &DRM_IOCTL_SYNCOBJ_RESET and &DRM_IOCTL_SYNCOBJ_SIGNAL can be used
  90  * to manipulate a syncobj from the host by resetting its pointer to NULL or
  91  * setting its pointer to a fence which is already signaled.
  92  *
  93  * With a timeline syncobj, all manipulation of the synobj's fence happens in
  94  * terms of a u64 value referring to point in the timeline. See
  95  * dma_fence_chain_find_seqno() to see how a given point is found in the
  96  * timeline.
  97  *
  98  * Note that applications should be careful to always use timeline set of
  99  * ioctl() when dealing with syncobj considered as timeline. Using a binary
 100  * set of ioctl() with a syncobj considered as timeline could result incorrect
 101  * synchronization. The use of binary syncobj is supported through the
 102  * timeline set of ioctl() by using a point value of 0, this will reproduce
 103  * the behavior of the binary set of ioctl() (for example replace the
 104  * syncobj's fence when signaling).
 105  *
 106  *
 107  * Host-side wait on syncobjs
 108  * --------------------------
 109  *
 110  * &DRM_IOCTL_SYNCOBJ_WAIT takes an array of syncobj handles and does a
 111  * host-side wait on all of the syncobj fences simultaneously.
 112  * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL is set, the wait ioctl will wait on
 113  * all of the syncobj fences to be signaled before it returns.
 114  * Otherwise, it returns once at least one syncobj fence has been signaled
 115  * and the index of a signaled fence is written back to the client.
 116  *
 117  * Unlike the enqueued GPU work dependencies which fail if they see a NULL
 118  * fence in a syncobj, if &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is set,
 119  * the host-side wait will first wait for the syncobj to receive a non-NULL
 120  * fence and then wait on that fence.
 121  * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is not set and any one of the
 122  * syncobjs in the array has a NULL fence, -EINVAL will be returned.
 123  * Assuming the syncobj starts off with a NULL fence, this allows a client
 124  * to do a host wait in one thread (or process) which waits on GPU work
 125  * submitted in another thread (or process) without having to manually
 126  * synchronize between the two.
 127  * This requirement is inherited from the Vulkan fence API.
 128  *
 129  * Similarly, &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT takes an array of syncobj
 130  * handles as well as an array of u64 points and does a host-side wait on all
 131  * of syncobj fences at the given points simultaneously.
 132  *
 133  * &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT also adds the ability to wait for a given
 134  * fence to materialize on the timeline without waiting for the fence to be
 135  * signaled by using the &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE flag. This
 136  * requirement is inherited from the wait-before-signal behavior required by
 137  * the Vulkan timeline semaphore API.
 138  *
 139  *
 140  * Import/export of syncobjs
 141  * -------------------------
 142  *
 143  * &DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE and &DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD
 144  * provide two mechanisms for import/export of syncobjs.
 145  *
 146  * The first lets the client import or export an entire syncobj to a file
 147  * descriptor.
 148  * These fd's are opaque and have no other use case, except passing the
 149  * syncobj between processes.
 150  * All exported file descriptors and any syncobj handles created as a
 151  * result of importing those file descriptors own a reference to the
 152  * same underlying struct &drm_syncobj and the syncobj can be used
 153  * persistently across all the processes with which it is shared.
 154  * The syncobj is freed only once the last reference is dropped.
 155  * Unlike dma-buf, importing a syncobj creates a new handle (with its own
 156  * reference) for every import instead of de-duplicating.
 157  * The primary use-case of this persistent import/export is for shared
 158  * Vulkan fences and semaphores.
 159  *
 160  * The second import/export mechanism, which is indicated by
 161  * &DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE or
 162  * &DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE lets the client
 163  * import/export the syncobj's current fence from/to a &sync_file.
 164  * When a syncobj is exported to a sync file, that sync file wraps the
 165  * sycnobj's fence at the time of export and any later signal or reset
 166  * operations on the syncobj will not affect the exported sync file.
 167  * When a sync file is imported into a syncobj, the syncobj's fence is set
 168  * to the fence wrapped by that sync file.
 169  * Because sync files are immutable, resetting or signaling the syncobj
 170  * will not affect any sync files whose fences have been imported into the
 171  * syncobj.
 172  *
 173  *
 174  * Import/export of timeline points in timeline syncobjs
 175  * -----------------------------------------------------
 176  *
 177  * &DRM_IOCTL_SYNCOBJ_TRANSFER provides a mechanism to transfer a struct
 178  * &dma_fence_chain of a syncobj at a given u64 point to another u64 point
 179  * into another syncobj.
 180  *
 181  * Note that if you want to transfer a struct &dma_fence_chain from a given
 182  * point on a timeline syncobj from/into a binary syncobj, you can use the
 183  * point 0 to mean take/replace the fence in the syncobj.
 184  */
 185
 186 #include <linux/anon_inodes.h>
 187 #include <linux/file.h>
 188 #include <linux/fs.h>
 189 #include <linux/sched/signal.h>
 190 #include <linux/sync_file.h>
 191 #include <linux/uaccess.h>
 192
 193 #include <drm/drm.h>
 194 #include <drm/drm_drv.h>
 195 #include <drm/drm_file.h>
 196 #include <drm/drm_gem.h>
 197 #include <drm/drm_print.h>
 198 #include <drm/drm_syncobj.h>
 199 #include <drm/drm_utils.h>
 200
 201 #include "drm_internal.h"
 202
 203 struct syncobj_wait_entry {
 204         struct list_head node;
 205         struct task_struct *task;
 206         struct dma_fence *fence;
 207         struct dma_fence_cb fence_cb;
 208         u64    point;
 209 };
 210
 211 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
 212                                       struct syncobj_wait_entry *wait);
 213
 214 /**
 215  * drm_syncobj_find - lookup and reference a sync object.
 216  * @file_private: drm file private pointer
 217  * @handle: sync object handle to lookup.
 218  *
 219  * Returns a reference to the syncobj pointed to by handle or NULL. The
 220  * reference must be released by calling drm_syncobj_put().
 221  */
 222 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
 223                                      u32 handle)
 224 {
 225         struct drm_syncobj *syncobj;
 226
 227         spin_lock(&file_private->syncobj_table_lock);
 228
 229         /* Check if we currently have a reference on the object */
 230         syncobj = idr_find(&file_private->syncobj_idr, handle);
 231         if (syncobj)
 232                 drm_syncobj_get(syncobj);
 233
 234         spin_unlock(&file_private->syncobj_table_lock);
 235
 236         return syncobj;
 237 }
 238 EXPORT_SYMBOL(drm_syncobj_find);
 239
 240 static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,
 241                                        struct syncobj_wait_entry *wait)
 242 {
 243         struct dma_fence *fence;
 244
 245         if (wait->fence)
 246                 return;
 247
 248         spin_lock(&syncobj->lock);
 249         /* We've already tried once to get a fence and failed.  Now that we
 250          * have the lock, try one more time just to be sure we don't add a
 251          * callback when a fence has already been set.
 252          */
 253         fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
 254         if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
 255                 dma_fence_put(fence);
 256                 list_add_tail(&wait->node, &syncobj->cb_list);
 257         } else if (!fence) {
 258                 wait->fence = dma_fence_get_stub();
 259         } else {
 260                 wait->fence = fence;
 261         }
 262         spin_unlock(&syncobj->lock);
 263 }
 264
 265 static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj,
 266                                     struct syncobj_wait_entry *wait)
 267 {
 268         if (!wait->node.next)
 269                 return;
 270
 271         spin_lock(&syncobj->lock);
 272         list_del_init(&wait->node);
 273         spin_unlock(&syncobj->lock);
 274 }
 275
 276 /**
 277  * drm_syncobj_add_point - add new timeline point to the syncobj
 278  * @syncobj: sync object to add timeline point do
 279  * @chain: chain node to use to add the point
 280  * @fence: fence to encapsulate in the chain node
 281  * @point: sequence number to use for the point
 282  *
 283  * Add the chain node as new timeline point to the syncobj.
 284  */
 285 void drm_syncobj_add_point(struct drm_syncobj *syncobj,
 286                            struct dma_fence_chain *chain,
 287                            struct dma_fence *fence,
 288                            uint64_t point)
 289 {
 290         struct syncobj_wait_entry *cur, *tmp;
 291         struct dma_fence *prev;
 292
 293         dma_fence_get(fence);
 294
 295         spin_lock(&syncobj->lock);
 296
 297         prev = drm_syncobj_fence_get(syncobj);
 298         /* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */
 299         if (prev && prev->seqno >= point)
 300                 DRM_DEBUG("You are adding an unorder point to timeline!\n");
 301         dma_fence_chain_init(chain, prev, fence, point);
 302         rcu_assign_pointer(syncobj->fence, &chain->base);
 303
 304         list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node)
 305                 syncobj_wait_syncobj_func(syncobj, cur);
 306         spin_unlock(&syncobj->lock);
 307
 308         /* Walk the chain once to trigger garbage collection */
 309         dma_fence_chain_for_each(fence, prev);
 310         dma_fence_put(prev);
 311 }
 312 EXPORT_SYMBOL(drm_syncobj_add_point);
 313
 314 /**
 315  * drm_syncobj_replace_fence - replace fence in a sync object.
 316  * @syncobj: Sync object to replace fence in
 317  * @fence: fence to install in sync file.
 318  *
 319  * This replaces the fence on a sync object.
 320  */
 321 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
 322                                struct dma_fence *fence)
 323 {
 324         struct dma_fence *old_fence;
 325         struct syncobj_wait_entry *cur, *tmp;
 326
 327         if (fence)
 328                 dma_fence_get(fence);
 329
 330         spin_lock(&syncobj->lock);
 331
 332         old_fence = rcu_dereference_protected(syncobj->fence,
 333                                               lockdep_is_held(&syncobj->lock));
 334         rcu_assign_pointer(syncobj->fence, fence);
 335
 336         if (fence != old_fence) {
 337                 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node)
 338                         syncobj_wait_syncobj_func(syncobj, cur);
 339         }
 340
 341         spin_unlock(&syncobj->lock);
 342
 343         dma_fence_put(old_fence);
 344 }
 345 EXPORT_SYMBOL(drm_syncobj_replace_fence);
 346
 347 /**
 348  * drm_syncobj_assign_null_handle - assign a stub fence to the sync object
 349  * @syncobj: sync object to assign the fence on
 350  *
 351  * Assign a already signaled stub fence to the sync object.
 352  */
 353 static void drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
 354 {
 355         struct dma_fence *fence = dma_fence_get_stub();
 356
 357         drm_syncobj_replace_fence(syncobj, fence);
 358         dma_fence_put(fence);
 359 }
 360
 361 /* 5s default for wait submission */
 362 #define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL
 363 /**
 364  * drm_syncobj_find_fence - lookup and reference the fence in a sync object
 365  * @file_private: drm file private pointer
 366  * @handle: sync object handle to lookup.
 367  * @point: timeline point
 368  * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not
 369  * @fence: out parameter for the fence
 370  *
 371  * This is just a convenience function that combines drm_syncobj_find() and
 372  * drm_syncobj_fence_get().
 373  *
 374  * Returns 0 on success or a negative error value on failure. On success @fence
 375  * contains a reference to the fence, which must be released by calling
 376  * dma_fence_put().
 377  */
 378 int drm_syncobj_find_fence(struct drm_file *file_private,
 379                            u32 handle, u64 point, u64 flags,
 380                            struct dma_fence **fence)
 381 {
 382         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 383         struct syncobj_wait_entry wait;
 384         u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT);
 385         int ret;
 386
 387         if (!syncobj)
 388                 return -ENOENT;
 389
 390         *fence = drm_syncobj_fence_get(syncobj);
 391
 392         if (*fence) {
 393                 ret = dma_fence_chain_find_seqno(fence, point);
 394                 if (!ret)
 395                         goto out;
 396                 dma_fence_put(*fence);
 397         } else {
 398                 ret = -EINVAL;
 399         }
 400
 401         if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
 402                 goto out;
 403
 404         memset(&wait, 0, sizeof(wait));
 405         wait.task = current;
 406         wait.point = point;
 407         drm_syncobj_fence_add_wait(syncobj, &wait);
 408
 409         do {
 410                 set_current_state(TASK_INTERRUPTIBLE);
 411                 if (wait.fence) {
 412                         ret = 0;
 413                         break;
 414                 }
 415                 if (timeout == 0) {
 416                         ret = -ETIME;
 417                         break;
 418                 }
 419
 420                 if (signal_pending(current)) {
 421                         ret = -ERESTARTSYS;
 422                         break;
 423                 }
 424
 425                 timeout = schedule_timeout(timeout);
 426         } while (1);
 427
 428         __set_current_state(TASK_RUNNING);
 429         *fence = wait.fence;
 430
 431         if (wait.node.next)
 432                 drm_syncobj_remove_wait(syncobj, &wait);
 433
 434 out:
 435         drm_syncobj_put(syncobj);
 436
 437         return ret;
 438 }
 439 EXPORT_SYMBOL(drm_syncobj_find_fence);
 440
 441 /**
 442  * drm_syncobj_free - free a sync object.
 443  * @kref: kref to free.
 444  *
 445  * Only to be called from kref_put in drm_syncobj_put.
 446  */
 447 void drm_syncobj_free(struct kref *kref)
 448 {
 449         struct drm_syncobj *syncobj = container_of(kref,
 450                                                    struct drm_syncobj,
 451                                                    refcount);
 452         drm_syncobj_replace_fence(syncobj, NULL);
 453         kfree(syncobj);
 454 }
 455 EXPORT_SYMBOL(drm_syncobj_free);
 456
 457 /**
 458  * drm_syncobj_create - create a new syncobj
 459  * @out_syncobj: returned syncobj
 460  * @flags: DRM_SYNCOBJ_* flags
 461  * @fence: if non-NULL, the syncobj will represent this fence
 462  *
 463  * This is the first function to create a sync object. After creating, drivers
 464  * probably want to make it available to userspace, either through
 465  * drm_syncobj_get_handle() or drm_syncobj_get_fd().
 466  *
 467  * Returns 0 on success or a negative error value on failure.
 468  */
 469 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
 470                        struct dma_fence *fence)
 471 {
 472         struct drm_syncobj *syncobj;
 473
 474         syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
 475         if (!syncobj)
 476                 return -ENOMEM;
 477
 478         kref_init(&syncobj->refcount);
 479         INIT_LIST_HEAD(&syncobj->cb_list);
 480         spin_lock_init(&syncobj->lock);
 481
 482         if (flags & DRM_SYNCOBJ_CREATE_SIGNALED)
 483                 drm_syncobj_assign_null_handle(syncobj);
 484
 485         if (fence)
 486                 drm_syncobj_replace_fence(syncobj, fence);
 487
 488         *out_syncobj = syncobj;
 489         return 0;
 490 }
 491 EXPORT_SYMBOL(drm_syncobj_create);
 492
 493 /**
 494  * drm_syncobj_get_handle - get a handle from a syncobj
 495  * @file_private: drm file private pointer
 496  * @syncobj: Sync object to export
 497  * @handle: out parameter with the new handle
 498  *
 499  * Exports a sync object created with drm_syncobj_create() as a handle on
 500  * @file_private to userspace.
 501  *
 502  * Returns 0 on success or a negative error value on failure.
 503  */
 504 int drm_syncobj_get_handle(struct drm_file *file_private,
 505                            struct drm_syncobj *syncobj, u32 *handle)
 506 {
 507         int ret;
 508
 509         /* take a reference to put in the idr */
 510         drm_syncobj_get(syncobj);
 511
 512         idr_preload(GFP_KERNEL);
 513         spin_lock(&file_private->syncobj_table_lock);
 514         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 515         spin_unlock(&file_private->syncobj_table_lock);
 516
 517         idr_preload_end();
 518
 519         if (ret < 0) {
 520                 drm_syncobj_put(syncobj);
 521                 return ret;
 522         }
 523
 524         *handle = ret;
 525         return 0;
 526 }
 527 EXPORT_SYMBOL(drm_syncobj_get_handle);
 528
 529 static int drm_syncobj_create_as_handle(struct drm_file *file_private,
 530                                         u32 *handle, uint32_t flags)
 531 {
 532         int ret;
 533         struct drm_syncobj *syncobj;
 534
 535         ret = drm_syncobj_create(&syncobj, flags, NULL);
 536         if (ret)
 537                 return ret;
 538
 539         ret = drm_syncobj_get_handle(file_private, syncobj, handle);
 540         drm_syncobj_put(syncobj);
 541         return ret;
 542 }
 543
 544 static int drm_syncobj_destroy(struct drm_file *file_private,
 545                                u32 handle)
 546 {
 547         struct drm_syncobj *syncobj;
 548
 549         spin_lock(&file_private->syncobj_table_lock);
 550         syncobj = idr_remove(&file_private->syncobj_idr, handle);
 551         spin_unlock(&file_private->syncobj_table_lock);
 552
 553         if (!syncobj)
 554                 return -EINVAL;
 555
 556         drm_syncobj_put(syncobj);
 557         return 0;
 558 }
 559
 560 static int drm_syncobj_file_release(struct inode *inode, struct file *file)
 561 {
 562         struct drm_syncobj *syncobj = file->private_data;
 563
 564         drm_syncobj_put(syncobj);
 565         return 0;
 566 }
 567
 568 static const struct file_operations drm_syncobj_file_fops = {
 569         .release = drm_syncobj_file_release,
 570 };
 571
 572 /**
 573  * drm_syncobj_get_fd - get a file descriptor from a syncobj
 574  * @syncobj: Sync object to export
 575  * @p_fd: out parameter with the new file descriptor
 576  *
 577  * Exports a sync object created with drm_syncobj_create() as a file descriptor.
 578  *
 579  * Returns 0 on success or a negative error value on failure.
 580  */
 581 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
 582 {
 583         struct file *file;
 584         int fd;
 585
 586         fd = get_unused_fd_flags(O_CLOEXEC);
 587         if (fd < 0)
 588                 return fd;
 589
 590         file = anon_inode_getfile("syncobj_file",
 591                                   &drm_syncobj_file_fops,
 592                                   syncobj, 0);
 593         if (IS_ERR(file)) {
 594                 put_unused_fd(fd);
 595                 return PTR_ERR(file);
 596         }
 597
 598         drm_syncobj_get(syncobj);
 599         fd_install(fd, file);
 600
 601         *p_fd = fd;
 602         return 0;
 603 }
 604 EXPORT_SYMBOL(drm_syncobj_get_fd);
 605
 606 static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
 607                                     u32 handle, int *p_fd)
 608 {
 609         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 610         int ret;
 611
 612         if (!syncobj)
 613                 return -EINVAL;
 614
 615         ret = drm_syncobj_get_fd(syncobj, p_fd);
 616         drm_syncobj_put(syncobj);
 617         return ret;
 618 }
 619
 620 static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
 621                                     int fd, u32 *handle)
 622 {
 623         struct drm_syncobj *syncobj;
 624         struct fd f = fdget(fd);
 625         int ret;
 626
 627         if (!f.file)
 628                 return -EINVAL;
 629
 630         if (f.file->f_op != &drm_syncobj_file_fops) {
 631                 fdput(f);
 632                 return -EINVAL;
 633         }
 634
 635         /* take a reference to put in the idr */
 636         syncobj = f.file->private_data;
 637         drm_syncobj_get(syncobj);
 638
 639         idr_preload(GFP_KERNEL);
 640         spin_lock(&file_private->syncobj_table_lock);
 641         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 642         spin_unlock(&file_private->syncobj_table_lock);
 643         idr_preload_end();
 644
 645         if (ret > 0) {
 646                 *handle = ret;
 647                 ret = 0;
 648         } else
 649                 drm_syncobj_put(syncobj);
 650
 651         fdput(f);
 652         return ret;
 653 }
 654
 655 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
 656                                               int fd, int handle)
 657 {
 658         struct dma_fence *fence = sync_file_get_fence(fd);
 659         struct drm_syncobj *syncobj;
 660
 661         if (!fence)
 662                 return -EINVAL;
 663
 664         syncobj = drm_syncobj_find(file_private, handle);
 665         if (!syncobj) {
 666                 dma_fence_put(fence);
 667                 return -ENOENT;
 668         }
 669
 670         drm_syncobj_replace_fence(syncobj, fence);
 671         dma_fence_put(fence);
 672         drm_syncobj_put(syncobj);
 673         return 0;
 674 }
 675
 676 static int drm_syncobj_export_sync_file(struct drm_file *file_private,
 677                                         int handle, int *p_fd)
 678 {
 679         int ret;
 680         struct dma_fence *fence;
 681         struct sync_file *sync_file;
 682         int fd = get_unused_fd_flags(O_CLOEXEC);
 683
 684         if (fd < 0)
 685                 return fd;
 686
 687         ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence);
 688         if (ret)
 689                 goto err_put_fd;
 690
 691         sync_file = sync_file_create(fence);
 692
 693         dma_fence_put(fence);
 694
 695         if (!sync_file) {
 696                 ret = -EINVAL;
 697                 goto err_put_fd;
 698         }
 699
 700         fd_install(fd, sync_file->file);
 701
 702         *p_fd = fd;
 703         return 0;
 704 err_put_fd:
 705         put_unused_fd(fd);
 706         return ret;
 707 }
 708 /**
 709  * drm_syncobj_open - initalizes syncobj file-private structures at devnode open time
 710  * @file_private: drm file-private structure to set up
 711  *
 712  * Called at device open time, sets up the structure for handling refcounting
 713  * of sync objects.
 714  */
 715 void
 716 drm_syncobj_open(struct drm_file *file_private)
 717 {
 718         idr_init_base(&file_private->syncobj_idr, 1);
 719         spin_lock_init(&file_private->syncobj_table_lock);
 720 }
 721
 722 static int
 723 drm_syncobj_release_handle(int id, void *ptr, void *data)
 724 {
 725         struct drm_syncobj *syncobj = ptr;
 726
 727         drm_syncobj_put(syncobj);
 728         return 0;
 729 }
 730
 731 /**
 732  * drm_syncobj_release - release file-private sync object resources
 733  * @file_private: drm file-private structure to clean up
 734  *
 735  * Called at close time when the filp is going away.
 736  *
 737  * Releases any remaining references on objects by this filp.
 738  */
 739 void
 740 drm_syncobj_release(struct drm_file *file_private)
 741 {
 742         idr_for_each(&file_private->syncobj_idr,
 743                      &drm_syncobj_release_handle, file_private);
 744         idr_destroy(&file_private->syncobj_idr);
 745 }
 746
 747 int
 748 drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
 749                          struct drm_file *file_private)
 750 {
 751         struct drm_syncobj_create *args = data;
 752
 753         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 754                 return -EOPNOTSUPP;
 755
 756         /* no valid flags yet */
 757         if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
 758                 return -EINVAL;
 759
 760         return drm_syncobj_create_as_handle(file_private,
 761                                             &args->handle, args->flags);
 762 }
 763
 764 int
 765 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
 766                           struct drm_file *file_private)
 767 {
 768         struct drm_syncobj_destroy *args = data;
 769
 770         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 771                 return -EOPNOTSUPP;
 772
 773         /* make sure padding is empty */
 774         if (args->pad)
 775                 return -EINVAL;
 776         return drm_syncobj_destroy(file_private, args->handle);
 777 }
 778
 779 int
 780 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
 781                                    struct drm_file *file_private)
 782 {
 783         struct drm_syncobj_handle *args = data;
 784
 785         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 786                 return -EOPNOTSUPP;
 787
 788         if (args->pad)
 789                 return -EINVAL;
 790
 791         if (args->flags != 0 &&
 792             args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 793                 return -EINVAL;
 794
 795         if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 796                 return drm_syncobj_export_sync_file(file_private, args->handle,
 797                                                     &args->fd);
 798
 799         return drm_syncobj_handle_to_fd(file_private, args->handle,
 800                                         &args->fd);
 801 }
 802
 803 int
 804 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
 805                                    struct drm_file *file_private)
 806 {
 807         struct drm_syncobj_handle *args = data;
 808
 809         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 810                 return -EOPNOTSUPP;
 811
 812         if (args->pad)
 813                 return -EINVAL;
 814
 815         if (args->flags != 0 &&
 816             args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 817                 return -EINVAL;
 818
 819         if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 820                 return drm_syncobj_import_sync_file_fence(file_private,
 821                                                           args->fd,
 822                                                           args->handle);
 823
 824         return drm_syncobj_fd_to_handle(file_private, args->fd,
 825                                         &args->handle);
 826 }
 827
 828 static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private,
 829                                             struct drm_syncobj_transfer *args)
 830 {
 831         struct drm_syncobj *timeline_syncobj = NULL;
 832         struct dma_fence *fence;
 833         struct dma_fence_chain *chain;
 834         int ret;
 835
 836         timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle);
 837         if (!timeline_syncobj) {
 838                 return -ENOENT;
 839         }
 840         ret = drm_syncobj_find_fence(file_private, args->src_handle,
 841                                      args->src_point, args->flags,
 842                                      &fence);
 843         if (ret)
 844                 goto err;
 845         chain = kzalloc(sizeof(struct dma_fence_chain), GFP_KERNEL);
 846         if (!chain) {
 847                 ret = -ENOMEM;
 848                 goto err1;
 849         }
 850         drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point);
 851 err1:
 852         dma_fence_put(fence);
 853 err:
 854         drm_syncobj_put(timeline_syncobj);
 855
 856         return ret;
 857 }
 858
 859 static int
 860 drm_syncobj_transfer_to_binary(struct drm_file *file_private,
 861                                struct drm_syncobj_transfer *args)
 862 {
 863         struct drm_syncobj *binary_syncobj = NULL;
 864         struct dma_fence *fence;
 865         int ret;
 866
 867         binary_syncobj = drm_syncobj_find(file_private, args->dst_handle);
 868         if (!binary_syncobj)
 869                 return -ENOENT;
 870         ret = drm_syncobj_find_fence(file_private, args->src_handle,
 871                                      args->src_point, args->flags, &fence);
 872         if (ret)
 873                 goto err;
 874         drm_syncobj_replace_fence(binary_syncobj, fence);
 875         dma_fence_put(fence);
 876 err:
 877         drm_syncobj_put(binary_syncobj);
 878
 879         return ret;
 880 }
 881 int
 882 drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data,
 883                            struct drm_file *file_private)
 884 {
 885         struct drm_syncobj_transfer *args = data;
 886         int ret;
 887
 888         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
 889                 return -EOPNOTSUPP;
 890
 891         if (args->pad)
 892                 return -EINVAL;
 893
 894         if (args->dst_point)
 895                 ret = drm_syncobj_transfer_to_timeline(file_private, args);
 896         else
 897                 ret = drm_syncobj_transfer_to_binary(file_private, args);
 898
 899         return ret;
 900 }
 901
 902 static void syncobj_wait_fence_func(struct dma_fence *fence,
 903                                     struct dma_fence_cb *cb)
 904 {
 905         struct syncobj_wait_entry *wait =
 906                 container_of(cb, struct syncobj_wait_entry, fence_cb);
 907
 908         wake_up_process(wait->task);
 909 }
 910
 911 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
 912                                       struct syncobj_wait_entry *wait)
 913 {
 914         struct dma_fence *fence;
 915
 916         /* This happens inside the syncobj lock */
 917         fence = rcu_dereference_protected(syncobj->fence,
 918                                           lockdep_is_held(&syncobj->lock));
 919         dma_fence_get(fence);
 920         if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
 921                 dma_fence_put(fence);
 922                 return;
 923         } else if (!fence) {
 924                 wait->fence = dma_fence_get_stub();
 925         } else {
 926                 wait->fence = fence;
 927         }
 928
 929         wake_up_process(wait->task);
 930         list_del_init(&wait->node);
 931 }
 932
 933 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
 934                                                   void __user *user_points,
 935                                                   uint32_t count,
 936                                                   uint32_t flags,
 937                                                   signed long timeout,
 938                                                   uint32_t *idx)
 939 {
 940         struct syncobj_wait_entry *entries;
 941         struct dma_fence *fence;
 942         uint64_t *points;
 943         uint32_t signaled_count, i;
 944
 945         points = kmalloc_array(count, sizeof(*points), GFP_KERNEL);
 946         if (points == NULL)
 947                 return -ENOMEM;
 948
 949         if (!user_points) {
 950                 memset(points, 0, count * sizeof(uint64_t));
 951
 952         } else if (copy_from_user(points, user_points,
 953                                   sizeof(uint64_t) * count)) {
 954                 timeout = -EFAULT;
 955                 goto err_free_points;
 956         }
 957
 958         entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
 959         if (!entries) {
 960                 timeout = -ENOMEM;
 961                 goto err_free_points;
 962         }
 963         /* Walk the list of sync objects and initialize entries.  We do
 964          * this up-front so that we can properly return -EINVAL if there is
 965          * a syncobj with a missing fence and then never have the chance of
 966          * returning -EINVAL again.
 967          */
 968         signaled_count = 0;
 969         for (i = 0; i < count; ++i) {
 970                 struct dma_fence *fence;
 971
 972                 entries[i].task = current;
 973                 entries[i].point = points[i];
 974                 fence = drm_syncobj_fence_get(syncobjs[i]);
 975                 if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) {
 976                         dma_fence_put(fence);
 977                         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 978                                 continue;
 979                         } else {
 980                                 timeout = -EINVAL;
 981                                 goto cleanup_entries;
 982                         }
 983                 }
 984
 985                 if (fence)
 986                         entries[i].fence = fence;
 987                 else
 988                         entries[i].fence = dma_fence_get_stub();
 989
 990                 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
 991                     dma_fence_is_signaled(entries[i].fence)) {
 992                         if (signaled_count == 0 && idx)
 993                                 *idx = i;
 994                         signaled_count++;
 995                 }
 996         }
 997
 998         if (signaled_count == count ||
 999             (signaled_count > 0 &&
1000              !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
1001                 goto cleanup_entries;
1002
1003         /* There's a very annoying laxness in the dma_fence API here, in
1004          * that backends are not required to automatically report when a
1005          * fence is signaled prior to fence->ops->enable_signaling() being
1006          * called.  So here if we fail to match signaled_count, we need to
1007          * fallthough and try a 0 timeout wait!
1008          */
1009
1010         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
1011                 for (i = 0; i < count; ++i)
1012                         drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]);
1013         }
1014
1015         do {
1016                 set_current_state(TASK_INTERRUPTIBLE);
1017
1018                 signaled_count = 0;
1019                 for (i = 0; i < count; ++i) {
1020                         fence = entries[i].fence;
1021                         if (!fence)
1022                                 continue;
1023
1024                         if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1025                             dma_fence_is_signaled(fence) ||
1026                             (!entries[i].fence_cb.func &&
1027                              dma_fence_add_callback(fence,
1028                                                     &entries[i].fence_cb,
1029                                                     syncobj_wait_fence_func))) {
1030                                 /* The fence has been signaled */
1031                                 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
1032                                         signaled_count++;
1033                                 } else {
1034                                         if (idx)
1035                                                 *idx = i;
1036                                         goto done_waiting;
1037                                 }
1038                         }
1039                 }
1040
1041                 if (signaled_count == count)
1042                         goto done_waiting;
1043
1044                 if (timeout == 0) {
1045                         timeout = -ETIME;
1046                         goto done_waiting;
1047                 }
1048
1049                 if (signal_pending(current)) {
1050                         timeout = -ERESTARTSYS;
1051                         goto done_waiting;
1052                 }
1053
1054                 timeout = schedule_timeout(timeout);
1055         } while (1);
1056
1057 done_waiting:
1058         __set_current_state(TASK_RUNNING);
1059
1060 cleanup_entries:
1061         for (i = 0; i < count; ++i) {
1062                 drm_syncobj_remove_wait(syncobjs[i], &entries[i]);
1063                 if (entries[i].fence_cb.func)
1064                         dma_fence_remove_callback(entries[i].fence,
1065                                                   &entries[i].fence_cb);
1066                 dma_fence_put(entries[i].fence);
1067         }
1068         kfree(entries);
1069
1070 err_free_points:
1071         kfree(points);
1072
1073         return timeout;
1074 }
1075
1076 /**
1077  * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
1078  *
1079  * @timeout_nsec: timeout nsec component in ns, 0 for poll
1080  *
1081  * Calculate the timeout in jiffies from an absolute time in sec/nsec.
1082  */
1083 signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
1084 {
1085         ktime_t abs_timeout, now;
1086         u64 timeout_ns, timeout_jiffies64;
1087
1088         /* make 0 timeout means poll - absolute 0 doesn't seem valid */
1089         if (timeout_nsec == 0)
1090                 return 0;
1091
1092         abs_timeout = ns_to_ktime(timeout_nsec);
1093         now = ktime_get();
1094
1095         if (!ktime_after(abs_timeout, now))
1096                 return 0;
1097
1098         timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
1099
1100         timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
1101         /*  clamp timeout to avoid infinite timeout */
1102         if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
1103                 return MAX_SCHEDULE_TIMEOUT - 1;
1104
1105         return timeout_jiffies64 + 1;
1106 }
1107 EXPORT_SYMBOL(drm_timeout_abs_to_jiffies);
1108
1109 static int drm_syncobj_array_wait(struct drm_device *dev,
1110                                   struct drm_file *file_private,
1111                                   struct drm_syncobj_wait *wait,
1112                                   struct drm_syncobj_timeline_wait *timeline_wait,
1113                                   struct drm_syncobj **syncobjs, bool timeline)
1114 {
1115         signed long timeout = 0;
1116         uint32_t first = ~0;
1117
1118         if (!timeline) {
1119                 timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
1120                 timeout = drm_syncobj_array_wait_timeout(syncobjs,
1121                                                          NULL,
1122                                                          wait->count_handles,
1123                                                          wait->flags,
1124                                                          timeout, &first);
1125                 if (timeout < 0)
1126                         return timeout;
1127                 wait->first_signaled = first;
1128         } else {
1129                 timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec);
1130                 timeout = drm_syncobj_array_wait_timeout(syncobjs,
1131                                                          u64_to_user_ptr(timeline_wait->points),
1132                                                          timeline_wait->count_handles,
1133                                                          timeline_wait->flags,
1134                                                          timeout, &first);
1135                 if (timeout < 0)
1136                         return timeout;
1137                 timeline_wait->first_signaled = first;
1138         }
1139         return 0;
1140 }
1141
1142 static int drm_syncobj_array_find(struct drm_file *file_private,
1143                                   void __user *user_handles,
1144                                   uint32_t count_handles,
1145                                   struct drm_syncobj ***syncobjs_out)
1146 {
1147         uint32_t i, *handles;
1148         struct drm_syncobj **syncobjs;
1149         int ret;
1150
1151         handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
1152         if (handles == NULL)
1153                 return -ENOMEM;
1154
1155         if (copy_from_user(handles, user_handles,
1156                            sizeof(uint32_t) * count_handles)) {
1157                 ret = -EFAULT;
1158                 goto err_free_handles;
1159         }
1160
1161         syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
1162         if (syncobjs == NULL) {
1163                 ret = -ENOMEM;
1164                 goto err_free_handles;
1165         }
1166
1167         for (i = 0; i < count_handles; i++) {
1168                 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
1169                 if (!syncobjs[i]) {
1170                         ret = -ENOENT;
1171                         goto err_put_syncobjs;
1172                 }
1173         }
1174
1175         kfree(handles);
1176         *syncobjs_out = syncobjs;
1177         return 0;
1178
1179 err_put_syncobjs:
1180         while (i-- > 0)
1181                 drm_syncobj_put(syncobjs[i]);
1182         kfree(syncobjs);
1183 err_free_handles:
1184         kfree(handles);
1185
1186         return ret;
1187 }
1188
1189 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
1190                                    uint32_t count)
1191 {
1192         uint32_t i;
1193
1194         for (i = 0; i < count; i++)
1195                 drm_syncobj_put(syncobjs[i]);
1196         kfree(syncobjs);
1197 }
1198
1199 int
1200 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
1201                        struct drm_file *file_private)
1202 {
1203         struct drm_syncobj_wait *args = data;
1204         struct drm_syncobj **syncobjs;
1205         int ret = 0;
1206
1207         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1208                 return -EOPNOTSUPP;
1209
1210         if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1211                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
1212                 return -EINVAL;
1213
1214         if (args->count_handles == 0)
1215                 return -EINVAL;
1216
1217         ret = drm_syncobj_array_find(file_private,
1218                                      u64_to_user_ptr(args->handles),
1219                                      args->count_handles,
1220                                      &syncobjs);
1221         if (ret < 0)
1222                 return ret;
1223
1224         ret = drm_syncobj_array_wait(dev, file_private,
1225                                      args, NULL, syncobjs, false);
1226
1227         drm_syncobj_array_free(syncobjs, args->count_handles);
1228
1229         return ret;
1230 }
1231
1232 int
1233 drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data,
1234                                 struct drm_file *file_private)
1235 {
1236         struct drm_syncobj_timeline_wait *args = data;
1237         struct drm_syncobj **syncobjs;
1238         int ret = 0;
1239
1240         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1241                 return -EOPNOTSUPP;
1242
1243         if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1244                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1245                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE))
1246                 return -EINVAL;
1247
1248         if (args->count_handles == 0)
1249                 return -EINVAL;
1250
1251         ret = drm_syncobj_array_find(file_private,
1252                                      u64_to_user_ptr(args->handles),
1253                                      args->count_handles,
1254                                      &syncobjs);
1255         if (ret < 0)
1256                 return ret;
1257
1258         ret = drm_syncobj_array_wait(dev, file_private,
1259                                      NULL, args, syncobjs, true);
1260
1261         drm_syncobj_array_free(syncobjs, args->count_handles);
1262
1263         return ret;
1264 }
1265
1266
1267 int
1268 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
1269                         struct drm_file *file_private)
1270 {
1271         struct drm_syncobj_array *args = data;
1272         struct drm_syncobj **syncobjs;
1273         uint32_t i;
1274         int ret;
1275
1276         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1277                 return -EOPNOTSUPP;
1278
1279         if (args->pad != 0)
1280                 return -EINVAL;
1281
1282         if (args->count_handles == 0)
1283                 return -EINVAL;
1284
1285         ret = drm_syncobj_array_find(file_private,
1286                                      u64_to_user_ptr(args->handles),
1287                                      args->count_handles,
1288                                      &syncobjs);
1289         if (ret < 0)
1290                 return ret;
1291
1292         for (i = 0; i < args->count_handles; i++)
1293                 drm_syncobj_replace_fence(syncobjs[i], NULL);
1294
1295         drm_syncobj_array_free(syncobjs, args->count_handles);
1296
1297         return 0;
1298 }
1299
1300 int
1301 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
1302                          struct drm_file *file_private)
1303 {
1304         struct drm_syncobj_array *args = data;
1305         struct drm_syncobj **syncobjs;
1306         uint32_t i;
1307         int ret;
1308
1309         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1310                 return -EOPNOTSUPP;
1311
1312         if (args->pad != 0)
1313                 return -EINVAL;
1314
1315         if (args->count_handles == 0)
1316                 return -EINVAL;
1317
1318         ret = drm_syncobj_array_find(file_private,
1319                                      u64_to_user_ptr(args->handles),
1320                                      args->count_handles,
1321                                      &syncobjs);
1322         if (ret < 0)
1323                 return ret;
1324
1325         for (i = 0; i < args->count_handles; i++)
1326                 drm_syncobj_assign_null_handle(syncobjs[i]);
1327
1328         drm_syncobj_array_free(syncobjs, args->count_handles);
1329
1330         return ret;
1331 }
1332
1333 int
1334 drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data,
1335                                   struct drm_file *file_private)
1336 {
1337         struct drm_syncobj_timeline_array *args = data;
1338         struct drm_syncobj **syncobjs;
1339         struct dma_fence_chain **chains;
1340         uint64_t *points;
1341         uint32_t i, j;
1342         int ret;
1343
1344         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1345                 return -EOPNOTSUPP;
1346
1347         if (args->flags != 0)
1348                 return -EINVAL;
1349
1350         if (args->count_handles == 0)
1351                 return -EINVAL;
1352
1353         ret = drm_syncobj_array_find(file_private,
1354                                      u64_to_user_ptr(args->handles),
1355                                      args->count_handles,
1356                                      &syncobjs);
1357         if (ret < 0)
1358                 return ret;
1359
1360         points = kmalloc_array(args->count_handles, sizeof(*points),
1361                                GFP_KERNEL);
1362         if (!points) {
1363                 ret = -ENOMEM;
1364                 goto out;
1365         }
1366         if (!u64_to_user_ptr(args->points)) {
1367                 memset(points, 0, args->count_handles * sizeof(uint64_t));
1368         } else if (copy_from_user(points, u64_to_user_ptr(args->points),
1369                                   sizeof(uint64_t) * args->count_handles)) {
1370                 ret = -EFAULT;
1371                 goto err_points;
1372         }
1373
1374         chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL);
1375         if (!chains) {
1376                 ret = -ENOMEM;
1377                 goto err_points;
1378         }
1379         for (i = 0; i < args->count_handles; i++) {
1380                 chains[i] = kzalloc(sizeof(struct dma_fence_chain), GFP_KERNEL);
1381                 if (!chains[i]) {
1382                         for (j = 0; j < i; j++)
1383                                 kfree(chains[j]);
1384                         ret = -ENOMEM;
1385                         goto err_chains;
1386                 }
1387         }
1388
1389         for (i = 0; i < args->count_handles; i++) {
1390                 struct dma_fence *fence = dma_fence_get_stub();
1391
1392                 drm_syncobj_add_point(syncobjs[i], chains[i],
1393                                       fence, points[i]);
1394                 dma_fence_put(fence);
1395         }
1396 err_chains:
1397         kfree(chains);
1398 err_points:
1399         kfree(points);
1400 out:
1401         drm_syncobj_array_free(syncobjs, args->count_handles);
1402
1403         return ret;
1404 }
1405
1406 int drm_syncobj_query_ioctl(struct drm_device *dev, void *data,
1407                             struct drm_file *file_private)
1408 {
1409         struct drm_syncobj_timeline_array *args = data;
1410         struct drm_syncobj **syncobjs;
1411         uint64_t __user *points = u64_to_user_ptr(args->points);
1412         uint32_t i;
1413         int ret;
1414
1415         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1416                 return -EOPNOTSUPP;
1417
1418         if (args->flags & ~DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED)
1419                 return -EINVAL;
1420
1421         if (args->count_handles == 0)
1422                 return -EINVAL;
1423
1424         ret = drm_syncobj_array_find(file_private,
1425                                      u64_to_user_ptr(args->handles),
1426                                      args->count_handles,
1427                                      &syncobjs);
1428         if (ret < 0)
1429                 return ret;
1430
1431         for (i = 0; i < args->count_handles; i++) {
1432                 struct dma_fence_chain *chain;
1433                 struct dma_fence *fence;
1434                 uint64_t point;
1435
1436                 fence = drm_syncobj_fence_get(syncobjs[i]);
1437                 chain = to_dma_fence_chain(fence);
1438                 if (chain) {
1439                         struct dma_fence *iter, *last_signaled =
1440                                 dma_fence_get(fence);
1441
1442                         if (args->flags &
1443                             DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) {
1444                                 point = fence->seqno;
1445                         } else {
1446                                 dma_fence_chain_for_each(iter, fence) {
1447                                         if (iter->context != fence->context) {
1448                                                 dma_fence_put(iter);
1449                                                 /* It is most likely that timeline has
1450                                                 * unorder points. */
1451                                                 break;
1452                                         }
1453                                         dma_fence_put(last_signaled);
1454                                         last_signaled = dma_fence_get(iter);
1455                                 }
1456                                 point = dma_fence_is_signaled(last_signaled) ?
1457                                         last_signaled->seqno :
1458                                         to_dma_fence_chain(last_signaled)->prev_seqno;
1459                         }
1460                         dma_fence_put(last_signaled);
1461                 } else {
1462                         point = 0;
1463                 }
1464                 dma_fence_put(fence);
1465                 ret = copy_to_user(&points[i], &point, sizeof(uint64_t));
1466                 ret = ret ? -EFAULT : 0;
1467                 if (ret)
1468                         break;
1469         }
1470         drm_syncobj_array_free(syncobjs, args->count_handles);
1471
1472         return ret;
1473 }