drivers/gpu/drm/drm_syncobj.c

   1 /*
   2  * Copyright 2017 Red Hat
   3  * Parts ported from amdgpu (fence wait code).
   4  * Copyright 2016 Advanced Micro Devices, Inc.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  23  * IN THE SOFTWARE.
  24  *
  25  * Authors:
  26  *
  27  */
  28
  29 /**
  30  * DOC: Overview
  31  *
  32  * DRM synchronisation objects (syncobj, see struct &drm_syncobj) provide a
  33  * container for a synchronization primitive which can be used by userspace
  34  * to explicitly synchronize GPU commands, can be shared between userspace
  35  * processes, and can be shared between different DRM drivers.
  36  * Their primary use-case is to implement Vulkan fences and semaphores.
  37  * The syncobj userspace API provides ioctls for several operations:
  38  *
  39  *  - Creation and destruction of syncobjs
  40  *  - Import and export of syncobjs to/from a syncobj file descriptor
  41  *  - Import and export a syncobj's underlying fence to/from a sync file
  42  *  - Reset a syncobj (set its fence to NULL)
  43  *  - Signal a syncobj (set a trivially signaled fence)
  44  *  - Wait for a syncobj's fence to appear and be signaled
  45  *
  46  * The syncobj userspace API also provides operations to manipulate a syncobj
  47  * in terms of a timeline of struct &dma_fence_chain rather than a single
  48  * struct &dma_fence, through the following operations:
  49  *
  50  *   - Signal a given point on the timeline
  51  *   - Wait for a given point to appear and/or be signaled
  52  *   - Import and export from/to a given point of a timeline
  53  *
  54  * At it's core, a syncobj is simply a wrapper around a pointer to a struct
  55  * &dma_fence which may be NULL.
  56  * When a syncobj is first created, its pointer is either NULL or a pointer
  57  * to an already signaled fence depending on whether the
  58  * &DRM_SYNCOBJ_CREATE_SIGNALED flag is passed to
  59  * &DRM_IOCTL_SYNCOBJ_CREATE.
  60  *
  61  * If the syncobj is considered as a binary (its state is either signaled or
  62  * unsignaled) primitive, when GPU work is enqueued in a DRM driver to signal
  63  * the syncobj, the syncobj's fence is replaced with a fence which will be
  64  * signaled by the completion of that work.
  65  * If the syncobj is considered as a timeline primitive, when GPU work is
  66  * enqueued in a DRM driver to signal the a given point of the syncobj, a new
  67  * struct &dma_fence_chain pointing to the DRM driver's fence and also
  68  * pointing to the previous fence that was in the syncobj. The new struct
  69  * &dma_fence_chain fence replace the syncobj's fence and will be signaled by
  70  * completion of the DRM driver's work and also any work associated with the
  71  * fence previously in the syncobj.
  72  *
  73  * When GPU work which waits on a syncobj is enqueued in a DRM driver, at the
  74  * time the work is enqueued, it waits on the syncobj's fence before
  75  * submitting the work to hardware. That fence is either :
  76  *
  77  *    - The syncobj's current fence if the syncobj is considered as a binary
  78  *      primitive.
  79  *    - The struct &dma_fence associated with a given point if the syncobj is
  80  *      considered as a timeline primitive.
  81  *
  82  * If the syncobj's fence is NULL or not present in the syncobj's timeline,
  83  * the enqueue operation is expected to fail.
  84  *
  85  * With binary syncobj, all manipulation of the syncobjs's fence happens in
  86  * terms of the current fence at the time the ioctl is called by userspace
  87  * regardless of whether that operation is an immediate host-side operation
  88  * (signal or reset) or or an operation which is enqueued in some driver
  89  * queue. &DRM_IOCTL_SYNCOBJ_RESET and &DRM_IOCTL_SYNCOBJ_SIGNAL can be used
  90  * to manipulate a syncobj from the host by resetting its pointer to NULL or
  91  * setting its pointer to a fence which is already signaled.
  92  *
  93  * With a timeline syncobj, all manipulation of the synobj's fence happens in
  94  * terms of a u64 value referring to point in the timeline. See
  95  * dma_fence_chain_find_seqno() to see how a given point is found in the
  96  * timeline.
  97  *
  98  * Note that applications should be careful to always use timeline set of
  99  * ioctl() when dealing with syncobj considered as timeline. Using a binary
 100  * set of ioctl() with a syncobj considered as timeline could result incorrect
 101  * synchronization. The use of binary syncobj is supported through the
 102  * timeline set of ioctl() by using a point value of 0, this will reproduce
 103  * the behavior of the binary set of ioctl() (for example replace the
 104  * syncobj's fence when signaling).
 105  *
 106  *
 107  * Host-side wait on syncobjs
 108  * --------------------------
 109  *
 110  * &DRM_IOCTL_SYNCOBJ_WAIT takes an array of syncobj handles and does a
 111  * host-side wait on all of the syncobj fences simultaneously.
 112  * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL is set, the wait ioctl will wait on
 113  * all of the syncobj fences to be signaled before it returns.
 114  * Otherwise, it returns once at least one syncobj fence has been signaled
 115  * and the index of a signaled fence is written back to the client.
 116  *
 117  * Unlike the enqueued GPU work dependencies which fail if they see a NULL
 118  * fence in a syncobj, if &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is set,
 119  * the host-side wait will first wait for the syncobj to receive a non-NULL
 120  * fence and then wait on that fence.
 121  * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is not set and any one of the
 122  * syncobjs in the array has a NULL fence, -EINVAL will be returned.
 123  * Assuming the syncobj starts off with a NULL fence, this allows a client
 124  * to do a host wait in one thread (or process) which waits on GPU work
 125  * submitted in another thread (or process) without having to manually
 126  * synchronize between the two.
 127  * This requirement is inherited from the Vulkan fence API.
 128  *
 129  * Similarly, &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT takes an array of syncobj
 130  * handles as well as an array of u64 points and does a host-side wait on all
 131  * of syncobj fences at the given points simultaneously.
 132  *
 133  * &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT also adds the ability to wait for a given
 134  * fence to materialize on the timeline without waiting for the fence to be
 135  * signaled by using the &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE flag. This
 136  * requirement is inherited from the wait-before-signal behavior required by
 137  * the Vulkan timeline semaphore API.
 138  *
 139  *
 140  * Import/export of syncobjs
 141  * -------------------------
 142  *
 143  * &DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE and &DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD
 144  * provide two mechanisms for import/export of syncobjs.
 145  *
 146  * The first lets the client import or export an entire syncobj to a file
 147  * descriptor.
 148  * These fd's are opaque and have no other use case, except passing the
 149  * syncobj between processes.
 150  * All exported file descriptors and any syncobj handles created as a
 151  * result of importing those file descriptors own a reference to the
 152  * same underlying struct &drm_syncobj and the syncobj can be used
 153  * persistently across all the processes with which it is shared.
 154  * The syncobj is freed only once the last reference is dropped.
 155  * Unlike dma-buf, importing a syncobj creates a new handle (with its own
 156  * reference) for every import instead of de-duplicating.
 157  * The primary use-case of this persistent import/export is for shared
 158  * Vulkan fences and semaphores.
 159  *
 160  * The second import/export mechanism, which is indicated by
 161  * &DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE or
 162  * &DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE lets the client
 163  * import/export the syncobj's current fence from/to a &sync_file.
 164  * When a syncobj is exported to a sync file, that sync file wraps the
 165  * sycnobj's fence at the time of export and any later signal or reset
 166  * operations on the syncobj will not affect the exported sync file.
 167  * When a sync file is imported into a syncobj, the syncobj's fence is set
 168  * to the fence wrapped by that sync file.
 169  * Because sync files are immutable, resetting or signaling the syncobj
 170  * will not affect any sync files whose fences have been imported into the
 171  * syncobj.
 172  *
 173  *
 174  * Import/export of timeline points in timeline syncobjs
 175  * -----------------------------------------------------
 176  *
 177  * &DRM_IOCTL_SYNCOBJ_TRANSFER provides a mechanism to transfer a struct
 178  * &dma_fence_chain of a syncobj at a given u64 point to another u64 point
 179  * into another syncobj.
 180  *
 181  * Note that if you want to transfer a struct &dma_fence_chain from a given
 182  * point on a timeline syncobj from/into a binary syncobj, you can use the
 183  * point 0 to mean take/replace the fence in the syncobj.
 184  */
 185
 186 #include <linux/anon_inodes.h>
 187 #include <linux/file.h>
 188 #include <linux/fs.h>
 189 #include <linux/sched/signal.h>
 190 #include <linux/sync_file.h>
 191 #include <linux/uaccess.h>
 192
 193 #include <drm/drm.h>
 194 #include <drm/drm_drv.h>
 195 #include <drm/drm_file.h>
 196 #include <drm/drm_gem.h>
 197 #include <drm/drm_print.h>
 198 #include <drm/drm_syncobj.h>
 199 #include <drm/drm_utils.h>
 200
 201 #include "drm_internal.h"
 202
 203 struct syncobj_wait_entry {
 204         struct list_head node;
 205         struct task_struct *task;
 206         struct dma_fence *fence;
 207         struct dma_fence_cb fence_cb;
 208         u64    point;
 209 };
 210
 211 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
 212                                       struct syncobj_wait_entry *wait);
 213
 214 /**
 215  * drm_syncobj_find - lookup and reference a sync object.
 216  * @file_private: drm file private pointer
 217  * @handle: sync object handle to lookup.
 218  *
 219  * Returns a reference to the syncobj pointed to by handle or NULL. The
 220  * reference must be released by calling drm_syncobj_put().
 221  */
 222 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
 223                                      u32 handle)
 224 {
 225         struct drm_syncobj *syncobj;
 226
 227         spin_lock(&file_private->syncobj_table_lock);
 228
 229         /* Check if we currently have a reference on the object */
 230         syncobj = idr_find(&file_private->syncobj_idr, handle);
 231         if (syncobj)
 232                 drm_syncobj_get(syncobj);
 233
 234         spin_unlock(&file_private->syncobj_table_lock);
 235
 236         return syncobj;
 237 }
 238 EXPORT_SYMBOL(drm_syncobj_find);
 239
 240 static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,
 241                                        struct syncobj_wait_entry *wait)
 242 {
 243         struct dma_fence *fence;
 244
 245         if (wait->fence)
 246                 return;
 247
 248         spin_lock(&syncobj->lock);
 249         /* We've already tried once to get a fence and failed.  Now that we
 250          * have the lock, try one more time just to be sure we don't add a
 251          * callback when a fence has already been set.
 252          */
 253         fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
 254         if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
 255                 dma_fence_put(fence);
 256                 list_add_tail(&wait->node, &syncobj->cb_list);
 257         } else if (!fence) {
 258                 wait->fence = dma_fence_get_stub();
 259         } else {
 260                 wait->fence = fence;
 261         }
 262         spin_unlock(&syncobj->lock);
 263 }
 264
 265 static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj,
 266                                     struct syncobj_wait_entry *wait)
 267 {
 268         if (!wait->node.next)
 269                 return;
 270
 271         spin_lock(&syncobj->lock);
 272         list_del_init(&wait->node);
 273         spin_unlock(&syncobj->lock);
 274 }
 275
 276 /**
 277  * drm_syncobj_add_point - add new timeline point to the syncobj
 278  * @syncobj: sync object to add timeline point do
 279  * @chain: chain node to use to add the point
 280  * @fence: fence to encapsulate in the chain node
 281  * @point: sequence number to use for the point
 282  *
 283  * Add the chain node as new timeline point to the syncobj.
 284  */
 285 void drm_syncobj_add_point(struct drm_syncobj *syncobj,
 286                            struct dma_fence_chain *chain,
 287                            struct dma_fence *fence,
 288                            uint64_t point)
 289 {
 290         struct syncobj_wait_entry *cur, *tmp;
 291         struct dma_fence *prev;
 292
 293         dma_fence_get(fence);
 294
 295         spin_lock(&syncobj->lock);
 296
 297         prev = drm_syncobj_fence_get(syncobj);
 298         /* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */
 299         if (prev && prev->seqno >= point)
 300                 DRM_DEBUG("You are adding an unorder point to timeline!\n");
 301         dma_fence_chain_init(chain, prev, fence, point);
 302         rcu_assign_pointer(syncobj->fence, &chain->base);
 303
 304         list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node)
 305                 syncobj_wait_syncobj_func(syncobj, cur);
 306         spin_unlock(&syncobj->lock);
 307
 308         /* Walk the chain once to trigger garbage collection */
 309         dma_fence_chain_for_each(fence, prev);
 310         dma_fence_put(prev);
 311 }
 312 EXPORT_SYMBOL(drm_syncobj_add_point);
 313
 314 /**
 315  * drm_syncobj_replace_fence - replace fence in a sync object.
 316  * @syncobj: Sync object to replace fence in
 317  * @fence: fence to install in sync file.
 318  *
 319  * This replaces the fence on a sync object.
 320  */
 321 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
 322                                struct dma_fence *fence)
 323 {
 324         struct dma_fence *old_fence;
 325         struct syncobj_wait_entry *cur, *tmp;
 326
 327         if (fence)
 328                 dma_fence_get(fence);
 329
 330         spin_lock(&syncobj->lock);
 331
 332         old_fence = rcu_dereference_protected(syncobj->fence,
 333                                               lockdep_is_held(&syncobj->lock));
 334         rcu_assign_pointer(syncobj->fence, fence);
 335
 336         if (fence != old_fence) {
 337                 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node)
 338                         syncobj_wait_syncobj_func(syncobj, cur);
 339         }
 340
 341         spin_unlock(&syncobj->lock);
 342
 343         dma_fence_put(old_fence);
 344 }
 345 EXPORT_SYMBOL(drm_syncobj_replace_fence);
 346
 347 /**
 348  * drm_syncobj_assign_null_handle - assign a stub fence to the sync object
 349  * @syncobj: sync object to assign the fence on
 350  *
 351  * Assign a already signaled stub fence to the sync object.
 352  */
 353 static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
 354 {
 355         struct dma_fence *fence = dma_fence_allocate_private_stub();
 356
 357         if (IS_ERR(fence))
 358                 return PTR_ERR(fence);
 359
 360         drm_syncobj_replace_fence(syncobj, fence);
 361         dma_fence_put(fence);
 362         return 0;
 363 }
 364
 365 /* 5s default for wait submission */
 366 #define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL
 367 /**
 368  * drm_syncobj_find_fence - lookup and reference the fence in a sync object
 369  * @file_private: drm file private pointer
 370  * @handle: sync object handle to lookup.
 371  * @point: timeline point
 372  * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not
 373  * @fence: out parameter for the fence
 374  *
 375  * This is just a convenience function that combines drm_syncobj_find() and
 376  * drm_syncobj_fence_get().
 377  *
 378  * Returns 0 on success or a negative error value on failure. On success @fence
 379  * contains a reference to the fence, which must be released by calling
 380  * dma_fence_put().
 381  */
 382 int drm_syncobj_find_fence(struct drm_file *file_private,
 383                            u32 handle, u64 point, u64 flags,
 384                            struct dma_fence **fence)
 385 {
 386         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 387         struct syncobj_wait_entry wait;
 388         u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT);
 389         int ret;
 390
 391         if (!syncobj)
 392                 return -ENOENT;
 393
 394         /* Waiting for userspace with locks help is illegal cause that can
 395          * trivial deadlock with page faults for example. Make lockdep complain
 396          * about it early on.
 397          */
 398         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 399                 might_sleep();
 400                 lockdep_assert_none_held_once();
 401         }
 402
 403         *fence = drm_syncobj_fence_get(syncobj);
 404
 405         if (*fence) {
 406                 ret = dma_fence_chain_find_seqno(fence, point);
 407                 if (!ret)
 408                         goto out;
 409                 dma_fence_put(*fence);
 410         } else {
 411                 ret = -EINVAL;
 412         }
 413
 414         if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
 415                 goto out;
 416
 417         memset(&wait, 0, sizeof(wait));
 418         wait.task = current;
 419         wait.point = point;
 420         drm_syncobj_fence_add_wait(syncobj, &wait);
 421
 422         do {
 423                 set_current_state(TASK_INTERRUPTIBLE);
 424                 if (wait.fence) {
 425                         ret = 0;
 426                         break;
 427                 }
 428                 if (timeout == 0) {
 429                         ret = -ETIME;
 430                         break;
 431                 }
 432
 433                 if (signal_pending(current)) {
 434                         ret = -ERESTARTSYS;
 435                         break;
 436                 }
 437
 438                 timeout = schedule_timeout(timeout);
 439         } while (1);
 440
 441         __set_current_state(TASK_RUNNING);
 442         *fence = wait.fence;
 443
 444         if (wait.node.next)
 445                 drm_syncobj_remove_wait(syncobj, &wait);
 446
 447 out:
 448         drm_syncobj_put(syncobj);
 449
 450         return ret;
 451 }
 452 EXPORT_SYMBOL(drm_syncobj_find_fence);
 453
 454 /**
 455  * drm_syncobj_free - free a sync object.
 456  * @kref: kref to free.
 457  *
 458  * Only to be called from kref_put in drm_syncobj_put.
 459  */
 460 void drm_syncobj_free(struct kref *kref)
 461 {
 462         struct drm_syncobj *syncobj = container_of(kref,
 463                                                    struct drm_syncobj,
 464                                                    refcount);
 465         drm_syncobj_replace_fence(syncobj, NULL);
 466         kfree(syncobj);
 467 }
 468 EXPORT_SYMBOL(drm_syncobj_free);
 469
 470 /**
 471  * drm_syncobj_create - create a new syncobj
 472  * @out_syncobj: returned syncobj
 473  * @flags: DRM_SYNCOBJ_* flags
 474  * @fence: if non-NULL, the syncobj will represent this fence
 475  *
 476  * This is the first function to create a sync object. After creating, drivers
 477  * probably want to make it available to userspace, either through
 478  * drm_syncobj_get_handle() or drm_syncobj_get_fd().
 479  *
 480  * Returns 0 on success or a negative error value on failure.
 481  */
 482 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
 483                        struct dma_fence *fence)
 484 {
 485         int ret;
 486         struct drm_syncobj *syncobj;
 487
 488         syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
 489         if (!syncobj)
 490                 return -ENOMEM;
 491
 492         kref_init(&syncobj->refcount);
 493         INIT_LIST_HEAD(&syncobj->cb_list);
 494         spin_lock_init(&syncobj->lock);
 495
 496         if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) {
 497                 ret = drm_syncobj_assign_null_handle(syncobj);
 498                 if (ret < 0) {
 499                         drm_syncobj_put(syncobj);
 500                         return ret;
 501                 }
 502         }
 503
 504         if (fence)
 505                 drm_syncobj_replace_fence(syncobj, fence);
 506
 507         *out_syncobj = syncobj;
 508         return 0;
 509 }
 510 EXPORT_SYMBOL(drm_syncobj_create);
 511
 512 /**
 513  * drm_syncobj_get_handle - get a handle from a syncobj
 514  * @file_private: drm file private pointer
 515  * @syncobj: Sync object to export
 516  * @handle: out parameter with the new handle
 517  *
 518  * Exports a sync object created with drm_syncobj_create() as a handle on
 519  * @file_private to userspace.
 520  *
 521  * Returns 0 on success or a negative error value on failure.
 522  */
 523 int drm_syncobj_get_handle(struct drm_file *file_private,
 524                            struct drm_syncobj *syncobj, u32 *handle)
 525 {
 526         int ret;
 527
 528         /* take a reference to put in the idr */
 529         drm_syncobj_get(syncobj);
 530
 531         idr_preload(GFP_KERNEL);
 532         spin_lock(&file_private->syncobj_table_lock);
 533         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 534         spin_unlock(&file_private->syncobj_table_lock);
 535
 536         idr_preload_end();
 537
 538         if (ret < 0) {
 539                 drm_syncobj_put(syncobj);
 540                 return ret;
 541         }
 542
 543         *handle = ret;
 544         return 0;
 545 }
 546 EXPORT_SYMBOL(drm_syncobj_get_handle);
 547
 548 static int drm_syncobj_create_as_handle(struct drm_file *file_private,
 549                                         u32 *handle, uint32_t flags)
 550 {
 551         int ret;
 552         struct drm_syncobj *syncobj;
 553
 554         ret = drm_syncobj_create(&syncobj, flags, NULL);
 555         if (ret)
 556                 return ret;
 557
 558         ret = drm_syncobj_get_handle(file_private, syncobj, handle);
 559         drm_syncobj_put(syncobj);
 560         return ret;
 561 }
 562
 563 static int drm_syncobj_destroy(struct drm_file *file_private,
 564                                u32 handle)
 565 {
 566         struct drm_syncobj *syncobj;
 567
 568         spin_lock(&file_private->syncobj_table_lock);
 569         syncobj = idr_remove(&file_private->syncobj_idr, handle);
 570         spin_unlock(&file_private->syncobj_table_lock);
 571
 572         if (!syncobj)
 573                 return -EINVAL;
 574
 575         drm_syncobj_put(syncobj);
 576         return 0;
 577 }
 578
 579 static int drm_syncobj_file_release(struct inode *inode, struct file *file)
 580 {
 581         struct drm_syncobj *syncobj = file->private_data;
 582
 583         drm_syncobj_put(syncobj);
 584         return 0;
 585 }
 586
 587 static const struct file_operations drm_syncobj_file_fops = {
 588         .release = drm_syncobj_file_release,
 589 };
 590
 591 /**
 592  * drm_syncobj_get_fd - get a file descriptor from a syncobj
 593  * @syncobj: Sync object to export
 594  * @p_fd: out parameter with the new file descriptor
 595  *
 596  * Exports a sync object created with drm_syncobj_create() as a file descriptor.
 597  *
 598  * Returns 0 on success or a negative error value on failure.
 599  */
 600 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
 601 {
 602         struct file *file;
 603         int fd;
 604
 605         fd = get_unused_fd_flags(O_CLOEXEC);
 606         if (fd < 0)
 607                 return fd;
 608
 609         file = anon_inode_getfile("syncobj_file",
 610                                   &drm_syncobj_file_fops,
 611                                   syncobj, 0);
 612         if (IS_ERR(file)) {
 613                 put_unused_fd(fd);
 614                 return PTR_ERR(file);
 615         }
 616
 617         drm_syncobj_get(syncobj);
 618         fd_install(fd, file);
 619
 620         *p_fd = fd;
 621         return 0;
 622 }
 623 EXPORT_SYMBOL(drm_syncobj_get_fd);
 624
 625 static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
 626                                     u32 handle, int *p_fd)
 627 {
 628         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 629         int ret;
 630
 631         if (!syncobj)
 632                 return -EINVAL;
 633
 634         ret = drm_syncobj_get_fd(syncobj, p_fd);
 635         drm_syncobj_put(syncobj);
 636         return ret;
 637 }
 638
 639 static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
 640                                     int fd, u32 *handle)
 641 {
 642         struct drm_syncobj *syncobj;
 643         struct fd f = fdget(fd);
 644         int ret;
 645
 646         if (!f.file)
 647                 return -EINVAL;
 648
 649         if (f.file->f_op != &drm_syncobj_file_fops) {
 650                 fdput(f);
 651                 return -EINVAL;
 652         }
 653
 654         /* take a reference to put in the idr */
 655         syncobj = f.file->private_data;
 656         drm_syncobj_get(syncobj);
 657
 658         idr_preload(GFP_KERNEL);
 659         spin_lock(&file_private->syncobj_table_lock);
 660         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 661         spin_unlock(&file_private->syncobj_table_lock);
 662         idr_preload_end();
 663
 664         if (ret > 0) {
 665                 *handle = ret;
 666                 ret = 0;
 667         } else
 668                 drm_syncobj_put(syncobj);
 669
 670         fdput(f);
 671         return ret;
 672 }
 673
 674 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
 675                                               int fd, int handle)
 676 {
 677         struct dma_fence *fence = sync_file_get_fence(fd);
 678         struct drm_syncobj *syncobj;
 679
 680         if (!fence)
 681                 return -EINVAL;
 682
 683         syncobj = drm_syncobj_find(file_private, handle);
 684         if (!syncobj) {
 685                 dma_fence_put(fence);
 686                 return -ENOENT;
 687         }
 688
 689         drm_syncobj_replace_fence(syncobj, fence);
 690         dma_fence_put(fence);
 691         drm_syncobj_put(syncobj);
 692         return 0;
 693 }
 694
 695 static int drm_syncobj_export_sync_file(struct drm_file *file_private,
 696                                         int handle, int *p_fd)
 697 {
 698         int ret;
 699         struct dma_fence *fence;
 700         struct sync_file *sync_file;
 701         int fd = get_unused_fd_flags(O_CLOEXEC);
 702
 703         if (fd < 0)
 704                 return fd;
 705
 706         ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence);
 707         if (ret)
 708                 goto err_put_fd;
 709
 710         sync_file = sync_file_create(fence);
 711
 712         dma_fence_put(fence);
 713
 714         if (!sync_file) {
 715                 ret = -EINVAL;
 716                 goto err_put_fd;
 717         }
 718
 719         fd_install(fd, sync_file->file);
 720
 721         *p_fd = fd;
 722         return 0;
 723 err_put_fd:
 724         put_unused_fd(fd);
 725         return ret;
 726 }
 727 /**
 728  * drm_syncobj_open - initalizes syncobj file-private structures at devnode open time
 729  * @file_private: drm file-private structure to set up
 730  *
 731  * Called at device open time, sets up the structure for handling refcounting
 732  * of sync objects.
 733  */
 734 void
 735 drm_syncobj_open(struct drm_file *file_private)
 736 {
 737         idr_init_base(&file_private->syncobj_idr, 1);
 738         spin_lock_init(&file_private->syncobj_table_lock);
 739 }
 740
 741 static int
 742 drm_syncobj_release_handle(int id, void *ptr, void *data)
 743 {
 744         struct drm_syncobj *syncobj = ptr;
 745
 746         drm_syncobj_put(syncobj);
 747         return 0;
 748 }
 749
 750 /**
 751  * drm_syncobj_release - release file-private sync object resources
 752  * @file_private: drm file-private structure to clean up
 753  *
 754  * Called at close time when the filp is going away.
 755  *
 756  * Releases any remaining references on objects by this filp.
 757  */
 758 void
 759 drm_syncobj_release(struct drm_file *file_private)
 760 {
 761         idr_for_each(&file_private->syncobj_idr,
 762                      &drm_syncobj_release_handle, file_private);
 763         idr_destroy(&file_private->syncobj_idr);
 764 }
 765
 766 int
 767 drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
 768                          struct drm_file *file_private)
 769 {
 770         struct drm_syncobj_create *args = data;
 771
 772         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 773                 return -EOPNOTSUPP;
 774
 775         /* no valid flags yet */
 776         if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
 777                 return -EINVAL;
 778
 779         return drm_syncobj_create_as_handle(file_private,
 780                                             &args->handle, args->flags);
 781 }
 782
 783 int
 784 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
 785                           struct drm_file *file_private)
 786 {
 787         struct drm_syncobj_destroy *args = data;
 788
 789         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 790                 return -EOPNOTSUPP;
 791
 792         /* make sure padding is empty */
 793         if (args->pad)
 794                 return -EINVAL;
 795         return drm_syncobj_destroy(file_private, args->handle);
 796 }
 797
 798 int
 799 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
 800                                    struct drm_file *file_private)
 801 {
 802         struct drm_syncobj_handle *args = data;
 803
 804         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 805                 return -EOPNOTSUPP;
 806
 807         if (args->pad)
 808                 return -EINVAL;
 809
 810         if (args->flags != 0 &&
 811             args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 812                 return -EINVAL;
 813
 814         if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 815                 return drm_syncobj_export_sync_file(file_private, args->handle,
 816                                                     &args->fd);
 817
 818         return drm_syncobj_handle_to_fd(file_private, args->handle,
 819                                         &args->fd);
 820 }
 821
 822 int
 823 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
 824                                    struct drm_file *file_private)
 825 {
 826         struct drm_syncobj_handle *args = data;
 827
 828         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 829                 return -EOPNOTSUPP;
 830
 831         if (args->pad)
 832                 return -EINVAL;
 833
 834         if (args->flags != 0 &&
 835             args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 836                 return -EINVAL;
 837
 838         if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 839                 return drm_syncobj_import_sync_file_fence(file_private,
 840                                                           args->fd,
 841                                                           args->handle);
 842
 843         return drm_syncobj_fd_to_handle(file_private, args->fd,
 844                                         &args->handle);
 845 }
 846
 847 static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private,
 848                                             struct drm_syncobj_transfer *args)
 849 {
 850         struct drm_syncobj *timeline_syncobj = NULL;
 851         struct dma_fence *fence;
 852         struct dma_fence_chain *chain;
 853         int ret;
 854
 855         timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle);
 856         if (!timeline_syncobj) {
 857                 return -ENOENT;
 858         }
 859         ret = drm_syncobj_find_fence(file_private, args->src_handle,
 860                                      args->src_point, args->flags,
 861                                      &fence);
 862         if (ret)
 863                 goto err;
 864         chain = kzalloc(sizeof(struct dma_fence_chain), GFP_KERNEL);
 865         if (!chain) {
 866                 ret = -ENOMEM;
 867                 goto err1;
 868         }
 869         drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point);
 870 err1:
 871         dma_fence_put(fence);
 872 err:
 873         drm_syncobj_put(timeline_syncobj);
 874
 875         return ret;
 876 }
 877
 878 static int
 879 drm_syncobj_transfer_to_binary(struct drm_file *file_private,
 880                                struct drm_syncobj_transfer *args)
 881 {
 882         struct drm_syncobj *binary_syncobj = NULL;
 883         struct dma_fence *fence;
 884         int ret;
 885
 886         binary_syncobj = drm_syncobj_find(file_private, args->dst_handle);
 887         if (!binary_syncobj)
 888                 return -ENOENT;
 889         ret = drm_syncobj_find_fence(file_private, args->src_handle,
 890                                      args->src_point, args->flags, &fence);
 891         if (ret)
 892                 goto err;
 893         drm_syncobj_replace_fence(binary_syncobj, fence);
 894         dma_fence_put(fence);
 895 err:
 896         drm_syncobj_put(binary_syncobj);
 897
 898         return ret;
 899 }
 900 int
 901 drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data,
 902                            struct drm_file *file_private)
 903 {
 904         struct drm_syncobj_transfer *args = data;
 905         int ret;
 906
 907         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
 908                 return -EOPNOTSUPP;
 909
 910         if (args->pad)
 911                 return -EINVAL;
 912
 913         if (args->dst_point)
 914                 ret = drm_syncobj_transfer_to_timeline(file_private, args);
 915         else
 916                 ret = drm_syncobj_transfer_to_binary(file_private, args);
 917
 918         return ret;
 919 }
 920
 921 static void syncobj_wait_fence_func(struct dma_fence *fence,
 922                                     struct dma_fence_cb *cb)
 923 {
 924         struct syncobj_wait_entry *wait =
 925                 container_of(cb, struct syncobj_wait_entry, fence_cb);
 926
 927         wake_up_process(wait->task);
 928 }
 929
 930 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
 931                                       struct syncobj_wait_entry *wait)
 932 {
 933         struct dma_fence *fence;
 934
 935         /* This happens inside the syncobj lock */
 936         fence = rcu_dereference_protected(syncobj->fence,
 937                                           lockdep_is_held(&syncobj->lock));
 938         dma_fence_get(fence);
 939         if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
 940                 dma_fence_put(fence);
 941                 return;
 942         } else if (!fence) {
 943                 wait->fence = dma_fence_get_stub();
 944         } else {
 945                 wait->fence = fence;
 946         }
 947
 948         wake_up_process(wait->task);
 949         list_del_init(&wait->node);
 950 }
 951
 952 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
 953                                                   void __user *user_points,
 954                                                   uint32_t count,
 955                                                   uint32_t flags,
 956                                                   signed long timeout,
 957                                                   uint32_t *idx)
 958 {
 959         struct syncobj_wait_entry *entries;
 960         struct dma_fence *fence;
 961         uint64_t *points;
 962         uint32_t signaled_count, i;
 963
 964         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)
 965                 lockdep_assert_none_held_once();
 966
 967         points = kmalloc_array(count, sizeof(*points), GFP_KERNEL);
 968         if (points == NULL)
 969                 return -ENOMEM;
 970
 971         if (!user_points) {
 972                 memset(points, 0, count * sizeof(uint64_t));
 973
 974         } else if (copy_from_user(points, user_points,
 975                                   sizeof(uint64_t) * count)) {
 976                 timeout = -EFAULT;
 977                 goto err_free_points;
 978         }
 979
 980         entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
 981         if (!entries) {
 982                 timeout = -ENOMEM;
 983                 goto err_free_points;
 984         }
 985         /* Walk the list of sync objects and initialize entries.  We do
 986          * this up-front so that we can properly return -EINVAL if there is
 987          * a syncobj with a missing fence and then never have the chance of
 988          * returning -EINVAL again.
 989          */
 990         signaled_count = 0;
 991         for (i = 0; i < count; ++i) {
 992                 struct dma_fence *fence;
 993
 994                 entries[i].task = current;
 995                 entries[i].point = points[i];
 996                 fence = drm_syncobj_fence_get(syncobjs[i]);
 997                 if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) {
 998                         dma_fence_put(fence);
 999                         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
1000                                 continue;
1001                         } else {
1002                                 timeout = -EINVAL;
1003                                 goto cleanup_entries;
1004                         }
1005                 }
1006
1007                 if (fence)
1008                         entries[i].fence = fence;
1009                 else
1010                         entries[i].fence = dma_fence_get_stub();
1011
1012                 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1013                     dma_fence_is_signaled(entries[i].fence)) {
1014                         if (signaled_count == 0 && idx)
1015                                 *idx = i;
1016                         signaled_count++;
1017                 }
1018         }
1019
1020         if (signaled_count == count ||
1021             (signaled_count > 0 &&
1022              !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
1023                 goto cleanup_entries;
1024
1025         /* There's a very annoying laxness in the dma_fence API here, in
1026          * that backends are not required to automatically report when a
1027          * fence is signaled prior to fence->ops->enable_signaling() being
1028          * called.  So here if we fail to match signaled_count, we need to
1029          * fallthough and try a 0 timeout wait!
1030          */
1031
1032         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
1033                 for (i = 0; i < count; ++i)
1034                         drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]);
1035         }
1036
1037         do {
1038                 set_current_state(TASK_INTERRUPTIBLE);
1039
1040                 signaled_count = 0;
1041                 for (i = 0; i < count; ++i) {
1042                         fence = entries[i].fence;
1043                         if (!fence)
1044                                 continue;
1045
1046                         if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1047                             dma_fence_is_signaled(fence) ||
1048                             (!entries[i].fence_cb.func &&
1049                              dma_fence_add_callback(fence,
1050                                                     &entries[i].fence_cb,
1051                                                     syncobj_wait_fence_func))) {
1052                                 /* The fence has been signaled */
1053                                 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
1054                                         signaled_count++;
1055                                 } else {
1056                                         if (idx)
1057                                                 *idx = i;
1058                                         goto done_waiting;
1059                                 }
1060                         }
1061                 }
1062
1063                 if (signaled_count == count)
1064                         goto done_waiting;
1065
1066                 if (timeout == 0) {
1067                         timeout = -ETIME;
1068                         goto done_waiting;
1069                 }
1070
1071                 if (signal_pending(current)) {
1072                         timeout = -ERESTARTSYS;
1073                         goto done_waiting;
1074                 }
1075
1076                 timeout = schedule_timeout(timeout);
1077         } while (1);
1078
1079 done_waiting:
1080         __set_current_state(TASK_RUNNING);
1081
1082 cleanup_entries:
1083         for (i = 0; i < count; ++i) {
1084                 drm_syncobj_remove_wait(syncobjs[i], &entries[i]);
1085                 if (entries[i].fence_cb.func)
1086                         dma_fence_remove_callback(entries[i].fence,
1087                                                   &entries[i].fence_cb);
1088                 dma_fence_put(entries[i].fence);
1089         }
1090         kfree(entries);
1091
1092 err_free_points:
1093         kfree(points);
1094
1095         return timeout;
1096 }
1097
1098 /**
1099  * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
1100  *
1101  * @timeout_nsec: timeout nsec component in ns, 0 for poll
1102  *
1103  * Calculate the timeout in jiffies from an absolute time in sec/nsec.
1104  */
1105 signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
1106 {
1107         ktime_t abs_timeout, now;
1108         u64 timeout_ns, timeout_jiffies64;
1109
1110         /* make 0 timeout means poll - absolute 0 doesn't seem valid */
1111         if (timeout_nsec == 0)
1112                 return 0;
1113
1114         abs_timeout = ns_to_ktime(timeout_nsec);
1115         now = ktime_get();
1116
1117         if (!ktime_after(abs_timeout, now))
1118                 return 0;
1119
1120         timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
1121
1122         timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
1123         /*  clamp timeout to avoid infinite timeout */
1124         if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
1125                 return MAX_SCHEDULE_TIMEOUT - 1;
1126
1127         return timeout_jiffies64 + 1;
1128 }
1129 EXPORT_SYMBOL(drm_timeout_abs_to_jiffies);
1130
1131 static int drm_syncobj_array_wait(struct drm_device *dev,
1132                                   struct drm_file *file_private,
1133                                   struct drm_syncobj_wait *wait,
1134                                   struct drm_syncobj_timeline_wait *timeline_wait,
1135                                   struct drm_syncobj **syncobjs, bool timeline)
1136 {
1137         signed long timeout = 0;
1138         uint32_t first = ~0;
1139
1140         if (!timeline) {
1141                 timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
1142                 timeout = drm_syncobj_array_wait_timeout(syncobjs,
1143                                                          NULL,
1144                                                          wait->count_handles,
1145                                                          wait->flags,
1146                                                          timeout, &first);
1147                 if (timeout < 0)
1148                         return timeout;
1149                 wait->first_signaled = first;
1150         } else {
1151                 timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec);
1152                 timeout = drm_syncobj_array_wait_timeout(syncobjs,
1153                                                          u64_to_user_ptr(timeline_wait->points),
1154                                                          timeline_wait->count_handles,
1155                                                          timeline_wait->flags,
1156                                                          timeout, &first);
1157                 if (timeout < 0)
1158                         return timeout;
1159                 timeline_wait->first_signaled = first;
1160         }
1161         return 0;
1162 }
1163
1164 static int drm_syncobj_array_find(struct drm_file *file_private,
1165                                   void __user *user_handles,
1166                                   uint32_t count_handles,
1167                                   struct drm_syncobj ***syncobjs_out)
1168 {
1169         uint32_t i, *handles;
1170         struct drm_syncobj **syncobjs;
1171         int ret;
1172
1173         handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
1174         if (handles == NULL)
1175                 return -ENOMEM;
1176
1177         if (copy_from_user(handles, user_handles,
1178                            sizeof(uint32_t) * count_handles)) {
1179                 ret = -EFAULT;
1180                 goto err_free_handles;
1181         }
1182
1183         syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
1184         if (syncobjs == NULL) {
1185                 ret = -ENOMEM;
1186                 goto err_free_handles;
1187         }
1188
1189         for (i = 0; i < count_handles; i++) {
1190                 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
1191                 if (!syncobjs[i]) {
1192                         ret = -ENOENT;
1193                         goto err_put_syncobjs;
1194                 }
1195         }
1196
1197         kfree(handles);
1198         *syncobjs_out = syncobjs;
1199         return 0;
1200
1201 err_put_syncobjs:
1202         while (i-- > 0)
1203                 drm_syncobj_put(syncobjs[i]);
1204         kfree(syncobjs);
1205 err_free_handles:
1206         kfree(handles);
1207
1208         return ret;
1209 }
1210
1211 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
1212                                    uint32_t count)
1213 {
1214         uint32_t i;
1215
1216         for (i = 0; i < count; i++)
1217                 drm_syncobj_put(syncobjs[i]);
1218         kfree(syncobjs);
1219 }
1220
1221 int
1222 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
1223                        struct drm_file *file_private)
1224 {
1225         struct drm_syncobj_wait *args = data;
1226         struct drm_syncobj **syncobjs;
1227         int ret = 0;
1228
1229         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1230                 return -EOPNOTSUPP;
1231
1232         if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1233                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
1234                 return -EINVAL;
1235
1236         if (args->count_handles == 0)
1237                 return -EINVAL;
1238
1239         ret = drm_syncobj_array_find(file_private,
1240                                      u64_to_user_ptr(args->handles),
1241                                      args->count_handles,
1242                                      &syncobjs);
1243         if (ret < 0)
1244                 return ret;
1245
1246         ret = drm_syncobj_array_wait(dev, file_private,
1247                                      args, NULL, syncobjs, false);
1248
1249         drm_syncobj_array_free(syncobjs, args->count_handles);
1250
1251         return ret;
1252 }
1253
1254 int
1255 drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data,
1256                                 struct drm_file *file_private)
1257 {
1258         struct drm_syncobj_timeline_wait *args = data;
1259         struct drm_syncobj **syncobjs;
1260         int ret = 0;
1261
1262         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1263                 return -EOPNOTSUPP;
1264
1265         if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1266                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1267                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE))
1268                 return -EINVAL;
1269
1270         if (args->count_handles == 0)
1271                 return -EINVAL;
1272
1273         ret = drm_syncobj_array_find(file_private,
1274                                      u64_to_user_ptr(args->handles),
1275                                      args->count_handles,
1276                                      &syncobjs);
1277         if (ret < 0)
1278                 return ret;
1279
1280         ret = drm_syncobj_array_wait(dev, file_private,
1281                                      NULL, args, syncobjs, true);
1282
1283         drm_syncobj_array_free(syncobjs, args->count_handles);
1284
1285         return ret;
1286 }
1287
1288
1289 int
1290 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
1291                         struct drm_file *file_private)
1292 {
1293         struct drm_syncobj_array *args = data;
1294         struct drm_syncobj **syncobjs;
1295         uint32_t i;
1296         int ret;
1297
1298         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1299                 return -EOPNOTSUPP;
1300
1301         if (args->pad != 0)
1302                 return -EINVAL;
1303
1304         if (args->count_handles == 0)
1305                 return -EINVAL;
1306
1307         ret = drm_syncobj_array_find(file_private,
1308                                      u64_to_user_ptr(args->handles),
1309                                      args->count_handles,
1310                                      &syncobjs);
1311         if (ret < 0)
1312                 return ret;
1313
1314         for (i = 0; i < args->count_handles; i++)
1315                 drm_syncobj_replace_fence(syncobjs[i], NULL);
1316
1317         drm_syncobj_array_free(syncobjs, args->count_handles);
1318
1319         return 0;
1320 }
1321
1322 int
1323 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
1324                          struct drm_file *file_private)
1325 {
1326         struct drm_syncobj_array *args = data;
1327         struct drm_syncobj **syncobjs;
1328         uint32_t i;
1329         int ret;
1330
1331         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1332                 return -EOPNOTSUPP;
1333
1334         if (args->pad != 0)
1335                 return -EINVAL;
1336
1337         if (args->count_handles == 0)
1338                 return -EINVAL;
1339
1340         ret = drm_syncobj_array_find(file_private,
1341                                      u64_to_user_ptr(args->handles),
1342                                      args->count_handles,
1343                                      &syncobjs);
1344         if (ret < 0)
1345                 return ret;
1346
1347         for (i = 0; i < args->count_handles; i++) {
1348                 ret = drm_syncobj_assign_null_handle(syncobjs[i]);
1349                 if (ret < 0)
1350                         break;
1351         }
1352
1353         drm_syncobj_array_free(syncobjs, args->count_handles);
1354
1355         return ret;
1356 }
1357
1358 int
1359 drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data,
1360                                   struct drm_file *file_private)
1361 {
1362         struct drm_syncobj_timeline_array *args = data;
1363         struct drm_syncobj **syncobjs;
1364         struct dma_fence_chain **chains;
1365         uint64_t *points;
1366         uint32_t i, j;
1367         int ret;
1368
1369         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1370                 return -EOPNOTSUPP;
1371
1372         if (args->flags != 0)
1373                 return -EINVAL;
1374
1375         if (args->count_handles == 0)
1376                 return -EINVAL;
1377
1378         ret = drm_syncobj_array_find(file_private,
1379                                      u64_to_user_ptr(args->handles),
1380                                      args->count_handles,
1381                                      &syncobjs);
1382         if (ret < 0)
1383                 return ret;
1384
1385         points = kmalloc_array(args->count_handles, sizeof(*points),
1386                                GFP_KERNEL);
1387         if (!points) {
1388                 ret = -ENOMEM;
1389                 goto out;
1390         }
1391         if (!u64_to_user_ptr(args->points)) {
1392                 memset(points, 0, args->count_handles * sizeof(uint64_t));
1393         } else if (copy_from_user(points, u64_to_user_ptr(args->points),
1394                                   sizeof(uint64_t) * args->count_handles)) {
1395                 ret = -EFAULT;
1396                 goto err_points;
1397         }
1398
1399         chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL);
1400         if (!chains) {
1401                 ret = -ENOMEM;
1402                 goto err_points;
1403         }
1404         for (i = 0; i < args->count_handles; i++) {
1405                 chains[i] = kzalloc(sizeof(struct dma_fence_chain), GFP_KERNEL);
1406                 if (!chains[i]) {
1407                         for (j = 0; j < i; j++)
1408                                 kfree(chains[j]);
1409                         ret = -ENOMEM;
1410                         goto err_chains;
1411                 }
1412         }
1413
1414         for (i = 0; i < args->count_handles; i++) {
1415                 struct dma_fence *fence = dma_fence_get_stub();
1416
1417                 drm_syncobj_add_point(syncobjs[i], chains[i],
1418                                       fence, points[i]);
1419                 dma_fence_put(fence);
1420         }
1421 err_chains:
1422         kfree(chains);
1423 err_points:
1424         kfree(points);
1425 out:
1426         drm_syncobj_array_free(syncobjs, args->count_handles);
1427
1428         return ret;
1429 }
1430
1431 int drm_syncobj_query_ioctl(struct drm_device *dev, void *data,
1432                             struct drm_file *file_private)
1433 {
1434         struct drm_syncobj_timeline_array *args = data;
1435         struct drm_syncobj **syncobjs;
1436         uint64_t __user *points = u64_to_user_ptr(args->points);
1437         uint32_t i;
1438         int ret;
1439
1440         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1441                 return -EOPNOTSUPP;
1442
1443         if (args->flags & ~DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED)
1444                 return -EINVAL;
1445
1446         if (args->count_handles == 0)
1447                 return -EINVAL;
1448
1449         ret = drm_syncobj_array_find(file_private,
1450                                      u64_to_user_ptr(args->handles),
1451                                      args->count_handles,
1452                                      &syncobjs);
1453         if (ret < 0)
1454                 return ret;
1455
1456         for (i = 0; i < args->count_handles; i++) {
1457                 struct dma_fence_chain *chain;
1458                 struct dma_fence *fence;
1459                 uint64_t point;
1460
1461                 fence = drm_syncobj_fence_get(syncobjs[i]);
1462                 chain = to_dma_fence_chain(fence);
1463                 if (chain) {
1464                         struct dma_fence *iter, *last_signaled =
1465                                 dma_fence_get(fence);
1466
1467                         if (args->flags &
1468                             DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) {
1469                                 point = fence->seqno;
1470                         } else {
1471                                 dma_fence_chain_for_each(iter, fence) {
1472                                         if (iter->context != fence->context) {
1473                                                 dma_fence_put(iter);
1474                                                 /* It is most likely that timeline has
1475                                                 * unorder points. */
1476                                                 break;
1477                                         }
1478                                         dma_fence_put(last_signaled);
1479                                         last_signaled = dma_fence_get(iter);
1480                                 }
1481                                 point = dma_fence_is_signaled(last_signaled) ?
1482                                         last_signaled->seqno :
1483                                         to_dma_fence_chain(last_signaled)->prev_seqno;
1484                         }
1485                         dma_fence_put(last_signaled);
1486                 } else {
1487                         point = 0;
1488                 }
1489                 dma_fence_put(fence);
1490                 ret = copy_to_user(&points[i], &point, sizeof(uint64_t));
1491                 ret = ret ? -EFAULT : 0;
1492                 if (ret)
1493                         break;
1494         }
1495         drm_syncobj_array_free(syncobjs, args->count_handles);
1496
1497         return ret;
1498 }