drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008-2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include <linux/dma-fence-array.h>
  29 #include <linux/kthread.h>
  30 #include <linux/dma-resv.h>
  31 #include <linux/shmem_fs.h>
  32 #include <linux/slab.h>
  33 #include <linux/stop_machine.h>
  34 #include <linux/swap.h>
  35 #include <linux/pci.h>
  36 #include <linux/dma-buf.h>
  37 #include <linux/mman.h>
  38
  39 #include <drm/drm_cache.h>
  40 #include <drm/drm_vma_manager.h>
  41
  42 #include "display/intel_display.h"
  43 #include "display/intel_frontbuffer.h"
  44
  45 #include "gem/i915_gem_clflush.h"
  46 #include "gem/i915_gem_context.h"
  47 #include "gem/i915_gem_ioctls.h"
  48 #include "gem/i915_gem_mman.h"
  49 #include "gem/i915_gem_pm.h"
  50 #include "gem/i915_gem_region.h"
  51 #include "gem/i915_gem_userptr.h"
  52 #include "gt/intel_engine_user.h"
  53 #include "gt/intel_gt.h"
  54 #include "gt/intel_gt_pm.h"
  55 #include "gt/intel_workarounds.h"
  56
  57 #include "i915_drv.h"
  58 #include "i915_file_private.h"
  59 #include "i915_trace.h"
  60 #include "i915_vgpu.h"
  61 #include "intel_pm.h"
  62
  63 static int
  64 insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size)
  65 {
  66         int err;
  67
  68         err = mutex_lock_interruptible(&ggtt->vm.mutex);
  69         if (err)
  70                 return err;
  71
  72         memset(node, 0, sizeof(*node));
  73         err = drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
  74                                           size, 0, I915_COLOR_UNEVICTABLE,
  75                                           0, ggtt->mappable_end,
  76                                           DRM_MM_INSERT_LOW);
  77
  78         mutex_unlock(&ggtt->vm.mutex);
  79
  80         return err;
  81 }
  82
  83 static void
  84 remove_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node)
  85 {
  86         mutex_lock(&ggtt->vm.mutex);
  87         drm_mm_remove_node(node);
  88         mutex_unlock(&ggtt->vm.mutex);
  89 }
  90
  91 int
  92 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  93                             struct drm_file *file)
  94 {
  95         struct drm_i915_private *i915 = to_i915(dev);
  96         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
  97         struct drm_i915_gem_get_aperture *args = data;
  98         struct i915_vma *vma;
  99         u64 pinned;
 100
 101         if (mutex_lock_interruptible(&ggtt->vm.mutex))
 102                 return -EINTR;
 103
 104         pinned = ggtt->vm.reserved;
 105         list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)
 106                 if (i915_vma_is_pinned(vma))
 107                         pinned += vma->node.size;
 108
 109         mutex_unlock(&ggtt->vm.mutex);
 110
 111         args->aper_size = ggtt->vm.total;
 112         args->aper_available_size = args->aper_size - pinned;
 113
 114         return 0;
 115 }
 116
 117 int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
 118                            unsigned long flags)
 119 {
 120         struct intel_runtime_pm *rpm = &to_i915(obj->base.dev)->runtime_pm;
 121         LIST_HEAD(still_in_list);
 122         intel_wakeref_t wakeref;
 123         struct i915_vma *vma;
 124         int ret;
 125
 126         assert_object_held(obj);
 127
 128         if (list_empty(&obj->vma.list))
 129                 return 0;
 130
 131         /*
 132          * As some machines use ACPI to handle runtime-resume callbacks, and
 133          * ACPI is quite kmalloc happy, we cannot resume beneath the vm->mutex
 134          * as they are required by the shrinker. Ergo, we wake the device up
 135          * first just in case.
 136          */
 137         wakeref = intel_runtime_pm_get(rpm);
 138
 139 try_again:
 140         ret = 0;
 141         spin_lock(&obj->vma.lock);
 142         while (!ret && (vma = list_first_entry_or_null(&obj->vma.list,
 143                                                        struct i915_vma,
 144                                                        obj_link))) {
 145                 list_move_tail(&vma->obj_link, &still_in_list);
 146                 if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK))
 147                         continue;
 148
 149                 if (flags & I915_GEM_OBJECT_UNBIND_TEST) {
 150                         ret = -EBUSY;
 151                         break;
 152                 }
 153
 154                 ret = -EAGAIN;
 155                 if (!i915_vm_tryget(vma->vm))
 156                         break;
 157
 158                 /* Prevent vma being freed by i915_vma_parked as we unbind */
 159                 vma = __i915_vma_get(vma);
 160                 spin_unlock(&obj->vma.lock);
 161
 162                 if (vma) {
 163                         bool vm_trylock = !!(flags & I915_GEM_OBJECT_UNBIND_VM_TRYLOCK);
 164                         ret = -EBUSY;
 165                         if (flags & I915_GEM_OBJECT_UNBIND_ASYNC) {
 166                                 assert_object_held(vma->obj);
 167                                 ret = i915_vma_unbind_async(vma, vm_trylock);
 168                         }
 169
 170                         if (ret == -EBUSY && (flags & I915_GEM_OBJECT_UNBIND_ACTIVE ||
 171                                               !i915_vma_is_active(vma))) {
 172                                 if (vm_trylock) {
 173                                         if (mutex_trylock(&vma->vm->mutex)) {
 174                                                 ret = __i915_vma_unbind(vma);
 175                                                 mutex_unlock(&vma->vm->mutex);
 176                                         } else {
 177                                                 ret = -EBUSY;
 178                                         }
 179                                 } else {
 180                                         ret = i915_vma_unbind(vma);
 181                                 }
 182                         }
 183
 184                         __i915_vma_put(vma);
 185                 }
 186
 187                 i915_vm_put(vma->vm);
 188                 spin_lock(&obj->vma.lock);
 189         }
 190         list_splice_init(&still_in_list, &obj->vma.list);
 191         spin_unlock(&obj->vma.lock);
 192
 193         if (ret == -EAGAIN && flags & I915_GEM_OBJECT_UNBIND_BARRIER) {
 194                 rcu_barrier(); /* flush the i915_vm_release() */
 195                 goto try_again;
 196         }
 197
 198         intel_runtime_pm_put(rpm, wakeref);
 199
 200         return ret;
 201 }
 202
 203 static int
 204 shmem_pread(struct page *page, int offset, int len, char __user *user_data,
 205             bool needs_clflush)
 206 {
 207         char *vaddr;
 208         int ret;
 209
 210         vaddr = kmap(page);
 211
 212         if (needs_clflush)
 213                 drm_clflush_virt_range(vaddr + offset, len);
 214
 215         ret = __copy_to_user(user_data, vaddr + offset, len);
 216
 217         kunmap(page);
 218
 219         return ret ? -EFAULT : 0;
 220 }
 221
 222 static int
 223 i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
 224                      struct drm_i915_gem_pread *args)
 225 {
 226         unsigned int needs_clflush;
 227         unsigned int idx, offset;
 228         char __user *user_data;
 229         u64 remain;
 230         int ret;
 231
 232         ret = i915_gem_object_lock_interruptible(obj, NULL);
 233         if (ret)
 234                 return ret;
 235
 236         ret = i915_gem_object_pin_pages(obj);
 237         if (ret)
 238                 goto err_unlock;
 239
 240         ret = i915_gem_object_prepare_read(obj, &needs_clflush);
 241         if (ret)
 242                 goto err_unpin;
 243
 244         i915_gem_object_finish_access(obj);
 245         i915_gem_object_unlock(obj);
 246
 247         remain = args->size;
 248         user_data = u64_to_user_ptr(args->data_ptr);
 249         offset = offset_in_page(args->offset);
 250         for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
 251                 struct page *page = i915_gem_object_get_page(obj, idx);
 252                 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
 253
 254                 ret = shmem_pread(page, offset, length, user_data,
 255                                   needs_clflush);
 256                 if (ret)
 257                         break;
 258
 259                 remain -= length;
 260                 user_data += length;
 261                 offset = 0;
 262         }
 263
 264         i915_gem_object_unpin_pages(obj);
 265         return ret;
 266
 267 err_unpin:
 268         i915_gem_object_unpin_pages(obj);
 269 err_unlock:
 270         i915_gem_object_unlock(obj);
 271         return ret;
 272 }
 273
 274 static inline bool
 275 gtt_user_read(struct io_mapping *mapping,
 276               loff_t base, int offset,
 277               char __user *user_data, int length)
 278 {
 279         void __iomem *vaddr;
 280         unsigned long unwritten;
 281
 282         /* We can use the cpu mem copy function because this is X86. */
 283         vaddr = io_mapping_map_atomic_wc(mapping, base);
 284         unwritten = __copy_to_user_inatomic(user_data,
 285                                             (void __force *)vaddr + offset,
 286                                             length);
 287         io_mapping_unmap_atomic(vaddr);
 288         if (unwritten) {
 289                 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
 290                 unwritten = copy_to_user(user_data,
 291                                          (void __force *)vaddr + offset,
 292                                          length);
 293                 io_mapping_unmap(vaddr);
 294         }
 295         return unwritten;
 296 }
 297
 298 static struct i915_vma *i915_gem_gtt_prepare(struct drm_i915_gem_object *obj,
 299                                              struct drm_mm_node *node,
 300                                              bool write)
 301 {
 302         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 303         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 304         struct i915_vma *vma;
 305         struct i915_gem_ww_ctx ww;
 306         int ret;
 307
 308         i915_gem_ww_ctx_init(&ww, true);
 309 retry:
 310         vma = ERR_PTR(-ENODEV);
 311         ret = i915_gem_object_lock(obj, &ww);
 312         if (ret)
 313                 goto err_ww;
 314
 315         ret = i915_gem_object_set_to_gtt_domain(obj, write);
 316         if (ret)
 317                 goto err_ww;
 318
 319         if (!i915_gem_object_is_tiled(obj))
 320                 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0,
 321                                                   PIN_MAPPABLE |
 322                                                   PIN_NONBLOCK /* NOWARN */ |
 323                                                   PIN_NOEVICT);
 324         if (vma == ERR_PTR(-EDEADLK)) {
 325                 ret = -EDEADLK;
 326                 goto err_ww;
 327         } else if (!IS_ERR(vma)) {
 328                 node->start = i915_ggtt_offset(vma);
 329                 node->flags = 0;
 330         } else {
 331                 ret = insert_mappable_node(ggtt, node, PAGE_SIZE);
 332                 if (ret)
 333                         goto err_ww;
 334                 GEM_BUG_ON(!drm_mm_node_allocated(node));
 335                 vma = NULL;
 336         }
 337
 338         ret = i915_gem_object_pin_pages(obj);
 339         if (ret) {
 340                 if (drm_mm_node_allocated(node)) {
 341                         ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);
 342                         remove_mappable_node(ggtt, node);
 343                 } else {
 344                         i915_vma_unpin(vma);
 345                 }
 346         }
 347
 348 err_ww:
 349         if (ret == -EDEADLK) {
 350                 ret = i915_gem_ww_ctx_backoff(&ww);
 351                 if (!ret)
 352                         goto retry;
 353         }
 354         i915_gem_ww_ctx_fini(&ww);
 355
 356         return ret ? ERR_PTR(ret) : vma;
 357 }
 358
 359 static void i915_gem_gtt_cleanup(struct drm_i915_gem_object *obj,
 360                                  struct drm_mm_node *node,
 361                                  struct i915_vma *vma)
 362 {
 363         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 364         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 365
 366         i915_gem_object_unpin_pages(obj);
 367         if (drm_mm_node_allocated(node)) {
 368                 ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);
 369                 remove_mappable_node(ggtt, node);
 370         } else {
 371                 i915_vma_unpin(vma);
 372         }
 373 }
 374
 375 static int
 376 i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
 377                    const struct drm_i915_gem_pread *args)
 378 {
 379         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 380         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 381         intel_wakeref_t wakeref;
 382         struct drm_mm_node node;
 383         void __user *user_data;
 384         struct i915_vma *vma;
 385         u64 remain, offset;
 386         int ret = 0;
 387
 388         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 389
 390         vma = i915_gem_gtt_prepare(obj, &node, false);
 391         if (IS_ERR(vma)) {
 392                 ret = PTR_ERR(vma);
 393                 goto out_rpm;
 394         }
 395
 396         user_data = u64_to_user_ptr(args->data_ptr);
 397         remain = args->size;
 398         offset = args->offset;
 399
 400         while (remain > 0) {
 401                 /* Operation in this page
 402                  *
 403                  * page_base = page offset within aperture
 404                  * page_offset = offset within page
 405                  * page_length = bytes to copy for this page
 406                  */
 407                 u32 page_base = node.start;
 408                 unsigned page_offset = offset_in_page(offset);
 409                 unsigned page_length = PAGE_SIZE - page_offset;
 410                 page_length = remain < page_length ? remain : page_length;
 411                 if (drm_mm_node_allocated(&node)) {
 412                         ggtt->vm.insert_page(&ggtt->vm,
 413                                              i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 414                                              node.start, I915_CACHE_NONE, 0);
 415                 } else {
 416                         page_base += offset & PAGE_MASK;
 417                 }
 418
 419                 if (gtt_user_read(&ggtt->iomap, page_base, page_offset,
 420                                   user_data, page_length)) {
 421                         ret = -EFAULT;
 422                         break;
 423                 }
 424
 425                 remain -= page_length;
 426                 user_data += page_length;
 427                 offset += page_length;
 428         }
 429
 430         i915_gem_gtt_cleanup(obj, &node, vma);
 431 out_rpm:
 432         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 433         return ret;
 434 }
 435
 436 /**
 437  * Reads data from the object referenced by handle.
 438  * @dev: drm device pointer
 439  * @data: ioctl data blob
 440  * @file: drm file pointer
 441  *
 442  * On error, the contents of *data are undefined.
 443  */
 444 int
 445 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 446                      struct drm_file *file)
 447 {
 448         struct drm_i915_private *i915 = to_i915(dev);
 449         struct drm_i915_gem_pread *args = data;
 450         struct drm_i915_gem_object *obj;
 451         int ret;
 452
 453         /* PREAD is disallowed for all platforms after TGL-LP.  This also
 454          * covers all platforms with local memory.
 455          */
 456         if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915))
 457                 return -EOPNOTSUPP;
 458
 459         if (args->size == 0)
 460                 return 0;
 461
 462         if (!access_ok(u64_to_user_ptr(args->data_ptr),
 463                        args->size))
 464                 return -EFAULT;
 465
 466         obj = i915_gem_object_lookup(file, args->handle);
 467         if (!obj)
 468                 return -ENOENT;
 469
 470         /* Bounds check source.  */
 471         if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
 472                 ret = -EINVAL;
 473                 goto out;
 474         }
 475
 476         trace_i915_gem_object_pread(obj, args->offset, args->size);
 477         ret = -ENODEV;
 478         if (obj->ops->pread)
 479                 ret = obj->ops->pread(obj, args);
 480         if (ret != -ENODEV)
 481                 goto out;
 482
 483         ret = i915_gem_object_wait(obj,
 484                                    I915_WAIT_INTERRUPTIBLE,
 485                                    MAX_SCHEDULE_TIMEOUT);
 486         if (ret)
 487                 goto out;
 488
 489         ret = i915_gem_shmem_pread(obj, args);
 490         if (ret == -EFAULT || ret == -ENODEV)
 491                 ret = i915_gem_gtt_pread(obj, args);
 492
 493 out:
 494         i915_gem_object_put(obj);
 495         return ret;
 496 }
 497
 498 /* This is the fast write path which cannot handle
 499  * page faults in the source data
 500  */
 501
 502 static inline bool
 503 ggtt_write(struct io_mapping *mapping,
 504            loff_t base, int offset,
 505            char __user *user_data, int length)
 506 {
 507         void __iomem *vaddr;
 508         unsigned long unwritten;
 509
 510         /* We can use the cpu mem copy function because this is X86. */
 511         vaddr = io_mapping_map_atomic_wc(mapping, base);
 512         unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset,
 513                                                       user_data, length);
 514         io_mapping_unmap_atomic(vaddr);
 515         if (unwritten) {
 516                 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
 517                 unwritten = copy_from_user((void __force *)vaddr + offset,
 518                                            user_data, length);
 519                 io_mapping_unmap(vaddr);
 520         }
 521
 522         return unwritten;
 523 }
 524
 525 /**
 526  * This is the fast pwrite path, where we copy the data directly from the
 527  * user into the GTT, uncached.
 528  * @obj: i915 GEM object
 529  * @args: pwrite arguments structure
 530  */
 531 static int
 532 i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
 533                          const struct drm_i915_gem_pwrite *args)
 534 {
 535         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 536         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 537         struct intel_runtime_pm *rpm = &i915->runtime_pm;
 538         intel_wakeref_t wakeref;
 539         struct drm_mm_node node;
 540         struct i915_vma *vma;
 541         u64 remain, offset;
 542         void __user *user_data;
 543         int ret = 0;
 544
 545         if (i915_gem_object_has_struct_page(obj)) {
 546                 /*
 547                  * Avoid waking the device up if we can fallback, as
 548                  * waking/resuming is very slow (worst-case 10-100 ms
 549                  * depending on PCI sleeps and our own resume time).
 550                  * This easily dwarfs any performance advantage from
 551                  * using the cache bypass of indirect GGTT access.
 552                  */
 553                 wakeref = intel_runtime_pm_get_if_in_use(rpm);
 554                 if (!wakeref)
 555                         return -EFAULT;
 556         } else {
 557                 /* No backing pages, no fallback, we must force GGTT access */
 558                 wakeref = intel_runtime_pm_get(rpm);
 559         }
 560
 561         vma = i915_gem_gtt_prepare(obj, &node, true);
 562         if (IS_ERR(vma)) {
 563                 ret = PTR_ERR(vma);
 564                 goto out_rpm;
 565         }
 566
 567         i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
 568
 569         user_data = u64_to_user_ptr(args->data_ptr);
 570         offset = args->offset;
 571         remain = args->size;
 572         while (remain) {
 573                 /* Operation in this page
 574                  *
 575                  * page_base = page offset within aperture
 576                  * page_offset = offset within page
 577                  * page_length = bytes to copy for this page
 578                  */
 579                 u32 page_base = node.start;
 580                 unsigned int page_offset = offset_in_page(offset);
 581                 unsigned int page_length = PAGE_SIZE - page_offset;
 582                 page_length = remain < page_length ? remain : page_length;
 583                 if (drm_mm_node_allocated(&node)) {
 584                         /* flush the write before we modify the GGTT */
 585                         intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 586                         ggtt->vm.insert_page(&ggtt->vm,
 587                                              i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 588                                              node.start, I915_CACHE_NONE, 0);
 589                         wmb(); /* flush modifications to the GGTT (insert_page) */
 590                 } else {
 591                         page_base += offset & PAGE_MASK;
 592                 }
 593                 /* If we get a fault while copying data, then (presumably) our
 594                  * source page isn't available.  Return the error and we'll
 595                  * retry in the slow path.
 596                  * If the object is non-shmem backed, we retry again with the
 597                  * path that handles page fault.
 598                  */
 599                 if (ggtt_write(&ggtt->iomap, page_base, page_offset,
 600                                user_data, page_length)) {
 601                         ret = -EFAULT;
 602                         break;
 603                 }
 604
 605                 remain -= page_length;
 606                 user_data += page_length;
 607                 offset += page_length;
 608         }
 609
 610         intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 611         i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
 612
 613         i915_gem_gtt_cleanup(obj, &node, vma);
 614 out_rpm:
 615         intel_runtime_pm_put(rpm, wakeref);
 616         return ret;
 617 }
 618
 619 /* Per-page copy function for the shmem pwrite fastpath.
 620  * Flushes invalid cachelines before writing to the target if
 621  * needs_clflush_before is set and flushes out any written cachelines after
 622  * writing if needs_clflush is set.
 623  */
 624 static int
 625 shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
 626              bool needs_clflush_before,
 627              bool needs_clflush_after)
 628 {
 629         char *vaddr;
 630         int ret;
 631
 632         vaddr = kmap(page);
 633
 634         if (needs_clflush_before)
 635                 drm_clflush_virt_range(vaddr + offset, len);
 636
 637         ret = __copy_from_user(vaddr + offset, user_data, len);
 638         if (!ret && needs_clflush_after)
 639                 drm_clflush_virt_range(vaddr + offset, len);
 640
 641         kunmap(page);
 642
 643         return ret ? -EFAULT : 0;
 644 }
 645
 646 static int
 647 i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
 648                       const struct drm_i915_gem_pwrite *args)
 649 {
 650         unsigned int partial_cacheline_write;
 651         unsigned int needs_clflush;
 652         unsigned int offset, idx;
 653         void __user *user_data;
 654         u64 remain;
 655         int ret;
 656
 657         ret = i915_gem_object_lock_interruptible(obj, NULL);
 658         if (ret)
 659                 return ret;
 660
 661         ret = i915_gem_object_pin_pages(obj);
 662         if (ret)
 663                 goto err_unlock;
 664
 665         ret = i915_gem_object_prepare_write(obj, &needs_clflush);
 666         if (ret)
 667                 goto err_unpin;
 668
 669         i915_gem_object_finish_access(obj);
 670         i915_gem_object_unlock(obj);
 671
 672         /* If we don't overwrite a cacheline completely we need to be
 673          * careful to have up-to-date data by first clflushing. Don't
 674          * overcomplicate things and flush the entire patch.
 675          */
 676         partial_cacheline_write = 0;
 677         if (needs_clflush & CLFLUSH_BEFORE)
 678                 partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1;
 679
 680         user_data = u64_to_user_ptr(args->data_ptr);
 681         remain = args->size;
 682         offset = offset_in_page(args->offset);
 683         for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
 684                 struct page *page = i915_gem_object_get_page(obj, idx);
 685                 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
 686
 687                 ret = shmem_pwrite(page, offset, length, user_data,
 688                                    (offset | length) & partial_cacheline_write,
 689                                    needs_clflush & CLFLUSH_AFTER);
 690                 if (ret)
 691                         break;
 692
 693                 remain -= length;
 694                 user_data += length;
 695                 offset = 0;
 696         }
 697
 698         i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
 699
 700         i915_gem_object_unpin_pages(obj);
 701         return ret;
 702
 703 err_unpin:
 704         i915_gem_object_unpin_pages(obj);
 705 err_unlock:
 706         i915_gem_object_unlock(obj);
 707         return ret;
 708 }
 709
 710 /**
 711  * Writes data to the object referenced by handle.
 712  * @dev: drm device
 713  * @data: ioctl data blob
 714  * @file: drm file
 715  *
 716  * On error, the contents of the buffer that were to be modified are undefined.
 717  */
 718 int
 719 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 720                       struct drm_file *file)
 721 {
 722         struct drm_i915_private *i915 = to_i915(dev);
 723         struct drm_i915_gem_pwrite *args = data;
 724         struct drm_i915_gem_object *obj;
 725         int ret;
 726
 727         /* PWRITE is disallowed for all platforms after TGL-LP.  This also
 728          * covers all platforms with local memory.
 729          */
 730         if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915))
 731                 return -EOPNOTSUPP;
 732
 733         if (args->size == 0)
 734                 return 0;
 735
 736         if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size))
 737                 return -EFAULT;
 738
 739         obj = i915_gem_object_lookup(file, args->handle);
 740         if (!obj)
 741                 return -ENOENT;
 742
 743         /* Bounds check destination. */
 744         if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
 745                 ret = -EINVAL;
 746                 goto err;
 747         }
 748
 749         /* Writes not allowed into this read-only object */
 750         if (i915_gem_object_is_readonly(obj)) {
 751                 ret = -EINVAL;
 752                 goto err;
 753         }
 754
 755         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 756
 757         ret = -ENODEV;
 758         if (obj->ops->pwrite)
 759                 ret = obj->ops->pwrite(obj, args);
 760         if (ret != -ENODEV)
 761                 goto err;
 762
 763         ret = i915_gem_object_wait(obj,
 764                                    I915_WAIT_INTERRUPTIBLE |
 765                                    I915_WAIT_ALL,
 766                                    MAX_SCHEDULE_TIMEOUT);
 767         if (ret)
 768                 goto err;
 769
 770         ret = -EFAULT;
 771         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 772          * it would end up going through the fenced access, and we'll get
 773          * different detiling behavior between reading and writing.
 774          * pread/pwrite currently are reading and writing from the CPU
 775          * perspective, requiring manual detiling by the client.
 776          */
 777         if (!i915_gem_object_has_struct_page(obj) ||
 778             i915_gem_cpu_write_needs_clflush(obj))
 779                 /* Note that the gtt paths might fail with non-page-backed user
 780                  * pointers (e.g. gtt mappings when moving data between
 781                  * textures). Fallback to the shmem path in that case.
 782                  */
 783                 ret = i915_gem_gtt_pwrite_fast(obj, args);
 784
 785         if (ret == -EFAULT || ret == -ENOSPC) {
 786                 if (i915_gem_object_has_struct_page(obj))
 787                         ret = i915_gem_shmem_pwrite(obj, args);
 788         }
 789
 790 err:
 791         i915_gem_object_put(obj);
 792         return ret;
 793 }
 794
 795 /**
 796  * Called when user space has done writes to this buffer
 797  * @dev: drm device
 798  * @data: ioctl data blob
 799  * @file: drm file
 800  */
 801 int
 802 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 803                          struct drm_file *file)
 804 {
 805         struct drm_i915_gem_sw_finish *args = data;
 806         struct drm_i915_gem_object *obj;
 807
 808         obj = i915_gem_object_lookup(file, args->handle);
 809         if (!obj)
 810                 return -ENOENT;
 811
 812         /*
 813          * Proxy objects are barred from CPU access, so there is no
 814          * need to ban sw_finish as it is a nop.
 815          */
 816
 817         /* Pinned buffers may be scanout, so flush the cache */
 818         i915_gem_object_flush_if_display(obj);
 819         i915_gem_object_put(obj);
 820
 821         return 0;
 822 }
 823
 824 void i915_gem_runtime_suspend(struct drm_i915_private *i915)
 825 {
 826         struct drm_i915_gem_object *obj, *on;
 827         int i;
 828
 829         /*
 830          * Only called during RPM suspend. All users of the userfault_list
 831          * must be holding an RPM wakeref to ensure that this can not
 832          * run concurrently with themselves (and use the struct_mutex for
 833          * protection between themselves).
 834          */
 835
 836         list_for_each_entry_safe(obj, on,
 837                                  &to_gt(i915)->ggtt->userfault_list, userfault_link)
 838                 __i915_gem_object_release_mmap_gtt(obj);
 839
 840         /*
 841          * The fence will be lost when the device powers down. If any were
 842          * in use by hardware (i.e. they are pinned), we should not be powering
 843          * down! All other fences will be reacquired by the user upon waking.
 844          */
 845         for (i = 0; i < to_gt(i915)->ggtt->num_fences; i++) {
 846                 struct i915_fence_reg *reg = &to_gt(i915)->ggtt->fence_regs[i];
 847
 848                 /*
 849                  * Ideally we want to assert that the fence register is not
 850                  * live at this point (i.e. that no piece of code will be
 851                  * trying to write through fence + GTT, as that both violates
 852                  * our tracking of activity and associated locking/barriers,
 853                  * but also is illegal given that the hw is powered down).
 854                  *
 855                  * Previously we used reg->pin_count as a "liveness" indicator.
 856                  * That is not sufficient, and we need a more fine-grained
 857                  * tool if we want to have a sanity check here.
 858                  */
 859
 860                 if (!reg->vma)
 861                         continue;
 862
 863                 GEM_BUG_ON(i915_vma_has_userfault(reg->vma));
 864                 reg->dirty = true;
 865         }
 866 }
 867
 868 static void discard_ggtt_vma(struct i915_vma *vma)
 869 {
 870         struct drm_i915_gem_object *obj = vma->obj;
 871
 872         spin_lock(&obj->vma.lock);
 873         if (!RB_EMPTY_NODE(&vma->obj_node)) {
 874                 rb_erase(&vma->obj_node, &obj->vma.tree);
 875                 RB_CLEAR_NODE(&vma->obj_node);
 876         }
 877         spin_unlock(&obj->vma.lock);
 878 }
 879
 880 struct i915_vma *
 881 i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj,
 882                             struct i915_gem_ww_ctx *ww,
 883                             const struct i915_ggtt_view *view,
 884                             u64 size, u64 alignment, u64 flags)
 885 {
 886         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 887         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 888         struct i915_vma *vma;
 889         int ret;
 890
 891         GEM_WARN_ON(!ww);
 892
 893         if (flags & PIN_MAPPABLE &&
 894             (!view || view->type == I915_GGTT_VIEW_NORMAL)) {
 895                 /*
 896                  * If the required space is larger than the available
 897                  * aperture, we will not able to find a slot for the
 898                  * object and unbinding the object now will be in
 899                  * vain. Worse, doing so may cause us to ping-pong
 900                  * the object in and out of the Global GTT and
 901                  * waste a lot of cycles under the mutex.
 902                  */
 903                 if (obj->base.size > ggtt->mappable_end)
 904                         return ERR_PTR(-E2BIG);
 905
 906                 /*
 907                  * If NONBLOCK is set the caller is optimistically
 908                  * trying to cache the full object within the mappable
 909                  * aperture, and *must* have a fallback in place for
 910                  * situations where we cannot bind the object. We
 911                  * can be a little more lax here and use the fallback
 912                  * more often to avoid costly migrations of ourselves
 913                  * and other objects within the aperture.
 914                  *
 915                  * Half-the-aperture is used as a simple heuristic.
 916                  * More interesting would to do search for a free
 917                  * block prior to making the commitment to unbind.
 918                  * That caters for the self-harm case, and with a
 919                  * little more heuristics (e.g. NOFAULT, NOEVICT)
 920                  * we could try to minimise harm to others.
 921                  */
 922                 if (flags & PIN_NONBLOCK &&
 923                     obj->base.size > ggtt->mappable_end / 2)
 924                         return ERR_PTR(-ENOSPC);
 925         }
 926
 927 new_vma:
 928         vma = i915_vma_instance(obj, &ggtt->vm, view);
 929         if (IS_ERR(vma))
 930                 return vma;
 931
 932         if (i915_vma_misplaced(vma, size, alignment, flags)) {
 933                 if (flags & PIN_NONBLOCK) {
 934                         if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))
 935                                 return ERR_PTR(-ENOSPC);
 936
 937                         if (flags & PIN_MAPPABLE &&
 938                             vma->fence_size > ggtt->mappable_end / 2)
 939                                 return ERR_PTR(-ENOSPC);
 940                 }
 941
 942                 if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)) {
 943                         discard_ggtt_vma(vma);
 944                         goto new_vma;
 945                 }
 946
 947                 ret = i915_vma_unbind(vma);
 948                 if (ret)
 949                         return ERR_PTR(ret);
 950         }
 951
 952         ret = i915_vma_pin_ww(vma, ww, size, alignment, flags | PIN_GLOBAL);
 953
 954         if (ret)
 955                 return ERR_PTR(ret);
 956
 957         if (vma->fence && !i915_gem_object_is_tiled(obj)) {
 958                 mutex_lock(&ggtt->vm.mutex);
 959                 i915_vma_revoke_fence(vma);
 960                 mutex_unlock(&ggtt->vm.mutex);
 961         }
 962
 963         ret = i915_vma_wait_for_bind(vma);
 964         if (ret) {
 965                 i915_vma_unpin(vma);
 966                 return ERR_PTR(ret);
 967         }
 968
 969         return vma;
 970 }
 971
 972 struct i915_vma * __must_check
 973 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
 974                          const struct i915_ggtt_view *view,
 975                          u64 size, u64 alignment, u64 flags)
 976 {
 977         struct i915_gem_ww_ctx ww;
 978         struct i915_vma *ret;
 979         int err;
 980
 981         for_i915_gem_ww(&ww, err, true) {
 982                 err = i915_gem_object_lock(obj, &ww);
 983                 if (err)
 984                         continue;
 985
 986                 ret = i915_gem_object_ggtt_pin_ww(obj, &ww, view, size,
 987                                                   alignment, flags);
 988                 if (IS_ERR(ret))
 989                         err = PTR_ERR(ret);
 990         }
 991
 992         return err ? ERR_PTR(err) : ret;
 993 }
 994
 995 int
 996 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 997                        struct drm_file *file_priv)
 998 {
 999         struct drm_i915_private *i915 = to_i915(dev);
1000         struct drm_i915_gem_madvise *args = data;
1001         struct drm_i915_gem_object *obj;
1002         int err;
1003
1004         switch (args->madv) {
1005         case I915_MADV_DONTNEED:
1006         case I915_MADV_WILLNEED:
1007             break;
1008         default:
1009             return -EINVAL;
1010         }
1011
1012         obj = i915_gem_object_lookup(file_priv, args->handle);
1013         if (!obj)
1014                 return -ENOENT;
1015
1016         err = i915_gem_object_lock_interruptible(obj, NULL);
1017         if (err)
1018                 goto out;
1019
1020         if (i915_gem_object_has_pages(obj) &&
1021             i915_gem_object_is_tiled(obj) &&
1022             i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
1023                 if (obj->mm.madv == I915_MADV_WILLNEED) {
1024                         GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj));
1025                         i915_gem_object_clear_tiling_quirk(obj);
1026                         i915_gem_object_make_shrinkable(obj);
1027                 }
1028                 if (args->madv == I915_MADV_WILLNEED) {
1029                         GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));
1030                         i915_gem_object_make_unshrinkable(obj);
1031                         i915_gem_object_set_tiling_quirk(obj);
1032                 }
1033         }
1034
1035         if (obj->mm.madv != __I915_MADV_PURGED) {
1036                 obj->mm.madv = args->madv;
1037                 if (obj->ops->adjust_lru)
1038                         obj->ops->adjust_lru(obj);
1039         }
1040
1041         if (i915_gem_object_has_pages(obj) ||
1042             i915_gem_object_has_self_managed_shrink_list(obj)) {
1043                 unsigned long flags;
1044
1045                 spin_lock_irqsave(&i915->mm.obj_lock, flags);
1046                 if (!list_empty(&obj->mm.link)) {
1047                         struct list_head *list;
1048
1049                         if (obj->mm.madv != I915_MADV_WILLNEED)
1050                                 list = &i915->mm.purge_list;
1051                         else
1052                                 list = &i915->mm.shrink_list;
1053                         list_move_tail(&obj->mm.link, list);
1054
1055                 }
1056                 spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
1057         }
1058
1059         /* if the object is no longer attached, discard its backing storage */
1060         if (obj->mm.madv == I915_MADV_DONTNEED &&
1061             !i915_gem_object_has_pages(obj))
1062                 i915_gem_object_truncate(obj);
1063
1064         args->retained = obj->mm.madv != __I915_MADV_PURGED;
1065
1066         i915_gem_object_unlock(obj);
1067 out:
1068         i915_gem_object_put(obj);
1069         return err;
1070 }
1071
1072 int i915_gem_init(struct drm_i915_private *dev_priv)
1073 {
1074         int ret;
1075
1076         /* We need to fallback to 4K pages if host doesn't support huge gtt. */
1077         if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv))
1078                 mkwrite_device_info(dev_priv)->page_sizes =
1079                         I915_GTT_PAGE_SIZE_4K;
1080
1081         ret = i915_gem_init_userptr(dev_priv);
1082         if (ret)
1083                 return ret;
1084
1085         intel_uc_fetch_firmwares(&to_gt(dev_priv)->uc);
1086         intel_wopcm_init(&dev_priv->wopcm);
1087
1088         ret = i915_init_ggtt(dev_priv);
1089         if (ret) {
1090                 GEM_BUG_ON(ret == -EIO);
1091                 goto err_unlock;
1092         }
1093
1094         /*
1095          * Despite its name intel_init_clock_gating applies both display
1096          * clock gating workarounds; GT mmio workarounds and the occasional
1097          * GT power context workaround. Worse, sometimes it includes a context
1098          * register workaround which we need to apply before we record the
1099          * default HW state for all contexts.
1100          *
1101          * FIXME: break up the workarounds and apply them at the right time!
1102          */
1103         intel_init_clock_gating(dev_priv);
1104
1105         ret = intel_gt_init(to_gt(dev_priv));
1106         if (ret)
1107                 goto err_unlock;
1108
1109         return 0;
1110
1111         /*
1112          * Unwinding is complicated by that we want to handle -EIO to mean
1113          * disable GPU submission but keep KMS alive. We want to mark the
1114          * HW as irrevisibly wedged, but keep enough state around that the
1115          * driver doesn't explode during runtime.
1116          */
1117 err_unlock:
1118         i915_gem_drain_workqueue(dev_priv);
1119
1120         if (ret != -EIO)
1121                 intel_uc_cleanup_firmwares(&to_gt(dev_priv)->uc);
1122
1123         if (ret == -EIO) {
1124                 /*
1125                  * Allow engines or uC initialisation to fail by marking the GPU
1126                  * as wedged. But we only want to do this when the GPU is angry,
1127                  * for all other failure, such as an allocation failure, bail.
1128                  */
1129                 if (!intel_gt_is_wedged(to_gt(dev_priv))) {
1130                         i915_probe_error(dev_priv,
1131                                          "Failed to initialize GPU, declaring it wedged!\n");
1132                         intel_gt_set_wedged(to_gt(dev_priv));
1133                 }
1134
1135                 /* Minimal basic recovery for KMS */
1136                 ret = i915_ggtt_enable_hw(dev_priv);
1137                 i915_ggtt_resume(to_gt(dev_priv)->ggtt);
1138                 intel_init_clock_gating(dev_priv);
1139         }
1140
1141         i915_gem_drain_freed_objects(dev_priv);
1142
1143         return ret;
1144 }
1145
1146 void i915_gem_driver_register(struct drm_i915_private *i915)
1147 {
1148         i915_gem_driver_register__shrinker(i915);
1149
1150         intel_engines_driver_register(i915);
1151 }
1152
1153 void i915_gem_driver_unregister(struct drm_i915_private *i915)
1154 {
1155         i915_gem_driver_unregister__shrinker(i915);
1156 }
1157
1158 void i915_gem_driver_remove(struct drm_i915_private *dev_priv)
1159 {
1160         intel_wakeref_auto_fini(&to_gt(dev_priv)->ggtt->userfault_wakeref);
1161
1162         i915_gem_suspend_late(dev_priv);
1163         intel_gt_driver_remove(to_gt(dev_priv));
1164         dev_priv->uabi_engines = RB_ROOT;
1165
1166         /* Flush any outstanding unpin_work. */
1167         i915_gem_drain_workqueue(dev_priv);
1168
1169         i915_gem_drain_freed_objects(dev_priv);
1170 }
1171
1172 void i915_gem_driver_release(struct drm_i915_private *dev_priv)
1173 {
1174         intel_gt_driver_release(to_gt(dev_priv));
1175
1176         intel_uc_cleanup_firmwares(&to_gt(dev_priv)->uc);
1177
1178         i915_gem_drain_freed_objects(dev_priv);
1179
1180         drm_WARN_ON(&dev_priv->drm, !list_empty(&dev_priv->gem.contexts.list));
1181 }
1182
1183 static void i915_gem_init__mm(struct drm_i915_private *i915)
1184 {
1185         spin_lock_init(&i915->mm.obj_lock);
1186
1187         init_llist_head(&i915->mm.free_list);
1188
1189         INIT_LIST_HEAD(&i915->mm.purge_list);
1190         INIT_LIST_HEAD(&i915->mm.shrink_list);
1191
1192         i915_gem_init__objects(i915);
1193 }
1194
1195 void i915_gem_init_early(struct drm_i915_private *dev_priv)
1196 {
1197         i915_gem_init__mm(dev_priv);
1198         i915_gem_init__contexts(dev_priv);
1199
1200         spin_lock_init(&dev_priv->fb_tracking.lock);
1201 }
1202
1203 void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
1204 {
1205         i915_gem_drain_freed_objects(dev_priv);
1206         GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list));
1207         GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count));
1208         drm_WARN_ON(&dev_priv->drm, dev_priv->mm.shrink_count);
1209 }
1210
1211 int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file)
1212 {
1213         struct drm_i915_file_private *file_priv;
1214         int ret;
1215
1216         DRM_DEBUG("\n");
1217
1218         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
1219         if (!file_priv)
1220                 return -ENOMEM;
1221
1222         file->driver_priv = file_priv;
1223         file_priv->dev_priv = i915;
1224         file_priv->file = file;
1225
1226         file_priv->bsd_engine = -1;
1227         file_priv->hang_timestamp = jiffies;
1228
1229         ret = i915_gem_context_open(i915, file);
1230         if (ret)
1231                 kfree(file_priv);
1232
1233         return ret;
1234 }
1235
1236 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1237 #include "selftests/mock_gem_device.c"
1238 #include "selftests/i915_gem.c"
1239 #endif