drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008-2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include <drm/drm_vma_manager.h>
  29 #include <linux/dma-fence-array.h>
  30 #include <linux/kthread.h>
  31 #include <linux/dma-resv.h>
  32 #include <linux/shmem_fs.h>
  33 #include <linux/slab.h>
  34 #include <linux/stop_machine.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37 #include <linux/dma-buf.h>
  38 #include <linux/mman.h>
  39
  40 #include "display/intel_display.h"
  41 #include "display/intel_frontbuffer.h"
  42
  43 #include "gem/i915_gem_clflush.h"
  44 #include "gem/i915_gem_context.h"
  45 #include "gem/i915_gem_ioctls.h"
  46 #include "gem/i915_gem_mman.h"
  47 #include "gem/i915_gem_region.h"
  48 #include "gt/intel_engine_user.h"
  49 #include "gt/intel_gt.h"
  50 #include "gt/intel_gt_pm.h"
  51 #include "gt/intel_workarounds.h"
  52
  53 #include "i915_drv.h"
  54 #include "i915_trace.h"
  55 #include "i915_vgpu.h"
  56
  57 #include "intel_pm.h"
  58
  59 static int
  60 insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size)
  61 {
  62         int err;
  63
  64         err = mutex_lock_interruptible(&ggtt->vm.mutex);
  65         if (err)
  66                 return err;
  67
  68         memset(node, 0, sizeof(*node));
  69         err = drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
  70                                           size, 0, I915_COLOR_UNEVICTABLE,
  71                                           0, ggtt->mappable_end,
  72                                           DRM_MM_INSERT_LOW);
  73
  74         mutex_unlock(&ggtt->vm.mutex);
  75
  76         return err;
  77 }
  78
  79 static void
  80 remove_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node)
  81 {
  82         mutex_lock(&ggtt->vm.mutex);
  83         drm_mm_remove_node(node);
  84         mutex_unlock(&ggtt->vm.mutex);
  85 }
  86
  87 int
  88 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  89                             struct drm_file *file)
  90 {
  91         struct drm_i915_private *i915 = to_i915(dev);
  92         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
  93         struct drm_i915_gem_get_aperture *args = data;
  94         struct i915_vma *vma;
  95         u64 pinned;
  96
  97         if (mutex_lock_interruptible(&ggtt->vm.mutex))
  98                 return -EINTR;
  99
 100         pinned = ggtt->vm.reserved;
 101         list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)
 102                 if (i915_vma_is_pinned(vma))
 103                         pinned += vma->node.size;
 104
 105         mutex_unlock(&ggtt->vm.mutex);
 106
 107         args->aper_size = ggtt->vm.total;
 108         args->aper_available_size = args->aper_size - pinned;
 109
 110         return 0;
 111 }
 112
 113 int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
 114                            unsigned long flags)
 115 {
 116         struct intel_runtime_pm *rpm = &to_i915(obj->base.dev)->runtime_pm;
 117         LIST_HEAD(still_in_list);
 118         intel_wakeref_t wakeref;
 119         struct i915_vma *vma;
 120         int ret;
 121
 122         if (list_empty(&obj->vma.list))
 123                 return 0;
 124
 125         /*
 126          * As some machines use ACPI to handle runtime-resume callbacks, and
 127          * ACPI is quite kmalloc happy, we cannot resume beneath the vm->mutex
 128          * as they are required by the shrinker. Ergo, we wake the device up
 129          * first just in case.
 130          */
 131         wakeref = intel_runtime_pm_get(rpm);
 132
 133 try_again:
 134         ret = 0;
 135         spin_lock(&obj->vma.lock);
 136         while (!ret && (vma = list_first_entry_or_null(&obj->vma.list,
 137                                                        struct i915_vma,
 138                                                        obj_link))) {
 139                 struct i915_address_space *vm = vma->vm;
 140
 141                 list_move_tail(&vma->obj_link, &still_in_list);
 142                 if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK))
 143                         continue;
 144
 145                 if (flags & I915_GEM_OBJECT_UNBIND_TEST) {
 146                         ret = -EBUSY;
 147                         break;
 148                 }
 149
 150                 ret = -EAGAIN;
 151                 if (!i915_vm_tryopen(vm))
 152                         break;
 153
 154                 /* Prevent vma being freed by i915_vma_parked as we unbind */
 155                 vma = __i915_vma_get(vma);
 156                 spin_unlock(&obj->vma.lock);
 157
 158                 if (vma) {
 159                         bool vm_trylock = !!(flags & I915_GEM_OBJECT_UNBIND_VM_TRYLOCK);
 160                         ret = -EBUSY;
 161                         if (flags & I915_GEM_OBJECT_UNBIND_ASYNC) {
 162                                 assert_object_held(vma->obj);
 163                                 ret = i915_vma_unbind_async(vma, vm_trylock);
 164                         }
 165
 166                         if (ret == -EBUSY && (flags & I915_GEM_OBJECT_UNBIND_ACTIVE ||
 167                                               !i915_vma_is_active(vma))) {
 168                                 if (vm_trylock) {
 169                                         if (mutex_trylock(&vma->vm->mutex)) {
 170                                                 ret = __i915_vma_unbind(vma);
 171                                                 mutex_unlock(&vma->vm->mutex);
 172                                         } else {
 173                                                 ret = -EBUSY;
 174                                         }
 175                                 } else {
 176                                         ret = i915_vma_unbind(vma);
 177                                 }
 178                         }
 179
 180                         __i915_vma_put(vma);
 181                 }
 182
 183                 i915_vm_close(vm);
 184                 spin_lock(&obj->vma.lock);
 185         }
 186         list_splice_init(&still_in_list, &obj->vma.list);
 187         spin_unlock(&obj->vma.lock);
 188
 189         if (ret == -EAGAIN && flags & I915_GEM_OBJECT_UNBIND_BARRIER) {
 190                 rcu_barrier(); /* flush the i915_vm_release() */
 191                 goto try_again;
 192         }
 193
 194         intel_runtime_pm_put(rpm, wakeref);
 195
 196         return ret;
 197 }
 198
 199 static int
 200 shmem_pread(struct page *page, int offset, int len, char __user *user_data,
 201             bool needs_clflush)
 202 {
 203         char *vaddr;
 204         int ret;
 205
 206         vaddr = kmap(page);
 207
 208         if (needs_clflush)
 209                 drm_clflush_virt_range(vaddr + offset, len);
 210
 211         ret = __copy_to_user(user_data, vaddr + offset, len);
 212
 213         kunmap(page);
 214
 215         return ret ? -EFAULT : 0;
 216 }
 217
 218 static int
 219 i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
 220                      struct drm_i915_gem_pread *args)
 221 {
 222         unsigned int needs_clflush;
 223         unsigned int idx, offset;
 224         char __user *user_data;
 225         u64 remain;
 226         int ret;
 227
 228         ret = i915_gem_object_lock_interruptible(obj, NULL);
 229         if (ret)
 230                 return ret;
 231
 232         ret = i915_gem_object_pin_pages(obj);
 233         if (ret)
 234                 goto err_unlock;
 235
 236         ret = i915_gem_object_prepare_read(obj, &needs_clflush);
 237         if (ret)
 238                 goto err_unpin;
 239
 240         i915_gem_object_finish_access(obj);
 241         i915_gem_object_unlock(obj);
 242
 243         remain = args->size;
 244         user_data = u64_to_user_ptr(args->data_ptr);
 245         offset = offset_in_page(args->offset);
 246         for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
 247                 struct page *page = i915_gem_object_get_page(obj, idx);
 248                 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
 249
 250                 ret = shmem_pread(page, offset, length, user_data,
 251                                   needs_clflush);
 252                 if (ret)
 253                         break;
 254
 255                 remain -= length;
 256                 user_data += length;
 257                 offset = 0;
 258         }
 259
 260         i915_gem_object_unpin_pages(obj);
 261         return ret;
 262
 263 err_unpin:
 264         i915_gem_object_unpin_pages(obj);
 265 err_unlock:
 266         i915_gem_object_unlock(obj);
 267         return ret;
 268 }
 269
 270 static inline bool
 271 gtt_user_read(struct io_mapping *mapping,
 272               loff_t base, int offset,
 273               char __user *user_data, int length)
 274 {
 275         void __iomem *vaddr;
 276         unsigned long unwritten;
 277
 278         /* We can use the cpu mem copy function because this is X86. */
 279         vaddr = io_mapping_map_atomic_wc(mapping, base);
 280         unwritten = __copy_to_user_inatomic(user_data,
 281                                             (void __force *)vaddr + offset,
 282                                             length);
 283         io_mapping_unmap_atomic(vaddr);
 284         if (unwritten) {
 285                 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
 286                 unwritten = copy_to_user(user_data,
 287                                          (void __force *)vaddr + offset,
 288                                          length);
 289                 io_mapping_unmap(vaddr);
 290         }
 291         return unwritten;
 292 }
 293
 294 static struct i915_vma *i915_gem_gtt_prepare(struct drm_i915_gem_object *obj,
 295                                              struct drm_mm_node *node,
 296                                              bool write)
 297 {
 298         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 299         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 300         struct i915_vma *vma;
 301         struct i915_gem_ww_ctx ww;
 302         int ret;
 303
 304         i915_gem_ww_ctx_init(&ww, true);
 305 retry:
 306         vma = ERR_PTR(-ENODEV);
 307         ret = i915_gem_object_lock(obj, &ww);
 308         if (ret)
 309                 goto err_ww;
 310
 311         ret = i915_gem_object_set_to_gtt_domain(obj, write);
 312         if (ret)
 313                 goto err_ww;
 314
 315         if (!i915_gem_object_is_tiled(obj))
 316                 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0,
 317                                                   PIN_MAPPABLE |
 318                                                   PIN_NONBLOCK /* NOWARN */ |
 319                                                   PIN_NOEVICT);
 320         if (vma == ERR_PTR(-EDEADLK)) {
 321                 ret = -EDEADLK;
 322                 goto err_ww;
 323         } else if (!IS_ERR(vma)) {
 324                 node->start = i915_ggtt_offset(vma);
 325                 node->flags = 0;
 326         } else {
 327                 ret = insert_mappable_node(ggtt, node, PAGE_SIZE);
 328                 if (ret)
 329                         goto err_ww;
 330                 GEM_BUG_ON(!drm_mm_node_allocated(node));
 331                 vma = NULL;
 332         }
 333
 334         ret = i915_gem_object_pin_pages(obj);
 335         if (ret) {
 336                 if (drm_mm_node_allocated(node)) {
 337                         ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);
 338                         remove_mappable_node(ggtt, node);
 339                 } else {
 340                         i915_vma_unpin(vma);
 341                 }
 342         }
 343
 344 err_ww:
 345         if (ret == -EDEADLK) {
 346                 ret = i915_gem_ww_ctx_backoff(&ww);
 347                 if (!ret)
 348                         goto retry;
 349         }
 350         i915_gem_ww_ctx_fini(&ww);
 351
 352         return ret ? ERR_PTR(ret) : vma;
 353 }
 354
 355 static void i915_gem_gtt_cleanup(struct drm_i915_gem_object *obj,
 356                                  struct drm_mm_node *node,
 357                                  struct i915_vma *vma)
 358 {
 359         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 360         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 361
 362         i915_gem_object_unpin_pages(obj);
 363         if (drm_mm_node_allocated(node)) {
 364                 ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);
 365                 remove_mappable_node(ggtt, node);
 366         } else {
 367                 i915_vma_unpin(vma);
 368         }
 369 }
 370
 371 static int
 372 i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
 373                    const struct drm_i915_gem_pread *args)
 374 {
 375         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 376         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 377         intel_wakeref_t wakeref;
 378         struct drm_mm_node node;
 379         void __user *user_data;
 380         struct i915_vma *vma;
 381         u64 remain, offset;
 382         int ret = 0;
 383
 384         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 385
 386         vma = i915_gem_gtt_prepare(obj, &node, false);
 387         if (IS_ERR(vma)) {
 388                 ret = PTR_ERR(vma);
 389                 goto out_rpm;
 390         }
 391
 392         user_data = u64_to_user_ptr(args->data_ptr);
 393         remain = args->size;
 394         offset = args->offset;
 395
 396         while (remain > 0) {
 397                 /* Operation in this page
 398                  *
 399                  * page_base = page offset within aperture
 400                  * page_offset = offset within page
 401                  * page_length = bytes to copy for this page
 402                  */
 403                 u32 page_base = node.start;
 404                 unsigned page_offset = offset_in_page(offset);
 405                 unsigned page_length = PAGE_SIZE - page_offset;
 406                 page_length = remain < page_length ? remain : page_length;
 407                 if (drm_mm_node_allocated(&node)) {
 408                         ggtt->vm.insert_page(&ggtt->vm,
 409                                              i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 410                                              node.start, I915_CACHE_NONE, 0);
 411                 } else {
 412                         page_base += offset & PAGE_MASK;
 413                 }
 414
 415                 if (gtt_user_read(&ggtt->iomap, page_base, page_offset,
 416                                   user_data, page_length)) {
 417                         ret = -EFAULT;
 418                         break;
 419                 }
 420
 421                 remain -= page_length;
 422                 user_data += page_length;
 423                 offset += page_length;
 424         }
 425
 426         i915_gem_gtt_cleanup(obj, &node, vma);
 427 out_rpm:
 428         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 429         return ret;
 430 }
 431
 432 /**
 433  * Reads data from the object referenced by handle.
 434  * @dev: drm device pointer
 435  * @data: ioctl data blob
 436  * @file: drm file pointer
 437  *
 438  * On error, the contents of *data are undefined.
 439  */
 440 int
 441 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 442                      struct drm_file *file)
 443 {
 444         struct drm_i915_private *i915 = to_i915(dev);
 445         struct drm_i915_gem_pread *args = data;
 446         struct drm_i915_gem_object *obj;
 447         int ret;
 448
 449         /* PREAD is disallowed for all platforms after TGL-LP.  This also
 450          * covers all platforms with local memory.
 451          */
 452         if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915))
 453                 return -EOPNOTSUPP;
 454
 455         if (args->size == 0)
 456                 return 0;
 457
 458         if (!access_ok(u64_to_user_ptr(args->data_ptr),
 459                        args->size))
 460                 return -EFAULT;
 461
 462         obj = i915_gem_object_lookup(file, args->handle);
 463         if (!obj)
 464                 return -ENOENT;
 465
 466         /* Bounds check source.  */
 467         if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
 468                 ret = -EINVAL;
 469                 goto out;
 470         }
 471
 472         trace_i915_gem_object_pread(obj, args->offset, args->size);
 473         ret = -ENODEV;
 474         if (obj->ops->pread)
 475                 ret = obj->ops->pread(obj, args);
 476         if (ret != -ENODEV)
 477                 goto out;
 478
 479         ret = i915_gem_object_wait(obj,
 480                                    I915_WAIT_INTERRUPTIBLE,
 481                                    MAX_SCHEDULE_TIMEOUT);
 482         if (ret)
 483                 goto out;
 484
 485         ret = i915_gem_shmem_pread(obj, args);
 486         if (ret == -EFAULT || ret == -ENODEV)
 487                 ret = i915_gem_gtt_pread(obj, args);
 488
 489 out:
 490         i915_gem_object_put(obj);
 491         return ret;
 492 }
 493
 494 /* This is the fast write path which cannot handle
 495  * page faults in the source data
 496  */
 497
 498 static inline bool
 499 ggtt_write(struct io_mapping *mapping,
 500            loff_t base, int offset,
 501            char __user *user_data, int length)
 502 {
 503         void __iomem *vaddr;
 504         unsigned long unwritten;
 505
 506         /* We can use the cpu mem copy function because this is X86. */
 507         vaddr = io_mapping_map_atomic_wc(mapping, base);
 508         unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset,
 509                                                       user_data, length);
 510         io_mapping_unmap_atomic(vaddr);
 511         if (unwritten) {
 512                 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
 513                 unwritten = copy_from_user((void __force *)vaddr + offset,
 514                                            user_data, length);
 515                 io_mapping_unmap(vaddr);
 516         }
 517
 518         return unwritten;
 519 }
 520
 521 /**
 522  * This is the fast pwrite path, where we copy the data directly from the
 523  * user into the GTT, uncached.
 524  * @obj: i915 GEM object
 525  * @args: pwrite arguments structure
 526  */
 527 static int
 528 i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
 529                          const struct drm_i915_gem_pwrite *args)
 530 {
 531         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 532         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 533         struct intel_runtime_pm *rpm = &i915->runtime_pm;
 534         intel_wakeref_t wakeref;
 535         struct drm_mm_node node;
 536         struct i915_vma *vma;
 537         u64 remain, offset;
 538         void __user *user_data;
 539         int ret = 0;
 540
 541         if (i915_gem_object_has_struct_page(obj)) {
 542                 /*
 543                  * Avoid waking the device up if we can fallback, as
 544                  * waking/resuming is very slow (worst-case 10-100 ms
 545                  * depending on PCI sleeps and our own resume time).
 546                  * This easily dwarfs any performance advantage from
 547                  * using the cache bypass of indirect GGTT access.
 548                  */
 549                 wakeref = intel_runtime_pm_get_if_in_use(rpm);
 550                 if (!wakeref)
 551                         return -EFAULT;
 552         } else {
 553                 /* No backing pages, no fallback, we must force GGTT access */
 554                 wakeref = intel_runtime_pm_get(rpm);
 555         }
 556
 557         vma = i915_gem_gtt_prepare(obj, &node, true);
 558         if (IS_ERR(vma)) {
 559                 ret = PTR_ERR(vma);
 560                 goto out_rpm;
 561         }
 562
 563         i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
 564
 565         user_data = u64_to_user_ptr(args->data_ptr);
 566         offset = args->offset;
 567         remain = args->size;
 568         while (remain) {
 569                 /* Operation in this page
 570                  *
 571                  * page_base = page offset within aperture
 572                  * page_offset = offset within page
 573                  * page_length = bytes to copy for this page
 574                  */
 575                 u32 page_base = node.start;
 576                 unsigned int page_offset = offset_in_page(offset);
 577                 unsigned int page_length = PAGE_SIZE - page_offset;
 578                 page_length = remain < page_length ? remain : page_length;
 579                 if (drm_mm_node_allocated(&node)) {
 580                         /* flush the write before we modify the GGTT */
 581                         intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 582                         ggtt->vm.insert_page(&ggtt->vm,
 583                                              i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 584                                              node.start, I915_CACHE_NONE, 0);
 585                         wmb(); /* flush modifications to the GGTT (insert_page) */
 586                 } else {
 587                         page_base += offset & PAGE_MASK;
 588                 }
 589                 /* If we get a fault while copying data, then (presumably) our
 590                  * source page isn't available.  Return the error and we'll
 591                  * retry in the slow path.
 592                  * If the object is non-shmem backed, we retry again with the
 593                  * path that handles page fault.
 594                  */
 595                 if (ggtt_write(&ggtt->iomap, page_base, page_offset,
 596                                user_data, page_length)) {
 597                         ret = -EFAULT;
 598                         break;
 599                 }
 600
 601                 remain -= page_length;
 602                 user_data += page_length;
 603                 offset += page_length;
 604         }
 605
 606         intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 607         i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
 608
 609         i915_gem_gtt_cleanup(obj, &node, vma);
 610 out_rpm:
 611         intel_runtime_pm_put(rpm, wakeref);
 612         return ret;
 613 }
 614
 615 /* Per-page copy function for the shmem pwrite fastpath.
 616  * Flushes invalid cachelines before writing to the target if
 617  * needs_clflush_before is set and flushes out any written cachelines after
 618  * writing if needs_clflush is set.
 619  */
 620 static int
 621 shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
 622              bool needs_clflush_before,
 623              bool needs_clflush_after)
 624 {
 625         char *vaddr;
 626         int ret;
 627
 628         vaddr = kmap(page);
 629
 630         if (needs_clflush_before)
 631                 drm_clflush_virt_range(vaddr + offset, len);
 632
 633         ret = __copy_from_user(vaddr + offset, user_data, len);
 634         if (!ret && needs_clflush_after)
 635                 drm_clflush_virt_range(vaddr + offset, len);
 636
 637         kunmap(page);
 638
 639         return ret ? -EFAULT : 0;
 640 }
 641
 642 static int
 643 i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
 644                       const struct drm_i915_gem_pwrite *args)
 645 {
 646         unsigned int partial_cacheline_write;
 647         unsigned int needs_clflush;
 648         unsigned int offset, idx;
 649         void __user *user_data;
 650         u64 remain;
 651         int ret;
 652
 653         ret = i915_gem_object_lock_interruptible(obj, NULL);
 654         if (ret)
 655                 return ret;
 656
 657         ret = i915_gem_object_pin_pages(obj);
 658         if (ret)
 659                 goto err_unlock;
 660
 661         ret = i915_gem_object_prepare_write(obj, &needs_clflush);
 662         if (ret)
 663                 goto err_unpin;
 664
 665         i915_gem_object_finish_access(obj);
 666         i915_gem_object_unlock(obj);
 667
 668         /* If we don't overwrite a cacheline completely we need to be
 669          * careful to have up-to-date data by first clflushing. Don't
 670          * overcomplicate things and flush the entire patch.
 671          */
 672         partial_cacheline_write = 0;
 673         if (needs_clflush & CLFLUSH_BEFORE)
 674                 partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1;
 675
 676         user_data = u64_to_user_ptr(args->data_ptr);
 677         remain = args->size;
 678         offset = offset_in_page(args->offset);
 679         for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
 680                 struct page *page = i915_gem_object_get_page(obj, idx);
 681                 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
 682
 683                 ret = shmem_pwrite(page, offset, length, user_data,
 684                                    (offset | length) & partial_cacheline_write,
 685                                    needs_clflush & CLFLUSH_AFTER);
 686                 if (ret)
 687                         break;
 688
 689                 remain -= length;
 690                 user_data += length;
 691                 offset = 0;
 692         }
 693
 694         i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
 695
 696         i915_gem_object_unpin_pages(obj);
 697         return ret;
 698
 699 err_unpin:
 700         i915_gem_object_unpin_pages(obj);
 701 err_unlock:
 702         i915_gem_object_unlock(obj);
 703         return ret;
 704 }
 705
 706 /**
 707  * Writes data to the object referenced by handle.
 708  * @dev: drm device
 709  * @data: ioctl data blob
 710  * @file: drm file
 711  *
 712  * On error, the contents of the buffer that were to be modified are undefined.
 713  */
 714 int
 715 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 716                       struct drm_file *file)
 717 {
 718         struct drm_i915_private *i915 = to_i915(dev);
 719         struct drm_i915_gem_pwrite *args = data;
 720         struct drm_i915_gem_object *obj;
 721         int ret;
 722
 723         /* PWRITE is disallowed for all platforms after TGL-LP.  This also
 724          * covers all platforms with local memory.
 725          */
 726         if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915))
 727                 return -EOPNOTSUPP;
 728
 729         if (args->size == 0)
 730                 return 0;
 731
 732         if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size))
 733                 return -EFAULT;
 734
 735         obj = i915_gem_object_lookup(file, args->handle);
 736         if (!obj)
 737                 return -ENOENT;
 738
 739         /* Bounds check destination. */
 740         if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
 741                 ret = -EINVAL;
 742                 goto err;
 743         }
 744
 745         /* Writes not allowed into this read-only object */
 746         if (i915_gem_object_is_readonly(obj)) {
 747                 ret = -EINVAL;
 748                 goto err;
 749         }
 750
 751         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 752
 753         ret = -ENODEV;
 754         if (obj->ops->pwrite)
 755                 ret = obj->ops->pwrite(obj, args);
 756         if (ret != -ENODEV)
 757                 goto err;
 758
 759         ret = i915_gem_object_wait(obj,
 760                                    I915_WAIT_INTERRUPTIBLE |
 761                                    I915_WAIT_ALL,
 762                                    MAX_SCHEDULE_TIMEOUT);
 763         if (ret)
 764                 goto err;
 765
 766         ret = -EFAULT;
 767         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 768          * it would end up going through the fenced access, and we'll get
 769          * different detiling behavior between reading and writing.
 770          * pread/pwrite currently are reading and writing from the CPU
 771          * perspective, requiring manual detiling by the client.
 772          */
 773         if (!i915_gem_object_has_struct_page(obj) ||
 774             i915_gem_cpu_write_needs_clflush(obj))
 775                 /* Note that the gtt paths might fail with non-page-backed user
 776                  * pointers (e.g. gtt mappings when moving data between
 777                  * textures). Fallback to the shmem path in that case.
 778                  */
 779                 ret = i915_gem_gtt_pwrite_fast(obj, args);
 780
 781         if (ret == -EFAULT || ret == -ENOSPC) {
 782                 if (i915_gem_object_has_struct_page(obj))
 783                         ret = i915_gem_shmem_pwrite(obj, args);
 784         }
 785
 786 err:
 787         i915_gem_object_put(obj);
 788         return ret;
 789 }
 790
 791 /**
 792  * Called when user space has done writes to this buffer
 793  * @dev: drm device
 794  * @data: ioctl data blob
 795  * @file: drm file
 796  */
 797 int
 798 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 799                          struct drm_file *file)
 800 {
 801         struct drm_i915_gem_sw_finish *args = data;
 802         struct drm_i915_gem_object *obj;
 803
 804         obj = i915_gem_object_lookup(file, args->handle);
 805         if (!obj)
 806                 return -ENOENT;
 807
 808         /*
 809          * Proxy objects are barred from CPU access, so there is no
 810          * need to ban sw_finish as it is a nop.
 811          */
 812
 813         /* Pinned buffers may be scanout, so flush the cache */
 814         i915_gem_object_flush_if_display(obj);
 815         i915_gem_object_put(obj);
 816
 817         return 0;
 818 }
 819
 820 void i915_gem_runtime_suspend(struct drm_i915_private *i915)
 821 {
 822         struct drm_i915_gem_object *obj, *on;
 823         int i;
 824
 825         /*
 826          * Only called during RPM suspend. All users of the userfault_list
 827          * must be holding an RPM wakeref to ensure that this can not
 828          * run concurrently with themselves (and use the struct_mutex for
 829          * protection between themselves).
 830          */
 831
 832         list_for_each_entry_safe(obj, on,
 833                                  &to_gt(i915)->ggtt->userfault_list, userfault_link)
 834                 __i915_gem_object_release_mmap_gtt(obj);
 835
 836         /*
 837          * The fence will be lost when the device powers down. If any were
 838          * in use by hardware (i.e. they are pinned), we should not be powering
 839          * down! All other fences will be reacquired by the user upon waking.
 840          */
 841         for (i = 0; i < to_gt(i915)->ggtt->num_fences; i++) {
 842                 struct i915_fence_reg *reg = &to_gt(i915)->ggtt->fence_regs[i];
 843
 844                 /*
 845                  * Ideally we want to assert that the fence register is not
 846                  * live at this point (i.e. that no piece of code will be
 847                  * trying to write through fence + GTT, as that both violates
 848                  * our tracking of activity and associated locking/barriers,
 849                  * but also is illegal given that the hw is powered down).
 850                  *
 851                  * Previously we used reg->pin_count as a "liveness" indicator.
 852                  * That is not sufficient, and we need a more fine-grained
 853                  * tool if we want to have a sanity check here.
 854                  */
 855
 856                 if (!reg->vma)
 857                         continue;
 858
 859                 GEM_BUG_ON(i915_vma_has_userfault(reg->vma));
 860                 reg->dirty = true;
 861         }
 862 }
 863
 864 static void discard_ggtt_vma(struct i915_vma *vma)
 865 {
 866         struct drm_i915_gem_object *obj = vma->obj;
 867
 868         spin_lock(&obj->vma.lock);
 869         if (!RB_EMPTY_NODE(&vma->obj_node)) {
 870                 rb_erase(&vma->obj_node, &obj->vma.tree);
 871                 RB_CLEAR_NODE(&vma->obj_node);
 872         }
 873         spin_unlock(&obj->vma.lock);
 874 }
 875
 876 struct i915_vma *
 877 i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj,
 878                             struct i915_gem_ww_ctx *ww,
 879                             const struct i915_ggtt_view *view,
 880                             u64 size, u64 alignment, u64 flags)
 881 {
 882         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 883         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 884         struct i915_vma *vma;
 885         int ret;
 886
 887         GEM_WARN_ON(!ww);
 888
 889         if (flags & PIN_MAPPABLE &&
 890             (!view || view->type == I915_GGTT_VIEW_NORMAL)) {
 891                 /*
 892                  * If the required space is larger than the available
 893                  * aperture, we will not able to find a slot for the
 894                  * object and unbinding the object now will be in
 895                  * vain. Worse, doing so may cause us to ping-pong
 896                  * the object in and out of the Global GTT and
 897                  * waste a lot of cycles under the mutex.
 898                  */
 899                 if (obj->base.size > ggtt->mappable_end)
 900                         return ERR_PTR(-E2BIG);
 901
 902                 /*
 903                  * If NONBLOCK is set the caller is optimistically
 904                  * trying to cache the full object within the mappable
 905                  * aperture, and *must* have a fallback in place for
 906                  * situations where we cannot bind the object. We
 907                  * can be a little more lax here and use the fallback
 908                  * more often to avoid costly migrations of ourselves
 909                  * and other objects within the aperture.
 910                  *
 911                  * Half-the-aperture is used as a simple heuristic.
 912                  * More interesting would to do search for a free
 913                  * block prior to making the commitment to unbind.
 914                  * That caters for the self-harm case, and with a
 915                  * little more heuristics (e.g. NOFAULT, NOEVICT)
 916                  * we could try to minimise harm to others.
 917                  */
 918                 if (flags & PIN_NONBLOCK &&
 919                     obj->base.size > ggtt->mappable_end / 2)
 920                         return ERR_PTR(-ENOSPC);
 921         }
 922
 923 new_vma:
 924         vma = i915_vma_instance(obj, &ggtt->vm, view);
 925         if (IS_ERR(vma))
 926                 return vma;
 927
 928         if (i915_vma_misplaced(vma, size, alignment, flags)) {
 929                 if (flags & PIN_NONBLOCK) {
 930                         if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))
 931                                 return ERR_PTR(-ENOSPC);
 932
 933                         if (flags & PIN_MAPPABLE &&
 934                             vma->fence_size > ggtt->mappable_end / 2)
 935                                 return ERR_PTR(-ENOSPC);
 936                 }
 937
 938                 if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)) {
 939                         discard_ggtt_vma(vma);
 940                         goto new_vma;
 941                 }
 942
 943                 ret = i915_vma_unbind(vma);
 944                 if (ret)
 945                         return ERR_PTR(ret);
 946         }
 947
 948         ret = i915_vma_pin_ww(vma, ww, size, alignment, flags | PIN_GLOBAL);
 949
 950         if (ret)
 951                 return ERR_PTR(ret);
 952
 953         if (vma->fence && !i915_gem_object_is_tiled(obj)) {
 954                 mutex_lock(&ggtt->vm.mutex);
 955                 i915_vma_revoke_fence(vma);
 956                 mutex_unlock(&ggtt->vm.mutex);
 957         }
 958
 959         ret = i915_vma_wait_for_bind(vma);
 960         if (ret) {
 961                 i915_vma_unpin(vma);
 962                 return ERR_PTR(ret);
 963         }
 964
 965         return vma;
 966 }
 967
 968 struct i915_vma * __must_check
 969 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
 970                          const struct i915_ggtt_view *view,
 971                          u64 size, u64 alignment, u64 flags)
 972 {
 973         struct i915_gem_ww_ctx ww;
 974         struct i915_vma *ret;
 975         int err;
 976
 977         for_i915_gem_ww(&ww, err, true) {
 978                 err = i915_gem_object_lock(obj, &ww);
 979                 if (err)
 980                         continue;
 981
 982                 ret = i915_gem_object_ggtt_pin_ww(obj, &ww, view, size,
 983                                                   alignment, flags);
 984                 if (IS_ERR(ret))
 985                         err = PTR_ERR(ret);
 986         }
 987
 988         return err ? ERR_PTR(err) : ret;
 989 }
 990
 991 int
 992 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 993                        struct drm_file *file_priv)
 994 {
 995         struct drm_i915_private *i915 = to_i915(dev);
 996         struct drm_i915_gem_madvise *args = data;
 997         struct drm_i915_gem_object *obj;
 998         int err;
 999
1000         switch (args->madv) {
1001         case I915_MADV_DONTNEED:
1002         case I915_MADV_WILLNEED:
1003             break;
1004         default:
1005             return -EINVAL;
1006         }
1007
1008         obj = i915_gem_object_lookup(file_priv, args->handle);
1009         if (!obj)
1010                 return -ENOENT;
1011
1012         err = i915_gem_object_lock_interruptible(obj, NULL);
1013         if (err)
1014                 goto out;
1015
1016         if (i915_gem_object_has_pages(obj) &&
1017             i915_gem_object_is_tiled(obj) &&
1018             i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
1019                 if (obj->mm.madv == I915_MADV_WILLNEED) {
1020                         GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj));
1021                         i915_gem_object_clear_tiling_quirk(obj);
1022                         i915_gem_object_make_shrinkable(obj);
1023                 }
1024                 if (args->madv == I915_MADV_WILLNEED) {
1025                         GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));
1026                         i915_gem_object_make_unshrinkable(obj);
1027                         i915_gem_object_set_tiling_quirk(obj);
1028                 }
1029         }
1030
1031         if (obj->mm.madv != __I915_MADV_PURGED) {
1032                 obj->mm.madv = args->madv;
1033                 if (obj->ops->adjust_lru)
1034                         obj->ops->adjust_lru(obj);
1035         }
1036
1037         if (i915_gem_object_has_pages(obj) ||
1038             i915_gem_object_has_self_managed_shrink_list(obj)) {
1039                 unsigned long flags;
1040
1041                 spin_lock_irqsave(&i915->mm.obj_lock, flags);
1042                 if (!list_empty(&obj->mm.link)) {
1043                         struct list_head *list;
1044
1045                         if (obj->mm.madv != I915_MADV_WILLNEED)
1046                                 list = &i915->mm.purge_list;
1047                         else
1048                                 list = &i915->mm.shrink_list;
1049                         list_move_tail(&obj->mm.link, list);
1050
1051                 }
1052                 spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
1053         }
1054
1055         /* if the object is no longer attached, discard its backing storage */
1056         if (obj->mm.madv == I915_MADV_DONTNEED &&
1057             !i915_gem_object_has_pages(obj))
1058                 i915_gem_object_truncate(obj);
1059
1060         args->retained = obj->mm.madv != __I915_MADV_PURGED;
1061
1062         i915_gem_object_unlock(obj);
1063 out:
1064         i915_gem_object_put(obj);
1065         return err;
1066 }
1067
1068 int i915_gem_init(struct drm_i915_private *dev_priv)
1069 {
1070         int ret;
1071
1072         /* We need to fallback to 4K pages if host doesn't support huge gtt. */
1073         if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv))
1074                 mkwrite_device_info(dev_priv)->page_sizes =
1075                         I915_GTT_PAGE_SIZE_4K;
1076
1077         ret = i915_gem_init_userptr(dev_priv);
1078         if (ret)
1079                 return ret;
1080
1081         intel_uc_fetch_firmwares(&to_gt(dev_priv)->uc);
1082         intel_wopcm_init(&dev_priv->wopcm);
1083
1084         ret = i915_init_ggtt(dev_priv);
1085         if (ret) {
1086                 GEM_BUG_ON(ret == -EIO);
1087                 goto err_unlock;
1088         }
1089
1090         /*
1091          * Despite its name intel_init_clock_gating applies both display
1092          * clock gating workarounds; GT mmio workarounds and the occasional
1093          * GT power context workaround. Worse, sometimes it includes a context
1094          * register workaround which we need to apply before we record the
1095          * default HW state for all contexts.
1096          *
1097          * FIXME: break up the workarounds and apply them at the right time!
1098          */
1099         intel_init_clock_gating(dev_priv);
1100
1101         ret = intel_gt_init(to_gt(dev_priv));
1102         if (ret)
1103                 goto err_unlock;
1104
1105         return 0;
1106
1107         /*
1108          * Unwinding is complicated by that we want to handle -EIO to mean
1109          * disable GPU submission but keep KMS alive. We want to mark the
1110          * HW as irrevisibly wedged, but keep enough state around that the
1111          * driver doesn't explode during runtime.
1112          */
1113 err_unlock:
1114         i915_gem_drain_workqueue(dev_priv);
1115
1116         if (ret != -EIO)
1117                 intel_uc_cleanup_firmwares(&to_gt(dev_priv)->uc);
1118
1119         if (ret == -EIO) {
1120                 /*
1121                  * Allow engines or uC initialisation to fail by marking the GPU
1122                  * as wedged. But we only want to do this when the GPU is angry,
1123                  * for all other failure, such as an allocation failure, bail.
1124                  */
1125                 if (!intel_gt_is_wedged(to_gt(dev_priv))) {
1126                         i915_probe_error(dev_priv,
1127                                          "Failed to initialize GPU, declaring it wedged!\n");
1128                         intel_gt_set_wedged(to_gt(dev_priv));
1129                 }
1130
1131                 /* Minimal basic recovery for KMS */
1132                 ret = i915_ggtt_enable_hw(dev_priv);
1133                 i915_ggtt_resume(to_gt(dev_priv)->ggtt);
1134                 intel_init_clock_gating(dev_priv);
1135         }
1136
1137         i915_gem_drain_freed_objects(dev_priv);
1138
1139         return ret;
1140 }
1141
1142 void i915_gem_driver_register(struct drm_i915_private *i915)
1143 {
1144         i915_gem_driver_register__shrinker(i915);
1145
1146         intel_engines_driver_register(i915);
1147 }
1148
1149 void i915_gem_driver_unregister(struct drm_i915_private *i915)
1150 {
1151         i915_gem_driver_unregister__shrinker(i915);
1152 }
1153
1154 void i915_gem_driver_remove(struct drm_i915_private *dev_priv)
1155 {
1156         intel_wakeref_auto_fini(&to_gt(dev_priv)->ggtt->userfault_wakeref);
1157
1158         i915_gem_suspend_late(dev_priv);
1159         intel_gt_driver_remove(to_gt(dev_priv));
1160         dev_priv->uabi_engines = RB_ROOT;
1161
1162         /* Flush any outstanding unpin_work. */
1163         i915_gem_drain_workqueue(dev_priv);
1164
1165         i915_gem_drain_freed_objects(dev_priv);
1166 }
1167
1168 void i915_gem_driver_release(struct drm_i915_private *dev_priv)
1169 {
1170         intel_gt_driver_release(to_gt(dev_priv));
1171
1172         intel_uc_cleanup_firmwares(&to_gt(dev_priv)->uc);
1173
1174         i915_gem_drain_freed_objects(dev_priv);
1175
1176         drm_WARN_ON(&dev_priv->drm, !list_empty(&dev_priv->gem.contexts.list));
1177 }
1178
1179 static void i915_gem_init__mm(struct drm_i915_private *i915)
1180 {
1181         spin_lock_init(&i915->mm.obj_lock);
1182
1183         init_llist_head(&i915->mm.free_list);
1184
1185         INIT_LIST_HEAD(&i915->mm.purge_list);
1186         INIT_LIST_HEAD(&i915->mm.shrink_list);
1187
1188         i915_gem_init__objects(i915);
1189 }
1190
1191 void i915_gem_init_early(struct drm_i915_private *dev_priv)
1192 {
1193         i915_gem_init__mm(dev_priv);
1194         i915_gem_init__contexts(dev_priv);
1195
1196         spin_lock_init(&dev_priv->fb_tracking.lock);
1197 }
1198
1199 void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
1200 {
1201         i915_gem_drain_freed_objects(dev_priv);
1202         GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list));
1203         GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count));
1204         drm_WARN_ON(&dev_priv->drm, dev_priv->mm.shrink_count);
1205 }
1206
1207 int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file)
1208 {
1209         struct drm_i915_file_private *file_priv;
1210         int ret;
1211
1212         DRM_DEBUG("\n");
1213
1214         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
1215         if (!file_priv)
1216                 return -ENOMEM;
1217
1218         file->driver_priv = file_priv;
1219         file_priv->dev_priv = i915;
1220         file_priv->file = file;
1221
1222         file_priv->bsd_engine = -1;
1223         file_priv->hang_timestamp = jiffies;
1224
1225         ret = i915_gem_context_open(i915, file);
1226         if (ret)
1227                 kfree(file_priv);
1228
1229         return ret;
1230 }
1231
1232 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1233 #include "selftests/mock_gem_device.c"
1234 #include "selftests/i915_gem.c"
1235 #endif