drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008-2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include <drm/drm_vma_manager.h>
  29 #include <linux/dma-fence-array.h>
  30 #include <linux/kthread.h>
  31 #include <linux/dma-resv.h>
  32 #include <linux/shmem_fs.h>
  33 #include <linux/slab.h>
  34 #include <linux/stop_machine.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37 #include <linux/dma-buf.h>
  38 #include <linux/mman.h>
  39
  40 #include "display/intel_display.h"
  41 #include "display/intel_frontbuffer.h"
  42
  43 #include "gem/i915_gem_clflush.h"
  44 #include "gem/i915_gem_context.h"
  45 #include "gem/i915_gem_ioctls.h"
  46 #include "gem/i915_gem_mman.h"
  47 #include "gem/i915_gem_region.h"
  48 #include "gt/intel_engine_user.h"
  49 #include "gt/intel_gt.h"
  50 #include "gt/intel_gt_pm.h"
  51 #include "gt/intel_workarounds.h"
  52
  53 #include "i915_drv.h"
  54 #include "i915_trace.h"
  55 #include "i915_vgpu.h"
  56
  57 #include "intel_pm.h"
  58
  59 static int
  60 insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size)
  61 {
  62         int err;
  63
  64         err = mutex_lock_interruptible(&ggtt->vm.mutex);
  65         if (err)
  66                 return err;
  67
  68         memset(node, 0, sizeof(*node));
  69         err = drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
  70                                           size, 0, I915_COLOR_UNEVICTABLE,
  71                                           0, ggtt->mappable_end,
  72                                           DRM_MM_INSERT_LOW);
  73
  74         mutex_unlock(&ggtt->vm.mutex);
  75
  76         return err;
  77 }
  78
  79 static void
  80 remove_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node)
  81 {
  82         mutex_lock(&ggtt->vm.mutex);
  83         drm_mm_remove_node(node);
  84         mutex_unlock(&ggtt->vm.mutex);
  85 }
  86
  87 int
  88 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  89                             struct drm_file *file)
  90 {
  91         struct i915_ggtt *ggtt = &to_i915(dev)->ggtt;
  92         struct drm_i915_gem_get_aperture *args = data;
  93         struct i915_vma *vma;
  94         u64 pinned;
  95
  96         if (mutex_lock_interruptible(&ggtt->vm.mutex))
  97                 return -EINTR;
  98
  99         pinned = ggtt->vm.reserved;
 100         list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)
 101                 if (i915_vma_is_pinned(vma))
 102                         pinned += vma->node.size;
 103
 104         mutex_unlock(&ggtt->vm.mutex);
 105
 106         args->aper_size = ggtt->vm.total;
 107         args->aper_available_size = args->aper_size - pinned;
 108
 109         return 0;
 110 }
 111
 112 int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
 113                            unsigned long flags)
 114 {
 115         struct intel_runtime_pm *rpm = &to_i915(obj->base.dev)->runtime_pm;
 116         LIST_HEAD(still_in_list);
 117         intel_wakeref_t wakeref;
 118         struct i915_vma *vma;
 119         int ret;
 120
 121         if (list_empty(&obj->vma.list))
 122                 return 0;
 123
 124         /*
 125          * As some machines use ACPI to handle runtime-resume callbacks, and
 126          * ACPI is quite kmalloc happy, we cannot resume beneath the vm->mutex
 127          * as they are required by the shrinker. Ergo, we wake the device up
 128          * first just in case.
 129          */
 130         wakeref = intel_runtime_pm_get(rpm);
 131
 132 try_again:
 133         ret = 0;
 134         spin_lock(&obj->vma.lock);
 135         while (!ret && (vma = list_first_entry_or_null(&obj->vma.list,
 136                                                        struct i915_vma,
 137                                                        obj_link))) {
 138                 struct i915_address_space *vm = vma->vm;
 139
 140                 list_move_tail(&vma->obj_link, &still_in_list);
 141                 if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK))
 142                         continue;
 143
 144                 if (flags & I915_GEM_OBJECT_UNBIND_TEST) {
 145                         ret = -EBUSY;
 146                         break;
 147                 }
 148
 149                 ret = -EAGAIN;
 150                 if (!i915_vm_tryopen(vm))
 151                         break;
 152
 153                 /* Prevent vma being freed by i915_vma_parked as we unbind */
 154                 vma = __i915_vma_get(vma);
 155                 spin_unlock(&obj->vma.lock);
 156
 157                 if (vma) {
 158                         ret = -EBUSY;
 159                         if (flags & I915_GEM_OBJECT_UNBIND_ACTIVE ||
 160                             !i915_vma_is_active(vma)) {
 161                                 if (flags & I915_GEM_OBJECT_UNBIND_VM_TRYLOCK) {
 162                                         if (mutex_trylock(&vma->vm->mutex)) {
 163                                                 ret = __i915_vma_unbind(vma);
 164                                                 mutex_unlock(&vma->vm->mutex);
 165                                         } else {
 166                                                 ret = -EBUSY;
 167                                         }
 168                                 } else {
 169                                         ret = i915_vma_unbind(vma);
 170                                 }
 171                         }
 172
 173                         __i915_vma_put(vma);
 174                 }
 175
 176                 i915_vm_close(vm);
 177                 spin_lock(&obj->vma.lock);
 178         }
 179         list_splice_init(&still_in_list, &obj->vma.list);
 180         spin_unlock(&obj->vma.lock);
 181
 182         if (ret == -EAGAIN && flags & I915_GEM_OBJECT_UNBIND_BARRIER) {
 183                 rcu_barrier(); /* flush the i915_vm_release() */
 184                 goto try_again;
 185         }
 186
 187         intel_runtime_pm_put(rpm, wakeref);
 188
 189         return ret;
 190 }
 191
 192 static int
 193 shmem_pread(struct page *page, int offset, int len, char __user *user_data,
 194             bool needs_clflush)
 195 {
 196         char *vaddr;
 197         int ret;
 198
 199         vaddr = kmap(page);
 200
 201         if (needs_clflush)
 202                 drm_clflush_virt_range(vaddr + offset, len);
 203
 204         ret = __copy_to_user(user_data, vaddr + offset, len);
 205
 206         kunmap(page);
 207
 208         return ret ? -EFAULT : 0;
 209 }
 210
 211 static int
 212 i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
 213                      struct drm_i915_gem_pread *args)
 214 {
 215         unsigned int needs_clflush;
 216         unsigned int idx, offset;
 217         char __user *user_data;
 218         u64 remain;
 219         int ret;
 220
 221         ret = i915_gem_object_lock_interruptible(obj, NULL);
 222         if (ret)
 223                 return ret;
 224
 225         ret = i915_gem_object_pin_pages(obj);
 226         if (ret)
 227                 goto err_unlock;
 228
 229         ret = i915_gem_object_prepare_read(obj, &needs_clflush);
 230         if (ret)
 231                 goto err_unpin;
 232
 233         i915_gem_object_finish_access(obj);
 234         i915_gem_object_unlock(obj);
 235
 236         remain = args->size;
 237         user_data = u64_to_user_ptr(args->data_ptr);
 238         offset = offset_in_page(args->offset);
 239         for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
 240                 struct page *page = i915_gem_object_get_page(obj, idx);
 241                 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
 242
 243                 ret = shmem_pread(page, offset, length, user_data,
 244                                   needs_clflush);
 245                 if (ret)
 246                         break;
 247
 248                 remain -= length;
 249                 user_data += length;
 250                 offset = 0;
 251         }
 252
 253         i915_gem_object_unpin_pages(obj);
 254         return ret;
 255
 256 err_unpin:
 257         i915_gem_object_unpin_pages(obj);
 258 err_unlock:
 259         i915_gem_object_unlock(obj);
 260         return ret;
 261 }
 262
 263 static inline bool
 264 gtt_user_read(struct io_mapping *mapping,
 265               loff_t base, int offset,
 266               char __user *user_data, int length)
 267 {
 268         void __iomem *vaddr;
 269         unsigned long unwritten;
 270
 271         /* We can use the cpu mem copy function because this is X86. */
 272         vaddr = io_mapping_map_atomic_wc(mapping, base);
 273         unwritten = __copy_to_user_inatomic(user_data,
 274                                             (void __force *)vaddr + offset,
 275                                             length);
 276         io_mapping_unmap_atomic(vaddr);
 277         if (unwritten) {
 278                 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
 279                 unwritten = copy_to_user(user_data,
 280                                          (void __force *)vaddr + offset,
 281                                          length);
 282                 io_mapping_unmap(vaddr);
 283         }
 284         return unwritten;
 285 }
 286
 287 static struct i915_vma *i915_gem_gtt_prepare(struct drm_i915_gem_object *obj,
 288                                              struct drm_mm_node *node,
 289                                              bool write)
 290 {
 291         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 292         struct i915_ggtt *ggtt = &i915->ggtt;
 293         struct i915_vma *vma;
 294         struct i915_gem_ww_ctx ww;
 295         int ret;
 296
 297         i915_gem_ww_ctx_init(&ww, true);
 298 retry:
 299         vma = ERR_PTR(-ENODEV);
 300         ret = i915_gem_object_lock(obj, &ww);
 301         if (ret)
 302                 goto err_ww;
 303
 304         ret = i915_gem_object_set_to_gtt_domain(obj, write);
 305         if (ret)
 306                 goto err_ww;
 307
 308         if (!i915_gem_object_is_tiled(obj))
 309                 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0,
 310                                                   PIN_MAPPABLE |
 311                                                   PIN_NONBLOCK /* NOWARN */ |
 312                                                   PIN_NOEVICT);
 313         if (vma == ERR_PTR(-EDEADLK)) {
 314                 ret = -EDEADLK;
 315                 goto err_ww;
 316         } else if (!IS_ERR(vma)) {
 317                 node->start = i915_ggtt_offset(vma);
 318                 node->flags = 0;
 319         } else {
 320                 ret = insert_mappable_node(ggtt, node, PAGE_SIZE);
 321                 if (ret)
 322                         goto err_ww;
 323                 GEM_BUG_ON(!drm_mm_node_allocated(node));
 324                 vma = NULL;
 325         }
 326
 327         ret = i915_gem_object_pin_pages(obj);
 328         if (ret) {
 329                 if (drm_mm_node_allocated(node)) {
 330                         ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);
 331                         remove_mappable_node(ggtt, node);
 332                 } else {
 333                         i915_vma_unpin(vma);
 334                 }
 335         }
 336
 337 err_ww:
 338         if (ret == -EDEADLK) {
 339                 ret = i915_gem_ww_ctx_backoff(&ww);
 340                 if (!ret)
 341                         goto retry;
 342         }
 343         i915_gem_ww_ctx_fini(&ww);
 344
 345         return ret ? ERR_PTR(ret) : vma;
 346 }
 347
 348 static void i915_gem_gtt_cleanup(struct drm_i915_gem_object *obj,
 349                                  struct drm_mm_node *node,
 350                                  struct i915_vma *vma)
 351 {
 352         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 353         struct i915_ggtt *ggtt = &i915->ggtt;
 354
 355         i915_gem_object_unpin_pages(obj);
 356         if (drm_mm_node_allocated(node)) {
 357                 ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);
 358                 remove_mappable_node(ggtt, node);
 359         } else {
 360                 i915_vma_unpin(vma);
 361         }
 362 }
 363
 364 static int
 365 i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
 366                    const struct drm_i915_gem_pread *args)
 367 {
 368         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 369         struct i915_ggtt *ggtt = &i915->ggtt;
 370         intel_wakeref_t wakeref;
 371         struct drm_mm_node node;
 372         void __user *user_data;
 373         struct i915_vma *vma;
 374         u64 remain, offset;
 375         int ret = 0;
 376
 377         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 378
 379         vma = i915_gem_gtt_prepare(obj, &node, false);
 380         if (IS_ERR(vma)) {
 381                 ret = PTR_ERR(vma);
 382                 goto out_rpm;
 383         }
 384
 385         user_data = u64_to_user_ptr(args->data_ptr);
 386         remain = args->size;
 387         offset = args->offset;
 388
 389         while (remain > 0) {
 390                 /* Operation in this page
 391                  *
 392                  * page_base = page offset within aperture
 393                  * page_offset = offset within page
 394                  * page_length = bytes to copy for this page
 395                  */
 396                 u32 page_base = node.start;
 397                 unsigned page_offset = offset_in_page(offset);
 398                 unsigned page_length = PAGE_SIZE - page_offset;
 399                 page_length = remain < page_length ? remain : page_length;
 400                 if (drm_mm_node_allocated(&node)) {
 401                         ggtt->vm.insert_page(&ggtt->vm,
 402                                              i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 403                                              node.start, I915_CACHE_NONE, 0);
 404                 } else {
 405                         page_base += offset & PAGE_MASK;
 406                 }
 407
 408                 if (gtt_user_read(&ggtt->iomap, page_base, page_offset,
 409                                   user_data, page_length)) {
 410                         ret = -EFAULT;
 411                         break;
 412                 }
 413
 414                 remain -= page_length;
 415                 user_data += page_length;
 416                 offset += page_length;
 417         }
 418
 419         i915_gem_gtt_cleanup(obj, &node, vma);
 420 out_rpm:
 421         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 422         return ret;
 423 }
 424
 425 /**
 426  * Reads data from the object referenced by handle.
 427  * @dev: drm device pointer
 428  * @data: ioctl data blob
 429  * @file: drm file pointer
 430  *
 431  * On error, the contents of *data are undefined.
 432  */
 433 int
 434 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 435                      struct drm_file *file)
 436 {
 437         struct drm_i915_private *i915 = to_i915(dev);
 438         struct drm_i915_gem_pread *args = data;
 439         struct drm_i915_gem_object *obj;
 440         int ret;
 441
 442         /* PREAD is disallowed for all platforms after TGL-LP.  This also
 443          * covers all platforms with local memory.
 444          */
 445         if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915))
 446                 return -EOPNOTSUPP;
 447
 448         if (args->size == 0)
 449                 return 0;
 450
 451         if (!access_ok(u64_to_user_ptr(args->data_ptr),
 452                        args->size))
 453                 return -EFAULT;
 454
 455         obj = i915_gem_object_lookup(file, args->handle);
 456         if (!obj)
 457                 return -ENOENT;
 458
 459         /* Bounds check source.  */
 460         if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
 461                 ret = -EINVAL;
 462                 goto out;
 463         }
 464
 465         trace_i915_gem_object_pread(obj, args->offset, args->size);
 466         ret = -ENODEV;
 467         if (obj->ops->pread)
 468                 ret = obj->ops->pread(obj, args);
 469         if (ret != -ENODEV)
 470                 goto out;
 471
 472         ret = -ENODEV;
 473         if (obj->ops->pread)
 474                 ret = obj->ops->pread(obj, args);
 475         if (ret != -ENODEV)
 476                 goto out;
 477
 478         ret = i915_gem_object_wait(obj,
 479                                    I915_WAIT_INTERRUPTIBLE,
 480                                    MAX_SCHEDULE_TIMEOUT);
 481         if (ret)
 482                 goto out;
 483
 484         ret = i915_gem_shmem_pread(obj, args);
 485         if (ret == -EFAULT || ret == -ENODEV)
 486                 ret = i915_gem_gtt_pread(obj, args);
 487
 488 out:
 489         i915_gem_object_put(obj);
 490         return ret;
 491 }
 492
 493 /* This is the fast write path which cannot handle
 494  * page faults in the source data
 495  */
 496
 497 static inline bool
 498 ggtt_write(struct io_mapping *mapping,
 499            loff_t base, int offset,
 500            char __user *user_data, int length)
 501 {
 502         void __iomem *vaddr;
 503         unsigned long unwritten;
 504
 505         /* We can use the cpu mem copy function because this is X86. */
 506         vaddr = io_mapping_map_atomic_wc(mapping, base);
 507         unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset,
 508                                                       user_data, length);
 509         io_mapping_unmap_atomic(vaddr);
 510         if (unwritten) {
 511                 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
 512                 unwritten = copy_from_user((void __force *)vaddr + offset,
 513                                            user_data, length);
 514                 io_mapping_unmap(vaddr);
 515         }
 516
 517         return unwritten;
 518 }
 519
 520 /**
 521  * This is the fast pwrite path, where we copy the data directly from the
 522  * user into the GTT, uncached.
 523  * @obj: i915 GEM object
 524  * @args: pwrite arguments structure
 525  */
 526 static int
 527 i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
 528                          const struct drm_i915_gem_pwrite *args)
 529 {
 530         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 531         struct i915_ggtt *ggtt = &i915->ggtt;
 532         struct intel_runtime_pm *rpm = &i915->runtime_pm;
 533         intel_wakeref_t wakeref;
 534         struct drm_mm_node node;
 535         struct i915_vma *vma;
 536         u64 remain, offset;
 537         void __user *user_data;
 538         int ret = 0;
 539
 540         if (i915_gem_object_has_struct_page(obj)) {
 541                 /*
 542                  * Avoid waking the device up if we can fallback, as
 543                  * waking/resuming is very slow (worst-case 10-100 ms
 544                  * depending on PCI sleeps and our own resume time).
 545                  * This easily dwarfs any performance advantage from
 546                  * using the cache bypass of indirect GGTT access.
 547                  */
 548                 wakeref = intel_runtime_pm_get_if_in_use(rpm);
 549                 if (!wakeref)
 550                         return -EFAULT;
 551         } else {
 552                 /* No backing pages, no fallback, we must force GGTT access */
 553                 wakeref = intel_runtime_pm_get(rpm);
 554         }
 555
 556         vma = i915_gem_gtt_prepare(obj, &node, true);
 557         if (IS_ERR(vma)) {
 558                 ret = PTR_ERR(vma);
 559                 goto out_rpm;
 560         }
 561
 562         i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
 563
 564         user_data = u64_to_user_ptr(args->data_ptr);
 565         offset = args->offset;
 566         remain = args->size;
 567         while (remain) {
 568                 /* Operation in this page
 569                  *
 570                  * page_base = page offset within aperture
 571                  * page_offset = offset within page
 572                  * page_length = bytes to copy for this page
 573                  */
 574                 u32 page_base = node.start;
 575                 unsigned int page_offset = offset_in_page(offset);
 576                 unsigned int page_length = PAGE_SIZE - page_offset;
 577                 page_length = remain < page_length ? remain : page_length;
 578                 if (drm_mm_node_allocated(&node)) {
 579                         /* flush the write before we modify the GGTT */
 580                         intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 581                         ggtt->vm.insert_page(&ggtt->vm,
 582                                              i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 583                                              node.start, I915_CACHE_NONE, 0);
 584                         wmb(); /* flush modifications to the GGTT (insert_page) */
 585                 } else {
 586                         page_base += offset & PAGE_MASK;
 587                 }
 588                 /* If we get a fault while copying data, then (presumably) our
 589                  * source page isn't available.  Return the error and we'll
 590                  * retry in the slow path.
 591                  * If the object is non-shmem backed, we retry again with the
 592                  * path that handles page fault.
 593                  */
 594                 if (ggtt_write(&ggtt->iomap, page_base, page_offset,
 595                                user_data, page_length)) {
 596                         ret = -EFAULT;
 597                         break;
 598                 }
 599
 600                 remain -= page_length;
 601                 user_data += page_length;
 602                 offset += page_length;
 603         }
 604
 605         intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 606         i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
 607
 608         i915_gem_gtt_cleanup(obj, &node, vma);
 609 out_rpm:
 610         intel_runtime_pm_put(rpm, wakeref);
 611         return ret;
 612 }
 613
 614 /* Per-page copy function for the shmem pwrite fastpath.
 615  * Flushes invalid cachelines before writing to the target if
 616  * needs_clflush_before is set and flushes out any written cachelines after
 617  * writing if needs_clflush is set.
 618  */
 619 static int
 620 shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
 621              bool needs_clflush_before,
 622              bool needs_clflush_after)
 623 {
 624         char *vaddr;
 625         int ret;
 626
 627         vaddr = kmap(page);
 628
 629         if (needs_clflush_before)
 630                 drm_clflush_virt_range(vaddr + offset, len);
 631
 632         ret = __copy_from_user(vaddr + offset, user_data, len);
 633         if (!ret && needs_clflush_after)
 634                 drm_clflush_virt_range(vaddr + offset, len);
 635
 636         kunmap(page);
 637
 638         return ret ? -EFAULT : 0;
 639 }
 640
 641 static int
 642 i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
 643                       const struct drm_i915_gem_pwrite *args)
 644 {
 645         unsigned int partial_cacheline_write;
 646         unsigned int needs_clflush;
 647         unsigned int offset, idx;
 648         void __user *user_data;
 649         u64 remain;
 650         int ret;
 651
 652         ret = i915_gem_object_lock_interruptible(obj, NULL);
 653         if (ret)
 654                 return ret;
 655
 656         ret = i915_gem_object_pin_pages(obj);
 657         if (ret)
 658                 goto err_unlock;
 659
 660         ret = i915_gem_object_prepare_write(obj, &needs_clflush);
 661         if (ret)
 662                 goto err_unpin;
 663
 664         i915_gem_object_finish_access(obj);
 665         i915_gem_object_unlock(obj);
 666
 667         /* If we don't overwrite a cacheline completely we need to be
 668          * careful to have up-to-date data by first clflushing. Don't
 669          * overcomplicate things and flush the entire patch.
 670          */
 671         partial_cacheline_write = 0;
 672         if (needs_clflush & CLFLUSH_BEFORE)
 673                 partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1;
 674
 675         user_data = u64_to_user_ptr(args->data_ptr);
 676         remain = args->size;
 677         offset = offset_in_page(args->offset);
 678         for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
 679                 struct page *page = i915_gem_object_get_page(obj, idx);
 680                 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
 681
 682                 ret = shmem_pwrite(page, offset, length, user_data,
 683                                    (offset | length) & partial_cacheline_write,
 684                                    needs_clflush & CLFLUSH_AFTER);
 685                 if (ret)
 686                         break;
 687
 688                 remain -= length;
 689                 user_data += length;
 690                 offset = 0;
 691         }
 692
 693         i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
 694
 695         i915_gem_object_unpin_pages(obj);
 696         return ret;
 697
 698 err_unpin:
 699         i915_gem_object_unpin_pages(obj);
 700 err_unlock:
 701         i915_gem_object_unlock(obj);
 702         return ret;
 703 }
 704
 705 /**
 706  * Writes data to the object referenced by handle.
 707  * @dev: drm device
 708  * @data: ioctl data blob
 709  * @file: drm file
 710  *
 711  * On error, the contents of the buffer that were to be modified are undefined.
 712  */
 713 int
 714 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 715                       struct drm_file *file)
 716 {
 717         struct drm_i915_private *i915 = to_i915(dev);
 718         struct drm_i915_gem_pwrite *args = data;
 719         struct drm_i915_gem_object *obj;
 720         int ret;
 721
 722         /* PWRITE is disallowed for all platforms after TGL-LP.  This also
 723          * covers all platforms with local memory.
 724          */
 725         if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915))
 726                 return -EOPNOTSUPP;
 727
 728         if (args->size == 0)
 729                 return 0;
 730
 731         if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size))
 732                 return -EFAULT;
 733
 734         obj = i915_gem_object_lookup(file, args->handle);
 735         if (!obj)
 736                 return -ENOENT;
 737
 738         /* Bounds check destination. */
 739         if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
 740                 ret = -EINVAL;
 741                 goto err;
 742         }
 743
 744         /* Writes not allowed into this read-only object */
 745         if (i915_gem_object_is_readonly(obj)) {
 746                 ret = -EINVAL;
 747                 goto err;
 748         }
 749
 750         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 751
 752         ret = -ENODEV;
 753         if (obj->ops->pwrite)
 754                 ret = obj->ops->pwrite(obj, args);
 755         if (ret != -ENODEV)
 756                 goto err;
 757
 758         ret = i915_gem_object_wait(obj,
 759                                    I915_WAIT_INTERRUPTIBLE |
 760                                    I915_WAIT_ALL,
 761                                    MAX_SCHEDULE_TIMEOUT);
 762         if (ret)
 763                 goto err;
 764
 765         ret = -EFAULT;
 766         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 767          * it would end up going through the fenced access, and we'll get
 768          * different detiling behavior between reading and writing.
 769          * pread/pwrite currently are reading and writing from the CPU
 770          * perspective, requiring manual detiling by the client.
 771          */
 772         if (!i915_gem_object_has_struct_page(obj) ||
 773             cpu_write_needs_clflush(obj))
 774                 /* Note that the gtt paths might fail with non-page-backed user
 775                  * pointers (e.g. gtt mappings when moving data between
 776                  * textures). Fallback to the shmem path in that case.
 777                  */
 778                 ret = i915_gem_gtt_pwrite_fast(obj, args);
 779
 780         if (ret == -EFAULT || ret == -ENOSPC) {
 781                 if (i915_gem_object_has_struct_page(obj))
 782                         ret = i915_gem_shmem_pwrite(obj, args);
 783         }
 784
 785 err:
 786         i915_gem_object_put(obj);
 787         return ret;
 788 }
 789
 790 /**
 791  * Called when user space has done writes to this buffer
 792  * @dev: drm device
 793  * @data: ioctl data blob
 794  * @file: drm file
 795  */
 796 int
 797 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 798                          struct drm_file *file)
 799 {
 800         struct drm_i915_gem_sw_finish *args = data;
 801         struct drm_i915_gem_object *obj;
 802
 803         obj = i915_gem_object_lookup(file, args->handle);
 804         if (!obj)
 805                 return -ENOENT;
 806
 807         /*
 808          * Proxy objects are barred from CPU access, so there is no
 809          * need to ban sw_finish as it is a nop.
 810          */
 811
 812         /* Pinned buffers may be scanout, so flush the cache */
 813         i915_gem_object_flush_if_display(obj);
 814         i915_gem_object_put(obj);
 815
 816         return 0;
 817 }
 818
 819 void i915_gem_runtime_suspend(struct drm_i915_private *i915)
 820 {
 821         struct drm_i915_gem_object *obj, *on;
 822         int i;
 823
 824         /*
 825          * Only called during RPM suspend. All users of the userfault_list
 826          * must be holding an RPM wakeref to ensure that this can not
 827          * run concurrently with themselves (and use the struct_mutex for
 828          * protection between themselves).
 829          */
 830
 831         list_for_each_entry_safe(obj, on,
 832                                  &i915->ggtt.userfault_list, userfault_link)
 833                 __i915_gem_object_release_mmap_gtt(obj);
 834
 835         /*
 836          * The fence will be lost when the device powers down. If any were
 837          * in use by hardware (i.e. they are pinned), we should not be powering
 838          * down! All other fences will be reacquired by the user upon waking.
 839          */
 840         for (i = 0; i < i915->ggtt.num_fences; i++) {
 841                 struct i915_fence_reg *reg = &i915->ggtt.fence_regs[i];
 842
 843                 /*
 844                  * Ideally we want to assert that the fence register is not
 845                  * live at this point (i.e. that no piece of code will be
 846                  * trying to write through fence + GTT, as that both violates
 847                  * our tracking of activity and associated locking/barriers,
 848                  * but also is illegal given that the hw is powered down).
 849                  *
 850                  * Previously we used reg->pin_count as a "liveness" indicator.
 851                  * That is not sufficient, and we need a more fine-grained
 852                  * tool if we want to have a sanity check here.
 853                  */
 854
 855                 if (!reg->vma)
 856                         continue;
 857
 858                 GEM_BUG_ON(i915_vma_has_userfault(reg->vma));
 859                 reg->dirty = true;
 860         }
 861 }
 862
 863 static void discard_ggtt_vma(struct i915_vma *vma)
 864 {
 865         struct drm_i915_gem_object *obj = vma->obj;
 866
 867         spin_lock(&obj->vma.lock);
 868         if (!RB_EMPTY_NODE(&vma->obj_node)) {
 869                 rb_erase(&vma->obj_node, &obj->vma.tree);
 870                 RB_CLEAR_NODE(&vma->obj_node);
 871         }
 872         spin_unlock(&obj->vma.lock);
 873 }
 874
 875 struct i915_vma *
 876 i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj,
 877                             struct i915_gem_ww_ctx *ww,
 878                             const struct i915_ggtt_view *view,
 879                             u64 size, u64 alignment, u64 flags)
 880 {
 881         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 882         struct i915_ggtt *ggtt = &i915->ggtt;
 883         struct i915_vma *vma;
 884         int ret;
 885
 886         if (flags & PIN_MAPPABLE &&
 887             (!view || view->type == I915_GGTT_VIEW_NORMAL)) {
 888                 /*
 889                  * If the required space is larger than the available
 890                  * aperture, we will not able to find a slot for the
 891                  * object and unbinding the object now will be in
 892                  * vain. Worse, doing so may cause us to ping-pong
 893                  * the object in and out of the Global GTT and
 894                  * waste a lot of cycles under the mutex.
 895                  */
 896                 if (obj->base.size > ggtt->mappable_end)
 897                         return ERR_PTR(-E2BIG);
 898
 899                 /*
 900                  * If NONBLOCK is set the caller is optimistically
 901                  * trying to cache the full object within the mappable
 902                  * aperture, and *must* have a fallback in place for
 903                  * situations where we cannot bind the object. We
 904                  * can be a little more lax here and use the fallback
 905                  * more often to avoid costly migrations of ourselves
 906                  * and other objects within the aperture.
 907                  *
 908                  * Half-the-aperture is used as a simple heuristic.
 909                  * More interesting would to do search for a free
 910                  * block prior to making the commitment to unbind.
 911                  * That caters for the self-harm case, and with a
 912                  * little more heuristics (e.g. NOFAULT, NOEVICT)
 913                  * we could try to minimise harm to others.
 914                  */
 915                 if (flags & PIN_NONBLOCK &&
 916                     obj->base.size > ggtt->mappable_end / 2)
 917                         return ERR_PTR(-ENOSPC);
 918         }
 919
 920 new_vma:
 921         vma = i915_vma_instance(obj, &ggtt->vm, view);
 922         if (IS_ERR(vma))
 923                 return vma;
 924
 925         if (i915_vma_misplaced(vma, size, alignment, flags)) {
 926                 if (flags & PIN_NONBLOCK) {
 927                         if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))
 928                                 return ERR_PTR(-ENOSPC);
 929
 930                         if (flags & PIN_MAPPABLE &&
 931                             vma->fence_size > ggtt->mappable_end / 2)
 932                                 return ERR_PTR(-ENOSPC);
 933                 }
 934
 935                 if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)) {
 936                         discard_ggtt_vma(vma);
 937                         goto new_vma;
 938                 }
 939
 940                 ret = i915_vma_unbind(vma);
 941                 if (ret)
 942                         return ERR_PTR(ret);
 943         }
 944
 945         if (ww)
 946                 ret = i915_vma_pin_ww(vma, ww, size, alignment, flags | PIN_GLOBAL);
 947         else
 948                 ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL);
 949
 950         if (ret)
 951                 return ERR_PTR(ret);
 952
 953         if (vma->fence && !i915_gem_object_is_tiled(obj)) {
 954                 mutex_lock(&ggtt->vm.mutex);
 955                 i915_vma_revoke_fence(vma);
 956                 mutex_unlock(&ggtt->vm.mutex);
 957         }
 958
 959         ret = i915_vma_wait_for_bind(vma);
 960         if (ret) {
 961                 i915_vma_unpin(vma);
 962                 return ERR_PTR(ret);
 963         }
 964
 965         return vma;
 966 }
 967
 968 int
 969 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 970                        struct drm_file *file_priv)
 971 {
 972         struct drm_i915_private *i915 = to_i915(dev);
 973         struct drm_i915_gem_madvise *args = data;
 974         struct drm_i915_gem_object *obj;
 975         int err;
 976
 977         switch (args->madv) {
 978         case I915_MADV_DONTNEED:
 979         case I915_MADV_WILLNEED:
 980             break;
 981         default:
 982             return -EINVAL;
 983         }
 984
 985         obj = i915_gem_object_lookup(file_priv, args->handle);
 986         if (!obj)
 987                 return -ENOENT;
 988
 989         err = i915_gem_object_lock_interruptible(obj, NULL);
 990         if (err)
 991                 goto out;
 992
 993         if (i915_gem_object_has_pages(obj) &&
 994             i915_gem_object_is_tiled(obj) &&
 995             i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
 996                 if (obj->mm.madv == I915_MADV_WILLNEED) {
 997                         GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj));
 998                         i915_gem_object_clear_tiling_quirk(obj);
 999                         i915_gem_object_make_shrinkable(obj);
1000                 }
1001                 if (args->madv == I915_MADV_WILLNEED) {
1002                         GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));
1003                         i915_gem_object_make_unshrinkable(obj);
1004                         i915_gem_object_set_tiling_quirk(obj);
1005                 }
1006         }
1007
1008         if (obj->mm.madv != __I915_MADV_PURGED)
1009                 obj->mm.madv = args->madv;
1010
1011         if (i915_gem_object_has_pages(obj)) {
1012                 unsigned long flags;
1013
1014                 spin_lock_irqsave(&i915->mm.obj_lock, flags);
1015                 if (!list_empty(&obj->mm.link)) {
1016                         struct list_head *list;
1017
1018                         if (obj->mm.madv != I915_MADV_WILLNEED)
1019                                 list = &i915->mm.purge_list;
1020                         else
1021                                 list = &i915->mm.shrink_list;
1022                         list_move_tail(&obj->mm.link, list);
1023
1024                 }
1025                 spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
1026         }
1027
1028         /* if the object is no longer attached, discard its backing storage */
1029         if (obj->mm.madv == I915_MADV_DONTNEED &&
1030             !i915_gem_object_has_pages(obj))
1031                 i915_gem_object_truncate(obj);
1032
1033         args->retained = obj->mm.madv != __I915_MADV_PURGED;
1034
1035         i915_gem_object_unlock(obj);
1036 out:
1037         i915_gem_object_put(obj);
1038         return err;
1039 }
1040
1041 int i915_gem_init(struct drm_i915_private *dev_priv)
1042 {
1043         int ret;
1044
1045         /* We need to fallback to 4K pages if host doesn't support huge gtt. */
1046         if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv))
1047                 mkwrite_device_info(dev_priv)->page_sizes =
1048                         I915_GTT_PAGE_SIZE_4K;
1049
1050         ret = i915_gem_init_userptr(dev_priv);
1051         if (ret)
1052                 return ret;
1053
1054         intel_uc_fetch_firmwares(&dev_priv->gt.uc);
1055         intel_wopcm_init(&dev_priv->wopcm);
1056
1057         ret = i915_init_ggtt(dev_priv);
1058         if (ret) {
1059                 GEM_BUG_ON(ret == -EIO);
1060                 goto err_unlock;
1061         }
1062
1063         /*
1064          * Despite its name intel_init_clock_gating applies both display
1065          * clock gating workarounds; GT mmio workarounds and the occasional
1066          * GT power context workaround. Worse, sometimes it includes a context
1067          * register workaround which we need to apply before we record the
1068          * default HW state for all contexts.
1069          *
1070          * FIXME: break up the workarounds and apply them at the right time!
1071          */
1072         intel_init_clock_gating(dev_priv);
1073
1074         ret = intel_gt_init(&dev_priv->gt);
1075         if (ret)
1076                 goto err_unlock;
1077
1078         return 0;
1079
1080         /*
1081          * Unwinding is complicated by that we want to handle -EIO to mean
1082          * disable GPU submission but keep KMS alive. We want to mark the
1083          * HW as irrevisibly wedged, but keep enough state around that the
1084          * driver doesn't explode during runtime.
1085          */
1086 err_unlock:
1087         i915_gem_drain_workqueue(dev_priv);
1088
1089         if (ret != -EIO)
1090                 intel_uc_cleanup_firmwares(&dev_priv->gt.uc);
1091
1092         if (ret == -EIO) {
1093                 /*
1094                  * Allow engines or uC initialisation to fail by marking the GPU
1095                  * as wedged. But we only want to do this when the GPU is angry,
1096                  * for all other failure, such as an allocation failure, bail.
1097                  */
1098                 if (!intel_gt_is_wedged(&dev_priv->gt)) {
1099                         i915_probe_error(dev_priv,
1100                                          "Failed to initialize GPU, declaring it wedged!\n");
1101                         intel_gt_set_wedged(&dev_priv->gt);
1102                 }
1103
1104                 /* Minimal basic recovery for KMS */
1105                 ret = i915_ggtt_enable_hw(dev_priv);
1106                 i915_ggtt_resume(&dev_priv->ggtt);
1107                 intel_init_clock_gating(dev_priv);
1108         }
1109
1110         i915_gem_drain_freed_objects(dev_priv);
1111
1112         return ret;
1113 }
1114
1115 void i915_gem_driver_register(struct drm_i915_private *i915)
1116 {
1117         i915_gem_driver_register__shrinker(i915);
1118
1119         intel_engines_driver_register(i915);
1120 }
1121
1122 void i915_gem_driver_unregister(struct drm_i915_private *i915)
1123 {
1124         i915_gem_driver_unregister__shrinker(i915);
1125 }
1126
1127 void i915_gem_driver_remove(struct drm_i915_private *dev_priv)
1128 {
1129         intel_wakeref_auto_fini(&dev_priv->ggtt.userfault_wakeref);
1130
1131         i915_gem_suspend_late(dev_priv);
1132         intel_gt_driver_remove(&dev_priv->gt);
1133         dev_priv->uabi_engines = RB_ROOT;
1134
1135         /* Flush any outstanding unpin_work. */
1136         i915_gem_drain_workqueue(dev_priv);
1137
1138         i915_gem_drain_freed_objects(dev_priv);
1139 }
1140
1141 void i915_gem_driver_release(struct drm_i915_private *dev_priv)
1142 {
1143         intel_gt_driver_release(&dev_priv->gt);
1144
1145         intel_wa_list_free(&dev_priv->gt_wa_list);
1146
1147         intel_uc_cleanup_firmwares(&dev_priv->gt.uc);
1148
1149         i915_gem_drain_freed_objects(dev_priv);
1150
1151         drm_WARN_ON(&dev_priv->drm, !list_empty(&dev_priv->gem.contexts.list));
1152 }
1153
1154 static void i915_gem_init__mm(struct drm_i915_private *i915)
1155 {
1156         spin_lock_init(&i915->mm.obj_lock);
1157
1158         init_llist_head(&i915->mm.free_list);
1159
1160         INIT_LIST_HEAD(&i915->mm.purge_list);
1161         INIT_LIST_HEAD(&i915->mm.shrink_list);
1162
1163         i915_gem_init__objects(i915);
1164 }
1165
1166 void i915_gem_init_early(struct drm_i915_private *dev_priv)
1167 {
1168         i915_gem_init__mm(dev_priv);
1169         i915_gem_init__contexts(dev_priv);
1170
1171         spin_lock_init(&dev_priv->fb_tracking.lock);
1172 }
1173
1174 void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
1175 {
1176         i915_gem_drain_freed_objects(dev_priv);
1177         GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list));
1178         GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count));
1179         drm_WARN_ON(&dev_priv->drm, dev_priv->mm.shrink_count);
1180 }
1181
1182 int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file)
1183 {
1184         struct drm_i915_file_private *file_priv;
1185         int ret;
1186
1187         DRM_DEBUG("\n");
1188
1189         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
1190         if (!file_priv)
1191                 return -ENOMEM;
1192
1193         file->driver_priv = file_priv;
1194         file_priv->dev_priv = i915;
1195         file_priv->file = file;
1196
1197         file_priv->bsd_engine = -1;
1198         file_priv->hang_timestamp = jiffies;
1199
1200         ret = i915_gem_context_open(i915, file);
1201         if (ret)
1202                 kfree(file_priv);
1203
1204         return ret;
1205 }
1206
1207 void i915_gem_ww_ctx_init(struct i915_gem_ww_ctx *ww, bool intr)
1208 {
1209         ww_acquire_init(&ww->ctx, &reservation_ww_class);
1210         INIT_LIST_HEAD(&ww->obj_list);
1211         ww->intr = intr;
1212         ww->contended = NULL;
1213 }
1214
1215 static void i915_gem_ww_ctx_unlock_all(struct i915_gem_ww_ctx *ww)
1216 {
1217         struct drm_i915_gem_object *obj;
1218
1219         while ((obj = list_first_entry_or_null(&ww->obj_list, struct drm_i915_gem_object, obj_link))) {
1220                 list_del(&obj->obj_link);
1221                 i915_gem_object_unlock(obj);
1222         }
1223 }
1224
1225 void i915_gem_ww_unlock_single(struct drm_i915_gem_object *obj)
1226 {
1227         list_del(&obj->obj_link);
1228         i915_gem_object_unlock(obj);
1229 }
1230
1231 void i915_gem_ww_ctx_fini(struct i915_gem_ww_ctx *ww)
1232 {
1233         i915_gem_ww_ctx_unlock_all(ww);
1234         WARN_ON(ww->contended);
1235         ww_acquire_fini(&ww->ctx);
1236 }
1237
1238 int __must_check i915_gem_ww_ctx_backoff(struct i915_gem_ww_ctx *ww)
1239 {
1240         int ret = 0;
1241
1242         if (WARN_ON(!ww->contended))
1243                 return -EINVAL;
1244
1245         i915_gem_ww_ctx_unlock_all(ww);
1246         if (ww->intr)
1247                 ret = dma_resv_lock_slow_interruptible(ww->contended->base.resv, &ww->ctx);
1248         else
1249                 dma_resv_lock_slow(ww->contended->base.resv, &ww->ctx);
1250
1251         if (!ret)
1252                 list_add_tail(&ww->contended->obj_link, &ww->obj_list);
1253
1254         ww->contended = NULL;
1255
1256         return ret;
1257 }
1258
1259 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1260 #include "selftests/mock_gem_device.c"
1261 #include "selftests/i915_gem.c"
1262 #endif