drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008-2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include <drm/drm_vma_manager.h>
  29 #include <drm/i915_drm.h>
  30 #include <linux/dma-fence-array.h>
  31 #include <linux/kthread.h>
  32 #include <linux/dma-resv.h>
  33 #include <linux/shmem_fs.h>
  34 #include <linux/slab.h>
  35 #include <linux/stop_machine.h>
  36 #include <linux/swap.h>
  37 #include <linux/pci.h>
  38 #include <linux/dma-buf.h>
  39 #include <linux/mman.h>
  40
  41 #include "display/intel_display.h"
  42 #include "display/intel_frontbuffer.h"
  43
  44 #include "gem/i915_gem_clflush.h"
  45 #include "gem/i915_gem_context.h"
  46 #include "gem/i915_gem_ioctls.h"
  47 #include "gem/i915_gem_pm.h"
  48 #include "gt/intel_engine_user.h"
  49 #include "gt/intel_gt.h"
  50 #include "gt/intel_gt_pm.h"
  51 #include "gt/intel_gt_requests.h"
  52 #include "gt/intel_mocs.h"
  53 #include "gt/intel_reset.h"
  54 #include "gt/intel_renderstate.h"
  55 #include "gt/intel_rps.h"
  56 #include "gt/intel_workarounds.h"
  57
  58 #include "i915_drv.h"
  59 #include "i915_scatterlist.h"
  60 #include "i915_trace.h"
  61 #include "i915_vgpu.h"
  62
  63 #include "intel_pm.h"
  64
  65 static int
  66 insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size)
  67 {
  68         int err;
  69
  70         err = mutex_lock_interruptible(&ggtt->vm.mutex);
  71         if (err)
  72                 return err;
  73
  74         memset(node, 0, sizeof(*node));
  75         err = drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
  76                                           size, 0, I915_COLOR_UNEVICTABLE,
  77                                           0, ggtt->mappable_end,
  78                                           DRM_MM_INSERT_LOW);
  79
  80         mutex_unlock(&ggtt->vm.mutex);
  81
  82         return err;
  83 }
  84
  85 static void
  86 remove_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node)
  87 {
  88         mutex_lock(&ggtt->vm.mutex);
  89         drm_mm_remove_node(node);
  90         mutex_unlock(&ggtt->vm.mutex);
  91 }
  92
  93 int
  94 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  95                             struct drm_file *file)
  96 {
  97         struct i915_ggtt *ggtt = &to_i915(dev)->ggtt;
  98         struct drm_i915_gem_get_aperture *args = data;
  99         struct i915_vma *vma;
 100         u64 pinned;
 101
 102         if (mutex_lock_interruptible(&ggtt->vm.mutex))
 103                 return -EINTR;
 104
 105         pinned = ggtt->vm.reserved;
 106         list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)
 107                 if (i915_vma_is_pinned(vma))
 108                         pinned += vma->node.size;
 109
 110         mutex_unlock(&ggtt->vm.mutex);
 111
 112         args->aper_size = ggtt->vm.total;
 113         args->aper_available_size = args->aper_size - pinned;
 114
 115         return 0;
 116 }
 117
 118 int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
 119                            unsigned long flags)
 120 {
 121         struct i915_vma *vma;
 122         LIST_HEAD(still_in_list);
 123         int ret = 0;
 124
 125         spin_lock(&obj->vma.lock);
 126         while (!ret && (vma = list_first_entry_or_null(&obj->vma.list,
 127                                                        struct i915_vma,
 128                                                        obj_link))) {
 129                 struct i915_address_space *vm = vma->vm;
 130
 131                 ret = -EBUSY;
 132                 if (!i915_vm_tryopen(vm))
 133                         break;
 134
 135                 list_move_tail(&vma->obj_link, &still_in_list);
 136                 spin_unlock(&obj->vma.lock);
 137
 138                 if (flags & I915_GEM_OBJECT_UNBIND_ACTIVE ||
 139                     !i915_vma_is_active(vma))
 140                         ret = i915_vma_unbind(vma);
 141
 142                 i915_vm_close(vm);
 143                 spin_lock(&obj->vma.lock);
 144         }
 145         list_splice(&still_in_list, &obj->vma.list);
 146         spin_unlock(&obj->vma.lock);
 147
 148         return ret;
 149 }
 150
 151 static int
 152 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
 153                      struct drm_i915_gem_pwrite *args,
 154                      struct drm_file *file)
 155 {
 156         void *vaddr = obj->phys_handle->vaddr + args->offset;
 157         char __user *user_data = u64_to_user_ptr(args->data_ptr);
 158
 159         /*
 160          * We manually control the domain here and pretend that it
 161          * remains coherent i.e. in the GTT domain, like shmem_pwrite.
 162          */
 163         intel_frontbuffer_invalidate(obj->frontbuffer, ORIGIN_CPU);
 164
 165         if (copy_from_user(vaddr, user_data, args->size))
 166                 return -EFAULT;
 167
 168         drm_clflush_virt_range(vaddr, args->size);
 169         intel_gt_chipset_flush(&to_i915(obj->base.dev)->gt);
 170
 171         intel_frontbuffer_flush(obj->frontbuffer, ORIGIN_CPU);
 172         return 0;
 173 }
 174
 175 static int
 176 i915_gem_create(struct drm_file *file,
 177                 struct drm_i915_private *dev_priv,
 178                 u64 *size_p,
 179                 u32 *handle_p)
 180 {
 181         struct drm_i915_gem_object *obj;
 182         u32 handle;
 183         u64 size;
 184         int ret;
 185
 186         size = round_up(*size_p, PAGE_SIZE);
 187         if (size == 0)
 188                 return -EINVAL;
 189
 190         /* Allocate the new object */
 191         obj = i915_gem_object_create_shmem(dev_priv, size);
 192         if (IS_ERR(obj))
 193                 return PTR_ERR(obj);
 194
 195         ret = drm_gem_handle_create(file, &obj->base, &handle);
 196         /* drop reference from allocate - handle holds it now */
 197         i915_gem_object_put(obj);
 198         if (ret)
 199                 return ret;
 200
 201         *handle_p = handle;
 202         *size_p = size;
 203         return 0;
 204 }
 205
 206 int
 207 i915_gem_dumb_create(struct drm_file *file,
 208                      struct drm_device *dev,
 209                      struct drm_mode_create_dumb *args)
 210 {
 211         int cpp = DIV_ROUND_UP(args->bpp, 8);
 212         u32 format;
 213
 214         switch (cpp) {
 215         case 1:
 216                 format = DRM_FORMAT_C8;
 217                 break;
 218         case 2:
 219                 format = DRM_FORMAT_RGB565;
 220                 break;
 221         case 4:
 222                 format = DRM_FORMAT_XRGB8888;
 223                 break;
 224         default:
 225                 return -EINVAL;
 226         }
 227
 228         /* have to work out size/pitch and return them */
 229         args->pitch = ALIGN(args->width * cpp, 64);
 230
 231         /* align stride to page size so that we can remap */
 232         if (args->pitch > intel_plane_fb_max_stride(to_i915(dev), format,
 233                                                     DRM_FORMAT_MOD_LINEAR))
 234                 args->pitch = ALIGN(args->pitch, 4096);
 235
 236         args->size = args->pitch * args->height;
 237         return i915_gem_create(file, to_i915(dev),
 238                                &args->size, &args->handle);
 239 }
 240
 241 /**
 242  * Creates a new mm object and returns a handle to it.
 243  * @dev: drm device pointer
 244  * @data: ioctl data blob
 245  * @file: drm file pointer
 246  */
 247 int
 248 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 249                       struct drm_file *file)
 250 {
 251         struct drm_i915_private *dev_priv = to_i915(dev);
 252         struct drm_i915_gem_create *args = data;
 253
 254         i915_gem_flush_free_objects(dev_priv);
 255
 256         return i915_gem_create(file, dev_priv,
 257                                &args->size, &args->handle);
 258 }
 259
 260 static int
 261 shmem_pread(struct page *page, int offset, int len, char __user *user_data,
 262             bool needs_clflush)
 263 {
 264         char *vaddr;
 265         int ret;
 266
 267         vaddr = kmap(page);
 268
 269         if (needs_clflush)
 270                 drm_clflush_virt_range(vaddr + offset, len);
 271
 272         ret = __copy_to_user(user_data, vaddr + offset, len);
 273
 274         kunmap(page);
 275
 276         return ret ? -EFAULT : 0;
 277 }
 278
 279 static int
 280 i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
 281                      struct drm_i915_gem_pread *args)
 282 {
 283         unsigned int needs_clflush;
 284         unsigned int idx, offset;
 285         struct dma_fence *fence;
 286         char __user *user_data;
 287         u64 remain;
 288         int ret;
 289
 290         ret = i915_gem_object_prepare_read(obj, &needs_clflush);
 291         if (ret)
 292                 return ret;
 293
 294         fence = i915_gem_object_lock_fence(obj);
 295         i915_gem_object_finish_access(obj);
 296         if (!fence)
 297                 return -ENOMEM;
 298
 299         remain = args->size;
 300         user_data = u64_to_user_ptr(args->data_ptr);
 301         offset = offset_in_page(args->offset);
 302         for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
 303                 struct page *page = i915_gem_object_get_page(obj, idx);
 304                 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
 305
 306                 ret = shmem_pread(page, offset, length, user_data,
 307                                   needs_clflush);
 308                 if (ret)
 309                         break;
 310
 311                 remain -= length;
 312                 user_data += length;
 313                 offset = 0;
 314         }
 315
 316         i915_gem_object_unlock_fence(obj, fence);
 317         return ret;
 318 }
 319
 320 static inline bool
 321 gtt_user_read(struct io_mapping *mapping,
 322               loff_t base, int offset,
 323               char __user *user_data, int length)
 324 {
 325         void __iomem *vaddr;
 326         unsigned long unwritten;
 327
 328         /* We can use the cpu mem copy function because this is X86. */
 329         vaddr = io_mapping_map_atomic_wc(mapping, base);
 330         unwritten = __copy_to_user_inatomic(user_data,
 331                                             (void __force *)vaddr + offset,
 332                                             length);
 333         io_mapping_unmap_atomic(vaddr);
 334         if (unwritten) {
 335                 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
 336                 unwritten = copy_to_user(user_data,
 337                                          (void __force *)vaddr + offset,
 338                                          length);
 339                 io_mapping_unmap(vaddr);
 340         }
 341         return unwritten;
 342 }
 343
 344 static int
 345 i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
 346                    const struct drm_i915_gem_pread *args)
 347 {
 348         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 349         struct i915_ggtt *ggtt = &i915->ggtt;
 350         intel_wakeref_t wakeref;
 351         struct drm_mm_node node;
 352         struct dma_fence *fence;
 353         void __user *user_data;
 354         struct i915_vma *vma;
 355         u64 remain, offset;
 356         int ret;
 357
 358         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 359         vma = ERR_PTR(-ENODEV);
 360         if (!i915_gem_object_is_tiled(obj))
 361                 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
 362                                                PIN_MAPPABLE |
 363                                                PIN_NONBLOCK /* NOWARN */ |
 364                                                PIN_NOEVICT);
 365         if (!IS_ERR(vma)) {
 366                 node.start = i915_ggtt_offset(vma);
 367                 node.flags = 0;
 368         } else {
 369                 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
 370                 if (ret)
 371                         goto out_rpm;
 372                 GEM_BUG_ON(!drm_mm_node_allocated(&node));
 373         }
 374
 375         ret = i915_gem_object_lock_interruptible(obj);
 376         if (ret)
 377                 goto out_unpin;
 378
 379         ret = i915_gem_object_set_to_gtt_domain(obj, false);
 380         if (ret) {
 381                 i915_gem_object_unlock(obj);
 382                 goto out_unpin;
 383         }
 384
 385         fence = i915_gem_object_lock_fence(obj);
 386         i915_gem_object_unlock(obj);
 387         if (!fence) {
 388                 ret = -ENOMEM;
 389                 goto out_unpin;
 390         }
 391
 392         user_data = u64_to_user_ptr(args->data_ptr);
 393         remain = args->size;
 394         offset = args->offset;
 395
 396         while (remain > 0) {
 397                 /* Operation in this page
 398                  *
 399                  * page_base = page offset within aperture
 400                  * page_offset = offset within page
 401                  * page_length = bytes to copy for this page
 402                  */
 403                 u32 page_base = node.start;
 404                 unsigned page_offset = offset_in_page(offset);
 405                 unsigned page_length = PAGE_SIZE - page_offset;
 406                 page_length = remain < page_length ? remain : page_length;
 407                 if (drm_mm_node_allocated(&node)) {
 408                         ggtt->vm.insert_page(&ggtt->vm,
 409                                              i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 410                                              node.start, I915_CACHE_NONE, 0);
 411                 } else {
 412                         page_base += offset & PAGE_MASK;
 413                 }
 414
 415                 if (gtt_user_read(&ggtt->iomap, page_base, page_offset,
 416                                   user_data, page_length)) {
 417                         ret = -EFAULT;
 418                         break;
 419                 }
 420
 421                 remain -= page_length;
 422                 user_data += page_length;
 423                 offset += page_length;
 424         }
 425
 426         i915_gem_object_unlock_fence(obj, fence);
 427 out_unpin:
 428         if (drm_mm_node_allocated(&node)) {
 429                 ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
 430                 remove_mappable_node(ggtt, &node);
 431         } else {
 432                 i915_vma_unpin(vma);
 433         }
 434 out_rpm:
 435         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 436         return ret;
 437 }
 438
 439 /**
 440  * Reads data from the object referenced by handle.
 441  * @dev: drm device pointer
 442  * @data: ioctl data blob
 443  * @file: drm file pointer
 444  *
 445  * On error, the contents of *data are undefined.
 446  */
 447 int
 448 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 449                      struct drm_file *file)
 450 {
 451         struct drm_i915_gem_pread *args = data;
 452         struct drm_i915_gem_object *obj;
 453         int ret;
 454
 455         if (args->size == 0)
 456                 return 0;
 457
 458         if (!access_ok(u64_to_user_ptr(args->data_ptr),
 459                        args->size))
 460                 return -EFAULT;
 461
 462         obj = i915_gem_object_lookup(file, args->handle);
 463         if (!obj)
 464                 return -ENOENT;
 465
 466         /* Bounds check source.  */
 467         if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
 468                 ret = -EINVAL;
 469                 goto out;
 470         }
 471
 472         trace_i915_gem_object_pread(obj, args->offset, args->size);
 473
 474         ret = i915_gem_object_wait(obj,
 475                                    I915_WAIT_INTERRUPTIBLE,
 476                                    MAX_SCHEDULE_TIMEOUT);
 477         if (ret)
 478                 goto out;
 479
 480         ret = i915_gem_object_pin_pages(obj);
 481         if (ret)
 482                 goto out;
 483
 484         ret = i915_gem_shmem_pread(obj, args);
 485         if (ret == -EFAULT || ret == -ENODEV)
 486                 ret = i915_gem_gtt_pread(obj, args);
 487
 488         i915_gem_object_unpin_pages(obj);
 489 out:
 490         i915_gem_object_put(obj);
 491         return ret;
 492 }
 493
 494 /* This is the fast write path which cannot handle
 495  * page faults in the source data
 496  */
 497
 498 static inline bool
 499 ggtt_write(struct io_mapping *mapping,
 500            loff_t base, int offset,
 501            char __user *user_data, int length)
 502 {
 503         void __iomem *vaddr;
 504         unsigned long unwritten;
 505
 506         /* We can use the cpu mem copy function because this is X86. */
 507         vaddr = io_mapping_map_atomic_wc(mapping, base);
 508         unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset,
 509                                                       user_data, length);
 510         io_mapping_unmap_atomic(vaddr);
 511         if (unwritten) {
 512                 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
 513                 unwritten = copy_from_user((void __force *)vaddr + offset,
 514                                            user_data, length);
 515                 io_mapping_unmap(vaddr);
 516         }
 517
 518         return unwritten;
 519 }
 520
 521 /**
 522  * This is the fast pwrite path, where we copy the data directly from the
 523  * user into the GTT, uncached.
 524  * @obj: i915 GEM object
 525  * @args: pwrite arguments structure
 526  */
 527 static int
 528 i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
 529                          const struct drm_i915_gem_pwrite *args)
 530 {
 531         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 532         struct i915_ggtt *ggtt = &i915->ggtt;
 533         struct intel_runtime_pm *rpm = &i915->runtime_pm;
 534         intel_wakeref_t wakeref;
 535         struct drm_mm_node node;
 536         struct dma_fence *fence;
 537         struct i915_vma *vma;
 538         u64 remain, offset;
 539         void __user *user_data;
 540         int ret;
 541
 542         if (i915_gem_object_has_struct_page(obj)) {
 543                 /*
 544                  * Avoid waking the device up if we can fallback, as
 545                  * waking/resuming is very slow (worst-case 10-100 ms
 546                  * depending on PCI sleeps and our own resume time).
 547                  * This easily dwarfs any performance advantage from
 548                  * using the cache bypass of indirect GGTT access.
 549                  */
 550                 wakeref = intel_runtime_pm_get_if_in_use(rpm);
 551                 if (!wakeref)
 552                         return -EFAULT;
 553         } else {
 554                 /* No backing pages, no fallback, we must force GGTT access */
 555                 wakeref = intel_runtime_pm_get(rpm);
 556         }
 557
 558         vma = ERR_PTR(-ENODEV);
 559         if (!i915_gem_object_is_tiled(obj))
 560                 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
 561                                                PIN_MAPPABLE |
 562                                                PIN_NONBLOCK /* NOWARN */ |
 563                                                PIN_NOEVICT);
 564         if (!IS_ERR(vma)) {
 565                 node.start = i915_ggtt_offset(vma);
 566                 node.flags = 0;
 567         } else {
 568                 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
 569                 if (ret)
 570                         goto out_rpm;
 571                 GEM_BUG_ON(!drm_mm_node_allocated(&node));
 572         }
 573
 574         ret = i915_gem_object_lock_interruptible(obj);
 575         if (ret)
 576                 goto out_unpin;
 577
 578         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 579         if (ret) {
 580                 i915_gem_object_unlock(obj);
 581                 goto out_unpin;
 582         }
 583
 584         fence = i915_gem_object_lock_fence(obj);
 585         i915_gem_object_unlock(obj);
 586         if (!fence) {
 587                 ret = -ENOMEM;
 588                 goto out_unpin;
 589         }
 590
 591         intel_frontbuffer_invalidate(obj->frontbuffer, ORIGIN_CPU);
 592
 593         user_data = u64_to_user_ptr(args->data_ptr);
 594         offset = args->offset;
 595         remain = args->size;
 596         while (remain) {
 597                 /* Operation in this page
 598                  *
 599                  * page_base = page offset within aperture
 600                  * page_offset = offset within page
 601                  * page_length = bytes to copy for this page
 602                  */
 603                 u32 page_base = node.start;
 604                 unsigned int page_offset = offset_in_page(offset);
 605                 unsigned int page_length = PAGE_SIZE - page_offset;
 606                 page_length = remain < page_length ? remain : page_length;
 607                 if (drm_mm_node_allocated(&node)) {
 608                         /* flush the write before we modify the GGTT */
 609                         intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 610                         ggtt->vm.insert_page(&ggtt->vm,
 611                                              i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 612                                              node.start, I915_CACHE_NONE, 0);
 613                         wmb(); /* flush modifications to the GGTT (insert_page) */
 614                 } else {
 615                         page_base += offset & PAGE_MASK;
 616                 }
 617                 /* If we get a fault while copying data, then (presumably) our
 618                  * source page isn't available.  Return the error and we'll
 619                  * retry in the slow path.
 620                  * If the object is non-shmem backed, we retry again with the
 621                  * path that handles page fault.
 622                  */
 623                 if (ggtt_write(&ggtt->iomap, page_base, page_offset,
 624                                user_data, page_length)) {
 625                         ret = -EFAULT;
 626                         break;
 627                 }
 628
 629                 remain -= page_length;
 630                 user_data += page_length;
 631                 offset += page_length;
 632         }
 633         intel_frontbuffer_flush(obj->frontbuffer, ORIGIN_CPU);
 634
 635         i915_gem_object_unlock_fence(obj, fence);
 636 out_unpin:
 637         intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 638         if (drm_mm_node_allocated(&node)) {
 639                 ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
 640                 remove_mappable_node(ggtt, &node);
 641         } else {
 642                 i915_vma_unpin(vma);
 643         }
 644 out_rpm:
 645         intel_runtime_pm_put(rpm, wakeref);
 646         return ret;
 647 }
 648
 649 /* Per-page copy function for the shmem pwrite fastpath.
 650  * Flushes invalid cachelines before writing to the target if
 651  * needs_clflush_before is set and flushes out any written cachelines after
 652  * writing if needs_clflush is set.
 653  */
 654 static int
 655 shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
 656              bool needs_clflush_before,
 657              bool needs_clflush_after)
 658 {
 659         char *vaddr;
 660         int ret;
 661
 662         vaddr = kmap(page);
 663
 664         if (needs_clflush_before)
 665                 drm_clflush_virt_range(vaddr + offset, len);
 666
 667         ret = __copy_from_user(vaddr + offset, user_data, len);
 668         if (!ret && needs_clflush_after)
 669                 drm_clflush_virt_range(vaddr + offset, len);
 670
 671         kunmap(page);
 672
 673         return ret ? -EFAULT : 0;
 674 }
 675
 676 static int
 677 i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
 678                       const struct drm_i915_gem_pwrite *args)
 679 {
 680         unsigned int partial_cacheline_write;
 681         unsigned int needs_clflush;
 682         unsigned int offset, idx;
 683         struct dma_fence *fence;
 684         void __user *user_data;
 685         u64 remain;
 686         int ret;
 687
 688         ret = i915_gem_object_prepare_write(obj, &needs_clflush);
 689         if (ret)
 690                 return ret;
 691
 692         fence = i915_gem_object_lock_fence(obj);
 693         i915_gem_object_finish_access(obj);
 694         if (!fence)
 695                 return -ENOMEM;
 696
 697         /* If we don't overwrite a cacheline completely we need to be
 698          * careful to have up-to-date data by first clflushing. Don't
 699          * overcomplicate things and flush the entire patch.
 700          */
 701         partial_cacheline_write = 0;
 702         if (needs_clflush & CLFLUSH_BEFORE)
 703                 partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1;
 704
 705         user_data = u64_to_user_ptr(args->data_ptr);
 706         remain = args->size;
 707         offset = offset_in_page(args->offset);
 708         for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
 709                 struct page *page = i915_gem_object_get_page(obj, idx);
 710                 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
 711
 712                 ret = shmem_pwrite(page, offset, length, user_data,
 713                                    (offset | length) & partial_cacheline_write,
 714                                    needs_clflush & CLFLUSH_AFTER);
 715                 if (ret)
 716                         break;
 717
 718                 remain -= length;
 719                 user_data += length;
 720                 offset = 0;
 721         }
 722
 723         intel_frontbuffer_flush(obj->frontbuffer, ORIGIN_CPU);
 724         i915_gem_object_unlock_fence(obj, fence);
 725
 726         return ret;
 727 }
 728
 729 /**
 730  * Writes data to the object referenced by handle.
 731  * @dev: drm device
 732  * @data: ioctl data blob
 733  * @file: drm file
 734  *
 735  * On error, the contents of the buffer that were to be modified are undefined.
 736  */
 737 int
 738 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 739                       struct drm_file *file)
 740 {
 741         struct drm_i915_gem_pwrite *args = data;
 742         struct drm_i915_gem_object *obj;
 743         int ret;
 744
 745         if (args->size == 0)
 746                 return 0;
 747
 748         if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size))
 749                 return -EFAULT;
 750
 751         obj = i915_gem_object_lookup(file, args->handle);
 752         if (!obj)
 753                 return -ENOENT;
 754
 755         /* Bounds check destination. */
 756         if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
 757                 ret = -EINVAL;
 758                 goto err;
 759         }
 760
 761         /* Writes not allowed into this read-only object */
 762         if (i915_gem_object_is_readonly(obj)) {
 763                 ret = -EINVAL;
 764                 goto err;
 765         }
 766
 767         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 768
 769         ret = -ENODEV;
 770         if (obj->ops->pwrite)
 771                 ret = obj->ops->pwrite(obj, args);
 772         if (ret != -ENODEV)
 773                 goto err;
 774
 775         ret = i915_gem_object_wait(obj,
 776                                    I915_WAIT_INTERRUPTIBLE |
 777                                    I915_WAIT_ALL,
 778                                    MAX_SCHEDULE_TIMEOUT);
 779         if (ret)
 780                 goto err;
 781
 782         ret = i915_gem_object_pin_pages(obj);
 783         if (ret)
 784                 goto err;
 785
 786         ret = -EFAULT;
 787         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 788          * it would end up going through the fenced access, and we'll get
 789          * different detiling behavior between reading and writing.
 790          * pread/pwrite currently are reading and writing from the CPU
 791          * perspective, requiring manual detiling by the client.
 792          */
 793         if (!i915_gem_object_has_struct_page(obj) ||
 794             cpu_write_needs_clflush(obj))
 795                 /* Note that the gtt paths might fail with non-page-backed user
 796                  * pointers (e.g. gtt mappings when moving data between
 797                  * textures). Fallback to the shmem path in that case.
 798                  */
 799                 ret = i915_gem_gtt_pwrite_fast(obj, args);
 800
 801         if (ret == -EFAULT || ret == -ENOSPC) {
 802                 if (obj->phys_handle)
 803                         ret = i915_gem_phys_pwrite(obj, args, file);
 804                 else
 805                         ret = i915_gem_shmem_pwrite(obj, args);
 806         }
 807
 808         i915_gem_object_unpin_pages(obj);
 809 err:
 810         i915_gem_object_put(obj);
 811         return ret;
 812 }
 813
 814 /**
 815  * Called when user space has done writes to this buffer
 816  * @dev: drm device
 817  * @data: ioctl data blob
 818  * @file: drm file
 819  */
 820 int
 821 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 822                          struct drm_file *file)
 823 {
 824         struct drm_i915_gem_sw_finish *args = data;
 825         struct drm_i915_gem_object *obj;
 826
 827         obj = i915_gem_object_lookup(file, args->handle);
 828         if (!obj)
 829                 return -ENOENT;
 830
 831         /*
 832          * Proxy objects are barred from CPU access, so there is no
 833          * need to ban sw_finish as it is a nop.
 834          */
 835
 836         /* Pinned buffers may be scanout, so flush the cache */
 837         i915_gem_object_flush_if_display(obj);
 838         i915_gem_object_put(obj);
 839
 840         return 0;
 841 }
 842
 843 void i915_gem_runtime_suspend(struct drm_i915_private *i915)
 844 {
 845         struct drm_i915_gem_object *obj, *on;
 846         int i;
 847
 848         /*
 849          * Only called during RPM suspend. All users of the userfault_list
 850          * must be holding an RPM wakeref to ensure that this can not
 851          * run concurrently with themselves (and use the struct_mutex for
 852          * protection between themselves).
 853          */
 854
 855         list_for_each_entry_safe(obj, on,
 856                                  &i915->ggtt.userfault_list, userfault_link)
 857                 __i915_gem_object_release_mmap(obj);
 858
 859         /*
 860          * The fence will be lost when the device powers down. If any were
 861          * in use by hardware (i.e. they are pinned), we should not be powering
 862          * down! All other fences will be reacquired by the user upon waking.
 863          */
 864         for (i = 0; i < i915->ggtt.num_fences; i++) {
 865                 struct i915_fence_reg *reg = &i915->ggtt.fence_regs[i];
 866
 867                 /*
 868                  * Ideally we want to assert that the fence register is not
 869                  * live at this point (i.e. that no piece of code will be
 870                  * trying to write through fence + GTT, as that both violates
 871                  * our tracking of activity and associated locking/barriers,
 872                  * but also is illegal given that the hw is powered down).
 873                  *
 874                  * Previously we used reg->pin_count as a "liveness" indicator.
 875                  * That is not sufficient, and we need a more fine-grained
 876                  * tool if we want to have a sanity check here.
 877                  */
 878
 879                 if (!reg->vma)
 880                         continue;
 881
 882                 GEM_BUG_ON(i915_vma_has_userfault(reg->vma));
 883                 reg->dirty = true;
 884         }
 885 }
 886
 887 struct i915_vma *
 888 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
 889                          const struct i915_ggtt_view *view,
 890                          u64 size,
 891                          u64 alignment,
 892                          u64 flags)
 893 {
 894         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 895         struct i915_address_space *vm = &dev_priv->ggtt.vm;
 896
 897         return i915_gem_object_pin(obj, vm, view, size, alignment,
 898                                    flags | PIN_GLOBAL);
 899 }
 900
 901 struct i915_vma *
 902 i915_gem_object_pin(struct drm_i915_gem_object *obj,
 903                     struct i915_address_space *vm,
 904                     const struct i915_ggtt_view *view,
 905                     u64 size,
 906                     u64 alignment,
 907                     u64 flags)
 908 {
 909         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 910         struct i915_vma *vma;
 911         int ret;
 912
 913         if (i915_gem_object_never_bind_ggtt(obj))
 914                 return ERR_PTR(-ENODEV);
 915
 916         if (flags & PIN_MAPPABLE &&
 917             (!view || view->type == I915_GGTT_VIEW_NORMAL)) {
 918                 /* If the required space is larger than the available
 919                  * aperture, we will not able to find a slot for the
 920                  * object and unbinding the object now will be in
 921                  * vain. Worse, doing so may cause us to ping-pong
 922                  * the object in and out of the Global GTT and
 923                  * waste a lot of cycles under the mutex.
 924                  */
 925                 if (obj->base.size > dev_priv->ggtt.mappable_end)
 926                         return ERR_PTR(-E2BIG);
 927
 928                 /* If NONBLOCK is set the caller is optimistically
 929                  * trying to cache the full object within the mappable
 930                  * aperture, and *must* have a fallback in place for
 931                  * situations where we cannot bind the object. We
 932                  * can be a little more lax here and use the fallback
 933                  * more often to avoid costly migrations of ourselves
 934                  * and other objects within the aperture.
 935                  *
 936                  * Half-the-aperture is used as a simple heuristic.
 937                  * More interesting would to do search for a free
 938                  * block prior to making the commitment to unbind.
 939                  * That caters for the self-harm case, and with a
 940                  * little more heuristics (e.g. NOFAULT, NOEVICT)
 941                  * we could try to minimise harm to others.
 942                  */
 943                 if (flags & PIN_NONBLOCK &&
 944                     obj->base.size > dev_priv->ggtt.mappable_end / 2)
 945                         return ERR_PTR(-ENOSPC);
 946         }
 947
 948         vma = i915_vma_instance(obj, vm, view);
 949         if (IS_ERR(vma))
 950                 return vma;
 951
 952         if (i915_vma_misplaced(vma, size, alignment, flags)) {
 953                 if (flags & PIN_NONBLOCK) {
 954                         if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))
 955                                 return ERR_PTR(-ENOSPC);
 956
 957                         if (flags & PIN_MAPPABLE &&
 958                             vma->fence_size > dev_priv->ggtt.mappable_end / 2)
 959                                 return ERR_PTR(-ENOSPC);
 960                 }
 961
 962                 ret = i915_vma_unbind(vma);
 963                 if (ret)
 964                         return ERR_PTR(ret);
 965         }
 966
 967         if (vma->fence && !i915_gem_object_is_tiled(obj)) {
 968                 mutex_lock(&vma->vm->mutex);
 969                 ret = i915_vma_revoke_fence(vma);
 970                 mutex_unlock(&vma->vm->mutex);
 971                 if (ret)
 972                         return ERR_PTR(ret);
 973         }
 974
 975         ret = i915_vma_pin(vma, size, alignment, flags);
 976         if (ret)
 977                 return ERR_PTR(ret);
 978
 979         return vma;
 980 }
 981
 982 int
 983 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 984                        struct drm_file *file_priv)
 985 {
 986         struct drm_i915_private *i915 = to_i915(dev);
 987         struct drm_i915_gem_madvise *args = data;
 988         struct drm_i915_gem_object *obj;
 989         int err;
 990
 991         switch (args->madv) {
 992         case I915_MADV_DONTNEED:
 993         case I915_MADV_WILLNEED:
 994             break;
 995         default:
 996             return -EINVAL;
 997         }
 998
 999         obj = i915_gem_object_lookup(file_priv, args->handle);
1000         if (!obj)
1001                 return -ENOENT;
1002
1003         err = mutex_lock_interruptible(&obj->mm.lock);
1004         if (err)
1005                 goto out;
1006
1007         if (i915_gem_object_has_pages(obj) &&
1008             i915_gem_object_is_tiled(obj) &&
1009             i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
1010                 if (obj->mm.madv == I915_MADV_WILLNEED) {
1011                         GEM_BUG_ON(!obj->mm.quirked);
1012                         __i915_gem_object_unpin_pages(obj);
1013                         obj->mm.quirked = false;
1014                 }
1015                 if (args->madv == I915_MADV_WILLNEED) {
1016                         GEM_BUG_ON(obj->mm.quirked);
1017                         __i915_gem_object_pin_pages(obj);
1018                         obj->mm.quirked = true;
1019                 }
1020         }
1021
1022         if (obj->mm.madv != __I915_MADV_PURGED)
1023                 obj->mm.madv = args->madv;
1024
1025         if (i915_gem_object_has_pages(obj)) {
1026                 struct list_head *list;
1027
1028                 if (i915_gem_object_is_shrinkable(obj)) {
1029                         unsigned long flags;
1030
1031                         spin_lock_irqsave(&i915->mm.obj_lock, flags);
1032
1033                         if (obj->mm.madv != I915_MADV_WILLNEED)
1034                                 list = &i915->mm.purge_list;
1035                         else
1036                                 list = &i915->mm.shrink_list;
1037                         list_move_tail(&obj->mm.link, list);
1038
1039                         spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
1040                 }
1041         }
1042
1043         /* if the object is no longer attached, discard its backing storage */
1044         if (obj->mm.madv == I915_MADV_DONTNEED &&
1045             !i915_gem_object_has_pages(obj))
1046                 i915_gem_object_truncate(obj);
1047
1048         args->retained = obj->mm.madv != __I915_MADV_PURGED;
1049         mutex_unlock(&obj->mm.lock);
1050
1051 out:
1052         i915_gem_object_put(obj);
1053         return err;
1054 }
1055
1056 static int __intel_engines_record_defaults(struct intel_gt *gt)
1057 {
1058         struct i915_request *requests[I915_NUM_ENGINES] = {};
1059         struct intel_engine_cs *engine;
1060         enum intel_engine_id id;
1061         int err = 0;
1062
1063         /*
1064          * As we reset the gpu during very early sanitisation, the current
1065          * register state on the GPU should reflect its defaults values.
1066          * We load a context onto the hw (with restore-inhibit), then switch
1067          * over to a second context to save that default register state. We
1068          * can then prime every new context with that state so they all start
1069          * from the same default HW values.
1070          */
1071
1072         for_each_engine(engine, gt, id) {
1073                 struct intel_context *ce;
1074                 struct i915_request *rq;
1075
1076                 /* We must be able to switch to something! */
1077                 GEM_BUG_ON(!engine->kernel_context);
1078                 engine->serial++; /* force the kernel context switch */
1079
1080                 ce = intel_context_create(engine->kernel_context->gem_context,
1081                                           engine);
1082                 if (IS_ERR(ce)) {
1083                         err = PTR_ERR(ce);
1084                         goto out;
1085                 }
1086
1087                 rq = intel_context_create_request(ce);
1088                 if (IS_ERR(rq)) {
1089                         err = PTR_ERR(rq);
1090                         intel_context_put(ce);
1091                         goto out;
1092                 }
1093
1094                 err = intel_engine_emit_ctx_wa(rq);
1095                 if (err)
1096                         goto err_rq;
1097
1098                 err = intel_renderstate_emit(rq);
1099                 if (err)
1100                         goto err_rq;
1101
1102 err_rq:
1103                 requests[id] = i915_request_get(rq);
1104                 i915_request_add(rq);
1105                 if (err)
1106                         goto out;
1107         }
1108
1109         /* Flush the default context image to memory, and enable powersaving. */
1110         if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) {
1111                 err = -EIO;
1112                 goto out;
1113         }
1114
1115         for (id = 0; id < ARRAY_SIZE(requests); id++) {
1116                 struct i915_request *rq;
1117                 struct i915_vma *state;
1118                 void *vaddr;
1119
1120                 rq = requests[id];
1121                 if (!rq)
1122                         continue;
1123
1124                 /* We want to be able to unbind the state from the GGTT */
1125                 GEM_BUG_ON(intel_context_is_pinned(rq->hw_context));
1126
1127                 state = rq->hw_context->state;
1128                 if (!state)
1129                         continue;
1130
1131                 /*
1132                  * As we will hold a reference to the logical state, it will
1133                  * not be torn down with the context, and importantly the
1134                  * object will hold onto its vma (making it possible for a
1135                  * stray GTT write to corrupt our defaults). Unmap the vma
1136                  * from the GTT to prevent such accidents and reclaim the
1137                  * space.
1138                  */
1139                 err = i915_vma_unbind(state);
1140                 if (err)
1141                         goto out;
1142
1143                 i915_gem_object_lock(state->obj);
1144                 err = i915_gem_object_set_to_cpu_domain(state->obj, false);
1145                 i915_gem_object_unlock(state->obj);
1146                 if (err)
1147                         goto out;
1148
1149                 i915_gem_object_set_cache_coherency(state->obj, I915_CACHE_LLC);
1150
1151                 /* Check we can acquire the image of the context state */
1152                 vaddr = i915_gem_object_pin_map(state->obj, I915_MAP_FORCE_WB);
1153                 if (IS_ERR(vaddr)) {
1154                         err = PTR_ERR(vaddr);
1155                         goto out;
1156                 }
1157
1158                 rq->engine->default_state = i915_gem_object_get(state->obj);
1159                 i915_gem_object_unpin_map(state->obj);
1160         }
1161
1162 out:
1163         /*
1164          * If we have to abandon now, we expect the engines to be idle
1165          * and ready to be torn-down. The quickest way we can accomplish
1166          * this is by declaring ourselves wedged.
1167          */
1168         if (err)
1169                 intel_gt_set_wedged(gt);
1170
1171         for (id = 0; id < ARRAY_SIZE(requests); id++) {
1172                 struct intel_context *ce;
1173                 struct i915_request *rq;
1174
1175                 rq = requests[id];
1176                 if (!rq)
1177                         continue;
1178
1179                 ce = rq->hw_context;
1180                 i915_request_put(rq);
1181                 intel_context_put(ce);
1182         }
1183         return err;
1184 }
1185
1186 static int intel_engines_verify_workarounds(struct intel_gt *gt)
1187 {
1188         struct intel_engine_cs *engine;
1189         enum intel_engine_id id;
1190         int err = 0;
1191
1192         if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
1193                 return 0;
1194
1195         for_each_engine(engine, gt, id) {
1196                 if (intel_engine_verify_workarounds(engine, "load"))
1197                         err = -EIO;
1198         }
1199
1200         return err;
1201 }
1202
1203 int i915_gem_init(struct drm_i915_private *dev_priv)
1204 {
1205         int ret;
1206
1207         /* We need to fallback to 4K pages if host doesn't support huge gtt. */
1208         if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv))
1209                 mkwrite_device_info(dev_priv)->page_sizes =
1210                         I915_GTT_PAGE_SIZE_4K;
1211
1212         intel_timelines_init(dev_priv);
1213
1214         ret = i915_gem_init_userptr(dev_priv);
1215         if (ret)
1216                 return ret;
1217
1218         intel_uc_fetch_firmwares(&dev_priv->gt.uc);
1219         intel_wopcm_init(&dev_priv->wopcm);
1220
1221         /* This is just a security blanket to placate dragons.
1222          * On some systems, we very sporadically observe that the first TLBs
1223          * used by the CS may be stale, despite us poking the TLB reset. If
1224          * we hold the forcewake during initialisation these problems
1225          * just magically go away.
1226          */
1227         intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
1228
1229         ret = i915_init_ggtt(dev_priv);
1230         if (ret) {
1231                 GEM_BUG_ON(ret == -EIO);
1232                 goto err_unlock;
1233         }
1234
1235         intel_gt_init(&dev_priv->gt);
1236
1237         ret = intel_engines_setup(&dev_priv->gt);
1238         if (ret) {
1239                 GEM_BUG_ON(ret == -EIO);
1240                 goto err_unlock;
1241         }
1242
1243         ret = i915_gem_init_contexts(dev_priv);
1244         if (ret) {
1245                 GEM_BUG_ON(ret == -EIO);
1246                 goto err_scratch;
1247         }
1248
1249         ret = intel_engines_init(&dev_priv->gt);
1250         if (ret) {
1251                 GEM_BUG_ON(ret == -EIO);
1252                 goto err_context;
1253         }
1254
1255         intel_uc_init(&dev_priv->gt.uc);
1256
1257         ret = intel_gt_init_hw(&dev_priv->gt);
1258         if (ret)
1259                 goto err_uc_init;
1260
1261         /* Only when the HW is re-initialised, can we replay the requests */
1262         ret = intel_gt_resume(&dev_priv->gt);
1263         if (ret)
1264                 goto err_init_hw;
1265
1266         /*
1267          * Despite its name intel_init_clock_gating applies both display
1268          * clock gating workarounds; GT mmio workarounds and the occasional
1269          * GT power context workaround. Worse, sometimes it includes a context
1270          * register workaround which we need to apply before we record the
1271          * default HW state for all contexts.
1272          *
1273          * FIXME: break up the workarounds and apply them at the right time!
1274          */
1275         intel_init_clock_gating(dev_priv);
1276
1277         ret = intel_engines_verify_workarounds(&dev_priv->gt);
1278         if (ret)
1279                 goto err_gt;
1280
1281         ret = __intel_engines_record_defaults(&dev_priv->gt);
1282         if (ret)
1283                 goto err_gt;
1284
1285         ret = i915_inject_probe_error(dev_priv, -ENODEV);
1286         if (ret)
1287                 goto err_gt;
1288
1289         ret = i915_inject_probe_error(dev_priv, -EIO);
1290         if (ret)
1291                 goto err_gt;
1292
1293         intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
1294
1295         return 0;
1296
1297         /*
1298          * Unwinding is complicated by that we want to handle -EIO to mean
1299          * disable GPU submission but keep KMS alive. We want to mark the
1300          * HW as irrevisibly wedged, but keep enough state around that the
1301          * driver doesn't explode during runtime.
1302          */
1303 err_gt:
1304         intel_gt_set_wedged_on_init(&dev_priv->gt);
1305         i915_gem_suspend(dev_priv);
1306         i915_gem_suspend_late(dev_priv);
1307
1308         i915_gem_drain_workqueue(dev_priv);
1309 err_init_hw:
1310         intel_uc_fini_hw(&dev_priv->gt.uc);
1311 err_uc_init:
1312         if (ret != -EIO) {
1313                 intel_uc_fini(&dev_priv->gt.uc);
1314                 intel_engines_cleanup(&dev_priv->gt);
1315         }
1316 err_context:
1317         if (ret != -EIO)
1318                 i915_gem_driver_release__contexts(dev_priv);
1319 err_scratch:
1320         intel_gt_driver_release(&dev_priv->gt);
1321 err_unlock:
1322         intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
1323
1324         if (ret != -EIO) {
1325                 intel_uc_cleanup_firmwares(&dev_priv->gt.uc);
1326                 i915_gem_cleanup_userptr(dev_priv);
1327                 intel_timelines_fini(dev_priv);
1328         }
1329
1330         if (ret == -EIO) {
1331                 /*
1332                  * Allow engines or uC initialisation to fail by marking the GPU
1333                  * as wedged. But we only want to do this when the GPU is angry,
1334                  * for all other failure, such as an allocation failure, bail.
1335                  */
1336                 if (!intel_gt_is_wedged(&dev_priv->gt)) {
1337                         i915_probe_error(dev_priv,
1338                                          "Failed to initialize GPU, declaring it wedged!\n");
1339                         intel_gt_set_wedged(&dev_priv->gt);
1340                 }
1341
1342                 /* Minimal basic recovery for KMS */
1343                 ret = i915_ggtt_enable_hw(dev_priv);
1344                 i915_gem_restore_gtt_mappings(dev_priv);
1345                 i915_gem_restore_fences(&dev_priv->ggtt);
1346                 intel_init_clock_gating(dev_priv);
1347         }
1348
1349         i915_gem_drain_freed_objects(dev_priv);
1350         return ret;
1351 }
1352
1353 void i915_gem_driver_register(struct drm_i915_private *i915)
1354 {
1355         i915_gem_driver_register__shrinker(i915);
1356
1357         intel_engines_driver_register(i915);
1358 }
1359
1360 void i915_gem_driver_unregister(struct drm_i915_private *i915)
1361 {
1362         i915_gem_driver_unregister__shrinker(i915);
1363 }
1364
1365 void i915_gem_driver_remove(struct drm_i915_private *dev_priv)
1366 {
1367         intel_wakeref_auto_fini(&dev_priv->ggtt.userfault_wakeref);
1368
1369         i915_gem_suspend_late(dev_priv);
1370         intel_gt_driver_remove(&dev_priv->gt);
1371
1372         /* Flush any outstanding unpin_work. */
1373         i915_gem_drain_workqueue(dev_priv);
1374
1375         intel_uc_fini_hw(&dev_priv->gt.uc);
1376         intel_uc_fini(&dev_priv->gt.uc);
1377
1378         i915_gem_drain_freed_objects(dev_priv);
1379 }
1380
1381 void i915_gem_driver_release(struct drm_i915_private *dev_priv)
1382 {
1383         intel_engines_cleanup(&dev_priv->gt);
1384         i915_gem_driver_release__contexts(dev_priv);
1385         intel_gt_driver_release(&dev_priv->gt);
1386
1387         intel_wa_list_free(&dev_priv->gt_wa_list);
1388
1389         intel_uc_cleanup_firmwares(&dev_priv->gt.uc);
1390         i915_gem_cleanup_userptr(dev_priv);
1391         intel_timelines_fini(dev_priv);
1392
1393         i915_gem_drain_freed_objects(dev_priv);
1394
1395         WARN_ON(!list_empty(&dev_priv->gem.contexts.list));
1396 }
1397
1398 static void i915_gem_init__mm(struct drm_i915_private *i915)
1399 {
1400         spin_lock_init(&i915->mm.obj_lock);
1401
1402         init_llist_head(&i915->mm.free_list);
1403
1404         INIT_LIST_HEAD(&i915->mm.purge_list);
1405         INIT_LIST_HEAD(&i915->mm.shrink_list);
1406
1407         i915_gem_init__objects(i915);
1408 }
1409
1410 void i915_gem_init_early(struct drm_i915_private *dev_priv)
1411 {
1412         i915_gem_init__mm(dev_priv);
1413
1414         spin_lock_init(&dev_priv->fb_tracking.lock);
1415 }
1416
1417 void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
1418 {
1419         i915_gem_drain_freed_objects(dev_priv);
1420         GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list));
1421         GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count));
1422         WARN_ON(dev_priv->mm.shrink_count);
1423 }
1424
1425 int i915_gem_freeze(struct drm_i915_private *dev_priv)
1426 {
1427         /* Discard all purgeable objects, let userspace recover those as
1428          * required after resuming.
1429          */
1430         i915_gem_shrink_all(dev_priv);
1431
1432         return 0;
1433 }
1434
1435 int i915_gem_freeze_late(struct drm_i915_private *i915)
1436 {
1437         struct drm_i915_gem_object *obj;
1438         intel_wakeref_t wakeref;
1439
1440         /*
1441          * Called just before we write the hibernation image.
1442          *
1443          * We need to update the domain tracking to reflect that the CPU
1444          * will be accessing all the pages to create and restore from the
1445          * hibernation, and so upon restoration those pages will be in the
1446          * CPU domain.
1447          *
1448          * To make sure the hibernation image contains the latest state,
1449          * we update that state just before writing out the image.
1450          *
1451          * To try and reduce the hibernation image, we manually shrink
1452          * the objects as well, see i915_gem_freeze()
1453          */
1454
1455         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1456
1457         i915_gem_shrink(i915, -1UL, NULL, ~0);
1458         i915_gem_drain_freed_objects(i915);
1459
1460         list_for_each_entry(obj, &i915->mm.shrink_list, mm.link) {
1461                 i915_gem_object_lock(obj);
1462                 WARN_ON(i915_gem_object_set_to_cpu_domain(obj, true));
1463                 i915_gem_object_unlock(obj);
1464         }
1465
1466         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1467
1468         return 0;
1469 }
1470
1471 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
1472 {
1473         struct drm_i915_file_private *file_priv = file->driver_priv;
1474         struct i915_request *request;
1475
1476         /* Clean up our request list when the client is going away, so that
1477          * later retire_requests won't dereference our soon-to-be-gone
1478          * file_priv.
1479          */
1480         spin_lock(&file_priv->mm.lock);
1481         list_for_each_entry(request, &file_priv->mm.request_list, client_link)
1482                 request->file_priv = NULL;
1483         spin_unlock(&file_priv->mm.lock);
1484 }
1485
1486 int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file)
1487 {
1488         struct drm_i915_file_private *file_priv;
1489         int ret;
1490
1491         DRM_DEBUG("\n");
1492
1493         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
1494         if (!file_priv)
1495                 return -ENOMEM;
1496
1497         file->driver_priv = file_priv;
1498         file_priv->dev_priv = i915;
1499         file_priv->file = file;
1500
1501         spin_lock_init(&file_priv->mm.lock);
1502         INIT_LIST_HEAD(&file_priv->mm.request_list);
1503
1504         file_priv->bsd_engine = -1;
1505         file_priv->hang_timestamp = jiffies;
1506
1507         ret = i915_gem_context_open(i915, file);
1508         if (ret)
1509                 kfree(file_priv);
1510
1511         return ret;
1512 }
1513
1514 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1515 #include "selftests/mock_gem_device.c"
1516 #include "selftests/i915_gem.c"
1517 #endif