drivers/gpu/drm/i915/gem/i915_gem_shmem.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2014-2016 Intel Corporation
   5  */
   6
   7 #include <linux/pagevec.h>
   8 #include <linux/shmem_fs.h>
   9 #include <linux/swap.h>
  10
  11 #include <drm/drm_cache.h>
  12
  13 #include "gem/i915_gem_region.h"
  14 #include "i915_drv.h"
  15 #include "i915_gem_object.h"
  16 #include "i915_gem_tiling.h"
  17 #include "i915_gemfs.h"
  18 #include "i915_scatterlist.h"
  19 #include "i915_trace.h"
  20
  21 /*
  22  * Move pages to appropriate lru and release the pagevec, decrementing the
  23  * ref count of those pages.
  24  */
  25 static void check_release_pagevec(struct pagevec *pvec)
  26 {
  27         check_move_unevictable_pages(pvec);
  28         __pagevec_release(pvec);
  29         cond_resched();
  30 }
  31
  32 void shmem_sg_free_table(struct sg_table *st, struct address_space *mapping,
  33                          bool dirty, bool backup)
  34 {
  35         struct sgt_iter sgt_iter;
  36         struct pagevec pvec;
  37         struct page *page;
  38
  39         mapping_clear_unevictable(mapping);
  40
  41         pagevec_init(&pvec);
  42         for_each_sgt_page(page, sgt_iter, st) {
  43                 if (dirty)
  44                         set_page_dirty(page);
  45
  46                 if (backup)
  47                         mark_page_accessed(page);
  48
  49                 if (!pagevec_add(&pvec, page))
  50                         check_release_pagevec(&pvec);
  51         }
  52         if (pagevec_count(&pvec))
  53                 check_release_pagevec(&pvec);
  54
  55         sg_free_table(st);
  56 }
  57
  58 int shmem_sg_alloc_table(struct drm_i915_private *i915, struct sg_table *st,
  59                          size_t size, struct intel_memory_region *mr,
  60                          struct address_space *mapping,
  61                          unsigned int max_segment)
  62 {
  63         const unsigned long page_count = size / PAGE_SIZE;
  64         unsigned long i;
  65         struct scatterlist *sg;
  66         struct page *page;
  67         unsigned long last_pfn = 0;     /* suppress gcc warning */
  68         gfp_t noreclaim;
  69         int ret;
  70
  71         /*
  72          * If there's no chance of allocating enough pages for the whole
  73          * object, bail early.
  74          */
  75         if (size > resource_size(&mr->region))
  76                 return -ENOMEM;
  77
  78         if (sg_alloc_table(st, page_count, GFP_KERNEL))
  79                 return -ENOMEM;
  80
  81         /*
  82          * Get the list of pages out of our struct file.  They'll be pinned
  83          * at this point until we release them.
  84          *
  85          * Fail silently without starting the shrinker
  86          */
  87         mapping_set_unevictable(mapping);
  88         noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM);
  89         noreclaim |= __GFP_NORETRY | __GFP_NOWARN;
  90
  91         sg = st->sgl;
  92         st->nents = 0;
  93         for (i = 0; i < page_count; i++) {
  94                 const unsigned int shrink[] = {
  95                         I915_SHRINK_BOUND | I915_SHRINK_UNBOUND,
  96                         0,
  97                 }, *s = shrink;
  98                 gfp_t gfp = noreclaim;
  99
 100                 do {
 101                         cond_resched();
 102                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
 103                         if (!IS_ERR(page))
 104                                 break;
 105
 106                         if (!*s) {
 107                                 ret = PTR_ERR(page);
 108                                 goto err_sg;
 109                         }
 110
 111                         i915_gem_shrink(NULL, i915, 2 * page_count, NULL, *s++);
 112
 113                         /*
 114                          * We've tried hard to allocate the memory by reaping
 115                          * our own buffer, now let the real VM do its job and
 116                          * go down in flames if truly OOM.
 117                          *
 118                          * However, since graphics tend to be disposable,
 119                          * defer the oom here by reporting the ENOMEM back
 120                          * to userspace.
 121                          */
 122                         if (!*s) {
 123                                 /* reclaim and warn, but no oom */
 124                                 gfp = mapping_gfp_mask(mapping);
 125
 126                                 /*
 127                                  * Our bo are always dirty and so we require
 128                                  * kswapd to reclaim our pages (direct reclaim
 129                                  * does not effectively begin pageout of our
 130                                  * buffers on its own). However, direct reclaim
 131                                  * only waits for kswapd when under allocation
 132                                  * congestion. So as a result __GFP_RECLAIM is
 133                                  * unreliable and fails to actually reclaim our
 134                                  * dirty pages -- unless you try over and over
 135                                  * again with !__GFP_NORETRY. However, we still
 136                                  * want to fail this allocation rather than
 137                                  * trigger the out-of-memory killer and for
 138                                  * this we want __GFP_RETRY_MAYFAIL.
 139                                  */
 140                                 gfp |= __GFP_RETRY_MAYFAIL;
 141                         }
 142                 } while (1);
 143
 144                 if (!i ||
 145                     sg->length >= max_segment ||
 146                     page_to_pfn(page) != last_pfn + 1) {
 147                         if (i)
 148                                 sg = sg_next(sg);
 149
 150                         st->nents++;
 151                         sg_set_page(sg, page, PAGE_SIZE, 0);
 152                 } else {
 153                         sg->length += PAGE_SIZE;
 154                 }
 155                 last_pfn = page_to_pfn(page);
 156
 157                 /* Check that the i965g/gm workaround works. */
 158                 GEM_BUG_ON(gfp & __GFP_DMA32 && last_pfn >= 0x00100000UL);
 159         }
 160         if (sg) /* loop terminated early; short sg table */
 161                 sg_mark_end(sg);
 162
 163         /* Trim unused sg entries to avoid wasting memory. */
 164         i915_sg_trim(st);
 165
 166         return 0;
 167 err_sg:
 168         sg_mark_end(sg);
 169         if (sg != st->sgl) {
 170                 shmem_sg_free_table(st, mapping, false, false);
 171         } else {
 172                 mapping_clear_unevictable(mapping);
 173                 sg_free_table(st);
 174         }
 175
 176         /*
 177          * shmemfs first checks if there is enough memory to allocate the page
 178          * and reports ENOSPC should there be insufficient, along with the usual
 179          * ENOMEM for a genuine allocation failure.
 180          *
 181          * We use ENOSPC in our driver to mean that we have run out of aperture
 182          * space and so want to translate the error from shmemfs back to our
 183          * usual understanding of ENOMEM.
 184          */
 185         if (ret == -ENOSPC)
 186                 ret = -ENOMEM;
 187
 188         return ret;
 189 }
 190
 191 static int shmem_get_pages(struct drm_i915_gem_object *obj)
 192 {
 193         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 194         struct intel_memory_region *mem = obj->mm.region;
 195         struct address_space *mapping = obj->base.filp->f_mapping;
 196         const unsigned long page_count = obj->base.size / PAGE_SIZE;
 197         unsigned int max_segment = i915_sg_segment_size();
 198         struct sg_table *st;
 199         struct sgt_iter sgt_iter;
 200         struct page *page;
 201         int ret;
 202
 203         /*
 204          * Assert that the object is not currently in any GPU domain. As it
 205          * wasn't in the GTT, there shouldn't be any way it could have been in
 206          * a GPU cache
 207          */
 208         GEM_BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
 209         GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
 210
 211 rebuild_st:
 212         st = kmalloc(sizeof(*st), GFP_KERNEL);
 213         if (!st)
 214                 return -ENOMEM;
 215
 216         ret = shmem_sg_alloc_table(i915, st, obj->base.size, mem, mapping,
 217                                    max_segment);
 218         if (ret)
 219                 goto err_st;
 220
 221         ret = i915_gem_gtt_prepare_pages(obj, st);
 222         if (ret) {
 223                 /*
 224                  * DMA remapping failed? One possible cause is that
 225                  * it could not reserve enough large entries, asking
 226                  * for PAGE_SIZE chunks instead may be helpful.
 227                  */
 228                 if (max_segment > PAGE_SIZE) {
 229                         for_each_sgt_page(page, sgt_iter, st)
 230                                 put_page(page);
 231                         sg_free_table(st);
 232                         kfree(st);
 233
 234                         max_segment = PAGE_SIZE;
 235                         goto rebuild_st;
 236                 } else {
 237                         dev_warn(i915->drm.dev,
 238                                  "Failed to DMA remap %lu pages\n",
 239                                  page_count);
 240                         goto err_pages;
 241                 }
 242         }
 243
 244         if (i915_gem_object_needs_bit17_swizzle(obj))
 245                 i915_gem_object_do_bit_17_swizzle(obj, st);
 246
 247         if (i915_gem_object_can_bypass_llc(obj))
 248                 obj->cache_dirty = true;
 249
 250         __i915_gem_object_set_pages(obj, st, i915_sg_dma_sizes(st->sgl));
 251
 252         return 0;
 253
 254 err_pages:
 255         shmem_sg_free_table(st, mapping, false, false);
 256         /*
 257          * shmemfs first checks if there is enough memory to allocate the page
 258          * and reports ENOSPC should there be insufficient, along with the usual
 259          * ENOMEM for a genuine allocation failure.
 260          *
 261          * We use ENOSPC in our driver to mean that we have run out of aperture
 262          * space and so want to translate the error from shmemfs back to our
 263          * usual understanding of ENOMEM.
 264          */
 265 err_st:
 266         if (ret == -ENOSPC)
 267                 ret = -ENOMEM;
 268
 269         kfree(st);
 270
 271         return ret;
 272 }
 273
 274 static int
 275 shmem_truncate(struct drm_i915_gem_object *obj)
 276 {
 277         /*
 278          * Our goal here is to return as much of the memory as
 279          * is possible back to the system as we are called from OOM.
 280          * To do this we must instruct the shmfs to drop all of its
 281          * backing pages, *now*.
 282          */
 283         shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
 284         obj->mm.madv = __I915_MADV_PURGED;
 285         obj->mm.pages = ERR_PTR(-EFAULT);
 286
 287         return 0;
 288 }
 289
 290 void __shmem_writeback(size_t size, struct address_space *mapping)
 291 {
 292         struct writeback_control wbc = {
 293                 .sync_mode = WB_SYNC_NONE,
 294                 .nr_to_write = SWAP_CLUSTER_MAX,
 295                 .range_start = 0,
 296                 .range_end = LLONG_MAX,
 297                 .for_reclaim = 1,
 298         };
 299         unsigned long i;
 300
 301         /*
 302          * Leave mmapings intact (GTT will have been revoked on unbinding,
 303          * leaving only CPU mmapings around) and add those pages to the LRU
 304          * instead of invoking writeback so they are aged and paged out
 305          * as normal.
 306          */
 307
 308         /* Begin writeback on each dirty page */
 309         for (i = 0; i < size >> PAGE_SHIFT; i++) {
 310                 struct page *page;
 311
 312                 page = find_lock_page(mapping, i);
 313                 if (!page)
 314                         continue;
 315
 316                 if (!page_mapped(page) && clear_page_dirty_for_io(page)) {
 317                         int ret;
 318
 319                         SetPageReclaim(page);
 320                         ret = mapping->a_ops->writepage(page, &wbc);
 321                         if (!PageWriteback(page))
 322                                 ClearPageReclaim(page);
 323                         if (!ret)
 324                                 goto put;
 325                 }
 326                 unlock_page(page);
 327 put:
 328                 put_page(page);
 329         }
 330 }
 331
 332 static void
 333 shmem_writeback(struct drm_i915_gem_object *obj)
 334 {
 335         __shmem_writeback(obj->base.size, obj->base.filp->f_mapping);
 336 }
 337
 338 static int shmem_shrink(struct drm_i915_gem_object *obj, unsigned int flags)
 339 {
 340         switch (obj->mm.madv) {
 341         case I915_MADV_DONTNEED:
 342                 return i915_gem_object_truncate(obj);
 343         case __I915_MADV_PURGED:
 344                 return 0;
 345         }
 346
 347         if (flags & I915_GEM_OBJECT_SHRINK_WRITEBACK)
 348                 shmem_writeback(obj);
 349
 350         return 0;
 351 }
 352
 353 void
 354 __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
 355                                 struct sg_table *pages,
 356                                 bool needs_clflush)
 357 {
 358         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 359
 360         GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
 361
 362         if (obj->mm.madv == I915_MADV_DONTNEED)
 363                 obj->mm.dirty = false;
 364
 365         if (needs_clflush &&
 366             (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0 &&
 367             !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
 368                 drm_clflush_sg(pages);
 369
 370         __start_cpu_write(obj);
 371         /*
 372          * On non-LLC platforms, force the flush-on-acquire if this is ever
 373          * swapped-in. Our async flush path is not trust worthy enough yet(and
 374          * happens in the wrong order), and with some tricks it's conceivable
 375          * for userspace to change the cache-level to I915_CACHE_NONE after the
 376          * pages are swapped-in, and since execbuf binds the object before doing
 377          * the async flush, we have a race window.
 378          */
 379         if (!HAS_LLC(i915))
 380                 obj->cache_dirty = true;
 381 }
 382
 383 void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj, struct sg_table *pages)
 384 {
 385         __i915_gem_object_release_shmem(obj, pages, true);
 386
 387         i915_gem_gtt_finish_pages(obj, pages);
 388
 389         if (i915_gem_object_needs_bit17_swizzle(obj))
 390                 i915_gem_object_save_bit_17_swizzle(obj, pages);
 391
 392         shmem_sg_free_table(pages, file_inode(obj->base.filp)->i_mapping,
 393                             obj->mm.dirty, obj->mm.madv == I915_MADV_WILLNEED);
 394         kfree(pages);
 395         obj->mm.dirty = false;
 396 }
 397
 398 static void
 399 shmem_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages)
 400 {
 401         if (likely(i915_gem_object_has_struct_page(obj)))
 402                 i915_gem_object_put_pages_shmem(obj, pages);
 403         else
 404                 i915_gem_object_put_pages_phys(obj, pages);
 405 }
 406
 407 static int
 408 shmem_pwrite(struct drm_i915_gem_object *obj,
 409              const struct drm_i915_gem_pwrite *arg)
 410 {
 411         struct address_space *mapping = obj->base.filp->f_mapping;
 412         const struct address_space_operations *aops = mapping->a_ops;
 413         char __user *user_data = u64_to_user_ptr(arg->data_ptr);
 414         u64 remain, offset;
 415         unsigned int pg;
 416
 417         /* Caller already validated user args */
 418         GEM_BUG_ON(!access_ok(user_data, arg->size));
 419
 420         if (!i915_gem_object_has_struct_page(obj))
 421                 return i915_gem_object_pwrite_phys(obj, arg);
 422
 423         /*
 424          * Before we instantiate/pin the backing store for our use, we
 425          * can prepopulate the shmemfs filp efficiently using a write into
 426          * the pagecache. We avoid the penalty of instantiating all the
 427          * pages, important if the user is just writing to a few and never
 428          * uses the object on the GPU, and using a direct write into shmemfs
 429          * allows it to avoid the cost of retrieving a page (either swapin
 430          * or clearing-before-use) before it is overwritten.
 431          */
 432         if (i915_gem_object_has_pages(obj))
 433                 return -ENODEV;
 434
 435         if (obj->mm.madv != I915_MADV_WILLNEED)
 436                 return -EFAULT;
 437
 438         /*
 439          * Before the pages are instantiated the object is treated as being
 440          * in the CPU domain. The pages will be clflushed as required before
 441          * use, and we can freely write into the pages directly. If userspace
 442          * races pwrite with any other operation; corruption will ensue -
 443          * that is userspace's prerogative!
 444          */
 445
 446         remain = arg->size;
 447         offset = arg->offset;
 448         pg = offset_in_page(offset);
 449
 450         do {
 451                 unsigned int len, unwritten;
 452                 struct page *page;
 453                 void *data, *vaddr;
 454                 int err;
 455                 char c;
 456
 457                 len = PAGE_SIZE - pg;
 458                 if (len > remain)
 459                         len = remain;
 460
 461                 /* Prefault the user page to reduce potential recursion */
 462                 err = __get_user(c, user_data);
 463                 if (err)
 464                         return err;
 465
 466                 err = __get_user(c, user_data + len - 1);
 467                 if (err)
 468                         return err;
 469
 470                 err = aops->write_begin(obj->base.filp, mapping, offset, len,
 471                                         &page, &data);
 472                 if (err < 0)
 473                         return err;
 474
 475                 vaddr = kmap_atomic(page);
 476                 unwritten = __copy_from_user_inatomic(vaddr + pg,
 477                                                       user_data,
 478                                                       len);
 479                 kunmap_atomic(vaddr);
 480
 481                 err = aops->write_end(obj->base.filp, mapping, offset, len,
 482                                       len - unwritten, page, data);
 483                 if (err < 0)
 484                         return err;
 485
 486                 /* We don't handle -EFAULT, leave it to the caller to check */
 487                 if (unwritten)
 488                         return -ENODEV;
 489
 490                 remain -= len;
 491                 user_data += len;
 492                 offset += len;
 493                 pg = 0;
 494         } while (remain);
 495
 496         return 0;
 497 }
 498
 499 static int
 500 shmem_pread(struct drm_i915_gem_object *obj,
 501             const struct drm_i915_gem_pread *arg)
 502 {
 503         if (!i915_gem_object_has_struct_page(obj))
 504                 return i915_gem_object_pread_phys(obj, arg);
 505
 506         return -ENODEV;
 507 }
 508
 509 static void shmem_release(struct drm_i915_gem_object *obj)
 510 {
 511         if (i915_gem_object_has_struct_page(obj))
 512                 i915_gem_object_release_memory_region(obj);
 513
 514         fput(obj->base.filp);
 515 }
 516
 517 const struct drm_i915_gem_object_ops i915_gem_shmem_ops = {
 518         .name = "i915_gem_object_shmem",
 519         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 520
 521         .get_pages = shmem_get_pages,
 522         .put_pages = shmem_put_pages,
 523         .truncate = shmem_truncate,
 524         .shrink = shmem_shrink,
 525
 526         .pwrite = shmem_pwrite,
 527         .pread = shmem_pread,
 528
 529         .release = shmem_release,
 530 };
 531
 532 static int __create_shmem(struct drm_i915_private *i915,
 533                           struct drm_gem_object *obj,
 534                           resource_size_t size)
 535 {
 536         unsigned long flags = VM_NORESERVE;
 537         struct file *filp;
 538
 539         drm_gem_private_object_init(&i915->drm, obj, size);
 540
 541         if (i915->mm.gemfs)
 542                 filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size,
 543                                                  flags);
 544         else
 545                 filp = shmem_file_setup("i915", size, flags);
 546         if (IS_ERR(filp))
 547                 return PTR_ERR(filp);
 548
 549         obj->filp = filp;
 550         return 0;
 551 }
 552
 553 static int shmem_object_init(struct intel_memory_region *mem,
 554                              struct drm_i915_gem_object *obj,
 555                              resource_size_t offset,
 556                              resource_size_t size,
 557                              resource_size_t page_size,
 558                              unsigned int flags)
 559 {
 560         static struct lock_class_key lock_class;
 561         struct drm_i915_private *i915 = mem->i915;
 562         struct address_space *mapping;
 563         unsigned int cache_level;
 564         gfp_t mask;
 565         int ret;
 566
 567         ret = __create_shmem(i915, &obj->base, size);
 568         if (ret)
 569                 return ret;
 570
 571         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
 572         if (IS_I965GM(i915) || IS_I965G(i915)) {
 573                 /* 965gm cannot relocate objects above 4GiB. */
 574                 mask &= ~__GFP_HIGHMEM;
 575                 mask |= __GFP_DMA32;
 576         }
 577
 578         mapping = obj->base.filp->f_mapping;
 579         mapping_set_gfp_mask(mapping, mask);
 580         GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM));
 581
 582         i915_gem_object_init(obj, &i915_gem_shmem_ops, &lock_class, 0);
 583         obj->mem_flags |= I915_BO_FLAG_STRUCT_PAGE;
 584         obj->write_domain = I915_GEM_DOMAIN_CPU;
 585         obj->read_domains = I915_GEM_DOMAIN_CPU;
 586
 587         if (HAS_LLC(i915))
 588                 /* On some devices, we can have the GPU use the LLC (the CPU
 589                  * cache) for about a 10% performance improvement
 590                  * compared to uncached.  Graphics requests other than
 591                  * display scanout are coherent with the CPU in
 592                  * accessing this cache.  This means in this mode we
 593                  * don't need to clflush on the CPU side, and on the
 594                  * GPU side we only need to flush internal caches to
 595                  * get data visible to the CPU.
 596                  *
 597                  * However, we maintain the display planes as UC, and so
 598                  * need to rebind when first used as such.
 599                  */
 600                 cache_level = I915_CACHE_LLC;
 601         else
 602                 cache_level = I915_CACHE_NONE;
 603
 604         i915_gem_object_set_cache_coherency(obj, cache_level);
 605
 606         i915_gem_object_init_memory_region(obj, mem);
 607
 608         return 0;
 609 }
 610
 611 struct drm_i915_gem_object *
 612 i915_gem_object_create_shmem(struct drm_i915_private *i915,
 613                              resource_size_t size)
 614 {
 615         return i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_SMEM],
 616                                              size, 0, 0);
 617 }
 618
 619 /* Allocate a new GEM object and fill it with the supplied data */
 620 struct drm_i915_gem_object *
 621 i915_gem_object_create_shmem_from_data(struct drm_i915_private *dev_priv,
 622                                        const void *data, resource_size_t size)
 623 {
 624         struct drm_i915_gem_object *obj;
 625         struct file *file;
 626         const struct address_space_operations *aops;
 627         resource_size_t offset;
 628         int err;
 629
 630         GEM_WARN_ON(IS_DGFX(dev_priv));
 631         obj = i915_gem_object_create_shmem(dev_priv, round_up(size, PAGE_SIZE));
 632         if (IS_ERR(obj))
 633                 return obj;
 634
 635         GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU);
 636
 637         file = obj->base.filp;
 638         aops = file->f_mapping->a_ops;
 639         offset = 0;
 640         do {
 641                 unsigned int len = min_t(typeof(size), size, PAGE_SIZE);
 642                 struct page *page;
 643                 void *pgdata, *vaddr;
 644
 645                 err = aops->write_begin(file, file->f_mapping, offset, len,
 646                                         &page, &pgdata);
 647                 if (err < 0)
 648                         goto fail;
 649
 650                 vaddr = kmap(page);
 651                 memcpy(vaddr, data, len);
 652                 kunmap(page);
 653
 654                 err = aops->write_end(file, file->f_mapping, offset, len, len,
 655                                       page, pgdata);
 656                 if (err < 0)
 657                         goto fail;
 658
 659                 size -= len;
 660                 data += len;
 661                 offset += len;
 662         } while (size);
 663
 664         return obj;
 665
 666 fail:
 667         i915_gem_object_put(obj);
 668         return ERR_PTR(err);
 669 }
 670
 671 static int init_shmem(struct intel_memory_region *mem)
 672 {
 673         int err;
 674
 675         err = i915_gemfs_init(mem->i915);
 676         if (err) {
 677                 DRM_NOTE("Unable to create a private tmpfs mount, hugepage support will be disabled(%d).\n",
 678                          err);
 679         }
 680
 681         intel_memory_region_set_name(mem, "system");
 682
 683         return 0; /* Don't error, we can simply fallback to the kernel mnt */
 684 }
 685
 686 static int release_shmem(struct intel_memory_region *mem)
 687 {
 688         i915_gemfs_fini(mem->i915);
 689         return 0;
 690 }
 691
 692 static const struct intel_memory_region_ops shmem_region_ops = {
 693         .init = init_shmem,
 694         .release = release_shmem,
 695         .init_object = shmem_object_init,
 696 };
 697
 698 struct intel_memory_region *i915_gem_shmem_setup(struct drm_i915_private *i915,
 699                                                  u16 type, u16 instance)
 700 {
 701         return intel_memory_region_create(i915, 0,
 702                                           totalram_pages() << PAGE_SHIFT,
 703                                           PAGE_SIZE, 0, 0,
 704                                           type, instance,
 705                                           &shmem_region_ops);
 706 }
 707
 708 bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj)
 709 {
 710         return obj->ops == &i915_gem_shmem_ops;
 711 }