Merge tag 'regmap-v5.18' of git://git.kernel.org/pub/scm/linux/kernel/git/broonie...
[linux-2.6-microblaze.git] / mm / shmem.c
1 /*
2  * Resizable virtual memory filesystem for Linux.
3  *
4  * Copyright (C) 2000 Linus Torvalds.
5  *               2000 Transmeta Corp.
6  *               2000-2001 Christoph Rohland
7  *               2000-2001 SAP AG
8  *               2002 Red Hat Inc.
9  * Copyright (C) 2002-2011 Hugh Dickins.
10  * Copyright (C) 2011 Google Inc.
11  * Copyright (C) 2002-2005 VERITAS Software Corporation.
12  * Copyright (C) 2004 Andi Kleen, SuSE Labs
13  *
14  * Extended attribute support for tmpfs:
15  * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
16  * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
17  *
18  * tiny-shmem:
19  * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
20  *
21  * This file is released under the GPL.
22  */
23
24 #include <linux/fs.h>
25 #include <linux/init.h>
26 #include <linux/vfs.h>
27 #include <linux/mount.h>
28 #include <linux/ramfs.h>
29 #include <linux/pagemap.h>
30 #include <linux/file.h>
31 #include <linux/mm.h>
32 #include <linux/random.h>
33 #include <linux/sched/signal.h>
34 #include <linux/export.h>
35 #include <linux/swap.h>
36 #include <linux/uio.h>
37 #include <linux/khugepaged.h>
38 #include <linux/hugetlb.h>
39 #include <linux/fs_parser.h>
40 #include <linux/swapfile.h>
41
42 static struct vfsmount *shm_mnt;
43
44 #ifdef CONFIG_SHMEM
45 /*
46  * This virtual memory filesystem is heavily based on the ramfs. It
47  * extends ramfs by the ability to use swap and honor resource limits
48  * which makes it a completely usable filesystem.
49  */
50
51 #include <linux/xattr.h>
52 #include <linux/exportfs.h>
53 #include <linux/posix_acl.h>
54 #include <linux/posix_acl_xattr.h>
55 #include <linux/mman.h>
56 #include <linux/string.h>
57 #include <linux/slab.h>
58 #include <linux/backing-dev.h>
59 #include <linux/shmem_fs.h>
60 #include <linux/writeback.h>
61 #include <linux/pagevec.h>
62 #include <linux/percpu_counter.h>
63 #include <linux/falloc.h>
64 #include <linux/splice.h>
65 #include <linux/security.h>
66 #include <linux/swapops.h>
67 #include <linux/mempolicy.h>
68 #include <linux/namei.h>
69 #include <linux/ctype.h>
70 #include <linux/migrate.h>
71 #include <linux/highmem.h>
72 #include <linux/seq_file.h>
73 #include <linux/magic.h>
74 #include <linux/syscalls.h>
75 #include <linux/fcntl.h>
76 #include <uapi/linux/memfd.h>
77 #include <linux/userfaultfd_k.h>
78 #include <linux/rmap.h>
79 #include <linux/uuid.h>
80
81 #include <linux/uaccess.h>
82
83 #include "internal.h"
84
85 #define BLOCKS_PER_PAGE  (PAGE_SIZE/512)
86 #define VM_ACCT(size)    (PAGE_ALIGN(size) >> PAGE_SHIFT)
87
88 /* Pretend that each entry is of this size in directory's i_size */
89 #define BOGO_DIRENT_SIZE 20
90
91 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */
92 #define SHORT_SYMLINK_LEN 128
93
94 /*
95  * shmem_fallocate communicates with shmem_fault or shmem_writepage via
96  * inode->i_private (with i_rwsem making sure that it has only one user at
97  * a time): we would prefer not to enlarge the shmem inode just for that.
98  */
99 struct shmem_falloc {
100         wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
101         pgoff_t start;          /* start of range currently being fallocated */
102         pgoff_t next;           /* the next page offset to be fallocated */
103         pgoff_t nr_falloced;    /* how many new pages have been fallocated */
104         pgoff_t nr_unswapped;   /* how often writepage refused to swap out */
105 };
106
107 struct shmem_options {
108         unsigned long long blocks;
109         unsigned long long inodes;
110         struct mempolicy *mpol;
111         kuid_t uid;
112         kgid_t gid;
113         umode_t mode;
114         bool full_inums;
115         int huge;
116         int seen;
117 #define SHMEM_SEEN_BLOCKS 1
118 #define SHMEM_SEEN_INODES 2
119 #define SHMEM_SEEN_HUGE 4
120 #define SHMEM_SEEN_INUMS 8
121 };
122
123 #ifdef CONFIG_TMPFS
124 static unsigned long shmem_default_max_blocks(void)
125 {
126         return totalram_pages() / 2;
127 }
128
129 static unsigned long shmem_default_max_inodes(void)
130 {
131         unsigned long nr_pages = totalram_pages();
132
133         return min(nr_pages - totalhigh_pages(), nr_pages / 2);
134 }
135 #endif
136
137 static int shmem_swapin_page(struct inode *inode, pgoff_t index,
138                              struct page **pagep, enum sgp_type sgp,
139                              gfp_t gfp, struct vm_area_struct *vma,
140                              vm_fault_t *fault_type);
141 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
142                 struct page **pagep, enum sgp_type sgp,
143                 gfp_t gfp, struct vm_area_struct *vma,
144                 struct vm_fault *vmf, vm_fault_t *fault_type);
145
146 int shmem_getpage(struct inode *inode, pgoff_t index,
147                 struct page **pagep, enum sgp_type sgp)
148 {
149         return shmem_getpage_gfp(inode, index, pagep, sgp,
150                 mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL);
151 }
152
153 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
154 {
155         return sb->s_fs_info;
156 }
157
158 /*
159  * shmem_file_setup pre-accounts the whole fixed size of a VM object,
160  * for shared memory and for shared anonymous (/dev/zero) mappings
161  * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
162  * consistent with the pre-accounting of private mappings ...
163  */
164 static inline int shmem_acct_size(unsigned long flags, loff_t size)
165 {
166         return (flags & VM_NORESERVE) ?
167                 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
168 }
169
170 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
171 {
172         if (!(flags & VM_NORESERVE))
173                 vm_unacct_memory(VM_ACCT(size));
174 }
175
176 static inline int shmem_reacct_size(unsigned long flags,
177                 loff_t oldsize, loff_t newsize)
178 {
179         if (!(flags & VM_NORESERVE)) {
180                 if (VM_ACCT(newsize) > VM_ACCT(oldsize))
181                         return security_vm_enough_memory_mm(current->mm,
182                                         VM_ACCT(newsize) - VM_ACCT(oldsize));
183                 else if (VM_ACCT(newsize) < VM_ACCT(oldsize))
184                         vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize));
185         }
186         return 0;
187 }
188
189 /*
190  * ... whereas tmpfs objects are accounted incrementally as
191  * pages are allocated, in order to allow large sparse files.
192  * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
193  * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
194  */
195 static inline int shmem_acct_block(unsigned long flags, long pages)
196 {
197         if (!(flags & VM_NORESERVE))
198                 return 0;
199
200         return security_vm_enough_memory_mm(current->mm,
201                         pages * VM_ACCT(PAGE_SIZE));
202 }
203
204 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
205 {
206         if (flags & VM_NORESERVE)
207                 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE));
208 }
209
210 static inline bool shmem_inode_acct_block(struct inode *inode, long pages)
211 {
212         struct shmem_inode_info *info = SHMEM_I(inode);
213         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
214
215         if (shmem_acct_block(info->flags, pages))
216                 return false;
217
218         if (sbinfo->max_blocks) {
219                 if (percpu_counter_compare(&sbinfo->used_blocks,
220                                            sbinfo->max_blocks - pages) > 0)
221                         goto unacct;
222                 percpu_counter_add(&sbinfo->used_blocks, pages);
223         }
224
225         return true;
226
227 unacct:
228         shmem_unacct_blocks(info->flags, pages);
229         return false;
230 }
231
232 static inline void shmem_inode_unacct_blocks(struct inode *inode, long pages)
233 {
234         struct shmem_inode_info *info = SHMEM_I(inode);
235         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
236
237         if (sbinfo->max_blocks)
238                 percpu_counter_sub(&sbinfo->used_blocks, pages);
239         shmem_unacct_blocks(info->flags, pages);
240 }
241
242 static const struct super_operations shmem_ops;
243 const struct address_space_operations shmem_aops;
244 static const struct file_operations shmem_file_operations;
245 static const struct inode_operations shmem_inode_operations;
246 static const struct inode_operations shmem_dir_inode_operations;
247 static const struct inode_operations shmem_special_inode_operations;
248 static const struct vm_operations_struct shmem_vm_ops;
249 static struct file_system_type shmem_fs_type;
250
251 bool vma_is_shmem(struct vm_area_struct *vma)
252 {
253         return vma->vm_ops == &shmem_vm_ops;
254 }
255
256 static LIST_HEAD(shmem_swaplist);
257 static DEFINE_MUTEX(shmem_swaplist_mutex);
258
259 /*
260  * shmem_reserve_inode() performs bookkeeping to reserve a shmem inode, and
261  * produces a novel ino for the newly allocated inode.
262  *
263  * It may also be called when making a hard link to permit the space needed by
264  * each dentry. However, in that case, no new inode number is needed since that
265  * internally draws from another pool of inode numbers (currently global
266  * get_next_ino()). This case is indicated by passing NULL as inop.
267  */
268 #define SHMEM_INO_BATCH 1024
269 static int shmem_reserve_inode(struct super_block *sb, ino_t *inop)
270 {
271         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
272         ino_t ino;
273
274         if (!(sb->s_flags & SB_KERNMOUNT)) {
275                 raw_spin_lock(&sbinfo->stat_lock);
276                 if (sbinfo->max_inodes) {
277                         if (!sbinfo->free_inodes) {
278                                 raw_spin_unlock(&sbinfo->stat_lock);
279                                 return -ENOSPC;
280                         }
281                         sbinfo->free_inodes--;
282                 }
283                 if (inop) {
284                         ino = sbinfo->next_ino++;
285                         if (unlikely(is_zero_ino(ino)))
286                                 ino = sbinfo->next_ino++;
287                         if (unlikely(!sbinfo->full_inums &&
288                                      ino > UINT_MAX)) {
289                                 /*
290                                  * Emulate get_next_ino uint wraparound for
291                                  * compatibility
292                                  */
293                                 if (IS_ENABLED(CONFIG_64BIT))
294                                         pr_warn("%s: inode number overflow on device %d, consider using inode64 mount option\n",
295                                                 __func__, MINOR(sb->s_dev));
296                                 sbinfo->next_ino = 1;
297                                 ino = sbinfo->next_ino++;
298                         }
299                         *inop = ino;
300                 }
301                 raw_spin_unlock(&sbinfo->stat_lock);
302         } else if (inop) {
303                 /*
304                  * __shmem_file_setup, one of our callers, is lock-free: it
305                  * doesn't hold stat_lock in shmem_reserve_inode since
306                  * max_inodes is always 0, and is called from potentially
307                  * unknown contexts. As such, use a per-cpu batched allocator
308                  * which doesn't require the per-sb stat_lock unless we are at
309                  * the batch boundary.
310                  *
311                  * We don't need to worry about inode{32,64} since SB_KERNMOUNT
312                  * shmem mounts are not exposed to userspace, so we don't need
313                  * to worry about things like glibc compatibility.
314                  */
315                 ino_t *next_ino;
316
317                 next_ino = per_cpu_ptr(sbinfo->ino_batch, get_cpu());
318                 ino = *next_ino;
319                 if (unlikely(ino % SHMEM_INO_BATCH == 0)) {
320                         raw_spin_lock(&sbinfo->stat_lock);
321                         ino = sbinfo->next_ino;
322                         sbinfo->next_ino += SHMEM_INO_BATCH;
323                         raw_spin_unlock(&sbinfo->stat_lock);
324                         if (unlikely(is_zero_ino(ino)))
325                                 ino++;
326                 }
327                 *inop = ino;
328                 *next_ino = ++ino;
329                 put_cpu();
330         }
331
332         return 0;
333 }
334
335 static void shmem_free_inode(struct super_block *sb)
336 {
337         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
338         if (sbinfo->max_inodes) {
339                 raw_spin_lock(&sbinfo->stat_lock);
340                 sbinfo->free_inodes++;
341                 raw_spin_unlock(&sbinfo->stat_lock);
342         }
343 }
344
345 /**
346  * shmem_recalc_inode - recalculate the block usage of an inode
347  * @inode: inode to recalc
348  *
349  * We have to calculate the free blocks since the mm can drop
350  * undirtied hole pages behind our back.
351  *
352  * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
353  * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
354  *
355  * It has to be called with the spinlock held.
356  */
357 static void shmem_recalc_inode(struct inode *inode)
358 {
359         struct shmem_inode_info *info = SHMEM_I(inode);
360         long freed;
361
362         freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
363         if (freed > 0) {
364                 info->alloced -= freed;
365                 inode->i_blocks -= freed * BLOCKS_PER_PAGE;
366                 shmem_inode_unacct_blocks(inode, freed);
367         }
368 }
369
370 bool shmem_charge(struct inode *inode, long pages)
371 {
372         struct shmem_inode_info *info = SHMEM_I(inode);
373         unsigned long flags;
374
375         if (!shmem_inode_acct_block(inode, pages))
376                 return false;
377
378         /* nrpages adjustment first, then shmem_recalc_inode() when balanced */
379         inode->i_mapping->nrpages += pages;
380
381         spin_lock_irqsave(&info->lock, flags);
382         info->alloced += pages;
383         inode->i_blocks += pages * BLOCKS_PER_PAGE;
384         shmem_recalc_inode(inode);
385         spin_unlock_irqrestore(&info->lock, flags);
386
387         return true;
388 }
389
390 void shmem_uncharge(struct inode *inode, long pages)
391 {
392         struct shmem_inode_info *info = SHMEM_I(inode);
393         unsigned long flags;
394
395         /* nrpages adjustment done by __delete_from_page_cache() or caller */
396
397         spin_lock_irqsave(&info->lock, flags);
398         info->alloced -= pages;
399         inode->i_blocks -= pages * BLOCKS_PER_PAGE;
400         shmem_recalc_inode(inode);
401         spin_unlock_irqrestore(&info->lock, flags);
402
403         shmem_inode_unacct_blocks(inode, pages);
404 }
405
406 /*
407  * Replace item expected in xarray by a new item, while holding xa_lock.
408  */
409 static int shmem_replace_entry(struct address_space *mapping,
410                         pgoff_t index, void *expected, void *replacement)
411 {
412         XA_STATE(xas, &mapping->i_pages, index);
413         void *item;
414
415         VM_BUG_ON(!expected);
416         VM_BUG_ON(!replacement);
417         item = xas_load(&xas);
418         if (item != expected)
419                 return -ENOENT;
420         xas_store(&xas, replacement);
421         return 0;
422 }
423
424 /*
425  * Sometimes, before we decide whether to proceed or to fail, we must check
426  * that an entry was not already brought back from swap by a racing thread.
427  *
428  * Checking page is not enough: by the time a SwapCache page is locked, it
429  * might be reused, and again be SwapCache, using the same swap as before.
430  */
431 static bool shmem_confirm_swap(struct address_space *mapping,
432                                pgoff_t index, swp_entry_t swap)
433 {
434         return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(swap);
435 }
436
437 /*
438  * Definitions for "huge tmpfs": tmpfs mounted with the huge= option
439  *
440  * SHMEM_HUGE_NEVER:
441  *      disables huge pages for the mount;
442  * SHMEM_HUGE_ALWAYS:
443  *      enables huge pages for the mount;
444  * SHMEM_HUGE_WITHIN_SIZE:
445  *      only allocate huge pages if the page will be fully within i_size,
446  *      also respect fadvise()/madvise() hints;
447  * SHMEM_HUGE_ADVISE:
448  *      only allocate huge pages if requested with fadvise()/madvise();
449  */
450
451 #define SHMEM_HUGE_NEVER        0
452 #define SHMEM_HUGE_ALWAYS       1
453 #define SHMEM_HUGE_WITHIN_SIZE  2
454 #define SHMEM_HUGE_ADVISE       3
455
456 /*
457  * Special values.
458  * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled:
459  *
460  * SHMEM_HUGE_DENY:
461  *      disables huge on shm_mnt and all mounts, for emergency use;
462  * SHMEM_HUGE_FORCE:
463  *      enables huge on shm_mnt and all mounts, w/o needing option, for testing;
464  *
465  */
466 #define SHMEM_HUGE_DENY         (-1)
467 #define SHMEM_HUGE_FORCE        (-2)
468
469 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
470 /* ifdef here to avoid bloating shmem.o when not necessary */
471
472 static int shmem_huge __read_mostly = SHMEM_HUGE_NEVER;
473
474 bool shmem_is_huge(struct vm_area_struct *vma,
475                    struct inode *inode, pgoff_t index)
476 {
477         loff_t i_size;
478
479         if (shmem_huge == SHMEM_HUGE_DENY)
480                 return false;
481         if (vma && ((vma->vm_flags & VM_NOHUGEPAGE) ||
482             test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags)))
483                 return false;
484         if (shmem_huge == SHMEM_HUGE_FORCE)
485                 return true;
486
487         switch (SHMEM_SB(inode->i_sb)->huge) {
488         case SHMEM_HUGE_ALWAYS:
489                 return true;
490         case SHMEM_HUGE_WITHIN_SIZE:
491                 index = round_up(index + 1, HPAGE_PMD_NR);
492                 i_size = round_up(i_size_read(inode), PAGE_SIZE);
493                 if (i_size >> PAGE_SHIFT >= index)
494                         return true;
495                 fallthrough;
496         case SHMEM_HUGE_ADVISE:
497                 if (vma && (vma->vm_flags & VM_HUGEPAGE))
498                         return true;
499                 fallthrough;
500         default:
501                 return false;
502         }
503 }
504
505 #if defined(CONFIG_SYSFS)
506 static int shmem_parse_huge(const char *str)
507 {
508         if (!strcmp(str, "never"))
509                 return SHMEM_HUGE_NEVER;
510         if (!strcmp(str, "always"))
511                 return SHMEM_HUGE_ALWAYS;
512         if (!strcmp(str, "within_size"))
513                 return SHMEM_HUGE_WITHIN_SIZE;
514         if (!strcmp(str, "advise"))
515                 return SHMEM_HUGE_ADVISE;
516         if (!strcmp(str, "deny"))
517                 return SHMEM_HUGE_DENY;
518         if (!strcmp(str, "force"))
519                 return SHMEM_HUGE_FORCE;
520         return -EINVAL;
521 }
522 #endif
523
524 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS)
525 static const char *shmem_format_huge(int huge)
526 {
527         switch (huge) {
528         case SHMEM_HUGE_NEVER:
529                 return "never";
530         case SHMEM_HUGE_ALWAYS:
531                 return "always";
532         case SHMEM_HUGE_WITHIN_SIZE:
533                 return "within_size";
534         case SHMEM_HUGE_ADVISE:
535                 return "advise";
536         case SHMEM_HUGE_DENY:
537                 return "deny";
538         case SHMEM_HUGE_FORCE:
539                 return "force";
540         default:
541                 VM_BUG_ON(1);
542                 return "bad_val";
543         }
544 }
545 #endif
546
547 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
548                 struct shrink_control *sc, unsigned long nr_to_split)
549 {
550         LIST_HEAD(list), *pos, *next;
551         LIST_HEAD(to_remove);
552         struct inode *inode;
553         struct shmem_inode_info *info;
554         struct page *page;
555         unsigned long batch = sc ? sc->nr_to_scan : 128;
556         int split = 0;
557
558         if (list_empty(&sbinfo->shrinklist))
559                 return SHRINK_STOP;
560
561         spin_lock(&sbinfo->shrinklist_lock);
562         list_for_each_safe(pos, next, &sbinfo->shrinklist) {
563                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
564
565                 /* pin the inode */
566                 inode = igrab(&info->vfs_inode);
567
568                 /* inode is about to be evicted */
569                 if (!inode) {
570                         list_del_init(&info->shrinklist);
571                         goto next;
572                 }
573
574                 /* Check if there's anything to gain */
575                 if (round_up(inode->i_size, PAGE_SIZE) ==
576                                 round_up(inode->i_size, HPAGE_PMD_SIZE)) {
577                         list_move(&info->shrinklist, &to_remove);
578                         goto next;
579                 }
580
581                 list_move(&info->shrinklist, &list);
582 next:
583                 sbinfo->shrinklist_len--;
584                 if (!--batch)
585                         break;
586         }
587         spin_unlock(&sbinfo->shrinklist_lock);
588
589         list_for_each_safe(pos, next, &to_remove) {
590                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
591                 inode = &info->vfs_inode;
592                 list_del_init(&info->shrinklist);
593                 iput(inode);
594         }
595
596         list_for_each_safe(pos, next, &list) {
597                 int ret;
598
599                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
600                 inode = &info->vfs_inode;
601
602                 if (nr_to_split && split >= nr_to_split)
603                         goto move_back;
604
605                 page = find_get_page(inode->i_mapping,
606                                 (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT);
607                 if (!page)
608                         goto drop;
609
610                 /* No huge page at the end of the file: nothing to split */
611                 if (!PageTransHuge(page)) {
612                         put_page(page);
613                         goto drop;
614                 }
615
616                 /*
617                  * Move the inode on the list back to shrinklist if we failed
618                  * to lock the page at this time.
619                  *
620                  * Waiting for the lock may lead to deadlock in the
621                  * reclaim path.
622                  */
623                 if (!trylock_page(page)) {
624                         put_page(page);
625                         goto move_back;
626                 }
627
628                 ret = split_huge_page(page);
629                 unlock_page(page);
630                 put_page(page);
631
632                 /* If split failed move the inode on the list back to shrinklist */
633                 if (ret)
634                         goto move_back;
635
636                 split++;
637 drop:
638                 list_del_init(&info->shrinklist);
639                 goto put;
640 move_back:
641                 /*
642                  * Make sure the inode is either on the global list or deleted
643                  * from any local list before iput() since it could be deleted
644                  * in another thread once we put the inode (then the local list
645                  * is corrupted).
646                  */
647                 spin_lock(&sbinfo->shrinklist_lock);
648                 list_move(&info->shrinklist, &sbinfo->shrinklist);
649                 sbinfo->shrinklist_len++;
650                 spin_unlock(&sbinfo->shrinklist_lock);
651 put:
652                 iput(inode);
653         }
654
655         return split;
656 }
657
658 static long shmem_unused_huge_scan(struct super_block *sb,
659                 struct shrink_control *sc)
660 {
661         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
662
663         if (!READ_ONCE(sbinfo->shrinklist_len))
664                 return SHRINK_STOP;
665
666         return shmem_unused_huge_shrink(sbinfo, sc, 0);
667 }
668
669 static long shmem_unused_huge_count(struct super_block *sb,
670                 struct shrink_control *sc)
671 {
672         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
673         return READ_ONCE(sbinfo->shrinklist_len);
674 }
675 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */
676
677 #define shmem_huge SHMEM_HUGE_DENY
678
679 bool shmem_is_huge(struct vm_area_struct *vma,
680                    struct inode *inode, pgoff_t index)
681 {
682         return false;
683 }
684
685 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
686                 struct shrink_control *sc, unsigned long nr_to_split)
687 {
688         return 0;
689 }
690 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
691
692 /*
693  * Like add_to_page_cache_locked, but error if expected item has gone.
694  */
695 static int shmem_add_to_page_cache(struct page *page,
696                                    struct address_space *mapping,
697                                    pgoff_t index, void *expected, gfp_t gfp,
698                                    struct mm_struct *charge_mm)
699 {
700         XA_STATE_ORDER(xas, &mapping->i_pages, index, compound_order(page));
701         unsigned long nr = compound_nr(page);
702         int error;
703
704         VM_BUG_ON_PAGE(PageTail(page), page);
705         VM_BUG_ON_PAGE(index != round_down(index, nr), page);
706         VM_BUG_ON_PAGE(!PageLocked(page), page);
707         VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
708         VM_BUG_ON(expected && PageTransHuge(page));
709
710         page_ref_add(page, nr);
711         page->mapping = mapping;
712         page->index = index;
713
714         if (!PageSwapCache(page)) {
715                 error = mem_cgroup_charge(page_folio(page), charge_mm, gfp);
716                 if (error) {
717                         if (PageTransHuge(page)) {
718                                 count_vm_event(THP_FILE_FALLBACK);
719                                 count_vm_event(THP_FILE_FALLBACK_CHARGE);
720                         }
721                         goto error;
722                 }
723         }
724         cgroup_throttle_swaprate(page, gfp);
725
726         do {
727                 xas_lock_irq(&xas);
728                 if (expected != xas_find_conflict(&xas)) {
729                         xas_set_err(&xas, -EEXIST);
730                         goto unlock;
731                 }
732                 if (expected && xas_find_conflict(&xas)) {
733                         xas_set_err(&xas, -EEXIST);
734                         goto unlock;
735                 }
736                 xas_store(&xas, page);
737                 if (xas_error(&xas))
738                         goto unlock;
739                 if (PageTransHuge(page)) {
740                         count_vm_event(THP_FILE_ALLOC);
741                         __mod_lruvec_page_state(page, NR_SHMEM_THPS, nr);
742                 }
743                 mapping->nrpages += nr;
744                 __mod_lruvec_page_state(page, NR_FILE_PAGES, nr);
745                 __mod_lruvec_page_state(page, NR_SHMEM, nr);
746 unlock:
747                 xas_unlock_irq(&xas);
748         } while (xas_nomem(&xas, gfp));
749
750         if (xas_error(&xas)) {
751                 error = xas_error(&xas);
752                 goto error;
753         }
754
755         return 0;
756 error:
757         page->mapping = NULL;
758         page_ref_sub(page, nr);
759         return error;
760 }
761
762 /*
763  * Like delete_from_page_cache, but substitutes swap for page.
764  */
765 static void shmem_delete_from_page_cache(struct page *page, void *radswap)
766 {
767         struct address_space *mapping = page->mapping;
768         int error;
769
770         VM_BUG_ON_PAGE(PageCompound(page), page);
771
772         xa_lock_irq(&mapping->i_pages);
773         error = shmem_replace_entry(mapping, page->index, page, radswap);
774         page->mapping = NULL;
775         mapping->nrpages--;
776         __dec_lruvec_page_state(page, NR_FILE_PAGES);
777         __dec_lruvec_page_state(page, NR_SHMEM);
778         xa_unlock_irq(&mapping->i_pages);
779         put_page(page);
780         BUG_ON(error);
781 }
782
783 /*
784  * Remove swap entry from page cache, free the swap and its page cache.
785  */
786 static int shmem_free_swap(struct address_space *mapping,
787                            pgoff_t index, void *radswap)
788 {
789         void *old;
790
791         old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0);
792         if (old != radswap)
793                 return -ENOENT;
794         free_swap_and_cache(radix_to_swp_entry(radswap));
795         return 0;
796 }
797
798 /*
799  * Determine (in bytes) how many of the shmem object's pages mapped by the
800  * given offsets are swapped out.
801  *
802  * This is safe to call without i_rwsem or the i_pages lock thanks to RCU,
803  * as long as the inode doesn't go away and racy results are not a problem.
804  */
805 unsigned long shmem_partial_swap_usage(struct address_space *mapping,
806                                                 pgoff_t start, pgoff_t end)
807 {
808         XA_STATE(xas, &mapping->i_pages, start);
809         struct page *page;
810         unsigned long swapped = 0;
811
812         rcu_read_lock();
813         xas_for_each(&xas, page, end - 1) {
814                 if (xas_retry(&xas, page))
815                         continue;
816                 if (xa_is_value(page))
817                         swapped++;
818
819                 if (need_resched()) {
820                         xas_pause(&xas);
821                         cond_resched_rcu();
822                 }
823         }
824
825         rcu_read_unlock();
826
827         return swapped << PAGE_SHIFT;
828 }
829
830 /*
831  * Determine (in bytes) how many of the shmem object's pages mapped by the
832  * given vma is swapped out.
833  *
834  * This is safe to call without i_rwsem or the i_pages lock thanks to RCU,
835  * as long as the inode doesn't go away and racy results are not a problem.
836  */
837 unsigned long shmem_swap_usage(struct vm_area_struct *vma)
838 {
839         struct inode *inode = file_inode(vma->vm_file);
840         struct shmem_inode_info *info = SHMEM_I(inode);
841         struct address_space *mapping = inode->i_mapping;
842         unsigned long swapped;
843
844         /* Be careful as we don't hold info->lock */
845         swapped = READ_ONCE(info->swapped);
846
847         /*
848          * The easier cases are when the shmem object has nothing in swap, or
849          * the vma maps it whole. Then we can simply use the stats that we
850          * already track.
851          */
852         if (!swapped)
853                 return 0;
854
855         if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
856                 return swapped << PAGE_SHIFT;
857
858         /* Here comes the more involved part */
859         return shmem_partial_swap_usage(mapping, vma->vm_pgoff,
860                                         vma->vm_pgoff + vma_pages(vma));
861 }
862
863 /*
864  * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
865  */
866 void shmem_unlock_mapping(struct address_space *mapping)
867 {
868         struct pagevec pvec;
869         pgoff_t index = 0;
870
871         pagevec_init(&pvec);
872         /*
873          * Minor point, but we might as well stop if someone else SHM_LOCKs it.
874          */
875         while (!mapping_unevictable(mapping)) {
876                 if (!pagevec_lookup(&pvec, mapping, &index))
877                         break;
878                 check_move_unevictable_pages(&pvec);
879                 pagevec_release(&pvec);
880                 cond_resched();
881         }
882 }
883
884 static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index)
885 {
886         struct folio *folio;
887         struct page *page;
888
889         /*
890          * At first avoid shmem_getpage(,,,SGP_READ): that fails
891          * beyond i_size, and reports fallocated pages as holes.
892          */
893         folio = __filemap_get_folio(inode->i_mapping, index,
894                                         FGP_ENTRY | FGP_LOCK, 0);
895         if (!xa_is_value(folio))
896                 return folio;
897         /*
898          * But read a page back from swap if any of it is within i_size
899          * (although in some cases this is just a waste of time).
900          */
901         page = NULL;
902         shmem_getpage(inode, index, &page, SGP_READ);
903         return page ? page_folio(page) : NULL;
904 }
905
906 /*
907  * Remove range of pages and swap entries from page cache, and free them.
908  * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
909  */
910 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
911                                                                  bool unfalloc)
912 {
913         struct address_space *mapping = inode->i_mapping;
914         struct shmem_inode_info *info = SHMEM_I(inode);
915         pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
916         pgoff_t end = (lend + 1) >> PAGE_SHIFT;
917         struct folio_batch fbatch;
918         pgoff_t indices[PAGEVEC_SIZE];
919         struct folio *folio;
920         bool same_folio;
921         long nr_swaps_freed = 0;
922         pgoff_t index;
923         int i;
924
925         if (lend == -1)
926                 end = -1;       /* unsigned, so actually very big */
927
928         if (info->fallocend > start && info->fallocend <= end && !unfalloc)
929                 info->fallocend = start;
930
931         folio_batch_init(&fbatch);
932         index = start;
933         while (index < end && find_lock_entries(mapping, index, end - 1,
934                         &fbatch, indices)) {
935                 for (i = 0; i < folio_batch_count(&fbatch); i++) {
936                         folio = fbatch.folios[i];
937
938                         index = indices[i];
939
940                         if (xa_is_value(folio)) {
941                                 if (unfalloc)
942                                         continue;
943                                 nr_swaps_freed += !shmem_free_swap(mapping,
944                                                                 index, folio);
945                                 continue;
946                         }
947                         index += folio_nr_pages(folio) - 1;
948
949                         if (!unfalloc || !folio_test_uptodate(folio))
950                                 truncate_inode_folio(mapping, folio);
951                         folio_unlock(folio);
952                 }
953                 folio_batch_remove_exceptionals(&fbatch);
954                 folio_batch_release(&fbatch);
955                 cond_resched();
956                 index++;
957         }
958
959         same_folio = (lstart >> PAGE_SHIFT) == (lend >> PAGE_SHIFT);
960         folio = shmem_get_partial_folio(inode, lstart >> PAGE_SHIFT);
961         if (folio) {
962                 same_folio = lend < folio_pos(folio) + folio_size(folio);
963                 folio_mark_dirty(folio);
964                 if (!truncate_inode_partial_folio(folio, lstart, lend)) {
965                         start = folio->index + folio_nr_pages(folio);
966                         if (same_folio)
967                                 end = folio->index;
968                 }
969                 folio_unlock(folio);
970                 folio_put(folio);
971                 folio = NULL;
972         }
973
974         if (!same_folio)
975                 folio = shmem_get_partial_folio(inode, lend >> PAGE_SHIFT);
976         if (folio) {
977                 folio_mark_dirty(folio);
978                 if (!truncate_inode_partial_folio(folio, lstart, lend))
979                         end = folio->index;
980                 folio_unlock(folio);
981                 folio_put(folio);
982         }
983
984         index = start;
985         while (index < end) {
986                 cond_resched();
987
988                 if (!find_get_entries(mapping, index, end - 1, &fbatch,
989                                 indices)) {
990                         /* If all gone or hole-punch or unfalloc, we're done */
991                         if (index == start || end != -1)
992                                 break;
993                         /* But if truncating, restart to make sure all gone */
994                         index = start;
995                         continue;
996                 }
997                 for (i = 0; i < folio_batch_count(&fbatch); i++) {
998                         folio = fbatch.folios[i];
999
1000                         index = indices[i];
1001                         if (xa_is_value(folio)) {
1002                                 if (unfalloc)
1003                                         continue;
1004                                 if (shmem_free_swap(mapping, index, folio)) {
1005                                         /* Swap was replaced by page: retry */
1006                                         index--;
1007                                         break;
1008                                 }
1009                                 nr_swaps_freed++;
1010                                 continue;
1011                         }
1012
1013                         folio_lock(folio);
1014
1015                         if (!unfalloc || !folio_test_uptodate(folio)) {
1016                                 if (folio_mapping(folio) != mapping) {
1017                                         /* Page was replaced by swap: retry */
1018                                         folio_unlock(folio);
1019                                         index--;
1020                                         break;
1021                                 }
1022                                 VM_BUG_ON_FOLIO(folio_test_writeback(folio),
1023                                                 folio);
1024                                 truncate_inode_folio(mapping, folio);
1025                         }
1026                         index = folio->index + folio_nr_pages(folio) - 1;
1027                         folio_unlock(folio);
1028                 }
1029                 folio_batch_remove_exceptionals(&fbatch);
1030                 folio_batch_release(&fbatch);
1031                 index++;
1032         }
1033
1034         spin_lock_irq(&info->lock);
1035         info->swapped -= nr_swaps_freed;
1036         shmem_recalc_inode(inode);
1037         spin_unlock_irq(&info->lock);
1038 }
1039
1040 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
1041 {
1042         shmem_undo_range(inode, lstart, lend, false);
1043         inode->i_ctime = inode->i_mtime = current_time(inode);
1044 }
1045 EXPORT_SYMBOL_GPL(shmem_truncate_range);
1046
1047 static int shmem_getattr(struct user_namespace *mnt_userns,
1048                          const struct path *path, struct kstat *stat,
1049                          u32 request_mask, unsigned int query_flags)
1050 {
1051         struct inode *inode = path->dentry->d_inode;
1052         struct shmem_inode_info *info = SHMEM_I(inode);
1053
1054         if (info->alloced - info->swapped != inode->i_mapping->nrpages) {
1055                 spin_lock_irq(&info->lock);
1056                 shmem_recalc_inode(inode);
1057                 spin_unlock_irq(&info->lock);
1058         }
1059         generic_fillattr(&init_user_ns, inode, stat);
1060
1061         if (shmem_is_huge(NULL, inode, 0))
1062                 stat->blksize = HPAGE_PMD_SIZE;
1063
1064         return 0;
1065 }
1066
1067 static int shmem_setattr(struct user_namespace *mnt_userns,
1068                          struct dentry *dentry, struct iattr *attr)
1069 {
1070         struct inode *inode = d_inode(dentry);
1071         struct shmem_inode_info *info = SHMEM_I(inode);
1072         int error;
1073
1074         error = setattr_prepare(&init_user_ns, dentry, attr);
1075         if (error)
1076                 return error;
1077
1078         if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
1079                 loff_t oldsize = inode->i_size;
1080                 loff_t newsize = attr->ia_size;
1081
1082                 /* protected by i_rwsem */
1083                 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
1084                     (newsize > oldsize && (info->seals & F_SEAL_GROW)))
1085                         return -EPERM;
1086
1087                 if (newsize != oldsize) {
1088                         error = shmem_reacct_size(SHMEM_I(inode)->flags,
1089                                         oldsize, newsize);
1090                         if (error)
1091                                 return error;
1092                         i_size_write(inode, newsize);
1093                         inode->i_ctime = inode->i_mtime = current_time(inode);
1094                 }
1095                 if (newsize <= oldsize) {
1096                         loff_t holebegin = round_up(newsize, PAGE_SIZE);
1097                         if (oldsize > holebegin)
1098                                 unmap_mapping_range(inode->i_mapping,
1099                                                         holebegin, 0, 1);
1100                         if (info->alloced)
1101                                 shmem_truncate_range(inode,
1102                                                         newsize, (loff_t)-1);
1103                         /* unmap again to remove racily COWed private pages */
1104                         if (oldsize > holebegin)
1105                                 unmap_mapping_range(inode->i_mapping,
1106                                                         holebegin, 0, 1);
1107                 }
1108         }
1109
1110         setattr_copy(&init_user_ns, inode, attr);
1111         if (attr->ia_valid & ATTR_MODE)
1112                 error = posix_acl_chmod(&init_user_ns, inode, inode->i_mode);
1113         return error;
1114 }
1115
1116 static void shmem_evict_inode(struct inode *inode)
1117 {
1118         struct shmem_inode_info *info = SHMEM_I(inode);
1119         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1120
1121         if (shmem_mapping(inode->i_mapping)) {
1122                 shmem_unacct_size(info->flags, inode->i_size);
1123                 inode->i_size = 0;
1124                 shmem_truncate_range(inode, 0, (loff_t)-1);
1125                 if (!list_empty(&info->shrinklist)) {
1126                         spin_lock(&sbinfo->shrinklist_lock);
1127                         if (!list_empty(&info->shrinklist)) {
1128                                 list_del_init(&info->shrinklist);
1129                                 sbinfo->shrinklist_len--;
1130                         }
1131                         spin_unlock(&sbinfo->shrinklist_lock);
1132                 }
1133                 while (!list_empty(&info->swaplist)) {
1134                         /* Wait while shmem_unuse() is scanning this inode... */
1135                         wait_var_event(&info->stop_eviction,
1136                                        !atomic_read(&info->stop_eviction));
1137                         mutex_lock(&shmem_swaplist_mutex);
1138                         /* ...but beware of the race if we peeked too early */
1139                         if (!atomic_read(&info->stop_eviction))
1140                                 list_del_init(&info->swaplist);
1141                         mutex_unlock(&shmem_swaplist_mutex);
1142                 }
1143         }
1144
1145         simple_xattrs_free(&info->xattrs);
1146         WARN_ON(inode->i_blocks);
1147         shmem_free_inode(inode->i_sb);
1148         clear_inode(inode);
1149 }
1150
1151 static int shmem_find_swap_entries(struct address_space *mapping,
1152                                    pgoff_t start, unsigned int nr_entries,
1153                                    struct page **entries, pgoff_t *indices,
1154                                    unsigned int type)
1155 {
1156         XA_STATE(xas, &mapping->i_pages, start);
1157         struct page *page;
1158         swp_entry_t entry;
1159         unsigned int ret = 0;
1160
1161         if (!nr_entries)
1162                 return 0;
1163
1164         rcu_read_lock();
1165         xas_for_each(&xas, page, ULONG_MAX) {
1166                 if (xas_retry(&xas, page))
1167                         continue;
1168
1169                 if (!xa_is_value(page))
1170                         continue;
1171
1172                 entry = radix_to_swp_entry(page);
1173                 if (swp_type(entry) != type)
1174                         continue;
1175
1176                 indices[ret] = xas.xa_index;
1177                 entries[ret] = page;
1178
1179                 if (need_resched()) {
1180                         xas_pause(&xas);
1181                         cond_resched_rcu();
1182                 }
1183                 if (++ret == nr_entries)
1184                         break;
1185         }
1186         rcu_read_unlock();
1187
1188         return ret;
1189 }
1190
1191 /*
1192  * Move the swapped pages for an inode to page cache. Returns the count
1193  * of pages swapped in, or the error in case of failure.
1194  */
1195 static int shmem_unuse_swap_entries(struct inode *inode, struct pagevec pvec,
1196                                     pgoff_t *indices)
1197 {
1198         int i = 0;
1199         int ret = 0;
1200         int error = 0;
1201         struct address_space *mapping = inode->i_mapping;
1202
1203         for (i = 0; i < pvec.nr; i++) {
1204                 struct page *page = pvec.pages[i];
1205
1206                 if (!xa_is_value(page))
1207                         continue;
1208                 error = shmem_swapin_page(inode, indices[i],
1209                                           &page, SGP_CACHE,
1210                                           mapping_gfp_mask(mapping),
1211                                           NULL, NULL);
1212                 if (error == 0) {
1213                         unlock_page(page);
1214                         put_page(page);
1215                         ret++;
1216                 }
1217                 if (error == -ENOMEM)
1218                         break;
1219                 error = 0;
1220         }
1221         return error ? error : ret;
1222 }
1223
1224 /*
1225  * If swap found in inode, free it and move page from swapcache to filecache.
1226  */
1227 static int shmem_unuse_inode(struct inode *inode, unsigned int type)
1228 {
1229         struct address_space *mapping = inode->i_mapping;
1230         pgoff_t start = 0;
1231         struct pagevec pvec;
1232         pgoff_t indices[PAGEVEC_SIZE];
1233         int ret = 0;
1234
1235         pagevec_init(&pvec);
1236         do {
1237                 unsigned int nr_entries = PAGEVEC_SIZE;
1238
1239                 pvec.nr = shmem_find_swap_entries(mapping, start, nr_entries,
1240                                                   pvec.pages, indices, type);
1241                 if (pvec.nr == 0) {
1242                         ret = 0;
1243                         break;
1244                 }
1245
1246                 ret = shmem_unuse_swap_entries(inode, pvec, indices);
1247                 if (ret < 0)
1248                         break;
1249
1250                 start = indices[pvec.nr - 1];
1251         } while (true);
1252
1253         return ret;
1254 }
1255
1256 /*
1257  * Read all the shared memory data that resides in the swap
1258  * device 'type' back into memory, so the swap device can be
1259  * unused.
1260  */
1261 int shmem_unuse(unsigned int type)
1262 {
1263         struct shmem_inode_info *info, *next;
1264         int error = 0;
1265
1266         if (list_empty(&shmem_swaplist))
1267                 return 0;
1268
1269         mutex_lock(&shmem_swaplist_mutex);
1270         list_for_each_entry_safe(info, next, &shmem_swaplist, swaplist) {
1271                 if (!info->swapped) {
1272                         list_del_init(&info->swaplist);
1273                         continue;
1274                 }
1275                 /*
1276                  * Drop the swaplist mutex while searching the inode for swap;
1277                  * but before doing so, make sure shmem_evict_inode() will not
1278                  * remove placeholder inode from swaplist, nor let it be freed
1279                  * (igrab() would protect from unlink, but not from unmount).
1280                  */
1281                 atomic_inc(&info->stop_eviction);
1282                 mutex_unlock(&shmem_swaplist_mutex);
1283
1284                 error = shmem_unuse_inode(&info->vfs_inode, type);
1285                 cond_resched();
1286
1287                 mutex_lock(&shmem_swaplist_mutex);
1288                 next = list_next_entry(info, swaplist);
1289                 if (!info->swapped)
1290                         list_del_init(&info->swaplist);
1291                 if (atomic_dec_and_test(&info->stop_eviction))
1292                         wake_up_var(&info->stop_eviction);
1293                 if (error)
1294                         break;
1295         }
1296         mutex_unlock(&shmem_swaplist_mutex);
1297
1298         return error;
1299 }
1300
1301 /*
1302  * Move the page from the page cache to the swap cache.
1303  */
1304 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1305 {
1306         struct shmem_inode_info *info;
1307         struct address_space *mapping;
1308         struct inode *inode;
1309         swp_entry_t swap;
1310         pgoff_t index;
1311
1312         /*
1313          * If /sys/kernel/mm/transparent_hugepage/shmem_enabled is "always" or
1314          * "force", drivers/gpu/drm/i915/gem/i915_gem_shmem.c gets huge pages,
1315          * and its shmem_writeback() needs them to be split when swapping.
1316          */
1317         if (PageTransCompound(page)) {
1318                 /* Ensure the subpages are still dirty */
1319                 SetPageDirty(page);
1320                 if (split_huge_page(page) < 0)
1321                         goto redirty;
1322                 ClearPageDirty(page);
1323         }
1324
1325         BUG_ON(!PageLocked(page));
1326         mapping = page->mapping;
1327         index = page->index;
1328         inode = mapping->host;
1329         info = SHMEM_I(inode);
1330         if (info->flags & VM_LOCKED)
1331                 goto redirty;
1332         if (!total_swap_pages)
1333                 goto redirty;
1334
1335         /*
1336          * Our capabilities prevent regular writeback or sync from ever calling
1337          * shmem_writepage; but a stacking filesystem might use ->writepage of
1338          * its underlying filesystem, in which case tmpfs should write out to
1339          * swap only in response to memory pressure, and not for the writeback
1340          * threads or sync.
1341          */
1342         if (!wbc->for_reclaim) {
1343                 WARN_ON_ONCE(1);        /* Still happens? Tell us about it! */
1344                 goto redirty;
1345         }
1346
1347         /*
1348          * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
1349          * value into swapfile.c, the only way we can correctly account for a
1350          * fallocated page arriving here is now to initialize it and write it.
1351          *
1352          * That's okay for a page already fallocated earlier, but if we have
1353          * not yet completed the fallocation, then (a) we want to keep track
1354          * of this page in case we have to undo it, and (b) it may not be a
1355          * good idea to continue anyway, once we're pushing into swap.  So
1356          * reactivate the page, and let shmem_fallocate() quit when too many.
1357          */
1358         if (!PageUptodate(page)) {
1359                 if (inode->i_private) {
1360                         struct shmem_falloc *shmem_falloc;
1361                         spin_lock(&inode->i_lock);
1362                         shmem_falloc = inode->i_private;
1363                         if (shmem_falloc &&
1364                             !shmem_falloc->waitq &&
1365                             index >= shmem_falloc->start &&
1366                             index < shmem_falloc->next)
1367                                 shmem_falloc->nr_unswapped++;
1368                         else
1369                                 shmem_falloc = NULL;
1370                         spin_unlock(&inode->i_lock);
1371                         if (shmem_falloc)
1372                                 goto redirty;
1373                 }
1374                 clear_highpage(page);
1375                 flush_dcache_page(page);
1376                 SetPageUptodate(page);
1377         }
1378
1379         swap = get_swap_page(page);
1380         if (!swap.val)
1381                 goto redirty;
1382
1383         /*
1384          * Add inode to shmem_unuse()'s list of swapped-out inodes,
1385          * if it's not already there.  Do it now before the page is
1386          * moved to swap cache, when its pagelock no longer protects
1387          * the inode from eviction.  But don't unlock the mutex until
1388          * we've incremented swapped, because shmem_unuse_inode() will
1389          * prune a !swapped inode from the swaplist under this mutex.
1390          */
1391         mutex_lock(&shmem_swaplist_mutex);
1392         if (list_empty(&info->swaplist))
1393                 list_add(&info->swaplist, &shmem_swaplist);
1394
1395         if (add_to_swap_cache(page, swap,
1396                         __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN,
1397                         NULL) == 0) {
1398                 spin_lock_irq(&info->lock);
1399                 shmem_recalc_inode(inode);
1400                 info->swapped++;
1401                 spin_unlock_irq(&info->lock);
1402
1403                 swap_shmem_alloc(swap);
1404                 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));
1405
1406                 mutex_unlock(&shmem_swaplist_mutex);
1407                 BUG_ON(page_mapped(page));
1408                 swap_writepage(page, wbc);
1409                 return 0;
1410         }
1411
1412         mutex_unlock(&shmem_swaplist_mutex);
1413         put_swap_page(page, swap);
1414 redirty:
1415         set_page_dirty(page);
1416         if (wbc->for_reclaim)
1417                 return AOP_WRITEPAGE_ACTIVATE;  /* Return with page locked */
1418         unlock_page(page);
1419         return 0;
1420 }
1421
1422 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
1423 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1424 {
1425         char buffer[64];
1426
1427         if (!mpol || mpol->mode == MPOL_DEFAULT)
1428                 return;         /* show nothing */
1429
1430         mpol_to_str(buffer, sizeof(buffer), mpol);
1431
1432         seq_printf(seq, ",mpol=%s", buffer);
1433 }
1434
1435 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1436 {
1437         struct mempolicy *mpol = NULL;
1438         if (sbinfo->mpol) {
1439                 raw_spin_lock(&sbinfo->stat_lock);      /* prevent replace/use races */
1440                 mpol = sbinfo->mpol;
1441                 mpol_get(mpol);
1442                 raw_spin_unlock(&sbinfo->stat_lock);
1443         }
1444         return mpol;
1445 }
1446 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */
1447 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1448 {
1449 }
1450 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1451 {
1452         return NULL;
1453 }
1454 #endif /* CONFIG_NUMA && CONFIG_TMPFS */
1455 #ifndef CONFIG_NUMA
1456 #define vm_policy vm_private_data
1457 #endif
1458
1459 static void shmem_pseudo_vma_init(struct vm_area_struct *vma,
1460                 struct shmem_inode_info *info, pgoff_t index)
1461 {
1462         /* Create a pseudo vma that just contains the policy */
1463         vma_init(vma, NULL);
1464         /* Bias interleave by inode number to distribute better across nodes */
1465         vma->vm_pgoff = index + info->vfs_inode.i_ino;
1466         vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index);
1467 }
1468
1469 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma)
1470 {
1471         /* Drop reference taken by mpol_shared_policy_lookup() */
1472         mpol_cond_put(vma->vm_policy);
1473 }
1474
1475 static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
1476                         struct shmem_inode_info *info, pgoff_t index)
1477 {
1478         struct vm_area_struct pvma;
1479         struct page *page;
1480         struct vm_fault vmf = {
1481                 .vma = &pvma,
1482         };
1483
1484         shmem_pseudo_vma_init(&pvma, info, index);
1485         page = swap_cluster_readahead(swap, gfp, &vmf);
1486         shmem_pseudo_vma_destroy(&pvma);
1487
1488         return page;
1489 }
1490
1491 /*
1492  * Make sure huge_gfp is always more limited than limit_gfp.
1493  * Some of the flags set permissions, while others set limitations.
1494  */
1495 static gfp_t limit_gfp_mask(gfp_t huge_gfp, gfp_t limit_gfp)
1496 {
1497         gfp_t allowflags = __GFP_IO | __GFP_FS | __GFP_RECLAIM;
1498         gfp_t denyflags = __GFP_NOWARN | __GFP_NORETRY;
1499         gfp_t zoneflags = limit_gfp & GFP_ZONEMASK;
1500         gfp_t result = huge_gfp & ~(allowflags | GFP_ZONEMASK);
1501
1502         /* Allow allocations only from the originally specified zones. */
1503         result |= zoneflags;
1504
1505         /*
1506          * Minimize the result gfp by taking the union with the deny flags,
1507          * and the intersection of the allow flags.
1508          */
1509         result |= (limit_gfp & denyflags);
1510         result |= (huge_gfp & limit_gfp) & allowflags;
1511
1512         return result;
1513 }
1514
1515 static struct page *shmem_alloc_hugepage(gfp_t gfp,
1516                 struct shmem_inode_info *info, pgoff_t index)
1517 {
1518         struct vm_area_struct pvma;
1519         struct address_space *mapping = info->vfs_inode.i_mapping;
1520         pgoff_t hindex;
1521         struct page *page;
1522
1523         hindex = round_down(index, HPAGE_PMD_NR);
1524         if (xa_find(&mapping->i_pages, &hindex, hindex + HPAGE_PMD_NR - 1,
1525                                                                 XA_PRESENT))
1526                 return NULL;
1527
1528         shmem_pseudo_vma_init(&pvma, info, hindex);
1529         page = alloc_pages_vma(gfp, HPAGE_PMD_ORDER, &pvma, 0, true);
1530         shmem_pseudo_vma_destroy(&pvma);
1531         if (page)
1532                 prep_transhuge_page(page);
1533         else
1534                 count_vm_event(THP_FILE_FALLBACK);
1535         return page;
1536 }
1537
1538 static struct page *shmem_alloc_page(gfp_t gfp,
1539                         struct shmem_inode_info *info, pgoff_t index)
1540 {
1541         struct vm_area_struct pvma;
1542         struct page *page;
1543
1544         shmem_pseudo_vma_init(&pvma, info, index);
1545         page = alloc_page_vma(gfp, &pvma, 0);
1546         shmem_pseudo_vma_destroy(&pvma);
1547
1548         return page;
1549 }
1550
1551 static struct page *shmem_alloc_and_acct_page(gfp_t gfp,
1552                 struct inode *inode,
1553                 pgoff_t index, bool huge)
1554 {
1555         struct shmem_inode_info *info = SHMEM_I(inode);
1556         struct page *page;
1557         int nr;
1558         int err = -ENOSPC;
1559
1560         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1561                 huge = false;
1562         nr = huge ? HPAGE_PMD_NR : 1;
1563
1564         if (!shmem_inode_acct_block(inode, nr))
1565                 goto failed;
1566
1567         if (huge)
1568                 page = shmem_alloc_hugepage(gfp, info, index);
1569         else
1570                 page = shmem_alloc_page(gfp, info, index);
1571         if (page) {
1572                 __SetPageLocked(page);
1573                 __SetPageSwapBacked(page);
1574                 return page;
1575         }
1576
1577         err = -ENOMEM;
1578         shmem_inode_unacct_blocks(inode, nr);
1579 failed:
1580         return ERR_PTR(err);
1581 }
1582
1583 /*
1584  * When a page is moved from swapcache to shmem filecache (either by the
1585  * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of
1586  * shmem_unuse_inode()), it may have been read in earlier from swap, in
1587  * ignorance of the mapping it belongs to.  If that mapping has special
1588  * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
1589  * we may need to copy to a suitable page before moving to filecache.
1590  *
1591  * In a future release, this may well be extended to respect cpuset and
1592  * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
1593  * but for now it is a simple matter of zone.
1594  */
1595 static bool shmem_should_replace_page(struct page *page, gfp_t gfp)
1596 {
1597         return page_zonenum(page) > gfp_zone(gfp);
1598 }
1599
1600 static int shmem_replace_page(struct page **pagep, gfp_t gfp,
1601                                 struct shmem_inode_info *info, pgoff_t index)
1602 {
1603         struct page *oldpage, *newpage;
1604         struct folio *old, *new;
1605         struct address_space *swap_mapping;
1606         swp_entry_t entry;
1607         pgoff_t swap_index;
1608         int error;
1609
1610         oldpage = *pagep;
1611         entry.val = page_private(oldpage);
1612         swap_index = swp_offset(entry);
1613         swap_mapping = page_mapping(oldpage);
1614
1615         /*
1616          * We have arrived here because our zones are constrained, so don't
1617          * limit chance of success by further cpuset and node constraints.
1618          */
1619         gfp &= ~GFP_CONSTRAINT_MASK;
1620         newpage = shmem_alloc_page(gfp, info, index);
1621         if (!newpage)
1622                 return -ENOMEM;
1623
1624         get_page(newpage);
1625         copy_highpage(newpage, oldpage);
1626         flush_dcache_page(newpage);
1627
1628         __SetPageLocked(newpage);
1629         __SetPageSwapBacked(newpage);
1630         SetPageUptodate(newpage);
1631         set_page_private(newpage, entry.val);
1632         SetPageSwapCache(newpage);
1633
1634         /*
1635          * Our caller will very soon move newpage out of swapcache, but it's
1636          * a nice clean interface for us to replace oldpage by newpage there.
1637          */
1638         xa_lock_irq(&swap_mapping->i_pages);
1639         error = shmem_replace_entry(swap_mapping, swap_index, oldpage, newpage);
1640         if (!error) {
1641                 old = page_folio(oldpage);
1642                 new = page_folio(newpage);
1643                 mem_cgroup_migrate(old, new);
1644                 __inc_lruvec_page_state(newpage, NR_FILE_PAGES);
1645                 __dec_lruvec_page_state(oldpage, NR_FILE_PAGES);
1646         }
1647         xa_unlock_irq(&swap_mapping->i_pages);
1648
1649         if (unlikely(error)) {
1650                 /*
1651                  * Is this possible?  I think not, now that our callers check
1652                  * both PageSwapCache and page_private after getting page lock;
1653                  * but be defensive.  Reverse old to newpage for clear and free.
1654                  */
1655                 oldpage = newpage;
1656         } else {
1657                 lru_cache_add(newpage);
1658                 *pagep = newpage;
1659         }
1660
1661         ClearPageSwapCache(oldpage);
1662         set_page_private(oldpage, 0);
1663
1664         unlock_page(oldpage);
1665         put_page(oldpage);
1666         put_page(oldpage);
1667         return error;
1668 }
1669
1670 /*
1671  * Swap in the page pointed to by *pagep.
1672  * Caller has to make sure that *pagep contains a valid swapped page.
1673  * Returns 0 and the page in pagep if success. On failure, returns the
1674  * error code and NULL in *pagep.
1675  */
1676 static int shmem_swapin_page(struct inode *inode, pgoff_t index,
1677                              struct page **pagep, enum sgp_type sgp,
1678                              gfp_t gfp, struct vm_area_struct *vma,
1679                              vm_fault_t *fault_type)
1680 {
1681         struct address_space *mapping = inode->i_mapping;
1682         struct shmem_inode_info *info = SHMEM_I(inode);
1683         struct mm_struct *charge_mm = vma ? vma->vm_mm : NULL;
1684         struct page *page;
1685         swp_entry_t swap;
1686         int error;
1687
1688         VM_BUG_ON(!*pagep || !xa_is_value(*pagep));
1689         swap = radix_to_swp_entry(*pagep);
1690         *pagep = NULL;
1691
1692         /* Look it up and read it in.. */
1693         page = lookup_swap_cache(swap, NULL, 0);
1694         if (!page) {
1695                 /* Or update major stats only when swapin succeeds?? */
1696                 if (fault_type) {
1697                         *fault_type |= VM_FAULT_MAJOR;
1698                         count_vm_event(PGMAJFAULT);
1699                         count_memcg_event_mm(charge_mm, PGMAJFAULT);
1700                 }
1701                 /* Here we actually start the io */
1702                 page = shmem_swapin(swap, gfp, info, index);
1703                 if (!page) {
1704                         error = -ENOMEM;
1705                         goto failed;
1706                 }
1707         }
1708
1709         /* We have to do this with page locked to prevent races */
1710         lock_page(page);
1711         if (!PageSwapCache(page) || page_private(page) != swap.val ||
1712             !shmem_confirm_swap(mapping, index, swap)) {
1713                 error = -EEXIST;
1714                 goto unlock;
1715         }
1716         if (!PageUptodate(page)) {
1717                 error = -EIO;
1718                 goto failed;
1719         }
1720         wait_on_page_writeback(page);
1721
1722         /*
1723          * Some architectures may have to restore extra metadata to the
1724          * physical page after reading from swap.
1725          */
1726         arch_swap_restore(swap, page);
1727
1728         if (shmem_should_replace_page(page, gfp)) {
1729                 error = shmem_replace_page(&page, gfp, info, index);
1730                 if (error)
1731                         goto failed;
1732         }
1733
1734         error = shmem_add_to_page_cache(page, mapping, index,
1735                                         swp_to_radix_entry(swap), gfp,
1736                                         charge_mm);
1737         if (error)
1738                 goto failed;
1739
1740         spin_lock_irq(&info->lock);
1741         info->swapped--;
1742         shmem_recalc_inode(inode);
1743         spin_unlock_irq(&info->lock);
1744
1745         if (sgp == SGP_WRITE)
1746                 mark_page_accessed(page);
1747
1748         delete_from_swap_cache(page);
1749         set_page_dirty(page);
1750         swap_free(swap);
1751
1752         *pagep = page;
1753         return 0;
1754 failed:
1755         if (!shmem_confirm_swap(mapping, index, swap))
1756                 error = -EEXIST;
1757 unlock:
1758         if (page) {
1759                 unlock_page(page);
1760                 put_page(page);
1761         }
1762
1763         return error;
1764 }
1765
1766 /*
1767  * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
1768  *
1769  * If we allocate a new one we do not mark it dirty. That's up to the
1770  * vm. If we swap it in we mark it dirty since we also free the swap
1771  * entry since a page cannot live in both the swap and page cache.
1772  *
1773  * vma, vmf, and fault_type are only supplied by shmem_fault:
1774  * otherwise they are NULL.
1775  */
1776 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
1777         struct page **pagep, enum sgp_type sgp, gfp_t gfp,
1778         struct vm_area_struct *vma, struct vm_fault *vmf,
1779                         vm_fault_t *fault_type)
1780 {
1781         struct address_space *mapping = inode->i_mapping;
1782         struct shmem_inode_info *info = SHMEM_I(inode);
1783         struct shmem_sb_info *sbinfo;
1784         struct mm_struct *charge_mm;
1785         struct page *page;
1786         pgoff_t hindex = index;
1787         gfp_t huge_gfp;
1788         int error;
1789         int once = 0;
1790         int alloced = 0;
1791
1792         if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
1793                 return -EFBIG;
1794 repeat:
1795         if (sgp <= SGP_CACHE &&
1796             ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1797                 return -EINVAL;
1798         }
1799
1800         sbinfo = SHMEM_SB(inode->i_sb);
1801         charge_mm = vma ? vma->vm_mm : NULL;
1802
1803         page = pagecache_get_page(mapping, index,
1804                                         FGP_ENTRY | FGP_HEAD | FGP_LOCK, 0);
1805
1806         if (page && vma && userfaultfd_minor(vma)) {
1807                 if (!xa_is_value(page)) {
1808                         unlock_page(page);
1809                         put_page(page);
1810                 }
1811                 *fault_type = handle_userfault(vmf, VM_UFFD_MINOR);
1812                 return 0;
1813         }
1814
1815         if (xa_is_value(page)) {
1816                 error = shmem_swapin_page(inode, index, &page,
1817                                           sgp, gfp, vma, fault_type);
1818                 if (error == -EEXIST)
1819                         goto repeat;
1820
1821                 *pagep = page;
1822                 return error;
1823         }
1824
1825         if (page) {
1826                 hindex = page->index;
1827                 if (sgp == SGP_WRITE)
1828                         mark_page_accessed(page);
1829                 if (PageUptodate(page))
1830                         goto out;
1831                 /* fallocated page */
1832                 if (sgp != SGP_READ)
1833                         goto clear;
1834                 unlock_page(page);
1835                 put_page(page);
1836         }
1837
1838         /*
1839          * SGP_READ: succeed on hole, with NULL page, letting caller zero.
1840          * SGP_NOALLOC: fail on hole, with NULL page, letting caller fail.
1841          */
1842         *pagep = NULL;
1843         if (sgp == SGP_READ)
1844                 return 0;
1845         if (sgp == SGP_NOALLOC)
1846                 return -ENOENT;
1847
1848         /*
1849          * Fast cache lookup and swap lookup did not find it: allocate.
1850          */
1851
1852         if (vma && userfaultfd_missing(vma)) {
1853                 *fault_type = handle_userfault(vmf, VM_UFFD_MISSING);
1854                 return 0;
1855         }
1856
1857         /* Never use a huge page for shmem_symlink() */
1858         if (S_ISLNK(inode->i_mode))
1859                 goto alloc_nohuge;
1860         if (!shmem_is_huge(vma, inode, index))
1861                 goto alloc_nohuge;
1862
1863         huge_gfp = vma_thp_gfp_mask(vma);
1864         huge_gfp = limit_gfp_mask(huge_gfp, gfp);
1865         page = shmem_alloc_and_acct_page(huge_gfp, inode, index, true);
1866         if (IS_ERR(page)) {
1867 alloc_nohuge:
1868                 page = shmem_alloc_and_acct_page(gfp, inode,
1869                                                  index, false);
1870         }
1871         if (IS_ERR(page)) {
1872                 int retry = 5;
1873
1874                 error = PTR_ERR(page);
1875                 page = NULL;
1876                 if (error != -ENOSPC)
1877                         goto unlock;
1878                 /*
1879                  * Try to reclaim some space by splitting a huge page
1880                  * beyond i_size on the filesystem.
1881                  */
1882                 while (retry--) {
1883                         int ret;
1884
1885                         ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
1886                         if (ret == SHRINK_STOP)
1887                                 break;
1888                         if (ret)
1889                                 goto alloc_nohuge;
1890                 }
1891                 goto unlock;
1892         }
1893
1894         if (PageTransHuge(page))
1895                 hindex = round_down(index, HPAGE_PMD_NR);
1896         else
1897                 hindex = index;
1898
1899         if (sgp == SGP_WRITE)
1900                 __SetPageReferenced(page);
1901
1902         error = shmem_add_to_page_cache(page, mapping, hindex,
1903                                         NULL, gfp & GFP_RECLAIM_MASK,
1904                                         charge_mm);
1905         if (error)
1906                 goto unacct;
1907         lru_cache_add(page);
1908
1909         spin_lock_irq(&info->lock);
1910         info->alloced += compound_nr(page);
1911         inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page);
1912         shmem_recalc_inode(inode);
1913         spin_unlock_irq(&info->lock);
1914         alloced = true;
1915
1916         if (PageTransHuge(page) &&
1917             DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
1918                         hindex + HPAGE_PMD_NR - 1) {
1919                 /*
1920                  * Part of the huge page is beyond i_size: subject
1921                  * to shrink under memory pressure.
1922                  */
1923                 spin_lock(&sbinfo->shrinklist_lock);
1924                 /*
1925                  * _careful to defend against unlocked access to
1926                  * ->shrink_list in shmem_unused_huge_shrink()
1927                  */
1928                 if (list_empty_careful(&info->shrinklist)) {
1929                         list_add_tail(&info->shrinklist,
1930                                       &sbinfo->shrinklist);
1931                         sbinfo->shrinklist_len++;
1932                 }
1933                 spin_unlock(&sbinfo->shrinklist_lock);
1934         }
1935
1936         /*
1937          * Let SGP_FALLOC use the SGP_WRITE optimization on a new page.
1938          */
1939         if (sgp == SGP_FALLOC)
1940                 sgp = SGP_WRITE;
1941 clear:
1942         /*
1943          * Let SGP_WRITE caller clear ends if write does not fill page;
1944          * but SGP_FALLOC on a page fallocated earlier must initialize
1945          * it now, lest undo on failure cancel our earlier guarantee.
1946          */
1947         if (sgp != SGP_WRITE && !PageUptodate(page)) {
1948                 int i;
1949
1950                 for (i = 0; i < compound_nr(page); i++) {
1951                         clear_highpage(page + i);
1952                         flush_dcache_page(page + i);
1953                 }
1954                 SetPageUptodate(page);
1955         }
1956
1957         /* Perhaps the file has been truncated since we checked */
1958         if (sgp <= SGP_CACHE &&
1959             ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1960                 if (alloced) {
1961                         ClearPageDirty(page);
1962                         delete_from_page_cache(page);
1963                         spin_lock_irq(&info->lock);
1964                         shmem_recalc_inode(inode);
1965                         spin_unlock_irq(&info->lock);
1966                 }
1967                 error = -EINVAL;
1968                 goto unlock;
1969         }
1970 out:
1971         *pagep = page + index - hindex;
1972         return 0;
1973
1974         /*
1975          * Error recovery.
1976          */
1977 unacct:
1978         shmem_inode_unacct_blocks(inode, compound_nr(page));
1979
1980         if (PageTransHuge(page)) {
1981                 unlock_page(page);
1982                 put_page(page);
1983                 goto alloc_nohuge;
1984         }
1985 unlock:
1986         if (page) {
1987                 unlock_page(page);
1988                 put_page(page);
1989         }
1990         if (error == -ENOSPC && !once++) {
1991                 spin_lock_irq(&info->lock);
1992                 shmem_recalc_inode(inode);
1993                 spin_unlock_irq(&info->lock);
1994                 goto repeat;
1995         }
1996         if (error == -EEXIST)
1997                 goto repeat;
1998         return error;
1999 }
2000
2001 /*
2002  * This is like autoremove_wake_function, but it removes the wait queue
2003  * entry unconditionally - even if something else had already woken the
2004  * target.
2005  */
2006 static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
2007 {
2008         int ret = default_wake_function(wait, mode, sync, key);
2009         list_del_init(&wait->entry);
2010         return ret;
2011 }
2012
2013 static vm_fault_t shmem_fault(struct vm_fault *vmf)
2014 {
2015         struct vm_area_struct *vma = vmf->vma;
2016         struct inode *inode = file_inode(vma->vm_file);
2017         gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
2018         int err;
2019         vm_fault_t ret = VM_FAULT_LOCKED;
2020
2021         /*
2022          * Trinity finds that probing a hole which tmpfs is punching can
2023          * prevent the hole-punch from ever completing: which in turn
2024          * locks writers out with its hold on i_rwsem.  So refrain from
2025          * faulting pages into the hole while it's being punched.  Although
2026          * shmem_undo_range() does remove the additions, it may be unable to
2027          * keep up, as each new page needs its own unmap_mapping_range() call,
2028          * and the i_mmap tree grows ever slower to scan if new vmas are added.
2029          *
2030          * It does not matter if we sometimes reach this check just before the
2031          * hole-punch begins, so that one fault then races with the punch:
2032          * we just need to make racing faults a rare case.
2033          *
2034          * The implementation below would be much simpler if we just used a
2035          * standard mutex or completion: but we cannot take i_rwsem in fault,
2036          * and bloating every shmem inode for this unlikely case would be sad.
2037          */
2038         if (unlikely(inode->i_private)) {
2039                 struct shmem_falloc *shmem_falloc;
2040
2041                 spin_lock(&inode->i_lock);
2042                 shmem_falloc = inode->i_private;
2043                 if (shmem_falloc &&
2044                     shmem_falloc->waitq &&
2045                     vmf->pgoff >= shmem_falloc->start &&
2046                     vmf->pgoff < shmem_falloc->next) {
2047                         struct file *fpin;
2048                         wait_queue_head_t *shmem_falloc_waitq;
2049                         DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function);
2050
2051                         ret = VM_FAULT_NOPAGE;
2052                         fpin = maybe_unlock_mmap_for_io(vmf, NULL);
2053                         if (fpin)
2054                                 ret = VM_FAULT_RETRY;
2055
2056                         shmem_falloc_waitq = shmem_falloc->waitq;
2057                         prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
2058                                         TASK_UNINTERRUPTIBLE);
2059                         spin_unlock(&inode->i_lock);
2060                         schedule();
2061
2062                         /*
2063                          * shmem_falloc_waitq points into the shmem_fallocate()
2064                          * stack of the hole-punching task: shmem_falloc_waitq
2065                          * is usually invalid by the time we reach here, but
2066                          * finish_wait() does not dereference it in that case;
2067                          * though i_lock needed lest racing with wake_up_all().
2068                          */
2069                         spin_lock(&inode->i_lock);
2070                         finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
2071                         spin_unlock(&inode->i_lock);
2072
2073                         if (fpin)
2074                                 fput(fpin);
2075                         return ret;
2076                 }
2077                 spin_unlock(&inode->i_lock);
2078         }
2079
2080         err = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, SGP_CACHE,
2081                                   gfp, vma, vmf, &ret);
2082         if (err)
2083                 return vmf_error(err);
2084         return ret;
2085 }
2086
2087 unsigned long shmem_get_unmapped_area(struct file *file,
2088                                       unsigned long uaddr, unsigned long len,
2089                                       unsigned long pgoff, unsigned long flags)
2090 {
2091         unsigned long (*get_area)(struct file *,
2092                 unsigned long, unsigned long, unsigned long, unsigned long);
2093         unsigned long addr;
2094         unsigned long offset;
2095         unsigned long inflated_len;
2096         unsigned long inflated_addr;
2097         unsigned long inflated_offset;
2098
2099         if (len > TASK_SIZE)
2100                 return -ENOMEM;
2101
2102         get_area = current->mm->get_unmapped_area;
2103         addr = get_area(file, uaddr, len, pgoff, flags);
2104
2105         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
2106                 return addr;
2107         if (IS_ERR_VALUE(addr))
2108                 return addr;
2109         if (addr & ~PAGE_MASK)
2110                 return addr;
2111         if (addr > TASK_SIZE - len)
2112                 return addr;
2113
2114         if (shmem_huge == SHMEM_HUGE_DENY)
2115                 return addr;
2116         if (len < HPAGE_PMD_SIZE)
2117                 return addr;
2118         if (flags & MAP_FIXED)
2119                 return addr;
2120         /*
2121          * Our priority is to support MAP_SHARED mapped hugely;
2122          * and support MAP_PRIVATE mapped hugely too, until it is COWed.
2123          * But if caller specified an address hint and we allocated area there
2124          * successfully, respect that as before.
2125          */
2126         if (uaddr == addr)
2127                 return addr;
2128
2129         if (shmem_huge != SHMEM_HUGE_FORCE) {
2130                 struct super_block *sb;
2131
2132                 if (file) {
2133                         VM_BUG_ON(file->f_op != &shmem_file_operations);
2134                         sb = file_inode(file)->i_sb;
2135                 } else {
2136                         /*
2137                          * Called directly from mm/mmap.c, or drivers/char/mem.c
2138                          * for "/dev/zero", to create a shared anonymous object.
2139                          */
2140                         if (IS_ERR(shm_mnt))
2141                                 return addr;
2142                         sb = shm_mnt->mnt_sb;
2143                 }
2144                 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER)
2145                         return addr;
2146         }
2147
2148         offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1);
2149         if (offset && offset + len < 2 * HPAGE_PMD_SIZE)
2150                 return addr;
2151         if ((addr & (HPAGE_PMD_SIZE-1)) == offset)
2152                 return addr;
2153
2154         inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE;
2155         if (inflated_len > TASK_SIZE)
2156                 return addr;
2157         if (inflated_len < len)
2158                 return addr;
2159
2160         inflated_addr = get_area(NULL, uaddr, inflated_len, 0, flags);
2161         if (IS_ERR_VALUE(inflated_addr))
2162                 return addr;
2163         if (inflated_addr & ~PAGE_MASK)
2164                 return addr;
2165
2166         inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1);
2167         inflated_addr += offset - inflated_offset;
2168         if (inflated_offset > offset)
2169                 inflated_addr += HPAGE_PMD_SIZE;
2170
2171         if (inflated_addr > TASK_SIZE - len)
2172                 return addr;
2173         return inflated_addr;
2174 }
2175
2176 #ifdef CONFIG_NUMA
2177 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
2178 {
2179         struct inode *inode = file_inode(vma->vm_file);
2180         return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
2181 }
2182
2183 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
2184                                           unsigned long addr)
2185 {
2186         struct inode *inode = file_inode(vma->vm_file);
2187         pgoff_t index;
2188
2189         index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2190         return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
2191 }
2192 #endif
2193
2194 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)
2195 {
2196         struct inode *inode = file_inode(file);
2197         struct shmem_inode_info *info = SHMEM_I(inode);
2198         int retval = -ENOMEM;
2199
2200         /*
2201          * What serializes the accesses to info->flags?
2202          * ipc_lock_object() when called from shmctl_do_lock(),
2203          * no serialization needed when called from shm_destroy().
2204          */
2205         if (lock && !(info->flags & VM_LOCKED)) {
2206                 if (!user_shm_lock(inode->i_size, ucounts))
2207                         goto out_nomem;
2208                 info->flags |= VM_LOCKED;
2209                 mapping_set_unevictable(file->f_mapping);
2210         }
2211         if (!lock && (info->flags & VM_LOCKED) && ucounts) {
2212                 user_shm_unlock(inode->i_size, ucounts);
2213                 info->flags &= ~VM_LOCKED;
2214                 mapping_clear_unevictable(file->f_mapping);
2215         }
2216         retval = 0;
2217
2218 out_nomem:
2219         return retval;
2220 }
2221
2222 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
2223 {
2224         struct shmem_inode_info *info = SHMEM_I(file_inode(file));
2225         int ret;
2226
2227         ret = seal_check_future_write(info->seals, vma);
2228         if (ret)
2229                 return ret;
2230
2231         /* arm64 - allow memory tagging on RAM-based files */
2232         vma->vm_flags |= VM_MTE_ALLOWED;
2233
2234         file_accessed(file);
2235         vma->vm_ops = &shmem_vm_ops;
2236         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
2237                         ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
2238                         (vma->vm_end & HPAGE_PMD_MASK)) {
2239                 khugepaged_enter(vma, vma->vm_flags);
2240         }
2241         return 0;
2242 }
2243
2244 static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
2245                                      umode_t mode, dev_t dev, unsigned long flags)
2246 {
2247         struct inode *inode;
2248         struct shmem_inode_info *info;
2249         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2250         ino_t ino;
2251
2252         if (shmem_reserve_inode(sb, &ino))
2253                 return NULL;
2254
2255         inode = new_inode(sb);
2256         if (inode) {
2257                 inode->i_ino = ino;
2258                 inode_init_owner(&init_user_ns, inode, dir, mode);
2259                 inode->i_blocks = 0;
2260                 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
2261                 inode->i_generation = prandom_u32();
2262                 info = SHMEM_I(inode);
2263                 memset(info, 0, (char *)inode - (char *)info);
2264                 spin_lock_init(&info->lock);
2265                 atomic_set(&info->stop_eviction, 0);
2266                 info->seals = F_SEAL_SEAL;
2267                 info->flags = flags & VM_NORESERVE;
2268                 INIT_LIST_HEAD(&info->shrinklist);
2269                 INIT_LIST_HEAD(&info->swaplist);
2270                 simple_xattrs_init(&info->xattrs);
2271                 cache_no_acl(inode);
2272                 mapping_set_large_folios(inode->i_mapping);
2273
2274                 switch (mode & S_IFMT) {
2275                 default:
2276                         inode->i_op = &shmem_special_inode_operations;
2277                         init_special_inode(inode, mode, dev);
2278                         break;
2279                 case S_IFREG:
2280                         inode->i_mapping->a_ops = &shmem_aops;
2281                         inode->i_op = &shmem_inode_operations;
2282                         inode->i_fop = &shmem_file_operations;
2283                         mpol_shared_policy_init(&info->policy,
2284                                                  shmem_get_sbmpol(sbinfo));
2285                         break;
2286                 case S_IFDIR:
2287                         inc_nlink(inode);
2288                         /* Some things misbehave if size == 0 on a directory */
2289                         inode->i_size = 2 * BOGO_DIRENT_SIZE;
2290                         inode->i_op = &shmem_dir_inode_operations;
2291                         inode->i_fop = &simple_dir_operations;
2292                         break;
2293                 case S_IFLNK:
2294                         /*
2295                          * Must not load anything in the rbtree,
2296                          * mpol_free_shared_policy will not be called.
2297                          */
2298                         mpol_shared_policy_init(&info->policy, NULL);
2299                         break;
2300                 }
2301
2302                 lockdep_annotate_inode_mutex_key(inode);
2303         } else
2304                 shmem_free_inode(sb);
2305         return inode;
2306 }
2307
2308 #ifdef CONFIG_USERFAULTFD
2309 int shmem_mfill_atomic_pte(struct mm_struct *dst_mm,
2310                            pmd_t *dst_pmd,
2311                            struct vm_area_struct *dst_vma,
2312                            unsigned long dst_addr,
2313                            unsigned long src_addr,
2314                            bool zeropage,
2315                            struct page **pagep)
2316 {
2317         struct inode *inode = file_inode(dst_vma->vm_file);
2318         struct shmem_inode_info *info = SHMEM_I(inode);
2319         struct address_space *mapping = inode->i_mapping;
2320         gfp_t gfp = mapping_gfp_mask(mapping);
2321         pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
2322         void *page_kaddr;
2323         struct page *page;
2324         int ret;
2325         pgoff_t max_off;
2326
2327         if (!shmem_inode_acct_block(inode, 1)) {
2328                 /*
2329                  * We may have got a page, returned -ENOENT triggering a retry,
2330                  * and now we find ourselves with -ENOMEM. Release the page, to
2331                  * avoid a BUG_ON in our caller.
2332                  */
2333                 if (unlikely(*pagep)) {
2334                         put_page(*pagep);
2335                         *pagep = NULL;
2336                 }
2337                 return -ENOMEM;
2338         }
2339
2340         if (!*pagep) {
2341                 ret = -ENOMEM;
2342                 page = shmem_alloc_page(gfp, info, pgoff);
2343                 if (!page)
2344                         goto out_unacct_blocks;
2345
2346                 if (!zeropage) {        /* COPY */
2347                         page_kaddr = kmap_atomic(page);
2348                         ret = copy_from_user(page_kaddr,
2349                                              (const void __user *)src_addr,
2350                                              PAGE_SIZE);
2351                         kunmap_atomic(page_kaddr);
2352
2353                         /* fallback to copy_from_user outside mmap_lock */
2354                         if (unlikely(ret)) {
2355                                 *pagep = page;
2356                                 ret = -ENOENT;
2357                                 /* don't free the page */
2358                                 goto out_unacct_blocks;
2359                         }
2360                 } else {                /* ZEROPAGE */
2361                         clear_highpage(page);
2362                 }
2363         } else {
2364                 page = *pagep;
2365                 *pagep = NULL;
2366         }
2367
2368         VM_BUG_ON(PageLocked(page));
2369         VM_BUG_ON(PageSwapBacked(page));
2370         __SetPageLocked(page);
2371         __SetPageSwapBacked(page);
2372         __SetPageUptodate(page);
2373
2374         ret = -EFAULT;
2375         max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2376         if (unlikely(pgoff >= max_off))
2377                 goto out_release;
2378
2379         ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL,
2380                                       gfp & GFP_RECLAIM_MASK, dst_mm);
2381         if (ret)
2382                 goto out_release;
2383
2384         ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
2385                                        page, true, false);
2386         if (ret)
2387                 goto out_delete_from_cache;
2388
2389         spin_lock_irq(&info->lock);
2390         info->alloced++;
2391         inode->i_blocks += BLOCKS_PER_PAGE;
2392         shmem_recalc_inode(inode);
2393         spin_unlock_irq(&info->lock);
2394
2395         unlock_page(page);
2396         return 0;
2397 out_delete_from_cache:
2398         delete_from_page_cache(page);
2399 out_release:
2400         unlock_page(page);
2401         put_page(page);
2402 out_unacct_blocks:
2403         shmem_inode_unacct_blocks(inode, 1);
2404         return ret;
2405 }
2406 #endif /* CONFIG_USERFAULTFD */
2407
2408 #ifdef CONFIG_TMPFS
2409 static const struct inode_operations shmem_symlink_inode_operations;
2410 static const struct inode_operations shmem_short_symlink_operations;
2411
2412 #ifdef CONFIG_TMPFS_XATTR
2413 static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
2414 #else
2415 #define shmem_initxattrs NULL
2416 #endif
2417
2418 static int
2419 shmem_write_begin(struct file *file, struct address_space *mapping,
2420                         loff_t pos, unsigned len, unsigned flags,
2421                         struct page **pagep, void **fsdata)
2422 {
2423         struct inode *inode = mapping->host;
2424         struct shmem_inode_info *info = SHMEM_I(inode);
2425         pgoff_t index = pos >> PAGE_SHIFT;
2426         int ret = 0;
2427
2428         /* i_rwsem is held by caller */
2429         if (unlikely(info->seals & (F_SEAL_GROW |
2430                                    F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))) {
2431                 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))
2432                         return -EPERM;
2433                 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size)
2434                         return -EPERM;
2435         }
2436
2437         ret = shmem_getpage(inode, index, pagep, SGP_WRITE);
2438
2439         if (ret)
2440                 return ret;
2441
2442         if (PageHWPoison(*pagep)) {
2443                 unlock_page(*pagep);
2444                 put_page(*pagep);
2445                 *pagep = NULL;
2446                 return -EIO;
2447         }
2448
2449         return 0;
2450 }
2451
2452 static int
2453 shmem_write_end(struct file *file, struct address_space *mapping,
2454                         loff_t pos, unsigned len, unsigned copied,
2455                         struct page *page, void *fsdata)
2456 {
2457         struct inode *inode = mapping->host;
2458
2459         if (pos + copied > inode->i_size)
2460                 i_size_write(inode, pos + copied);
2461
2462         if (!PageUptodate(page)) {
2463                 struct page *head = compound_head(page);
2464                 if (PageTransCompound(page)) {
2465                         int i;
2466
2467                         for (i = 0; i < HPAGE_PMD_NR; i++) {
2468                                 if (head + i == page)
2469                                         continue;
2470                                 clear_highpage(head + i);
2471                                 flush_dcache_page(head + i);
2472                         }
2473                 }
2474                 if (copied < PAGE_SIZE) {
2475                         unsigned from = pos & (PAGE_SIZE - 1);
2476                         zero_user_segments(page, 0, from,
2477                                         from + copied, PAGE_SIZE);
2478                 }
2479                 SetPageUptodate(head);
2480         }
2481         set_page_dirty(page);
2482         unlock_page(page);
2483         put_page(page);
2484
2485         return copied;
2486 }
2487
2488 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
2489 {
2490         struct file *file = iocb->ki_filp;
2491         struct inode *inode = file_inode(file);
2492         struct address_space *mapping = inode->i_mapping;
2493         pgoff_t index;
2494         unsigned long offset;
2495         enum sgp_type sgp = SGP_READ;
2496         int error = 0;
2497         ssize_t retval = 0;
2498         loff_t *ppos = &iocb->ki_pos;
2499
2500         /*
2501          * Might this read be for a stacking filesystem?  Then when reading
2502          * holes of a sparse file, we actually need to allocate those pages,
2503          * and even mark them dirty, so it cannot exceed the max_blocks limit.
2504          */
2505         if (!iter_is_iovec(to))
2506                 sgp = SGP_CACHE;
2507
2508         index = *ppos >> PAGE_SHIFT;
2509         offset = *ppos & ~PAGE_MASK;
2510
2511         for (;;) {
2512                 struct page *page = NULL;
2513                 pgoff_t end_index;
2514                 unsigned long nr, ret;
2515                 loff_t i_size = i_size_read(inode);
2516
2517                 end_index = i_size >> PAGE_SHIFT;
2518                 if (index > end_index)
2519                         break;
2520                 if (index == end_index) {
2521                         nr = i_size & ~PAGE_MASK;
2522                         if (nr <= offset)
2523                                 break;
2524                 }
2525
2526                 error = shmem_getpage(inode, index, &page, sgp);
2527                 if (error) {
2528                         if (error == -EINVAL)
2529                                 error = 0;
2530                         break;
2531                 }
2532                 if (page) {
2533                         if (sgp == SGP_CACHE)
2534                                 set_page_dirty(page);
2535                         unlock_page(page);
2536
2537                         if (PageHWPoison(page)) {
2538                                 put_page(page);
2539                                 error = -EIO;
2540                                 break;
2541                         }
2542                 }
2543
2544                 /*
2545                  * We must evaluate after, since reads (unlike writes)
2546                  * are called without i_rwsem protection against truncate
2547                  */
2548                 nr = PAGE_SIZE;
2549                 i_size = i_size_read(inode);
2550                 end_index = i_size >> PAGE_SHIFT;
2551                 if (index == end_index) {
2552                         nr = i_size & ~PAGE_MASK;
2553                         if (nr <= offset) {
2554                                 if (page)
2555                                         put_page(page);
2556                                 break;
2557                         }
2558                 }
2559                 nr -= offset;
2560
2561                 if (page) {
2562                         /*
2563                          * If users can be writing to this page using arbitrary
2564                          * virtual addresses, take care about potential aliasing
2565                          * before reading the page on the kernel side.
2566                          */
2567                         if (mapping_writably_mapped(mapping))
2568                                 flush_dcache_page(page);
2569                         /*
2570                          * Mark the page accessed if we read the beginning.
2571                          */
2572                         if (!offset)
2573                                 mark_page_accessed(page);
2574                 } else {
2575                         page = ZERO_PAGE(0);
2576                         get_page(page);
2577                 }
2578
2579                 /*
2580                  * Ok, we have the page, and it's up-to-date, so
2581                  * now we can copy it to user space...
2582                  */
2583                 ret = copy_page_to_iter(page, offset, nr, to);
2584                 retval += ret;
2585                 offset += ret;
2586                 index += offset >> PAGE_SHIFT;
2587                 offset &= ~PAGE_MASK;
2588
2589                 put_page(page);
2590                 if (!iov_iter_count(to))
2591                         break;
2592                 if (ret < nr) {
2593                         error = -EFAULT;
2594                         break;
2595                 }
2596                 cond_resched();
2597         }
2598
2599         *ppos = ((loff_t) index << PAGE_SHIFT) + offset;
2600         file_accessed(file);
2601         return retval ? retval : error;
2602 }
2603
2604 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
2605 {
2606         struct address_space *mapping = file->f_mapping;
2607         struct inode *inode = mapping->host;
2608
2609         if (whence != SEEK_DATA && whence != SEEK_HOLE)
2610                 return generic_file_llseek_size(file, offset, whence,
2611                                         MAX_LFS_FILESIZE, i_size_read(inode));
2612         if (offset < 0)
2613                 return -ENXIO;
2614
2615         inode_lock(inode);
2616         /* We're holding i_rwsem so we can access i_size directly */
2617         offset = mapping_seek_hole_data(mapping, offset, inode->i_size, whence);
2618         if (offset >= 0)
2619                 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
2620         inode_unlock(inode);
2621         return offset;
2622 }
2623
2624 static long shmem_fallocate(struct file *file, int mode, loff_t offset,
2625                                                          loff_t len)
2626 {
2627         struct inode *inode = file_inode(file);
2628         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
2629         struct shmem_inode_info *info = SHMEM_I(inode);
2630         struct shmem_falloc shmem_falloc;
2631         pgoff_t start, index, end, undo_fallocend;
2632         int error;
2633
2634         if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
2635                 return -EOPNOTSUPP;
2636
2637         inode_lock(inode);
2638
2639         if (mode & FALLOC_FL_PUNCH_HOLE) {
2640                 struct address_space *mapping = file->f_mapping;
2641                 loff_t unmap_start = round_up(offset, PAGE_SIZE);
2642                 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
2643                 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
2644
2645                 /* protected by i_rwsem */
2646                 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) {
2647                         error = -EPERM;
2648                         goto out;
2649                 }
2650
2651                 shmem_falloc.waitq = &shmem_falloc_waitq;
2652                 shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT;
2653                 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
2654                 spin_lock(&inode->i_lock);
2655                 inode->i_private = &shmem_falloc;
2656                 spin_unlock(&inode->i_lock);
2657
2658                 if ((u64)unmap_end > (u64)unmap_start)
2659                         unmap_mapping_range(mapping, unmap_start,
2660                                             1 + unmap_end - unmap_start, 0);
2661                 shmem_truncate_range(inode, offset, offset + len - 1);
2662                 /* No need to unmap again: hole-punching leaves COWed pages */
2663
2664                 spin_lock(&inode->i_lock);
2665                 inode->i_private = NULL;
2666                 wake_up_all(&shmem_falloc_waitq);
2667                 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head));
2668                 spin_unlock(&inode->i_lock);
2669                 error = 0;
2670                 goto out;
2671         }
2672
2673         /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
2674         error = inode_newsize_ok(inode, offset + len);
2675         if (error)
2676                 goto out;
2677
2678         if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
2679                 error = -EPERM;
2680                 goto out;
2681         }
2682
2683         start = offset >> PAGE_SHIFT;
2684         end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
2685         /* Try to avoid a swapstorm if len is impossible to satisfy */
2686         if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
2687                 error = -ENOSPC;
2688                 goto out;
2689         }
2690
2691         shmem_falloc.waitq = NULL;
2692         shmem_falloc.start = start;
2693         shmem_falloc.next  = start;
2694         shmem_falloc.nr_falloced = 0;
2695         shmem_falloc.nr_unswapped = 0;
2696         spin_lock(&inode->i_lock);
2697         inode->i_private = &shmem_falloc;
2698         spin_unlock(&inode->i_lock);
2699
2700         /*
2701          * info->fallocend is only relevant when huge pages might be
2702          * involved: to prevent split_huge_page() freeing fallocated
2703          * pages when FALLOC_FL_KEEP_SIZE committed beyond i_size.
2704          */
2705         undo_fallocend = info->fallocend;
2706         if (info->fallocend < end)
2707                 info->fallocend = end;
2708
2709         for (index = start; index < end; ) {
2710                 struct page *page;
2711
2712                 /*
2713                  * Good, the fallocate(2) manpage permits EINTR: we may have
2714                  * been interrupted because we are using up too much memory.
2715                  */
2716                 if (signal_pending(current))
2717                         error = -EINTR;
2718                 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
2719                         error = -ENOMEM;
2720                 else
2721                         error = shmem_getpage(inode, index, &page, SGP_FALLOC);
2722                 if (error) {
2723                         info->fallocend = undo_fallocend;
2724                         /* Remove the !PageUptodate pages we added */
2725                         if (index > start) {
2726                                 shmem_undo_range(inode,
2727                                     (loff_t)start << PAGE_SHIFT,
2728                                     ((loff_t)index << PAGE_SHIFT) - 1, true);
2729                         }
2730                         goto undone;
2731                 }
2732
2733                 index++;
2734                 /*
2735                  * Here is a more important optimization than it appears:
2736                  * a second SGP_FALLOC on the same huge page will clear it,
2737                  * making it PageUptodate and un-undoable if we fail later.
2738                  */
2739                 if (PageTransCompound(page)) {
2740                         index = round_up(index, HPAGE_PMD_NR);
2741                         /* Beware 32-bit wraparound */
2742                         if (!index)
2743                                 index--;
2744                 }
2745
2746                 /*
2747                  * Inform shmem_writepage() how far we have reached.
2748                  * No need for lock or barrier: we have the page lock.
2749                  */
2750                 if (!PageUptodate(page))
2751                         shmem_falloc.nr_falloced += index - shmem_falloc.next;
2752                 shmem_falloc.next = index;
2753
2754                 /*
2755                  * If !PageUptodate, leave it that way so that freeable pages
2756                  * can be recognized if we need to rollback on error later.
2757                  * But set_page_dirty so that memory pressure will swap rather
2758                  * than free the pages we are allocating (and SGP_CACHE pages
2759                  * might still be clean: we now need to mark those dirty too).
2760                  */
2761                 set_page_dirty(page);
2762                 unlock_page(page);
2763                 put_page(page);
2764                 cond_resched();
2765         }
2766
2767         if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
2768                 i_size_write(inode, offset + len);
2769         inode->i_ctime = current_time(inode);
2770 undone:
2771         spin_lock(&inode->i_lock);
2772         inode->i_private = NULL;
2773         spin_unlock(&inode->i_lock);
2774 out:
2775         inode_unlock(inode);
2776         return error;
2777 }
2778
2779 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
2780 {
2781         struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
2782
2783         buf->f_type = TMPFS_MAGIC;
2784         buf->f_bsize = PAGE_SIZE;
2785         buf->f_namelen = NAME_MAX;
2786         if (sbinfo->max_blocks) {
2787                 buf->f_blocks = sbinfo->max_blocks;
2788                 buf->f_bavail =
2789                 buf->f_bfree  = sbinfo->max_blocks -
2790                                 percpu_counter_sum(&sbinfo->used_blocks);
2791         }
2792         if (sbinfo->max_inodes) {
2793                 buf->f_files = sbinfo->max_inodes;
2794                 buf->f_ffree = sbinfo->free_inodes;
2795         }
2796         /* else leave those fields 0 like simple_statfs */
2797
2798         buf->f_fsid = uuid_to_fsid(dentry->d_sb->s_uuid.b);
2799
2800         return 0;
2801 }
2802
2803 /*
2804  * File creation. Allocate an inode, and we're done..
2805  */
2806 static int
2807 shmem_mknod(struct user_namespace *mnt_userns, struct inode *dir,
2808             struct dentry *dentry, umode_t mode, dev_t dev)
2809 {
2810         struct inode *inode;
2811         int error = -ENOSPC;
2812
2813         inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
2814         if (inode) {
2815                 error = simple_acl_create(dir, inode);
2816                 if (error)
2817                         goto out_iput;
2818                 error = security_inode_init_security(inode, dir,
2819                                                      &dentry->d_name,
2820                                                      shmem_initxattrs, NULL);
2821                 if (error && error != -EOPNOTSUPP)
2822                         goto out_iput;
2823
2824                 error = 0;
2825                 dir->i_size += BOGO_DIRENT_SIZE;
2826                 dir->i_ctime = dir->i_mtime = current_time(dir);
2827                 d_instantiate(dentry, inode);
2828                 dget(dentry); /* Extra count - pin the dentry in core */
2829         }
2830         return error;
2831 out_iput:
2832         iput(inode);
2833         return error;
2834 }
2835
2836 static int
2837 shmem_tmpfile(struct user_namespace *mnt_userns, struct inode *dir,
2838               struct dentry *dentry, umode_t mode)
2839 {
2840         struct inode *inode;
2841         int error = -ENOSPC;
2842
2843         inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE);
2844         if (inode) {
2845                 error = security_inode_init_security(inode, dir,
2846                                                      NULL,
2847                                                      shmem_initxattrs, NULL);
2848                 if (error && error != -EOPNOTSUPP)
2849                         goto out_iput;
2850                 error = simple_acl_create(dir, inode);
2851                 if (error)
2852                         goto out_iput;
2853                 d_tmpfile(dentry, inode);
2854         }
2855         return error;
2856 out_iput:
2857         iput(inode);
2858         return error;
2859 }
2860
2861 static int shmem_mkdir(struct user_namespace *mnt_userns, struct inode *dir,
2862                        struct dentry *dentry, umode_t mode)
2863 {
2864         int error;
2865
2866         if ((error = shmem_mknod(&init_user_ns, dir, dentry,
2867                                  mode | S_IFDIR, 0)))
2868                 return error;
2869         inc_nlink(dir);
2870         return 0;
2871 }
2872
2873 static int shmem_create(struct user_namespace *mnt_userns, struct inode *dir,
2874                         struct dentry *dentry, umode_t mode, bool excl)
2875 {
2876         return shmem_mknod(&init_user_ns, dir, dentry, mode | S_IFREG, 0);
2877 }
2878
2879 /*
2880  * Link a file..
2881  */
2882 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
2883 {
2884         struct inode *inode = d_inode(old_dentry);
2885         int ret = 0;
2886
2887         /*
2888          * No ordinary (disk based) filesystem counts links as inodes;
2889          * but each new link needs a new dentry, pinning lowmem, and
2890          * tmpfs dentries cannot be pruned until they are unlinked.
2891          * But if an O_TMPFILE file is linked into the tmpfs, the
2892          * first link must skip that, to get the accounting right.
2893          */
2894         if (inode->i_nlink) {
2895                 ret = shmem_reserve_inode(inode->i_sb, NULL);
2896                 if (ret)
2897                         goto out;
2898         }
2899
2900         dir->i_size += BOGO_DIRENT_SIZE;
2901         inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
2902         inc_nlink(inode);
2903         ihold(inode);   /* New dentry reference */
2904         dget(dentry);           /* Extra pinning count for the created dentry */
2905         d_instantiate(dentry, inode);
2906 out:
2907         return ret;
2908 }
2909
2910 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
2911 {
2912         struct inode *inode = d_inode(dentry);
2913
2914         if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
2915                 shmem_free_inode(inode->i_sb);
2916
2917         dir->i_size -= BOGO_DIRENT_SIZE;
2918         inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
2919         drop_nlink(inode);
2920         dput(dentry);   /* Undo the count from "create" - this does all the work */
2921         return 0;
2922 }
2923
2924 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
2925 {
2926         if (!simple_empty(dentry))
2927                 return -ENOTEMPTY;
2928
2929         drop_nlink(d_inode(dentry));
2930         drop_nlink(dir);
2931         return shmem_unlink(dir, dentry);
2932 }
2933
2934 static int shmem_whiteout(struct user_namespace *mnt_userns,
2935                           struct inode *old_dir, struct dentry *old_dentry)
2936 {
2937         struct dentry *whiteout;
2938         int error;
2939
2940         whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
2941         if (!whiteout)
2942                 return -ENOMEM;
2943
2944         error = shmem_mknod(&init_user_ns, old_dir, whiteout,
2945                             S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
2946         dput(whiteout);
2947         if (error)
2948                 return error;
2949
2950         /*
2951          * Cheat and hash the whiteout while the old dentry is still in
2952          * place, instead of playing games with FS_RENAME_DOES_D_MOVE.
2953          *
2954          * d_lookup() will consistently find one of them at this point,
2955          * not sure which one, but that isn't even important.
2956          */
2957         d_rehash(whiteout);
2958         return 0;
2959 }
2960
2961 /*
2962  * The VFS layer already does all the dentry stuff for rename,
2963  * we just have to decrement the usage count for the target if
2964  * it exists so that the VFS layer correctly free's it when it
2965  * gets overwritten.
2966  */
2967 static int shmem_rename2(struct user_namespace *mnt_userns,
2968                          struct inode *old_dir, struct dentry *old_dentry,
2969                          struct inode *new_dir, struct dentry *new_dentry,
2970                          unsigned int flags)
2971 {
2972         struct inode *inode = d_inode(old_dentry);
2973         int they_are_dirs = S_ISDIR(inode->i_mode);
2974
2975         if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
2976                 return -EINVAL;
2977
2978         if (flags & RENAME_EXCHANGE)
2979                 return simple_rename_exchange(old_dir, old_dentry, new_dir, new_dentry);
2980
2981         if (!simple_empty(new_dentry))
2982                 return -ENOTEMPTY;
2983
2984         if (flags & RENAME_WHITEOUT) {
2985                 int error;
2986
2987                 error = shmem_whiteout(&init_user_ns, old_dir, old_dentry);
2988                 if (error)
2989                         return error;
2990         }
2991
2992         if (d_really_is_positive(new_dentry)) {
2993                 (void) shmem_unlink(new_dir, new_dentry);
2994                 if (they_are_dirs) {
2995                         drop_nlink(d_inode(new_dentry));
2996                         drop_nlink(old_dir);
2997                 }
2998         } else if (they_are_dirs) {
2999                 drop_nlink(old_dir);
3000                 inc_nlink(new_dir);
3001         }
3002
3003         old_dir->i_size -= BOGO_DIRENT_SIZE;
3004         new_dir->i_size += BOGO_DIRENT_SIZE;
3005         old_dir->i_ctime = old_dir->i_mtime =
3006         new_dir->i_ctime = new_dir->i_mtime =
3007         inode->i_ctime = current_time(old_dir);
3008         return 0;
3009 }
3010
3011 static int shmem_symlink(struct user_namespace *mnt_userns, struct inode *dir,
3012                          struct dentry *dentry, const char *symname)
3013 {
3014         int error;
3015         int len;
3016         struct inode *inode;
3017         struct page *page;
3018
3019         len = strlen(symname) + 1;
3020         if (len > PAGE_SIZE)
3021                 return -ENAMETOOLONG;
3022
3023         inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK | 0777, 0,
3024                                 VM_NORESERVE);
3025         if (!inode)
3026                 return -ENOSPC;
3027
3028         error = security_inode_init_security(inode, dir, &dentry->d_name,
3029                                              shmem_initxattrs, NULL);
3030         if (error && error != -EOPNOTSUPP) {
3031                 iput(inode);
3032                 return error;
3033         }
3034
3035         inode->i_size = len-1;
3036         if (len <= SHORT_SYMLINK_LEN) {
3037                 inode->i_link = kmemdup(symname, len, GFP_KERNEL);
3038                 if (!inode->i_link) {
3039                         iput(inode);
3040                         return -ENOMEM;
3041                 }
3042                 inode->i_op = &shmem_short_symlink_operations;
3043         } else {
3044                 inode_nohighmem(inode);
3045                 error = shmem_getpage(inode, 0, &page, SGP_WRITE);
3046                 if (error) {
3047                         iput(inode);
3048                         return error;
3049                 }
3050                 inode->i_mapping->a_ops = &shmem_aops;
3051                 inode->i_op = &shmem_symlink_inode_operations;
3052                 memcpy(page_address(page), symname, len);
3053                 SetPageUptodate(page);
3054                 set_page_dirty(page);
3055                 unlock_page(page);
3056                 put_page(page);
3057         }
3058         dir->i_size += BOGO_DIRENT_SIZE;
3059         dir->i_ctime = dir->i_mtime = current_time(dir);
3060         d_instantiate(dentry, inode);
3061         dget(dentry);
3062         return 0;
3063 }
3064
3065 static void shmem_put_link(void *arg)
3066 {
3067         mark_page_accessed(arg);
3068         put_page(arg);
3069 }
3070
3071 static const char *shmem_get_link(struct dentry *dentry,
3072                                   struct inode *inode,
3073                                   struct delayed_call *done)
3074 {
3075         struct page *page = NULL;
3076         int error;
3077         if (!dentry) {
3078                 page = find_get_page(inode->i_mapping, 0);
3079                 if (!page)
3080                         return ERR_PTR(-ECHILD);
3081                 if (PageHWPoison(page) ||
3082                     !PageUptodate(page)) {
3083                         put_page(page);
3084                         return ERR_PTR(-ECHILD);
3085                 }
3086         } else {
3087                 error = shmem_getpage(inode, 0, &page, SGP_READ);
3088                 if (error)
3089                         return ERR_PTR(error);
3090                 if (!page)
3091                         return ERR_PTR(-ECHILD);
3092                 if (PageHWPoison(page)) {
3093                         unlock_page(page);
3094                         put_page(page);
3095                         return ERR_PTR(-ECHILD);
3096                 }
3097                 unlock_page(page);
3098         }
3099         set_delayed_call(done, shmem_put_link, page);
3100         return page_address(page);
3101 }
3102
3103 #ifdef CONFIG_TMPFS_XATTR
3104 /*
3105  * Superblocks without xattr inode operations may get some security.* xattr
3106  * support from the LSM "for free". As soon as we have any other xattrs
3107  * like ACLs, we also need to implement the security.* handlers at
3108  * filesystem level, though.
3109  */
3110
3111 /*
3112  * Callback for security_inode_init_security() for acquiring xattrs.
3113  */
3114 static int shmem_initxattrs(struct inode *inode,
3115                             const struct xattr *xattr_array,
3116                             void *fs_info)
3117 {
3118         struct shmem_inode_info *info = SHMEM_I(inode);
3119         const struct xattr *xattr;
3120         struct simple_xattr *new_xattr;
3121         size_t len;
3122
3123         for (xattr = xattr_array; xattr->name != NULL; xattr++) {
3124                 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
3125                 if (!new_xattr)
3126                         return -ENOMEM;
3127
3128                 len = strlen(xattr->name) + 1;
3129                 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
3130                                           GFP_KERNEL);
3131                 if (!new_xattr->name) {
3132                         kvfree(new_xattr);
3133                         return -ENOMEM;
3134                 }
3135
3136                 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
3137                        XATTR_SECURITY_PREFIX_LEN);
3138                 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
3139                        xattr->name, len);
3140
3141                 simple_xattr_list_add(&info->xattrs, new_xattr);
3142         }
3143
3144         return 0;
3145 }
3146
3147 static int shmem_xattr_handler_get(const struct xattr_handler *handler,
3148                                    struct dentry *unused, struct inode *inode,
3149                                    const char *name, void *buffer, size_t size)
3150 {
3151         struct shmem_inode_info *info = SHMEM_I(inode);
3152
3153         name = xattr_full_name(handler, name);
3154         return simple_xattr_get(&info->xattrs, name, buffer, size);
3155 }
3156
3157 static int shmem_xattr_handler_set(const struct xattr_handler *handler,
3158                                    struct user_namespace *mnt_userns,
3159                                    struct dentry *unused, struct inode *inode,
3160                                    const char *name, const void *value,
3161                                    size_t size, int flags)
3162 {
3163         struct shmem_inode_info *info = SHMEM_I(inode);
3164
3165         name = xattr_full_name(handler, name);
3166         return simple_xattr_set(&info->xattrs, name, value, size, flags, NULL);
3167 }
3168
3169 static const struct xattr_handler shmem_security_xattr_handler = {
3170         .prefix = XATTR_SECURITY_PREFIX,
3171         .get = shmem_xattr_handler_get,
3172         .set = shmem_xattr_handler_set,
3173 };
3174
3175 static const struct xattr_handler shmem_trusted_xattr_handler = {
3176         .prefix = XATTR_TRUSTED_PREFIX,
3177         .get = shmem_xattr_handler_get,
3178         .set = shmem_xattr_handler_set,
3179 };
3180
3181 static const struct xattr_handler *shmem_xattr_handlers[] = {
3182 #ifdef CONFIG_TMPFS_POSIX_ACL
3183         &posix_acl_access_xattr_handler,
3184         &posix_acl_default_xattr_handler,
3185 #endif
3186         &shmem_security_xattr_handler,
3187         &shmem_trusted_xattr_handler,
3188         NULL
3189 };
3190
3191 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
3192 {
3193         struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3194         return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size);
3195 }
3196 #endif /* CONFIG_TMPFS_XATTR */
3197
3198 static const struct inode_operations shmem_short_symlink_operations = {
3199         .get_link       = simple_get_link,
3200 #ifdef CONFIG_TMPFS_XATTR
3201         .listxattr      = shmem_listxattr,
3202 #endif
3203 };
3204
3205 static const struct inode_operations shmem_symlink_inode_operations = {
3206         .get_link       = shmem_get_link,
3207 #ifdef CONFIG_TMPFS_XATTR
3208         .listxattr      = shmem_listxattr,
3209 #endif
3210 };
3211
3212 static struct dentry *shmem_get_parent(struct dentry *child)
3213 {
3214         return ERR_PTR(-ESTALE);
3215 }
3216
3217 static int shmem_match(struct inode *ino, void *vfh)
3218 {
3219         __u32 *fh = vfh;
3220         __u64 inum = fh[2];
3221         inum = (inum << 32) | fh[1];
3222         return ino->i_ino == inum && fh[0] == ino->i_generation;
3223 }
3224
3225 /* Find any alias of inode, but prefer a hashed alias */
3226 static struct dentry *shmem_find_alias(struct inode *inode)
3227 {
3228         struct dentry *alias = d_find_alias(inode);
3229
3230         return alias ?: d_find_any_alias(inode);
3231 }
3232
3233
3234 static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
3235                 struct fid *fid, int fh_len, int fh_type)
3236 {
3237         struct inode *inode;
3238         struct dentry *dentry = NULL;
3239         u64 inum;
3240
3241         if (fh_len < 3)
3242                 return NULL;
3243
3244         inum = fid->raw[2];
3245         inum = (inum << 32) | fid->raw[1];
3246
3247         inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
3248                         shmem_match, fid->raw);
3249         if (inode) {
3250                 dentry = shmem_find_alias(inode);
3251                 iput(inode);
3252         }
3253
3254         return dentry;
3255 }
3256
3257 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
3258                                 struct inode *parent)
3259 {
3260         if (*len < 3) {
3261                 *len = 3;
3262                 return FILEID_INVALID;
3263         }
3264
3265         if (inode_unhashed(inode)) {
3266                 /* Unfortunately insert_inode_hash is not idempotent,
3267                  * so as we hash inodes here rather than at creation
3268                  * time, we need a lock to ensure we only try
3269                  * to do it once
3270                  */
3271                 static DEFINE_SPINLOCK(lock);
3272                 spin_lock(&lock);
3273                 if (inode_unhashed(inode))
3274                         __insert_inode_hash(inode,
3275                                             inode->i_ino + inode->i_generation);
3276                 spin_unlock(&lock);
3277         }
3278
3279         fh[0] = inode->i_generation;
3280         fh[1] = inode->i_ino;
3281         fh[2] = ((__u64)inode->i_ino) >> 32;
3282
3283         *len = 3;
3284         return 1;
3285 }
3286
3287 static const struct export_operations shmem_export_ops = {
3288         .get_parent     = shmem_get_parent,
3289         .encode_fh      = shmem_encode_fh,
3290         .fh_to_dentry   = shmem_fh_to_dentry,
3291 };
3292
3293 enum shmem_param {
3294         Opt_gid,
3295         Opt_huge,
3296         Opt_mode,
3297         Opt_mpol,
3298         Opt_nr_blocks,
3299         Opt_nr_inodes,
3300         Opt_size,
3301         Opt_uid,
3302         Opt_inode32,
3303         Opt_inode64,
3304 };
3305
3306 static const struct constant_table shmem_param_enums_huge[] = {
3307         {"never",       SHMEM_HUGE_NEVER },
3308         {"always",      SHMEM_HUGE_ALWAYS },
3309         {"within_size", SHMEM_HUGE_WITHIN_SIZE },
3310         {"advise",      SHMEM_HUGE_ADVISE },
3311         {}
3312 };
3313
3314 const struct fs_parameter_spec shmem_fs_parameters[] = {
3315         fsparam_u32   ("gid",           Opt_gid),
3316         fsparam_enum  ("huge",          Opt_huge,  shmem_param_enums_huge),
3317         fsparam_u32oct("mode",          Opt_mode),
3318         fsparam_string("mpol",          Opt_mpol),
3319         fsparam_string("nr_blocks",     Opt_nr_blocks),
3320         fsparam_string("nr_inodes",     Opt_nr_inodes),
3321         fsparam_string("size",          Opt_size),
3322         fsparam_u32   ("uid",           Opt_uid),
3323         fsparam_flag  ("inode32",       Opt_inode32),
3324         fsparam_flag  ("inode64",       Opt_inode64),
3325         {}
3326 };
3327
3328 static int shmem_parse_one(struct fs_context *fc, struct fs_parameter *param)
3329 {
3330         struct shmem_options *ctx = fc->fs_private;
3331         struct fs_parse_result result;
3332         unsigned long long size;
3333         char *rest;
3334         int opt;
3335
3336         opt = fs_parse(fc, shmem_fs_parameters, param, &result);
3337         if (opt < 0)
3338                 return opt;
3339
3340         switch (opt) {
3341         case Opt_size:
3342                 size = memparse(param->string, &rest);
3343                 if (*rest == '%') {
3344                         size <<= PAGE_SHIFT;
3345                         size *= totalram_pages();
3346                         do_div(size, 100);
3347                         rest++;
3348                 }
3349                 if (*rest)
3350                         goto bad_value;
3351                 ctx->blocks = DIV_ROUND_UP(size, PAGE_SIZE);
3352                 ctx->seen |= SHMEM_SEEN_BLOCKS;
3353                 break;
3354         case Opt_nr_blocks:
3355                 ctx->blocks = memparse(param->string, &rest);
3356                 if (*rest)
3357                         goto bad_value;
3358                 ctx->seen |= SHMEM_SEEN_BLOCKS;
3359                 break;
3360         case Opt_nr_inodes:
3361                 ctx->inodes = memparse(param->string, &rest);
3362                 if (*rest)
3363                         goto bad_value;
3364                 ctx->seen |= SHMEM_SEEN_INODES;
3365                 break;
3366         case Opt_mode:
3367                 ctx->mode = result.uint_32 & 07777;
3368                 break;
3369         case Opt_uid:
3370                 ctx->uid = make_kuid(current_user_ns(), result.uint_32);
3371                 if (!uid_valid(ctx->uid))
3372                         goto bad_value;
3373                 break;
3374         case Opt_gid:
3375                 ctx->gid = make_kgid(current_user_ns(), result.uint_32);
3376                 if (!gid_valid(ctx->gid))
3377                         goto bad_value;
3378                 break;
3379         case Opt_huge:
3380                 ctx->huge = result.uint_32;
3381                 if (ctx->huge != SHMEM_HUGE_NEVER &&
3382                     !(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
3383                       has_transparent_hugepage()))
3384                         goto unsupported_parameter;
3385                 ctx->seen |= SHMEM_SEEN_HUGE;
3386                 break;
3387         case Opt_mpol:
3388                 if (IS_ENABLED(CONFIG_NUMA)) {
3389                         mpol_put(ctx->mpol);
3390                         ctx->mpol = NULL;
3391                         if (mpol_parse_str(param->string, &ctx->mpol))
3392                                 goto bad_value;
3393                         break;
3394                 }
3395                 goto unsupported_parameter;
3396         case Opt_inode32:
3397                 ctx->full_inums = false;
3398                 ctx->seen |= SHMEM_SEEN_INUMS;
3399                 break;
3400         case Opt_inode64:
3401                 if (sizeof(ino_t) < 8) {
3402                         return invalfc(fc,
3403                                        "Cannot use inode64 with <64bit inums in kernel\n");
3404                 }
3405                 ctx->full_inums = true;
3406                 ctx->seen |= SHMEM_SEEN_INUMS;
3407                 break;
3408         }
3409         return 0;
3410
3411 unsupported_parameter:
3412         return invalfc(fc, "Unsupported parameter '%s'", param->key);
3413 bad_value:
3414         return invalfc(fc, "Bad value for '%s'", param->key);
3415 }
3416
3417 static int shmem_parse_options(struct fs_context *fc, void *data)
3418 {
3419         char *options = data;
3420
3421         if (options) {
3422                 int err = security_sb_eat_lsm_opts(options, &fc->security);
3423                 if (err)
3424                         return err;
3425         }
3426
3427         while (options != NULL) {
3428                 char *this_char = options;
3429                 for (;;) {
3430                         /*
3431                          * NUL-terminate this option: unfortunately,
3432                          * mount options form a comma-separated list,
3433                          * but mpol's nodelist may also contain commas.
3434                          */
3435                         options = strchr(options, ',');
3436                         if (options == NULL)
3437                                 break;
3438                         options++;
3439                         if (!isdigit(*options)) {
3440                                 options[-1] = '\0';
3441                                 break;
3442                         }
3443                 }
3444                 if (*this_char) {
3445                         char *value = strchr(this_char, '=');
3446                         size_t len = 0;
3447                         int err;
3448
3449                         if (value) {
3450                                 *value++ = '\0';
3451                                 len = strlen(value);
3452                         }
3453                         err = vfs_parse_fs_string(fc, this_char, value, len);
3454                         if (err < 0)
3455                                 return err;
3456                 }
3457         }
3458         return 0;
3459 }
3460
3461 /*
3462  * Reconfigure a shmem filesystem.
3463  *
3464  * Note that we disallow change from limited->unlimited blocks/inodes while any
3465  * are in use; but we must separately disallow unlimited->limited, because in
3466  * that case we have no record of how much is already in use.
3467  */
3468 static int shmem_reconfigure(struct fs_context *fc)
3469 {
3470         struct shmem_options *ctx = fc->fs_private;
3471         struct shmem_sb_info *sbinfo = SHMEM_SB(fc->root->d_sb);
3472         unsigned long inodes;
3473         struct mempolicy *mpol = NULL;
3474         const char *err;
3475
3476         raw_spin_lock(&sbinfo->stat_lock);
3477         inodes = sbinfo->max_inodes - sbinfo->free_inodes;
3478         if ((ctx->seen & SHMEM_SEEN_BLOCKS) && ctx->blocks) {
3479                 if (!sbinfo->max_blocks) {
3480                         err = "Cannot retroactively limit size";
3481                         goto out;
3482                 }
3483                 if (percpu_counter_compare(&sbinfo->used_blocks,
3484                                            ctx->blocks) > 0) {
3485                         err = "Too small a size for current use";
3486                         goto out;
3487                 }
3488         }
3489         if ((ctx->seen & SHMEM_SEEN_INODES) && ctx->inodes) {
3490                 if (!sbinfo->max_inodes) {
3491                         err = "Cannot retroactively limit inodes";
3492                         goto out;
3493                 }
3494                 if (ctx->inodes < inodes) {
3495                         err = "Too few inodes for current use";
3496                         goto out;
3497                 }
3498         }
3499
3500         if ((ctx->seen & SHMEM_SEEN_INUMS) && !ctx->full_inums &&
3501             sbinfo->next_ino > UINT_MAX) {
3502                 err = "Current inum too high to switch to 32-bit inums";
3503                 goto out;
3504         }
3505
3506         if (ctx->seen & SHMEM_SEEN_HUGE)
3507                 sbinfo->huge = ctx->huge;
3508         if (ctx->seen & SHMEM_SEEN_INUMS)
3509                 sbinfo->full_inums = ctx->full_inums;
3510         if (ctx->seen & SHMEM_SEEN_BLOCKS)
3511                 sbinfo->max_blocks  = ctx->blocks;
3512         if (ctx->seen & SHMEM_SEEN_INODES) {
3513                 sbinfo->max_inodes  = ctx->inodes;
3514                 sbinfo->free_inodes = ctx->inodes - inodes;
3515         }
3516
3517         /*
3518          * Preserve previous mempolicy unless mpol remount option was specified.
3519          */
3520         if (ctx->mpol) {
3521                 mpol = sbinfo->mpol;
3522                 sbinfo->mpol = ctx->mpol;       /* transfers initial ref */
3523                 ctx->mpol = NULL;
3524         }
3525         raw_spin_unlock(&sbinfo->stat_lock);
3526         mpol_put(mpol);
3527         return 0;
3528 out:
3529         raw_spin_unlock(&sbinfo->stat_lock);
3530         return invalfc(fc, "%s", err);
3531 }
3532
3533 static int shmem_show_options(struct seq_file *seq, struct dentry *root)
3534 {
3535         struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
3536
3537         if (sbinfo->max_blocks != shmem_default_max_blocks())
3538                 seq_printf(seq, ",size=%luk",
3539                         sbinfo->max_blocks << (PAGE_SHIFT - 10));
3540         if (sbinfo->max_inodes != shmem_default_max_inodes())
3541                 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
3542         if (sbinfo->mode != (0777 | S_ISVTX))
3543                 seq_printf(seq, ",mode=%03ho", sbinfo->mode);
3544         if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
3545                 seq_printf(seq, ",uid=%u",
3546                                 from_kuid_munged(&init_user_ns, sbinfo->uid));
3547         if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
3548                 seq_printf(seq, ",gid=%u",
3549                                 from_kgid_munged(&init_user_ns, sbinfo->gid));
3550
3551         /*
3552          * Showing inode{64,32} might be useful even if it's the system default,
3553          * since then people don't have to resort to checking both here and
3554          * /proc/config.gz to confirm 64-bit inums were successfully applied
3555          * (which may not even exist if IKCONFIG_PROC isn't enabled).
3556          *
3557          * We hide it when inode64 isn't the default and we are using 32-bit
3558          * inodes, since that probably just means the feature isn't even under
3559          * consideration.
3560          *
3561          * As such:
3562          *
3563          *                     +-----------------+-----------------+
3564          *                     | TMPFS_INODE64=y | TMPFS_INODE64=n |
3565          *  +------------------+-----------------+-----------------+
3566          *  | full_inums=true  | show            | show            |
3567          *  | full_inums=false | show            | hide            |
3568          *  +------------------+-----------------+-----------------+
3569          *
3570          */
3571         if (IS_ENABLED(CONFIG_TMPFS_INODE64) || sbinfo->full_inums)
3572                 seq_printf(seq, ",inode%d", (sbinfo->full_inums ? 64 : 32));
3573 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3574         /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */
3575         if (sbinfo->huge)
3576                 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge));
3577 #endif
3578         shmem_show_mpol(seq, sbinfo->mpol);
3579         return 0;
3580 }
3581
3582 #endif /* CONFIG_TMPFS */
3583
3584 static void shmem_put_super(struct super_block *sb)
3585 {
3586         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
3587
3588         free_percpu(sbinfo->ino_batch);
3589         percpu_counter_destroy(&sbinfo->used_blocks);
3590         mpol_put(sbinfo->mpol);
3591         kfree(sbinfo);
3592         sb->s_fs_info = NULL;
3593 }
3594
3595 static int shmem_fill_super(struct super_block *sb, struct fs_context *fc)
3596 {
3597         struct shmem_options *ctx = fc->fs_private;
3598         struct inode *inode;
3599         struct shmem_sb_info *sbinfo;
3600
3601         /* Round up to L1_CACHE_BYTES to resist false sharing */
3602         sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
3603                                 L1_CACHE_BYTES), GFP_KERNEL);
3604         if (!sbinfo)
3605                 return -ENOMEM;
3606
3607         sb->s_fs_info = sbinfo;
3608
3609 #ifdef CONFIG_TMPFS
3610         /*
3611          * Per default we only allow half of the physical ram per
3612          * tmpfs instance, limiting inodes to one per page of lowmem;
3613          * but the internal instance is left unlimited.
3614          */
3615         if (!(sb->s_flags & SB_KERNMOUNT)) {
3616                 if (!(ctx->seen & SHMEM_SEEN_BLOCKS))
3617                         ctx->blocks = shmem_default_max_blocks();
3618                 if (!(ctx->seen & SHMEM_SEEN_INODES))
3619                         ctx->inodes = shmem_default_max_inodes();
3620                 if (!(ctx->seen & SHMEM_SEEN_INUMS))
3621                         ctx->full_inums = IS_ENABLED(CONFIG_TMPFS_INODE64);
3622         } else {
3623                 sb->s_flags |= SB_NOUSER;
3624         }
3625         sb->s_export_op = &shmem_export_ops;
3626         sb->s_flags |= SB_NOSEC;
3627 #else
3628         sb->s_flags |= SB_NOUSER;
3629 #endif
3630         sbinfo->max_blocks = ctx->blocks;
3631         sbinfo->free_inodes = sbinfo->max_inodes = ctx->inodes;
3632         if (sb->s_flags & SB_KERNMOUNT) {
3633                 sbinfo->ino_batch = alloc_percpu(ino_t);
3634                 if (!sbinfo->ino_batch)
3635                         goto failed;
3636         }
3637         sbinfo->uid = ctx->uid;
3638         sbinfo->gid = ctx->gid;
3639         sbinfo->full_inums = ctx->full_inums;
3640         sbinfo->mode = ctx->mode;
3641         sbinfo->huge = ctx->huge;
3642         sbinfo->mpol = ctx->mpol;
3643         ctx->mpol = NULL;
3644
3645         raw_spin_lock_init(&sbinfo->stat_lock);
3646         if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL))
3647                 goto failed;
3648         spin_lock_init(&sbinfo->shrinklist_lock);
3649         INIT_LIST_HEAD(&sbinfo->shrinklist);
3650
3651         sb->s_maxbytes = MAX_LFS_FILESIZE;
3652         sb->s_blocksize = PAGE_SIZE;
3653         sb->s_blocksize_bits = PAGE_SHIFT;
3654         sb->s_magic = TMPFS_MAGIC;
3655         sb->s_op = &shmem_ops;
3656         sb->s_time_gran = 1;
3657 #ifdef CONFIG_TMPFS_XATTR
3658         sb->s_xattr = shmem_xattr_handlers;
3659 #endif
3660 #ifdef CONFIG_TMPFS_POSIX_ACL
3661         sb->s_flags |= SB_POSIXACL;
3662 #endif
3663         uuid_gen(&sb->s_uuid);
3664
3665         inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
3666         if (!inode)
3667                 goto failed;
3668         inode->i_uid = sbinfo->uid;
3669         inode->i_gid = sbinfo->gid;
3670         sb->s_root = d_make_root(inode);
3671         if (!sb->s_root)
3672                 goto failed;
3673         return 0;
3674
3675 failed:
3676         shmem_put_super(sb);
3677         return -ENOMEM;
3678 }
3679
3680 static int shmem_get_tree(struct fs_context *fc)
3681 {
3682         return get_tree_nodev(fc, shmem_fill_super);
3683 }
3684
3685 static void shmem_free_fc(struct fs_context *fc)
3686 {
3687         struct shmem_options *ctx = fc->fs_private;
3688
3689         if (ctx) {
3690                 mpol_put(ctx->mpol);
3691                 kfree(ctx);
3692         }
3693 }
3694
3695 static const struct fs_context_operations shmem_fs_context_ops = {
3696         .free                   = shmem_free_fc,
3697         .get_tree               = shmem_get_tree,
3698 #ifdef CONFIG_TMPFS
3699         .parse_monolithic       = shmem_parse_options,
3700         .parse_param            = shmem_parse_one,
3701         .reconfigure            = shmem_reconfigure,
3702 #endif
3703 };
3704
3705 static struct kmem_cache *shmem_inode_cachep;
3706
3707 static struct inode *shmem_alloc_inode(struct super_block *sb)
3708 {
3709         struct shmem_inode_info *info;
3710         info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
3711         if (!info)
3712                 return NULL;
3713         return &info->vfs_inode;
3714 }
3715
3716 static void shmem_free_in_core_inode(struct inode *inode)
3717 {
3718         if (S_ISLNK(inode->i_mode))
3719                 kfree(inode->i_link);
3720         kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
3721 }
3722
3723 static void shmem_destroy_inode(struct inode *inode)
3724 {
3725         if (S_ISREG(inode->i_mode))
3726                 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
3727 }
3728
3729 static void shmem_init_inode(void *foo)
3730 {
3731         struct shmem_inode_info *info = foo;
3732         inode_init_once(&info->vfs_inode);
3733 }
3734
3735 static void shmem_init_inodecache(void)
3736 {
3737         shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
3738                                 sizeof(struct shmem_inode_info),
3739                                 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode);
3740 }
3741
3742 static void shmem_destroy_inodecache(void)
3743 {
3744         kmem_cache_destroy(shmem_inode_cachep);
3745 }
3746
3747 /* Keep the page in page cache instead of truncating it */
3748 static int shmem_error_remove_page(struct address_space *mapping,
3749                                    struct page *page)
3750 {
3751         return 0;
3752 }
3753
3754 const struct address_space_operations shmem_aops = {
3755         .writepage      = shmem_writepage,
3756         .set_page_dirty = __set_page_dirty_no_writeback,
3757 #ifdef CONFIG_TMPFS
3758         .write_begin    = shmem_write_begin,
3759         .write_end      = shmem_write_end,
3760 #endif
3761 #ifdef CONFIG_MIGRATION
3762         .migratepage    = migrate_page,
3763 #endif
3764         .error_remove_page = shmem_error_remove_page,
3765 };
3766 EXPORT_SYMBOL(shmem_aops);
3767
3768 static const struct file_operations shmem_file_operations = {
3769         .mmap           = shmem_mmap,
3770         .get_unmapped_area = shmem_get_unmapped_area,
3771 #ifdef CONFIG_TMPFS
3772         .llseek         = shmem_file_llseek,
3773         .read_iter      = shmem_file_read_iter,
3774         .write_iter     = generic_file_write_iter,
3775         .fsync          = noop_fsync,
3776         .splice_read    = generic_file_splice_read,
3777         .splice_write   = iter_file_splice_write,
3778         .fallocate      = shmem_fallocate,
3779 #endif
3780 };
3781
3782 static const struct inode_operations shmem_inode_operations = {
3783         .getattr        = shmem_getattr,
3784         .setattr        = shmem_setattr,
3785 #ifdef CONFIG_TMPFS_XATTR
3786         .listxattr      = shmem_listxattr,
3787         .set_acl        = simple_set_acl,
3788 #endif
3789 };
3790
3791 static const struct inode_operations shmem_dir_inode_operations = {
3792 #ifdef CONFIG_TMPFS
3793         .create         = shmem_create,
3794         .lookup         = simple_lookup,
3795         .link           = shmem_link,
3796         .unlink         = shmem_unlink,
3797         .symlink        = shmem_symlink,
3798         .mkdir          = shmem_mkdir,
3799         .rmdir          = shmem_rmdir,
3800         .mknod          = shmem_mknod,
3801         .rename         = shmem_rename2,
3802         .tmpfile        = shmem_tmpfile,
3803 #endif
3804 #ifdef CONFIG_TMPFS_XATTR
3805         .listxattr      = shmem_listxattr,
3806 #endif
3807 #ifdef CONFIG_TMPFS_POSIX_ACL
3808         .setattr        = shmem_setattr,
3809         .set_acl        = simple_set_acl,
3810 #endif
3811 };
3812
3813 static const struct inode_operations shmem_special_inode_operations = {
3814 #ifdef CONFIG_TMPFS_XATTR
3815         .listxattr      = shmem_listxattr,
3816 #endif
3817 #ifdef CONFIG_TMPFS_POSIX_ACL
3818         .setattr        = shmem_setattr,
3819         .set_acl        = simple_set_acl,
3820 #endif
3821 };
3822
3823 static const struct super_operations shmem_ops = {
3824         .alloc_inode    = shmem_alloc_inode,
3825         .free_inode     = shmem_free_in_core_inode,
3826         .destroy_inode  = shmem_destroy_inode,
3827 #ifdef CONFIG_TMPFS
3828         .statfs         = shmem_statfs,
3829         .show_options   = shmem_show_options,
3830 #endif
3831         .evict_inode    = shmem_evict_inode,
3832         .drop_inode     = generic_delete_inode,
3833         .put_super      = shmem_put_super,
3834 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3835         .nr_cached_objects      = shmem_unused_huge_count,
3836         .free_cached_objects    = shmem_unused_huge_scan,
3837 #endif
3838 };
3839
3840 static const struct vm_operations_struct shmem_vm_ops = {
3841         .fault          = shmem_fault,
3842         .map_pages      = filemap_map_pages,
3843 #ifdef CONFIG_NUMA
3844         .set_policy     = shmem_set_policy,
3845         .get_policy     = shmem_get_policy,
3846 #endif
3847 };
3848
3849 int shmem_init_fs_context(struct fs_context *fc)
3850 {
3851         struct shmem_options *ctx;
3852
3853         ctx = kzalloc(sizeof(struct shmem_options), GFP_KERNEL);
3854         if (!ctx)
3855                 return -ENOMEM;
3856
3857         ctx->mode = 0777 | S_ISVTX;
3858         ctx->uid = current_fsuid();
3859         ctx->gid = current_fsgid();
3860
3861         fc->fs_private = ctx;
3862         fc->ops = &shmem_fs_context_ops;
3863         return 0;
3864 }
3865
3866 static struct file_system_type shmem_fs_type = {
3867         .owner          = THIS_MODULE,
3868         .name           = "tmpfs",
3869         .init_fs_context = shmem_init_fs_context,
3870 #ifdef CONFIG_TMPFS
3871         .parameters     = shmem_fs_parameters,
3872 #endif
3873         .kill_sb        = kill_litter_super,
3874         .fs_flags       = FS_USERNS_MOUNT,
3875 };
3876
3877 int __init shmem_init(void)
3878 {
3879         int error;
3880
3881         shmem_init_inodecache();
3882
3883         error = register_filesystem(&shmem_fs_type);
3884         if (error) {
3885                 pr_err("Could not register tmpfs\n");
3886                 goto out2;
3887         }
3888
3889         shm_mnt = kern_mount(&shmem_fs_type);
3890         if (IS_ERR(shm_mnt)) {
3891                 error = PTR_ERR(shm_mnt);
3892                 pr_err("Could not kern_mount tmpfs\n");
3893                 goto out1;
3894         }
3895
3896 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3897         if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY)
3898                 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
3899         else
3900                 shmem_huge = SHMEM_HUGE_NEVER; /* just in case it was patched */
3901 #endif
3902         return 0;
3903
3904 out1:
3905         unregister_filesystem(&shmem_fs_type);
3906 out2:
3907         shmem_destroy_inodecache();
3908         shm_mnt = ERR_PTR(error);
3909         return error;
3910 }
3911
3912 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS)
3913 static ssize_t shmem_enabled_show(struct kobject *kobj,
3914                                   struct kobj_attribute *attr, char *buf)
3915 {
3916         static const int values[] = {
3917                 SHMEM_HUGE_ALWAYS,
3918                 SHMEM_HUGE_WITHIN_SIZE,
3919                 SHMEM_HUGE_ADVISE,
3920                 SHMEM_HUGE_NEVER,
3921                 SHMEM_HUGE_DENY,
3922                 SHMEM_HUGE_FORCE,
3923         };
3924         int len = 0;
3925         int i;
3926
3927         for (i = 0; i < ARRAY_SIZE(values); i++) {
3928                 len += sysfs_emit_at(buf, len,
3929                                      shmem_huge == values[i] ? "%s[%s]" : "%s%s",
3930                                      i ? " " : "",
3931                                      shmem_format_huge(values[i]));
3932         }
3933
3934         len += sysfs_emit_at(buf, len, "\n");
3935
3936         return len;
3937 }
3938
3939 static ssize_t shmem_enabled_store(struct kobject *kobj,
3940                 struct kobj_attribute *attr, const char *buf, size_t count)
3941 {
3942         char tmp[16];
3943         int huge;
3944
3945         if (count + 1 > sizeof(tmp))
3946                 return -EINVAL;
3947         memcpy(tmp, buf, count);
3948         tmp[count] = '\0';
3949         if (count && tmp[count - 1] == '\n')
3950                 tmp[count - 1] = '\0';
3951
3952         huge = shmem_parse_huge(tmp);
3953         if (huge == -EINVAL)
3954                 return -EINVAL;
3955         if (!has_transparent_hugepage() &&
3956                         huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY)
3957                 return -EINVAL;
3958
3959         shmem_huge = huge;
3960         if (shmem_huge > SHMEM_HUGE_DENY)
3961                 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
3962         return count;
3963 }
3964
3965 struct kobj_attribute shmem_enabled_attr =
3966         __ATTR(shmem_enabled, 0644, shmem_enabled_show, shmem_enabled_store);
3967 #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_SYSFS */
3968
3969 #else /* !CONFIG_SHMEM */
3970
3971 /*
3972  * tiny-shmem: simple shmemfs and tmpfs using ramfs code
3973  *
3974  * This is intended for small system where the benefits of the full
3975  * shmem code (swap-backed and resource-limited) are outweighed by
3976  * their complexity. On systems without swap this code should be
3977  * effectively equivalent, but much lighter weight.
3978  */
3979
3980 static struct file_system_type shmem_fs_type = {
3981         .name           = "tmpfs",
3982         .init_fs_context = ramfs_init_fs_context,
3983         .parameters     = ramfs_fs_parameters,
3984         .kill_sb        = kill_litter_super,
3985         .fs_flags       = FS_USERNS_MOUNT,
3986 };
3987
3988 int __init shmem_init(void)
3989 {
3990         BUG_ON(register_filesystem(&shmem_fs_type) != 0);
3991
3992         shm_mnt = kern_mount(&shmem_fs_type);
3993         BUG_ON(IS_ERR(shm_mnt));
3994
3995         return 0;
3996 }
3997
3998 int shmem_unuse(unsigned int type)
3999 {
4000         return 0;
4001 }
4002
4003 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)
4004 {
4005         return 0;
4006 }
4007
4008 void shmem_unlock_mapping(struct address_space *mapping)
4009 {
4010 }
4011
4012 #ifdef CONFIG_MMU
4013 unsigned long shmem_get_unmapped_area(struct file *file,
4014                                       unsigned long addr, unsigned long len,
4015                                       unsigned long pgoff, unsigned long flags)
4016 {
4017         return current->mm->get_unmapped_area(file, addr, len, pgoff, flags);
4018 }
4019 #endif
4020
4021 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
4022 {
4023         truncate_inode_pages_range(inode->i_mapping, lstart, lend);
4024 }
4025 EXPORT_SYMBOL_GPL(shmem_truncate_range);
4026
4027 #define shmem_vm_ops                            generic_file_vm_ops
4028 #define shmem_file_operations                   ramfs_file_operations
4029 #define shmem_get_inode(sb, dir, mode, dev, flags)      ramfs_get_inode(sb, dir, mode, dev)
4030 #define shmem_acct_size(flags, size)            0
4031 #define shmem_unacct_size(flags, size)          do {} while (0)
4032
4033 #endif /* CONFIG_SHMEM */
4034
4035 /* common code */
4036
4037 static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name, loff_t size,
4038                                        unsigned long flags, unsigned int i_flags)
4039 {
4040         struct inode *inode;
4041         struct file *res;
4042
4043         if (IS_ERR(mnt))
4044                 return ERR_CAST(mnt);
4045
4046         if (size < 0 || size > MAX_LFS_FILESIZE)
4047                 return ERR_PTR(-EINVAL);
4048
4049         if (shmem_acct_size(flags, size))
4050                 return ERR_PTR(-ENOMEM);
4051
4052         inode = shmem_get_inode(mnt->mnt_sb, NULL, S_IFREG | S_IRWXUGO, 0,
4053                                 flags);
4054         if (unlikely(!inode)) {
4055                 shmem_unacct_size(flags, size);
4056                 return ERR_PTR(-ENOSPC);
4057         }
4058         inode->i_flags |= i_flags;
4059         inode->i_size = size;
4060         clear_nlink(inode);     /* It is unlinked */
4061         res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size));
4062         if (!IS_ERR(res))
4063                 res = alloc_file_pseudo(inode, mnt, name, O_RDWR,
4064                                 &shmem_file_operations);
4065         if (IS_ERR(res))
4066                 iput(inode);
4067         return res;
4068 }
4069
4070 /**
4071  * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be
4072  *      kernel internal.  There will be NO LSM permission checks against the
4073  *      underlying inode.  So users of this interface must do LSM checks at a
4074  *      higher layer.  The users are the big_key and shm implementations.  LSM
4075  *      checks are provided at the key or shm level rather than the inode.
4076  * @name: name for dentry (to be seen in /proc/<pid>/maps
4077  * @size: size to be set for the file
4078  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4079  */
4080 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags)
4081 {
4082         return __shmem_file_setup(shm_mnt, name, size, flags, S_PRIVATE);
4083 }
4084
4085 /**
4086  * shmem_file_setup - get an unlinked file living in tmpfs
4087  * @name: name for dentry (to be seen in /proc/<pid>/maps
4088  * @size: size to be set for the file
4089  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4090  */
4091 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
4092 {
4093         return __shmem_file_setup(shm_mnt, name, size, flags, 0);
4094 }
4095 EXPORT_SYMBOL_GPL(shmem_file_setup);
4096
4097 /**
4098  * shmem_file_setup_with_mnt - get an unlinked file living in tmpfs
4099  * @mnt: the tmpfs mount where the file will be created
4100  * @name: name for dentry (to be seen in /proc/<pid>/maps
4101  * @size: size to be set for the file
4102  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4103  */
4104 struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name,
4105                                        loff_t size, unsigned long flags)
4106 {
4107         return __shmem_file_setup(mnt, name, size, flags, 0);
4108 }
4109 EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt);
4110
4111 /**
4112  * shmem_zero_setup - setup a shared anonymous mapping
4113  * @vma: the vma to be mmapped is prepared by do_mmap
4114  */
4115 int shmem_zero_setup(struct vm_area_struct *vma)
4116 {
4117         struct file *file;
4118         loff_t size = vma->vm_end - vma->vm_start;
4119
4120         /*
4121          * Cloning a new file under mmap_lock leads to a lock ordering conflict
4122          * between XFS directory reading and selinux: since this file is only
4123          * accessible to the user through its mapping, use S_PRIVATE flag to
4124          * bypass file security, in the same way as shmem_kernel_file_setup().
4125          */
4126         file = shmem_kernel_file_setup("dev/zero", size, vma->vm_flags);
4127         if (IS_ERR(file))
4128                 return PTR_ERR(file);
4129
4130         if (vma->vm_file)
4131                 fput(vma->vm_file);
4132         vma->vm_file = file;
4133         vma->vm_ops = &shmem_vm_ops;
4134
4135         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
4136                         ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
4137                         (vma->vm_end & HPAGE_PMD_MASK)) {
4138                 khugepaged_enter(vma, vma->vm_flags);
4139         }
4140
4141         return 0;
4142 }
4143
4144 /**
4145  * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
4146  * @mapping:    the page's address_space
4147  * @index:      the page index
4148  * @gfp:        the page allocator flags to use if allocating
4149  *
4150  * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
4151  * with any new page allocations done using the specified allocation flags.
4152  * But read_cache_page_gfp() uses the ->readpage() method: which does not
4153  * suit tmpfs, since it may have pages in swapcache, and needs to find those
4154  * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
4155  *
4156  * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
4157  * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
4158  */
4159 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
4160                                          pgoff_t index, gfp_t gfp)
4161 {
4162 #ifdef CONFIG_SHMEM
4163         struct inode *inode = mapping->host;
4164         struct page *page;
4165         int error;
4166
4167         BUG_ON(!shmem_mapping(mapping));
4168         error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE,
4169                                   gfp, NULL, NULL, NULL);
4170         if (error)
4171                 return ERR_PTR(error);
4172
4173         unlock_page(page);
4174         if (PageHWPoison(page)) {
4175                 put_page(page);
4176                 return ERR_PTR(-EIO);
4177         }
4178
4179         return page;
4180 #else
4181         /*
4182          * The tiny !SHMEM case uses ramfs without swap
4183          */
4184         return read_cache_page_gfp(mapping, index, gfp);
4185 #endif
4186 }
4187 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);