Merge tag 'hexagon-5.13-0' of git://git.kernel.org/pub/scm/linux/kernel/git/bcain...
[linux-2.6-microblaze.git] / mm / hugetlb.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Generic hugetlb support.
4  * (C) Nadia Yvette Chambers, April 2004
5  */
6 #include <linux/list.h>
7 #include <linux/init.h>
8 #include <linux/mm.h>
9 #include <linux/seq_file.h>
10 #include <linux/sysctl.h>
11 #include <linux/highmem.h>
12 #include <linux/mmu_notifier.h>
13 #include <linux/nodemask.h>
14 #include <linux/pagemap.h>
15 #include <linux/mempolicy.h>
16 #include <linux/compiler.h>
17 #include <linux/cpuset.h>
18 #include <linux/mutex.h>
19 #include <linux/memblock.h>
20 #include <linux/sysfs.h>
21 #include <linux/slab.h>
22 #include <linux/sched/mm.h>
23 #include <linux/mmdebug.h>
24 #include <linux/sched/signal.h>
25 #include <linux/rmap.h>
26 #include <linux/string_helpers.h>
27 #include <linux/swap.h>
28 #include <linux/swapops.h>
29 #include <linux/jhash.h>
30 #include <linux/numa.h>
31 #include <linux/llist.h>
32 #include <linux/cma.h>
33
34 #include <asm/page.h>
35 #include <asm/pgalloc.h>
36 #include <asm/tlb.h>
37
38 #include <linux/io.h>
39 #include <linux/hugetlb.h>
40 #include <linux/hugetlb_cgroup.h>
41 #include <linux/node.h>
42 #include <linux/page_owner.h>
43 #include "internal.h"
44
45 int hugetlb_max_hstate __read_mostly;
46 unsigned int default_hstate_idx;
47 struct hstate hstates[HUGE_MAX_HSTATE];
48
49 #ifdef CONFIG_CMA
50 static struct cma *hugetlb_cma[MAX_NUMNODES];
51 #endif
52 static unsigned long hugetlb_cma_size __initdata;
53
54 /*
55  * Minimum page order among possible hugepage sizes, set to a proper value
56  * at boot time.
57  */
58 static unsigned int minimum_order __read_mostly = UINT_MAX;
59
60 __initdata LIST_HEAD(huge_boot_pages);
61
62 /* for command line parsing */
63 static struct hstate * __initdata parsed_hstate;
64 static unsigned long __initdata default_hstate_max_huge_pages;
65 static bool __initdata parsed_valid_hugepagesz = true;
66 static bool __initdata parsed_default_hugepagesz;
67
68 /*
69  * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
70  * free_huge_pages, and surplus_huge_pages.
71  */
72 DEFINE_SPINLOCK(hugetlb_lock);
73
74 /*
75  * Serializes faults on the same logical page.  This is used to
76  * prevent spurious OOMs when the hugepage pool is fully utilized.
77  */
78 static int num_fault_mutexes;
79 struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;
80
81 /* Forward declaration */
82 static int hugetlb_acct_memory(struct hstate *h, long delta);
83
84 static inline bool subpool_is_free(struct hugepage_subpool *spool)
85 {
86         if (spool->count)
87                 return false;
88         if (spool->max_hpages != -1)
89                 return spool->used_hpages == 0;
90         if (spool->min_hpages != -1)
91                 return spool->rsv_hpages == spool->min_hpages;
92
93         return true;
94 }
95
96 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool,
97                                                 unsigned long irq_flags)
98 {
99         spin_unlock_irqrestore(&spool->lock, irq_flags);
100
101         /* If no pages are used, and no other handles to the subpool
102          * remain, give up any reservations based on minimum size and
103          * free the subpool */
104         if (subpool_is_free(spool)) {
105                 if (spool->min_hpages != -1)
106                         hugetlb_acct_memory(spool->hstate,
107                                                 -spool->min_hpages);
108                 kfree(spool);
109         }
110 }
111
112 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
113                                                 long min_hpages)
114 {
115         struct hugepage_subpool *spool;
116
117         spool = kzalloc(sizeof(*spool), GFP_KERNEL);
118         if (!spool)
119                 return NULL;
120
121         spin_lock_init(&spool->lock);
122         spool->count = 1;
123         spool->max_hpages = max_hpages;
124         spool->hstate = h;
125         spool->min_hpages = min_hpages;
126
127         if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
128                 kfree(spool);
129                 return NULL;
130         }
131         spool->rsv_hpages = min_hpages;
132
133         return spool;
134 }
135
136 void hugepage_put_subpool(struct hugepage_subpool *spool)
137 {
138         unsigned long flags;
139
140         spin_lock_irqsave(&spool->lock, flags);
141         BUG_ON(!spool->count);
142         spool->count--;
143         unlock_or_release_subpool(spool, flags);
144 }
145
146 /*
147  * Subpool accounting for allocating and reserving pages.
148  * Return -ENOMEM if there are not enough resources to satisfy the
149  * request.  Otherwise, return the number of pages by which the
150  * global pools must be adjusted (upward).  The returned value may
151  * only be different than the passed value (delta) in the case where
152  * a subpool minimum size must be maintained.
153  */
154 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
155                                       long delta)
156 {
157         long ret = delta;
158
159         if (!spool)
160                 return ret;
161
162         spin_lock_irq(&spool->lock);
163
164         if (spool->max_hpages != -1) {          /* maximum size accounting */
165                 if ((spool->used_hpages + delta) <= spool->max_hpages)
166                         spool->used_hpages += delta;
167                 else {
168                         ret = -ENOMEM;
169                         goto unlock_ret;
170                 }
171         }
172
173         /* minimum size accounting */
174         if (spool->min_hpages != -1 && spool->rsv_hpages) {
175                 if (delta > spool->rsv_hpages) {
176                         /*
177                          * Asking for more reserves than those already taken on
178                          * behalf of subpool.  Return difference.
179                          */
180                         ret = delta - spool->rsv_hpages;
181                         spool->rsv_hpages = 0;
182                 } else {
183                         ret = 0;        /* reserves already accounted for */
184                         spool->rsv_hpages -= delta;
185                 }
186         }
187
188 unlock_ret:
189         spin_unlock_irq(&spool->lock);
190         return ret;
191 }
192
193 /*
194  * Subpool accounting for freeing and unreserving pages.
195  * Return the number of global page reservations that must be dropped.
196  * The return value may only be different than the passed value (delta)
197  * in the case where a subpool minimum size must be maintained.
198  */
199 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
200                                        long delta)
201 {
202         long ret = delta;
203         unsigned long flags;
204
205         if (!spool)
206                 return delta;
207
208         spin_lock_irqsave(&spool->lock, flags);
209
210         if (spool->max_hpages != -1)            /* maximum size accounting */
211                 spool->used_hpages -= delta;
212
213          /* minimum size accounting */
214         if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) {
215                 if (spool->rsv_hpages + delta <= spool->min_hpages)
216                         ret = 0;
217                 else
218                         ret = spool->rsv_hpages + delta - spool->min_hpages;
219
220                 spool->rsv_hpages += delta;
221                 if (spool->rsv_hpages > spool->min_hpages)
222                         spool->rsv_hpages = spool->min_hpages;
223         }
224
225         /*
226          * If hugetlbfs_put_super couldn't free spool due to an outstanding
227          * quota reference, free it now.
228          */
229         unlock_or_release_subpool(spool, flags);
230
231         return ret;
232 }
233
234 static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
235 {
236         return HUGETLBFS_SB(inode->i_sb)->spool;
237 }
238
239 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
240 {
241         return subpool_inode(file_inode(vma->vm_file));
242 }
243
244 /* Helper that removes a struct file_region from the resv_map cache and returns
245  * it for use.
246  */
247 static struct file_region *
248 get_file_region_entry_from_cache(struct resv_map *resv, long from, long to)
249 {
250         struct file_region *nrg = NULL;
251
252         VM_BUG_ON(resv->region_cache_count <= 0);
253
254         resv->region_cache_count--;
255         nrg = list_first_entry(&resv->region_cache, struct file_region, link);
256         list_del(&nrg->link);
257
258         nrg->from = from;
259         nrg->to = to;
260
261         return nrg;
262 }
263
264 static void copy_hugetlb_cgroup_uncharge_info(struct file_region *nrg,
265                                               struct file_region *rg)
266 {
267 #ifdef CONFIG_CGROUP_HUGETLB
268         nrg->reservation_counter = rg->reservation_counter;
269         nrg->css = rg->css;
270         if (rg->css)
271                 css_get(rg->css);
272 #endif
273 }
274
275 /* Helper that records hugetlb_cgroup uncharge info. */
276 static void record_hugetlb_cgroup_uncharge_info(struct hugetlb_cgroup *h_cg,
277                                                 struct hstate *h,
278                                                 struct resv_map *resv,
279                                                 struct file_region *nrg)
280 {
281 #ifdef CONFIG_CGROUP_HUGETLB
282         if (h_cg) {
283                 nrg->reservation_counter =
284                         &h_cg->rsvd_hugepage[hstate_index(h)];
285                 nrg->css = &h_cg->css;
286                 /*
287                  * The caller will hold exactly one h_cg->css reference for the
288                  * whole contiguous reservation region. But this area might be
289                  * scattered when there are already some file_regions reside in
290                  * it. As a result, many file_regions may share only one css
291                  * reference. In order to ensure that one file_region must hold
292                  * exactly one h_cg->css reference, we should do css_get for
293                  * each file_region and leave the reference held by caller
294                  * untouched.
295                  */
296                 css_get(&h_cg->css);
297                 if (!resv->pages_per_hpage)
298                         resv->pages_per_hpage = pages_per_huge_page(h);
299                 /* pages_per_hpage should be the same for all entries in
300                  * a resv_map.
301                  */
302                 VM_BUG_ON(resv->pages_per_hpage != pages_per_huge_page(h));
303         } else {
304                 nrg->reservation_counter = NULL;
305                 nrg->css = NULL;
306         }
307 #endif
308 }
309
310 static void put_uncharge_info(struct file_region *rg)
311 {
312 #ifdef CONFIG_CGROUP_HUGETLB
313         if (rg->css)
314                 css_put(rg->css);
315 #endif
316 }
317
318 static bool has_same_uncharge_info(struct file_region *rg,
319                                    struct file_region *org)
320 {
321 #ifdef CONFIG_CGROUP_HUGETLB
322         return rg && org &&
323                rg->reservation_counter == org->reservation_counter &&
324                rg->css == org->css;
325
326 #else
327         return true;
328 #endif
329 }
330
331 static void coalesce_file_region(struct resv_map *resv, struct file_region *rg)
332 {
333         struct file_region *nrg = NULL, *prg = NULL;
334
335         prg = list_prev_entry(rg, link);
336         if (&prg->link != &resv->regions && prg->to == rg->from &&
337             has_same_uncharge_info(prg, rg)) {
338                 prg->to = rg->to;
339
340                 list_del(&rg->link);
341                 put_uncharge_info(rg);
342                 kfree(rg);
343
344                 rg = prg;
345         }
346
347         nrg = list_next_entry(rg, link);
348         if (&nrg->link != &resv->regions && nrg->from == rg->to &&
349             has_same_uncharge_info(nrg, rg)) {
350                 nrg->from = rg->from;
351
352                 list_del(&rg->link);
353                 put_uncharge_info(rg);
354                 kfree(rg);
355         }
356 }
357
358 static inline long
359 hugetlb_resv_map_add(struct resv_map *map, struct file_region *rg, long from,
360                      long to, struct hstate *h, struct hugetlb_cgroup *cg,
361                      long *regions_needed)
362 {
363         struct file_region *nrg;
364
365         if (!regions_needed) {
366                 nrg = get_file_region_entry_from_cache(map, from, to);
367                 record_hugetlb_cgroup_uncharge_info(cg, h, map, nrg);
368                 list_add(&nrg->link, rg->link.prev);
369                 coalesce_file_region(map, nrg);
370         } else
371                 *regions_needed += 1;
372
373         return to - from;
374 }
375
376 /*
377  * Must be called with resv->lock held.
378  *
379  * Calling this with regions_needed != NULL will count the number of pages
380  * to be added but will not modify the linked list. And regions_needed will
381  * indicate the number of file_regions needed in the cache to carry out to add
382  * the regions for this range.
383  */
384 static long add_reservation_in_range(struct resv_map *resv, long f, long t,
385                                      struct hugetlb_cgroup *h_cg,
386                                      struct hstate *h, long *regions_needed)
387 {
388         long add = 0;
389         struct list_head *head = &resv->regions;
390         long last_accounted_offset = f;
391         struct file_region *rg = NULL, *trg = NULL;
392
393         if (regions_needed)
394                 *regions_needed = 0;
395
396         /* In this loop, we essentially handle an entry for the range
397          * [last_accounted_offset, rg->from), at every iteration, with some
398          * bounds checking.
399          */
400         list_for_each_entry_safe(rg, trg, head, link) {
401                 /* Skip irrelevant regions that start before our range. */
402                 if (rg->from < f) {
403                         /* If this region ends after the last accounted offset,
404                          * then we need to update last_accounted_offset.
405                          */
406                         if (rg->to > last_accounted_offset)
407                                 last_accounted_offset = rg->to;
408                         continue;
409                 }
410
411                 /* When we find a region that starts beyond our range, we've
412                  * finished.
413                  */
414                 if (rg->from >= t)
415                         break;
416
417                 /* Add an entry for last_accounted_offset -> rg->from, and
418                  * update last_accounted_offset.
419                  */
420                 if (rg->from > last_accounted_offset)
421                         add += hugetlb_resv_map_add(resv, rg,
422                                                     last_accounted_offset,
423                                                     rg->from, h, h_cg,
424                                                     regions_needed);
425
426                 last_accounted_offset = rg->to;
427         }
428
429         /* Handle the case where our range extends beyond
430          * last_accounted_offset.
431          */
432         if (last_accounted_offset < t)
433                 add += hugetlb_resv_map_add(resv, rg, last_accounted_offset,
434                                             t, h, h_cg, regions_needed);
435
436         VM_BUG_ON(add < 0);
437         return add;
438 }
439
440 /* Must be called with resv->lock acquired. Will drop lock to allocate entries.
441  */
442 static int allocate_file_region_entries(struct resv_map *resv,
443                                         int regions_needed)
444         __must_hold(&resv->lock)
445 {
446         struct list_head allocated_regions;
447         int to_allocate = 0, i = 0;
448         struct file_region *trg = NULL, *rg = NULL;
449
450         VM_BUG_ON(regions_needed < 0);
451
452         INIT_LIST_HEAD(&allocated_regions);
453
454         /*
455          * Check for sufficient descriptors in the cache to accommodate
456          * the number of in progress add operations plus regions_needed.
457          *
458          * This is a while loop because when we drop the lock, some other call
459          * to region_add or region_del may have consumed some region_entries,
460          * so we keep looping here until we finally have enough entries for
461          * (adds_in_progress + regions_needed).
462          */
463         while (resv->region_cache_count <
464                (resv->adds_in_progress + regions_needed)) {
465                 to_allocate = resv->adds_in_progress + regions_needed -
466                               resv->region_cache_count;
467
468                 /* At this point, we should have enough entries in the cache
469                  * for all the existings adds_in_progress. We should only be
470                  * needing to allocate for regions_needed.
471                  */
472                 VM_BUG_ON(resv->region_cache_count < resv->adds_in_progress);
473
474                 spin_unlock(&resv->lock);
475                 for (i = 0; i < to_allocate; i++) {
476                         trg = kmalloc(sizeof(*trg), GFP_KERNEL);
477                         if (!trg)
478                                 goto out_of_memory;
479                         list_add(&trg->link, &allocated_regions);
480                 }
481
482                 spin_lock(&resv->lock);
483
484                 list_splice(&allocated_regions, &resv->region_cache);
485                 resv->region_cache_count += to_allocate;
486         }
487
488         return 0;
489
490 out_of_memory:
491         list_for_each_entry_safe(rg, trg, &allocated_regions, link) {
492                 list_del(&rg->link);
493                 kfree(rg);
494         }
495         return -ENOMEM;
496 }
497
498 /*
499  * Add the huge page range represented by [f, t) to the reserve
500  * map.  Regions will be taken from the cache to fill in this range.
501  * Sufficient regions should exist in the cache due to the previous
502  * call to region_chg with the same range, but in some cases the cache will not
503  * have sufficient entries due to races with other code doing region_add or
504  * region_del.  The extra needed entries will be allocated.
505  *
506  * regions_needed is the out value provided by a previous call to region_chg.
507  *
508  * Return the number of new huge pages added to the map.  This number is greater
509  * than or equal to zero.  If file_region entries needed to be allocated for
510  * this operation and we were not able to allocate, it returns -ENOMEM.
511  * region_add of regions of length 1 never allocate file_regions and cannot
512  * fail; region_chg will always allocate at least 1 entry and a region_add for
513  * 1 page will only require at most 1 entry.
514  */
515 static long region_add(struct resv_map *resv, long f, long t,
516                        long in_regions_needed, struct hstate *h,
517                        struct hugetlb_cgroup *h_cg)
518 {
519         long add = 0, actual_regions_needed = 0;
520
521         spin_lock(&resv->lock);
522 retry:
523
524         /* Count how many regions are actually needed to execute this add. */
525         add_reservation_in_range(resv, f, t, NULL, NULL,
526                                  &actual_regions_needed);
527
528         /*
529          * Check for sufficient descriptors in the cache to accommodate
530          * this add operation. Note that actual_regions_needed may be greater
531          * than in_regions_needed, as the resv_map may have been modified since
532          * the region_chg call. In this case, we need to make sure that we
533          * allocate extra entries, such that we have enough for all the
534          * existing adds_in_progress, plus the excess needed for this
535          * operation.
536          */
537         if (actual_regions_needed > in_regions_needed &&
538             resv->region_cache_count <
539                     resv->adds_in_progress +
540                             (actual_regions_needed - in_regions_needed)) {
541                 /* region_add operation of range 1 should never need to
542                  * allocate file_region entries.
543                  */
544                 VM_BUG_ON(t - f <= 1);
545
546                 if (allocate_file_region_entries(
547                             resv, actual_regions_needed - in_regions_needed)) {
548                         return -ENOMEM;
549                 }
550
551                 goto retry;
552         }
553
554         add = add_reservation_in_range(resv, f, t, h_cg, h, NULL);
555
556         resv->adds_in_progress -= in_regions_needed;
557
558         spin_unlock(&resv->lock);
559         return add;
560 }
561
562 /*
563  * Examine the existing reserve map and determine how many
564  * huge pages in the specified range [f, t) are NOT currently
565  * represented.  This routine is called before a subsequent
566  * call to region_add that will actually modify the reserve
567  * map to add the specified range [f, t).  region_chg does
568  * not change the number of huge pages represented by the
569  * map.  A number of new file_region structures is added to the cache as a
570  * placeholder, for the subsequent region_add call to use. At least 1
571  * file_region structure is added.
572  *
573  * out_regions_needed is the number of regions added to the
574  * resv->adds_in_progress.  This value needs to be provided to a follow up call
575  * to region_add or region_abort for proper accounting.
576  *
577  * Returns the number of huge pages that need to be added to the existing
578  * reservation map for the range [f, t).  This number is greater or equal to
579  * zero.  -ENOMEM is returned if a new file_region structure or cache entry
580  * is needed and can not be allocated.
581  */
582 static long region_chg(struct resv_map *resv, long f, long t,
583                        long *out_regions_needed)
584 {
585         long chg = 0;
586
587         spin_lock(&resv->lock);
588
589         /* Count how many hugepages in this range are NOT represented. */
590         chg = add_reservation_in_range(resv, f, t, NULL, NULL,
591                                        out_regions_needed);
592
593         if (*out_regions_needed == 0)
594                 *out_regions_needed = 1;
595
596         if (allocate_file_region_entries(resv, *out_regions_needed))
597                 return -ENOMEM;
598
599         resv->adds_in_progress += *out_regions_needed;
600
601         spin_unlock(&resv->lock);
602         return chg;
603 }
604
605 /*
606  * Abort the in progress add operation.  The adds_in_progress field
607  * of the resv_map keeps track of the operations in progress between
608  * calls to region_chg and region_add.  Operations are sometimes
609  * aborted after the call to region_chg.  In such cases, region_abort
610  * is called to decrement the adds_in_progress counter. regions_needed
611  * is the value returned by the region_chg call, it is used to decrement
612  * the adds_in_progress counter.
613  *
614  * NOTE: The range arguments [f, t) are not needed or used in this
615  * routine.  They are kept to make reading the calling code easier as
616  * arguments will match the associated region_chg call.
617  */
618 static void region_abort(struct resv_map *resv, long f, long t,
619                          long regions_needed)
620 {
621         spin_lock(&resv->lock);
622         VM_BUG_ON(!resv->region_cache_count);
623         resv->adds_in_progress -= regions_needed;
624         spin_unlock(&resv->lock);
625 }
626
627 /*
628  * Delete the specified range [f, t) from the reserve map.  If the
629  * t parameter is LONG_MAX, this indicates that ALL regions after f
630  * should be deleted.  Locate the regions which intersect [f, t)
631  * and either trim, delete or split the existing regions.
632  *
633  * Returns the number of huge pages deleted from the reserve map.
634  * In the normal case, the return value is zero or more.  In the
635  * case where a region must be split, a new region descriptor must
636  * be allocated.  If the allocation fails, -ENOMEM will be returned.
637  * NOTE: If the parameter t == LONG_MAX, then we will never split
638  * a region and possibly return -ENOMEM.  Callers specifying
639  * t == LONG_MAX do not need to check for -ENOMEM error.
640  */
641 static long region_del(struct resv_map *resv, long f, long t)
642 {
643         struct list_head *head = &resv->regions;
644         struct file_region *rg, *trg;
645         struct file_region *nrg = NULL;
646         long del = 0;
647
648 retry:
649         spin_lock(&resv->lock);
650         list_for_each_entry_safe(rg, trg, head, link) {
651                 /*
652                  * Skip regions before the range to be deleted.  file_region
653                  * ranges are normally of the form [from, to).  However, there
654                  * may be a "placeholder" entry in the map which is of the form
655                  * (from, to) with from == to.  Check for placeholder entries
656                  * at the beginning of the range to be deleted.
657                  */
658                 if (rg->to <= f && (rg->to != rg->from || rg->to != f))
659                         continue;
660
661                 if (rg->from >= t)
662                         break;
663
664                 if (f > rg->from && t < rg->to) { /* Must split region */
665                         /*
666                          * Check for an entry in the cache before dropping
667                          * lock and attempting allocation.
668                          */
669                         if (!nrg &&
670                             resv->region_cache_count > resv->adds_in_progress) {
671                                 nrg = list_first_entry(&resv->region_cache,
672                                                         struct file_region,
673                                                         link);
674                                 list_del(&nrg->link);
675                                 resv->region_cache_count--;
676                         }
677
678                         if (!nrg) {
679                                 spin_unlock(&resv->lock);
680                                 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
681                                 if (!nrg)
682                                         return -ENOMEM;
683                                 goto retry;
684                         }
685
686                         del += t - f;
687                         hugetlb_cgroup_uncharge_file_region(
688                                 resv, rg, t - f, false);
689
690                         /* New entry for end of split region */
691                         nrg->from = t;
692                         nrg->to = rg->to;
693
694                         copy_hugetlb_cgroup_uncharge_info(nrg, rg);
695
696                         INIT_LIST_HEAD(&nrg->link);
697
698                         /* Original entry is trimmed */
699                         rg->to = f;
700
701                         list_add(&nrg->link, &rg->link);
702                         nrg = NULL;
703                         break;
704                 }
705
706                 if (f <= rg->from && t >= rg->to) { /* Remove entire region */
707                         del += rg->to - rg->from;
708                         hugetlb_cgroup_uncharge_file_region(resv, rg,
709                                                             rg->to - rg->from, true);
710                         list_del(&rg->link);
711                         kfree(rg);
712                         continue;
713                 }
714
715                 if (f <= rg->from) {    /* Trim beginning of region */
716                         hugetlb_cgroup_uncharge_file_region(resv, rg,
717                                                             t - rg->from, false);
718
719                         del += t - rg->from;
720                         rg->from = t;
721                 } else {                /* Trim end of region */
722                         hugetlb_cgroup_uncharge_file_region(resv, rg,
723                                                             rg->to - f, false);
724
725                         del += rg->to - f;
726                         rg->to = f;
727                 }
728         }
729
730         spin_unlock(&resv->lock);
731         kfree(nrg);
732         return del;
733 }
734
735 /*
736  * A rare out of memory error was encountered which prevented removal of
737  * the reserve map region for a page.  The huge page itself was free'ed
738  * and removed from the page cache.  This routine will adjust the subpool
739  * usage count, and the global reserve count if needed.  By incrementing
740  * these counts, the reserve map entry which could not be deleted will
741  * appear as a "reserved" entry instead of simply dangling with incorrect
742  * counts.
743  */
744 void hugetlb_fix_reserve_counts(struct inode *inode)
745 {
746         struct hugepage_subpool *spool = subpool_inode(inode);
747         long rsv_adjust;
748         bool reserved = false;
749
750         rsv_adjust = hugepage_subpool_get_pages(spool, 1);
751         if (rsv_adjust > 0) {
752                 struct hstate *h = hstate_inode(inode);
753
754                 if (!hugetlb_acct_memory(h, 1))
755                         reserved = true;
756         } else if (!rsv_adjust) {
757                 reserved = true;
758         }
759
760         if (!reserved)
761                 pr_warn("hugetlb: Huge Page Reserved count may go negative.\n");
762 }
763
764 /*
765  * Count and return the number of huge pages in the reserve map
766  * that intersect with the range [f, t).
767  */
768 static long region_count(struct resv_map *resv, long f, long t)
769 {
770         struct list_head *head = &resv->regions;
771         struct file_region *rg;
772         long chg = 0;
773
774         spin_lock(&resv->lock);
775         /* Locate each segment we overlap with, and count that overlap. */
776         list_for_each_entry(rg, head, link) {
777                 long seg_from;
778                 long seg_to;
779
780                 if (rg->to <= f)
781                         continue;
782                 if (rg->from >= t)
783                         break;
784
785                 seg_from = max(rg->from, f);
786                 seg_to = min(rg->to, t);
787
788                 chg += seg_to - seg_from;
789         }
790         spin_unlock(&resv->lock);
791
792         return chg;
793 }
794
795 /*
796  * Convert the address within this vma to the page offset within
797  * the mapping, in pagecache page units; huge pages here.
798  */
799 static pgoff_t vma_hugecache_offset(struct hstate *h,
800                         struct vm_area_struct *vma, unsigned long address)
801 {
802         return ((address - vma->vm_start) >> huge_page_shift(h)) +
803                         (vma->vm_pgoff >> huge_page_order(h));
804 }
805
806 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
807                                      unsigned long address)
808 {
809         return vma_hugecache_offset(hstate_vma(vma), vma, address);
810 }
811 EXPORT_SYMBOL_GPL(linear_hugepage_index);
812
813 /*
814  * Return the size of the pages allocated when backing a VMA. In the majority
815  * cases this will be same size as used by the page table entries.
816  */
817 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
818 {
819         if (vma->vm_ops && vma->vm_ops->pagesize)
820                 return vma->vm_ops->pagesize(vma);
821         return PAGE_SIZE;
822 }
823 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
824
825 /*
826  * Return the page size being used by the MMU to back a VMA. In the majority
827  * of cases, the page size used by the kernel matches the MMU size. On
828  * architectures where it differs, an architecture-specific 'strong'
829  * version of this symbol is required.
830  */
831 __weak unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
832 {
833         return vma_kernel_pagesize(vma);
834 }
835
836 /*
837  * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
838  * bits of the reservation map pointer, which are always clear due to
839  * alignment.
840  */
841 #define HPAGE_RESV_OWNER    (1UL << 0)
842 #define HPAGE_RESV_UNMAPPED (1UL << 1)
843 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
844
845 /*
846  * These helpers are used to track how many pages are reserved for
847  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
848  * is guaranteed to have their future faults succeed.
849  *
850  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
851  * the reserve counters are updated with the hugetlb_lock held. It is safe
852  * to reset the VMA at fork() time as it is not in use yet and there is no
853  * chance of the global counters getting corrupted as a result of the values.
854  *
855  * The private mapping reservation is represented in a subtly different
856  * manner to a shared mapping.  A shared mapping has a region map associated
857  * with the underlying file, this region map represents the backing file
858  * pages which have ever had a reservation assigned which this persists even
859  * after the page is instantiated.  A private mapping has a region map
860  * associated with the original mmap which is attached to all VMAs which
861  * reference it, this region map represents those offsets which have consumed
862  * reservation ie. where pages have been instantiated.
863  */
864 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
865 {
866         return (unsigned long)vma->vm_private_data;
867 }
868
869 static void set_vma_private_data(struct vm_area_struct *vma,
870                                                         unsigned long value)
871 {
872         vma->vm_private_data = (void *)value;
873 }
874
875 static void
876 resv_map_set_hugetlb_cgroup_uncharge_info(struct resv_map *resv_map,
877                                           struct hugetlb_cgroup *h_cg,
878                                           struct hstate *h)
879 {
880 #ifdef CONFIG_CGROUP_HUGETLB
881         if (!h_cg || !h) {
882                 resv_map->reservation_counter = NULL;
883                 resv_map->pages_per_hpage = 0;
884                 resv_map->css = NULL;
885         } else {
886                 resv_map->reservation_counter =
887                         &h_cg->rsvd_hugepage[hstate_index(h)];
888                 resv_map->pages_per_hpage = pages_per_huge_page(h);
889                 resv_map->css = &h_cg->css;
890         }
891 #endif
892 }
893
894 struct resv_map *resv_map_alloc(void)
895 {
896         struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
897         struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL);
898
899         if (!resv_map || !rg) {
900                 kfree(resv_map);
901                 kfree(rg);
902                 return NULL;
903         }
904
905         kref_init(&resv_map->refs);
906         spin_lock_init(&resv_map->lock);
907         INIT_LIST_HEAD(&resv_map->regions);
908
909         resv_map->adds_in_progress = 0;
910         /*
911          * Initialize these to 0. On shared mappings, 0's here indicate these
912          * fields don't do cgroup accounting. On private mappings, these will be
913          * re-initialized to the proper values, to indicate that hugetlb cgroup
914          * reservations are to be un-charged from here.
915          */
916         resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, NULL, NULL);
917
918         INIT_LIST_HEAD(&resv_map->region_cache);
919         list_add(&rg->link, &resv_map->region_cache);
920         resv_map->region_cache_count = 1;
921
922         return resv_map;
923 }
924
925 void resv_map_release(struct kref *ref)
926 {
927         struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
928         struct list_head *head = &resv_map->region_cache;
929         struct file_region *rg, *trg;
930
931         /* Clear out any active regions before we release the map. */
932         region_del(resv_map, 0, LONG_MAX);
933
934         /* ... and any entries left in the cache */
935         list_for_each_entry_safe(rg, trg, head, link) {
936                 list_del(&rg->link);
937                 kfree(rg);
938         }
939
940         VM_BUG_ON(resv_map->adds_in_progress);
941
942         kfree(resv_map);
943 }
944
945 static inline struct resv_map *inode_resv_map(struct inode *inode)
946 {
947         /*
948          * At inode evict time, i_mapping may not point to the original
949          * address space within the inode.  This original address space
950          * contains the pointer to the resv_map.  So, always use the
951          * address space embedded within the inode.
952          * The VERY common case is inode->mapping == &inode->i_data but,
953          * this may not be true for device special inodes.
954          */
955         return (struct resv_map *)(&inode->i_data)->private_data;
956 }
957
958 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
959 {
960         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
961         if (vma->vm_flags & VM_MAYSHARE) {
962                 struct address_space *mapping = vma->vm_file->f_mapping;
963                 struct inode *inode = mapping->host;
964
965                 return inode_resv_map(inode);
966
967         } else {
968                 return (struct resv_map *)(get_vma_private_data(vma) &
969                                                         ~HPAGE_RESV_MASK);
970         }
971 }
972
973 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
974 {
975         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
976         VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
977
978         set_vma_private_data(vma, (get_vma_private_data(vma) &
979                                 HPAGE_RESV_MASK) | (unsigned long)map);
980 }
981
982 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
983 {
984         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
985         VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
986
987         set_vma_private_data(vma, get_vma_private_data(vma) | flags);
988 }
989
990 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
991 {
992         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
993
994         return (get_vma_private_data(vma) & flag) != 0;
995 }
996
997 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
998 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
999 {
1000         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1001         if (!(vma->vm_flags & VM_MAYSHARE))
1002                 vma->vm_private_data = (void *)0;
1003 }
1004
1005 /* Returns true if the VMA has associated reserve pages */
1006 static bool vma_has_reserves(struct vm_area_struct *vma, long chg)
1007 {
1008         if (vma->vm_flags & VM_NORESERVE) {
1009                 /*
1010                  * This address is already reserved by other process(chg == 0),
1011                  * so, we should decrement reserved count. Without decrementing,
1012                  * reserve count remains after releasing inode, because this
1013                  * allocated page will go into page cache and is regarded as
1014                  * coming from reserved pool in releasing step.  Currently, we
1015                  * don't have any other solution to deal with this situation
1016                  * properly, so add work-around here.
1017                  */
1018                 if (vma->vm_flags & VM_MAYSHARE && chg == 0)
1019                         return true;
1020                 else
1021                         return false;
1022         }
1023
1024         /* Shared mappings always use reserves */
1025         if (vma->vm_flags & VM_MAYSHARE) {
1026                 /*
1027                  * We know VM_NORESERVE is not set.  Therefore, there SHOULD
1028                  * be a region map for all pages.  The only situation where
1029                  * there is no region map is if a hole was punched via
1030                  * fallocate.  In this case, there really are no reserves to
1031                  * use.  This situation is indicated if chg != 0.
1032                  */
1033                 if (chg)
1034                         return false;
1035                 else
1036                         return true;
1037         }
1038
1039         /*
1040          * Only the process that called mmap() has reserves for
1041          * private mappings.
1042          */
1043         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1044                 /*
1045                  * Like the shared case above, a hole punch or truncate
1046                  * could have been performed on the private mapping.
1047                  * Examine the value of chg to determine if reserves
1048                  * actually exist or were previously consumed.
1049                  * Very Subtle - The value of chg comes from a previous
1050                  * call to vma_needs_reserves().  The reserve map for
1051                  * private mappings has different (opposite) semantics
1052                  * than that of shared mappings.  vma_needs_reserves()
1053                  * has already taken this difference in semantics into
1054                  * account.  Therefore, the meaning of chg is the same
1055                  * as in the shared case above.  Code could easily be
1056                  * combined, but keeping it separate draws attention to
1057                  * subtle differences.
1058                  */
1059                 if (chg)
1060                         return false;
1061                 else
1062                         return true;
1063         }
1064
1065         return false;
1066 }
1067
1068 static void enqueue_huge_page(struct hstate *h, struct page *page)
1069 {
1070         int nid = page_to_nid(page);
1071
1072         lockdep_assert_held(&hugetlb_lock);
1073         list_move(&page->lru, &h->hugepage_freelists[nid]);
1074         h->free_huge_pages++;
1075         h->free_huge_pages_node[nid]++;
1076         SetHPageFreed(page);
1077 }
1078
1079 static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid)
1080 {
1081         struct page *page;
1082         bool pin = !!(current->flags & PF_MEMALLOC_PIN);
1083
1084         lockdep_assert_held(&hugetlb_lock);
1085         list_for_each_entry(page, &h->hugepage_freelists[nid], lru) {
1086                 if (pin && !is_pinnable_page(page))
1087                         continue;
1088
1089                 if (PageHWPoison(page))
1090                         continue;
1091
1092                 list_move(&page->lru, &h->hugepage_activelist);
1093                 set_page_refcounted(page);
1094                 ClearHPageFreed(page);
1095                 h->free_huge_pages--;
1096                 h->free_huge_pages_node[nid]--;
1097                 return page;
1098         }
1099
1100         return NULL;
1101 }
1102
1103 static struct page *dequeue_huge_page_nodemask(struct hstate *h, gfp_t gfp_mask, int nid,
1104                 nodemask_t *nmask)
1105 {
1106         unsigned int cpuset_mems_cookie;
1107         struct zonelist *zonelist;
1108         struct zone *zone;
1109         struct zoneref *z;
1110         int node = NUMA_NO_NODE;
1111
1112         zonelist = node_zonelist(nid, gfp_mask);
1113
1114 retry_cpuset:
1115         cpuset_mems_cookie = read_mems_allowed_begin();
1116         for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nmask) {
1117                 struct page *page;
1118
1119                 if (!cpuset_zone_allowed(zone, gfp_mask))
1120                         continue;
1121                 /*
1122                  * no need to ask again on the same node. Pool is node rather than
1123                  * zone aware
1124                  */
1125                 if (zone_to_nid(zone) == node)
1126                         continue;
1127                 node = zone_to_nid(zone);
1128
1129                 page = dequeue_huge_page_node_exact(h, node);
1130                 if (page)
1131                         return page;
1132         }
1133         if (unlikely(read_mems_allowed_retry(cpuset_mems_cookie)))
1134                 goto retry_cpuset;
1135
1136         return NULL;
1137 }
1138
1139 static struct page *dequeue_huge_page_vma(struct hstate *h,
1140                                 struct vm_area_struct *vma,
1141                                 unsigned long address, int avoid_reserve,
1142                                 long chg)
1143 {
1144         struct page *page;
1145         struct mempolicy *mpol;
1146         gfp_t gfp_mask;
1147         nodemask_t *nodemask;
1148         int nid;
1149
1150         /*
1151          * A child process with MAP_PRIVATE mappings created by their parent
1152          * have no page reserves. This check ensures that reservations are
1153          * not "stolen". The child may still get SIGKILLed
1154          */
1155         if (!vma_has_reserves(vma, chg) &&
1156                         h->free_huge_pages - h->resv_huge_pages == 0)
1157                 goto err;
1158
1159         /* If reserves cannot be used, ensure enough pages are in the pool */
1160         if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
1161                 goto err;
1162
1163         gfp_mask = htlb_alloc_mask(h);
1164         nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
1165         page = dequeue_huge_page_nodemask(h, gfp_mask, nid, nodemask);
1166         if (page && !avoid_reserve && vma_has_reserves(vma, chg)) {
1167                 SetHPageRestoreReserve(page);
1168                 h->resv_huge_pages--;
1169         }
1170
1171         mpol_cond_put(mpol);
1172         return page;
1173
1174 err:
1175         return NULL;
1176 }
1177
1178 /*
1179  * common helper functions for hstate_next_node_to_{alloc|free}.
1180  * We may have allocated or freed a huge page based on a different
1181  * nodes_allowed previously, so h->next_node_to_{alloc|free} might
1182  * be outside of *nodes_allowed.  Ensure that we use an allowed
1183  * node for alloc or free.
1184  */
1185 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
1186 {
1187         nid = next_node_in(nid, *nodes_allowed);
1188         VM_BUG_ON(nid >= MAX_NUMNODES);
1189
1190         return nid;
1191 }
1192
1193 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
1194 {
1195         if (!node_isset(nid, *nodes_allowed))
1196                 nid = next_node_allowed(nid, nodes_allowed);
1197         return nid;
1198 }
1199
1200 /*
1201  * returns the previously saved node ["this node"] from which to
1202  * allocate a persistent huge page for the pool and advance the
1203  * next node from which to allocate, handling wrap at end of node
1204  * mask.
1205  */
1206 static int hstate_next_node_to_alloc(struct hstate *h,
1207                                         nodemask_t *nodes_allowed)
1208 {
1209         int nid;
1210
1211         VM_BUG_ON(!nodes_allowed);
1212
1213         nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
1214         h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
1215
1216         return nid;
1217 }
1218
1219 /*
1220  * helper for remove_pool_huge_page() - return the previously saved
1221  * node ["this node"] from which to free a huge page.  Advance the
1222  * next node id whether or not we find a free huge page to free so
1223  * that the next attempt to free addresses the next node.
1224  */
1225 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
1226 {
1227         int nid;
1228
1229         VM_BUG_ON(!nodes_allowed);
1230
1231         nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
1232         h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
1233
1234         return nid;
1235 }
1236
1237 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)           \
1238         for (nr_nodes = nodes_weight(*mask);                            \
1239                 nr_nodes > 0 &&                                         \
1240                 ((node = hstate_next_node_to_alloc(hs, mask)) || 1);    \
1241                 nr_nodes--)
1242
1243 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask)            \
1244         for (nr_nodes = nodes_weight(*mask);                            \
1245                 nr_nodes > 0 &&                                         \
1246                 ((node = hstate_next_node_to_free(hs, mask)) || 1);     \
1247                 nr_nodes--)
1248
1249 #ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE
1250 static void destroy_compound_gigantic_page(struct page *page,
1251                                         unsigned int order)
1252 {
1253         int i;
1254         int nr_pages = 1 << order;
1255         struct page *p = page + 1;
1256
1257         atomic_set(compound_mapcount_ptr(page), 0);
1258         atomic_set(compound_pincount_ptr(page), 0);
1259
1260         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1261                 clear_compound_head(p);
1262                 set_page_refcounted(p);
1263         }
1264
1265         set_compound_order(page, 0);
1266         page[1].compound_nr = 0;
1267         __ClearPageHead(page);
1268 }
1269
1270 static void free_gigantic_page(struct page *page, unsigned int order)
1271 {
1272         /*
1273          * If the page isn't allocated using the cma allocator,
1274          * cma_release() returns false.
1275          */
1276 #ifdef CONFIG_CMA
1277         if (cma_release(hugetlb_cma[page_to_nid(page)], page, 1 << order))
1278                 return;
1279 #endif
1280
1281         free_contig_range(page_to_pfn(page), 1 << order);
1282 }
1283
1284 #ifdef CONFIG_CONTIG_ALLOC
1285 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
1286                 int nid, nodemask_t *nodemask)
1287 {
1288         unsigned long nr_pages = pages_per_huge_page(h);
1289         if (nid == NUMA_NO_NODE)
1290                 nid = numa_mem_id();
1291
1292 #ifdef CONFIG_CMA
1293         {
1294                 struct page *page;
1295                 int node;
1296
1297                 if (hugetlb_cma[nid]) {
1298                         page = cma_alloc(hugetlb_cma[nid], nr_pages,
1299                                         huge_page_order(h), true);
1300                         if (page)
1301                                 return page;
1302                 }
1303
1304                 if (!(gfp_mask & __GFP_THISNODE)) {
1305                         for_each_node_mask(node, *nodemask) {
1306                                 if (node == nid || !hugetlb_cma[node])
1307                                         continue;
1308
1309                                 page = cma_alloc(hugetlb_cma[node], nr_pages,
1310                                                 huge_page_order(h), true);
1311                                 if (page)
1312                                         return page;
1313                         }
1314                 }
1315         }
1316 #endif
1317
1318         return alloc_contig_pages(nr_pages, gfp_mask, nid, nodemask);
1319 }
1320
1321 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
1322 static void prep_compound_gigantic_page(struct page *page, unsigned int order);
1323 #else /* !CONFIG_CONTIG_ALLOC */
1324 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
1325                                         int nid, nodemask_t *nodemask)
1326 {
1327         return NULL;
1328 }
1329 #endif /* CONFIG_CONTIG_ALLOC */
1330
1331 #else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE */
1332 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
1333                                         int nid, nodemask_t *nodemask)
1334 {
1335         return NULL;
1336 }
1337 static inline void free_gigantic_page(struct page *page, unsigned int order) { }
1338 static inline void destroy_compound_gigantic_page(struct page *page,
1339                                                 unsigned int order) { }
1340 #endif
1341
1342 /*
1343  * Remove hugetlb page from lists, and update dtor so that page appears
1344  * as just a compound page.  A reference is held on the page.
1345  *
1346  * Must be called with hugetlb lock held.
1347  */
1348 static void remove_hugetlb_page(struct hstate *h, struct page *page,
1349                                                         bool adjust_surplus)
1350 {
1351         int nid = page_to_nid(page);
1352
1353         VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);
1354         VM_BUG_ON_PAGE(hugetlb_cgroup_from_page_rsvd(page), page);
1355
1356         lockdep_assert_held(&hugetlb_lock);
1357         if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
1358                 return;
1359
1360         list_del(&page->lru);
1361
1362         if (HPageFreed(page)) {
1363                 h->free_huge_pages--;
1364                 h->free_huge_pages_node[nid]--;
1365         }
1366         if (adjust_surplus) {
1367                 h->surplus_huge_pages--;
1368                 h->surplus_huge_pages_node[nid]--;
1369         }
1370
1371         set_page_refcounted(page);
1372         set_compound_page_dtor(page, NULL_COMPOUND_DTOR);
1373
1374         h->nr_huge_pages--;
1375         h->nr_huge_pages_node[nid]--;
1376 }
1377
1378 static void update_and_free_page(struct hstate *h, struct page *page)
1379 {
1380         int i;
1381         struct page *subpage = page;
1382
1383         if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
1384                 return;
1385
1386         for (i = 0; i < pages_per_huge_page(h);
1387              i++, subpage = mem_map_next(subpage, page, i)) {
1388                 subpage->flags &= ~(1 << PG_locked | 1 << PG_error |
1389                                 1 << PG_referenced | 1 << PG_dirty |
1390                                 1 << PG_active | 1 << PG_private |
1391                                 1 << PG_writeback);
1392         }
1393         if (hstate_is_gigantic(h)) {
1394                 destroy_compound_gigantic_page(page, huge_page_order(h));
1395                 free_gigantic_page(page, huge_page_order(h));
1396         } else {
1397                 __free_pages(page, huge_page_order(h));
1398         }
1399 }
1400
1401 static void update_and_free_pages_bulk(struct hstate *h, struct list_head *list)
1402 {
1403         struct page *page, *t_page;
1404
1405         list_for_each_entry_safe(page, t_page, list, lru) {
1406                 update_and_free_page(h, page);
1407                 cond_resched();
1408         }
1409 }
1410
1411 struct hstate *size_to_hstate(unsigned long size)
1412 {
1413         struct hstate *h;
1414
1415         for_each_hstate(h) {
1416                 if (huge_page_size(h) == size)
1417                         return h;
1418         }
1419         return NULL;
1420 }
1421
1422 void free_huge_page(struct page *page)
1423 {
1424         /*
1425          * Can't pass hstate in here because it is called from the
1426          * compound page destructor.
1427          */
1428         struct hstate *h = page_hstate(page);
1429         int nid = page_to_nid(page);
1430         struct hugepage_subpool *spool = hugetlb_page_subpool(page);
1431         bool restore_reserve;
1432         unsigned long flags;
1433
1434         VM_BUG_ON_PAGE(page_count(page), page);
1435         VM_BUG_ON_PAGE(page_mapcount(page), page);
1436
1437         hugetlb_set_page_subpool(page, NULL);
1438         page->mapping = NULL;
1439         restore_reserve = HPageRestoreReserve(page);
1440         ClearHPageRestoreReserve(page);
1441
1442         /*
1443          * If HPageRestoreReserve was set on page, page allocation consumed a
1444          * reservation.  If the page was associated with a subpool, there
1445          * would have been a page reserved in the subpool before allocation
1446          * via hugepage_subpool_get_pages().  Since we are 'restoring' the
1447          * reservation, do not call hugepage_subpool_put_pages() as this will
1448          * remove the reserved page from the subpool.
1449          */
1450         if (!restore_reserve) {
1451                 /*
1452                  * A return code of zero implies that the subpool will be
1453                  * under its minimum size if the reservation is not restored
1454                  * after page is free.  Therefore, force restore_reserve
1455                  * operation.
1456                  */
1457                 if (hugepage_subpool_put_pages(spool, 1) == 0)
1458                         restore_reserve = true;
1459         }
1460
1461         spin_lock_irqsave(&hugetlb_lock, flags);
1462         ClearHPageMigratable(page);
1463         hugetlb_cgroup_uncharge_page(hstate_index(h),
1464                                      pages_per_huge_page(h), page);
1465         hugetlb_cgroup_uncharge_page_rsvd(hstate_index(h),
1466                                           pages_per_huge_page(h), page);
1467         if (restore_reserve)
1468                 h->resv_huge_pages++;
1469
1470         if (HPageTemporary(page)) {
1471                 remove_hugetlb_page(h, page, false);
1472                 spin_unlock_irqrestore(&hugetlb_lock, flags);
1473                 update_and_free_page(h, page);
1474         } else if (h->surplus_huge_pages_node[nid]) {
1475                 /* remove the page from active list */
1476                 remove_hugetlb_page(h, page, true);
1477                 spin_unlock_irqrestore(&hugetlb_lock, flags);
1478                 update_and_free_page(h, page);
1479         } else {
1480                 arch_clear_hugepage_flags(page);
1481                 enqueue_huge_page(h, page);
1482                 spin_unlock_irqrestore(&hugetlb_lock, flags);
1483         }
1484 }
1485
1486 /*
1487  * Must be called with the hugetlb lock held
1488  */
1489 static void __prep_account_new_huge_page(struct hstate *h, int nid)
1490 {
1491         lockdep_assert_held(&hugetlb_lock);
1492         h->nr_huge_pages++;
1493         h->nr_huge_pages_node[nid]++;
1494 }
1495
1496 static void __prep_new_huge_page(struct page *page)
1497 {
1498         INIT_LIST_HEAD(&page->lru);
1499         set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1500         hugetlb_set_page_subpool(page, NULL);
1501         set_hugetlb_cgroup(page, NULL);
1502         set_hugetlb_cgroup_rsvd(page, NULL);
1503 }
1504
1505 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
1506 {
1507         __prep_new_huge_page(page);
1508         spin_lock_irq(&hugetlb_lock);
1509         __prep_account_new_huge_page(h, nid);
1510         spin_unlock_irq(&hugetlb_lock);
1511 }
1512
1513 static void prep_compound_gigantic_page(struct page *page, unsigned int order)
1514 {
1515         int i;
1516         int nr_pages = 1 << order;
1517         struct page *p = page + 1;
1518
1519         /* we rely on prep_new_huge_page to set the destructor */
1520         set_compound_order(page, order);
1521         __ClearPageReserved(page);
1522         __SetPageHead(page);
1523         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1524                 /*
1525                  * For gigantic hugepages allocated through bootmem at
1526                  * boot, it's safer to be consistent with the not-gigantic
1527                  * hugepages and clear the PG_reserved bit from all tail pages
1528                  * too.  Otherwise drivers using get_user_pages() to access tail
1529                  * pages may get the reference counting wrong if they see
1530                  * PG_reserved set on a tail page (despite the head page not
1531                  * having PG_reserved set).  Enforcing this consistency between
1532                  * head and tail pages allows drivers to optimize away a check
1533                  * on the head page when they need know if put_page() is needed
1534                  * after get_user_pages().
1535                  */
1536                 __ClearPageReserved(p);
1537                 set_page_count(p, 0);
1538                 set_compound_head(p, page);
1539         }
1540         atomic_set(compound_mapcount_ptr(page), -1);
1541         atomic_set(compound_pincount_ptr(page), 0);
1542 }
1543
1544 /*
1545  * PageHuge() only returns true for hugetlbfs pages, but not for normal or
1546  * transparent huge pages.  See the PageTransHuge() documentation for more
1547  * details.
1548  */
1549 int PageHuge(struct page *page)
1550 {
1551         if (!PageCompound(page))
1552                 return 0;
1553
1554         page = compound_head(page);
1555         return page[1].compound_dtor == HUGETLB_PAGE_DTOR;
1556 }
1557 EXPORT_SYMBOL_GPL(PageHuge);
1558
1559 /*
1560  * PageHeadHuge() only returns true for hugetlbfs head page, but not for
1561  * normal or transparent huge pages.
1562  */
1563 int PageHeadHuge(struct page *page_head)
1564 {
1565         if (!PageHead(page_head))
1566                 return 0;
1567
1568         return page_head[1].compound_dtor == HUGETLB_PAGE_DTOR;
1569 }
1570
1571 /*
1572  * Find and lock address space (mapping) in write mode.
1573  *
1574  * Upon entry, the page is locked which means that page_mapping() is
1575  * stable.  Due to locking order, we can only trylock_write.  If we can
1576  * not get the lock, simply return NULL to caller.
1577  */
1578 struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage)
1579 {
1580         struct address_space *mapping = page_mapping(hpage);
1581
1582         if (!mapping)
1583                 return mapping;
1584
1585         if (i_mmap_trylock_write(mapping))
1586                 return mapping;
1587
1588         return NULL;
1589 }
1590
1591 pgoff_t __basepage_index(struct page *page)
1592 {
1593         struct page *page_head = compound_head(page);
1594         pgoff_t index = page_index(page_head);
1595         unsigned long compound_idx;
1596
1597         if (!PageHuge(page_head))
1598                 return page_index(page);
1599
1600         if (compound_order(page_head) >= MAX_ORDER)
1601                 compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
1602         else
1603                 compound_idx = page - page_head;
1604
1605         return (index << compound_order(page_head)) + compound_idx;
1606 }
1607
1608 static struct page *alloc_buddy_huge_page(struct hstate *h,
1609                 gfp_t gfp_mask, int nid, nodemask_t *nmask,
1610                 nodemask_t *node_alloc_noretry)
1611 {
1612         int order = huge_page_order(h);
1613         struct page *page;
1614         bool alloc_try_hard = true;
1615
1616         /*
1617          * By default we always try hard to allocate the page with
1618          * __GFP_RETRY_MAYFAIL flag.  However, if we are allocating pages in
1619          * a loop (to adjust global huge page counts) and previous allocation
1620          * failed, do not continue to try hard on the same node.  Use the
1621          * node_alloc_noretry bitmap to manage this state information.
1622          */
1623         if (node_alloc_noretry && node_isset(nid, *node_alloc_noretry))
1624                 alloc_try_hard = false;
1625         gfp_mask |= __GFP_COMP|__GFP_NOWARN;
1626         if (alloc_try_hard)
1627                 gfp_mask |= __GFP_RETRY_MAYFAIL;
1628         if (nid == NUMA_NO_NODE)
1629                 nid = numa_mem_id();
1630         page = __alloc_pages(gfp_mask, order, nid, nmask);
1631         if (page)
1632                 __count_vm_event(HTLB_BUDDY_PGALLOC);
1633         else
1634                 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1635
1636         /*
1637          * If we did not specify __GFP_RETRY_MAYFAIL, but still got a page this
1638          * indicates an overall state change.  Clear bit so that we resume
1639          * normal 'try hard' allocations.
1640          */
1641         if (node_alloc_noretry && page && !alloc_try_hard)
1642                 node_clear(nid, *node_alloc_noretry);
1643
1644         /*
1645          * If we tried hard to get a page but failed, set bit so that
1646          * subsequent attempts will not try as hard until there is an
1647          * overall state change.
1648          */
1649         if (node_alloc_noretry && !page && alloc_try_hard)
1650                 node_set(nid, *node_alloc_noretry);
1651
1652         return page;
1653 }
1654
1655 /*
1656  * Common helper to allocate a fresh hugetlb page. All specific allocators
1657  * should use this function to get new hugetlb pages
1658  */
1659 static struct page *alloc_fresh_huge_page(struct hstate *h,
1660                 gfp_t gfp_mask, int nid, nodemask_t *nmask,
1661                 nodemask_t *node_alloc_noretry)
1662 {
1663         struct page *page;
1664
1665         if (hstate_is_gigantic(h))
1666                 page = alloc_gigantic_page(h, gfp_mask, nid, nmask);
1667         else
1668                 page = alloc_buddy_huge_page(h, gfp_mask,
1669                                 nid, nmask, node_alloc_noretry);
1670         if (!page)
1671                 return NULL;
1672
1673         if (hstate_is_gigantic(h))
1674                 prep_compound_gigantic_page(page, huge_page_order(h));
1675         prep_new_huge_page(h, page, page_to_nid(page));
1676
1677         return page;
1678 }
1679
1680 /*
1681  * Allocates a fresh page to the hugetlb allocator pool in the node interleaved
1682  * manner.
1683  */
1684 static int alloc_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
1685                                 nodemask_t *node_alloc_noretry)
1686 {
1687         struct page *page;
1688         int nr_nodes, node;
1689         gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
1690
1691         for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1692                 page = alloc_fresh_huge_page(h, gfp_mask, node, nodes_allowed,
1693                                                 node_alloc_noretry);
1694                 if (page)
1695                         break;
1696         }
1697
1698         if (!page)
1699                 return 0;
1700
1701         put_page(page); /* free it into the hugepage allocator */
1702
1703         return 1;
1704 }
1705
1706 /*
1707  * Remove huge page from pool from next node to free.  Attempt to keep
1708  * persistent huge pages more or less balanced over allowed nodes.
1709  * This routine only 'removes' the hugetlb page.  The caller must make
1710  * an additional call to free the page to low level allocators.
1711  * Called with hugetlb_lock locked.
1712  */
1713 static struct page *remove_pool_huge_page(struct hstate *h,
1714                                                 nodemask_t *nodes_allowed,
1715                                                  bool acct_surplus)
1716 {
1717         int nr_nodes, node;
1718         struct page *page = NULL;
1719
1720         lockdep_assert_held(&hugetlb_lock);
1721         for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1722                 /*
1723                  * If we're returning unused surplus pages, only examine
1724                  * nodes with surplus pages.
1725                  */
1726                 if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
1727                     !list_empty(&h->hugepage_freelists[node])) {
1728                         page = list_entry(h->hugepage_freelists[node].next,
1729                                           struct page, lru);
1730                         remove_hugetlb_page(h, page, acct_surplus);
1731                         break;
1732                 }
1733         }
1734
1735         return page;
1736 }
1737
1738 /*
1739  * Dissolve a given free hugepage into free buddy pages. This function does
1740  * nothing for in-use hugepages and non-hugepages.
1741  * This function returns values like below:
1742  *
1743  *  -EBUSY: failed to dissolved free hugepages or the hugepage is in-use
1744  *          (allocated or reserved.)
1745  *       0: successfully dissolved free hugepages or the page is not a
1746  *          hugepage (considered as already dissolved)
1747  */
1748 int dissolve_free_huge_page(struct page *page)
1749 {
1750         int rc = -EBUSY;
1751
1752 retry:
1753         /* Not to disrupt normal path by vainly holding hugetlb_lock */
1754         if (!PageHuge(page))
1755                 return 0;
1756
1757         spin_lock_irq(&hugetlb_lock);
1758         if (!PageHuge(page)) {
1759                 rc = 0;
1760                 goto out;
1761         }
1762
1763         if (!page_count(page)) {
1764                 struct page *head = compound_head(page);
1765                 struct hstate *h = page_hstate(head);
1766                 if (h->free_huge_pages - h->resv_huge_pages == 0)
1767                         goto out;
1768
1769                 /*
1770                  * We should make sure that the page is already on the free list
1771                  * when it is dissolved.
1772                  */
1773                 if (unlikely(!HPageFreed(head))) {
1774                         spin_unlock_irq(&hugetlb_lock);
1775                         cond_resched();
1776
1777                         /*
1778                          * Theoretically, we should return -EBUSY when we
1779                          * encounter this race. In fact, we have a chance
1780                          * to successfully dissolve the page if we do a
1781                          * retry. Because the race window is quite small.
1782                          * If we seize this opportunity, it is an optimization
1783                          * for increasing the success rate of dissolving page.
1784                          */
1785                         goto retry;
1786                 }
1787
1788                 /*
1789                  * Move PageHWPoison flag from head page to the raw error page,
1790                  * which makes any subpages rather than the error page reusable.
1791                  */
1792                 if (PageHWPoison(head) && page != head) {
1793                         SetPageHWPoison(page);
1794                         ClearPageHWPoison(head);
1795                 }
1796                 remove_hugetlb_page(h, page, false);
1797                 h->max_huge_pages--;
1798                 spin_unlock_irq(&hugetlb_lock);
1799                 update_and_free_page(h, head);
1800                 return 0;
1801         }
1802 out:
1803         spin_unlock_irq(&hugetlb_lock);
1804         return rc;
1805 }
1806
1807 /*
1808  * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
1809  * make specified memory blocks removable from the system.
1810  * Note that this will dissolve a free gigantic hugepage completely, if any
1811  * part of it lies within the given range.
1812  * Also note that if dissolve_free_huge_page() returns with an error, all
1813  * free hugepages that were dissolved before that error are lost.
1814  */
1815 int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
1816 {
1817         unsigned long pfn;
1818         struct page *page;
1819         int rc = 0;
1820
1821         if (!hugepages_supported())
1822                 return rc;
1823
1824         for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order) {
1825                 page = pfn_to_page(pfn);
1826                 rc = dissolve_free_huge_page(page);
1827                 if (rc)
1828                         break;
1829         }
1830
1831         return rc;
1832 }
1833
1834 /*
1835  * Allocates a fresh surplus page from the page allocator.
1836  */
1837 static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask,
1838                 int nid, nodemask_t *nmask)
1839 {
1840         struct page *page = NULL;
1841
1842         if (hstate_is_gigantic(h))
1843                 return NULL;
1844
1845         spin_lock_irq(&hugetlb_lock);
1846         if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages)
1847                 goto out_unlock;
1848         spin_unlock_irq(&hugetlb_lock);
1849
1850         page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL);
1851         if (!page)
1852                 return NULL;
1853
1854         spin_lock_irq(&hugetlb_lock);
1855         /*
1856          * We could have raced with the pool size change.
1857          * Double check that and simply deallocate the new page
1858          * if we would end up overcommiting the surpluses. Abuse
1859          * temporary page to workaround the nasty free_huge_page
1860          * codeflow
1861          */
1862         if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
1863                 SetHPageTemporary(page);
1864                 spin_unlock_irq(&hugetlb_lock);
1865                 put_page(page);
1866                 return NULL;
1867         } else {
1868                 h->surplus_huge_pages++;
1869                 h->surplus_huge_pages_node[page_to_nid(page)]++;
1870         }
1871
1872 out_unlock:
1873         spin_unlock_irq(&hugetlb_lock);
1874
1875         return page;
1876 }
1877
1878 static struct page *alloc_migrate_huge_page(struct hstate *h, gfp_t gfp_mask,
1879                                      int nid, nodemask_t *nmask)
1880 {
1881         struct page *page;
1882
1883         if (hstate_is_gigantic(h))
1884                 return NULL;
1885
1886         page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL);
1887         if (!page)
1888                 return NULL;
1889
1890         /*
1891          * We do not account these pages as surplus because they are only
1892          * temporary and will be released properly on the last reference
1893          */
1894         SetHPageTemporary(page);
1895
1896         return page;
1897 }
1898
1899 /*
1900  * Use the VMA's mpolicy to allocate a huge page from the buddy.
1901  */
1902 static
1903 struct page *alloc_buddy_huge_page_with_mpol(struct hstate *h,
1904                 struct vm_area_struct *vma, unsigned long addr)
1905 {
1906         struct page *page;
1907         struct mempolicy *mpol;
1908         gfp_t gfp_mask = htlb_alloc_mask(h);
1909         int nid;
1910         nodemask_t *nodemask;
1911
1912         nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask);
1913         page = alloc_surplus_huge_page(h, gfp_mask, nid, nodemask);
1914         mpol_cond_put(mpol);
1915
1916         return page;
1917 }
1918
1919 /* page migration callback function */
1920 struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid,
1921                 nodemask_t *nmask, gfp_t gfp_mask)
1922 {
1923         spin_lock_irq(&hugetlb_lock);
1924         if (h->free_huge_pages - h->resv_huge_pages > 0) {
1925                 struct page *page;
1926
1927                 page = dequeue_huge_page_nodemask(h, gfp_mask, preferred_nid, nmask);
1928                 if (page) {
1929                         spin_unlock_irq(&hugetlb_lock);
1930                         return page;
1931                 }
1932         }
1933         spin_unlock_irq(&hugetlb_lock);
1934
1935         return alloc_migrate_huge_page(h, gfp_mask, preferred_nid, nmask);
1936 }
1937
1938 /* mempolicy aware migration callback */
1939 struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma,
1940                 unsigned long address)
1941 {
1942         struct mempolicy *mpol;
1943         nodemask_t *nodemask;
1944         struct page *page;
1945         gfp_t gfp_mask;
1946         int node;
1947
1948         gfp_mask = htlb_alloc_mask(h);
1949         node = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
1950         page = alloc_huge_page_nodemask(h, node, nodemask, gfp_mask);
1951         mpol_cond_put(mpol);
1952
1953         return page;
1954 }
1955
1956 /*
1957  * Increase the hugetlb pool such that it can accommodate a reservation
1958  * of size 'delta'.
1959  */
1960 static int gather_surplus_pages(struct hstate *h, long delta)
1961         __must_hold(&hugetlb_lock)
1962 {
1963         struct list_head surplus_list;
1964         struct page *page, *tmp;
1965         int ret;
1966         long i;
1967         long needed, allocated;
1968         bool alloc_ok = true;
1969
1970         lockdep_assert_held(&hugetlb_lock);
1971         needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
1972         if (needed <= 0) {
1973                 h->resv_huge_pages += delta;
1974                 return 0;
1975         }
1976
1977         allocated = 0;
1978         INIT_LIST_HEAD(&surplus_list);
1979
1980         ret = -ENOMEM;
1981 retry:
1982         spin_unlock_irq(&hugetlb_lock);
1983         for (i = 0; i < needed; i++) {
1984                 page = alloc_surplus_huge_page(h, htlb_alloc_mask(h),
1985                                 NUMA_NO_NODE, NULL);
1986                 if (!page) {
1987                         alloc_ok = false;
1988                         break;
1989                 }
1990                 list_add(&page->lru, &surplus_list);
1991                 cond_resched();
1992         }
1993         allocated += i;
1994
1995         /*
1996          * After retaking hugetlb_lock, we need to recalculate 'needed'
1997          * because either resv_huge_pages or free_huge_pages may have changed.
1998          */
1999         spin_lock_irq(&hugetlb_lock);
2000         needed = (h->resv_huge_pages + delta) -
2001                         (h->free_huge_pages + allocated);
2002         if (needed > 0) {
2003                 if (alloc_ok)
2004                         goto retry;
2005                 /*
2006                  * We were not able to allocate enough pages to
2007                  * satisfy the entire reservation so we free what
2008                  * we've allocated so far.
2009                  */
2010                 goto free;
2011         }
2012         /*
2013          * The surplus_list now contains _at_least_ the number of extra pages
2014          * needed to accommodate the reservation.  Add the appropriate number
2015          * of pages to the hugetlb pool and free the extras back to the buddy
2016          * allocator.  Commit the entire reservation here to prevent another
2017          * process from stealing the pages as they are added to the pool but
2018          * before they are reserved.
2019          */
2020         needed += allocated;
2021         h->resv_huge_pages += delta;
2022         ret = 0;
2023
2024         /* Free the needed pages to the hugetlb pool */
2025         list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
2026                 int zeroed;
2027
2028                 if ((--needed) < 0)
2029                         break;
2030                 /*
2031                  * This page is now managed by the hugetlb allocator and has
2032                  * no users -- drop the buddy allocator's reference.
2033                  */
2034                 zeroed = put_page_testzero(page);
2035                 VM_BUG_ON_PAGE(!zeroed, page);
2036                 enqueue_huge_page(h, page);
2037         }
2038 free:
2039         spin_unlock_irq(&hugetlb_lock);
2040
2041         /* Free unnecessary surplus pages to the buddy allocator */
2042         list_for_each_entry_safe(page, tmp, &surplus_list, lru)
2043                 put_page(page);
2044         spin_lock_irq(&hugetlb_lock);
2045
2046         return ret;
2047 }
2048
2049 /*
2050  * This routine has two main purposes:
2051  * 1) Decrement the reservation count (resv_huge_pages) by the value passed
2052  *    in unused_resv_pages.  This corresponds to the prior adjustments made
2053  *    to the associated reservation map.
2054  * 2) Free any unused surplus pages that may have been allocated to satisfy
2055  *    the reservation.  As many as unused_resv_pages may be freed.
2056  */
2057 static void return_unused_surplus_pages(struct hstate *h,
2058                                         unsigned long unused_resv_pages)
2059 {
2060         unsigned long nr_pages;
2061         struct page *page;
2062         LIST_HEAD(page_list);
2063
2064         lockdep_assert_held(&hugetlb_lock);
2065         /* Uncommit the reservation */
2066         h->resv_huge_pages -= unused_resv_pages;
2067
2068         /* Cannot return gigantic pages currently */
2069         if (hstate_is_gigantic(h))
2070                 goto out;
2071
2072         /*
2073          * Part (or even all) of the reservation could have been backed
2074          * by pre-allocated pages. Only free surplus pages.
2075          */
2076         nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
2077
2078         /*
2079          * We want to release as many surplus pages as possible, spread
2080          * evenly across all nodes with memory. Iterate across these nodes
2081          * until we can no longer free unreserved surplus pages. This occurs
2082          * when the nodes with surplus pages have no free pages.
2083          * remove_pool_huge_page() will balance the freed pages across the
2084          * on-line nodes with memory and will handle the hstate accounting.
2085          */
2086         while (nr_pages--) {
2087                 page = remove_pool_huge_page(h, &node_states[N_MEMORY], 1);
2088                 if (!page)
2089                         goto out;
2090
2091                 list_add(&page->lru, &page_list);
2092         }
2093
2094 out:
2095         spin_unlock_irq(&hugetlb_lock);
2096         update_and_free_pages_bulk(h, &page_list);
2097         spin_lock_irq(&hugetlb_lock);
2098 }
2099
2100
2101 /*
2102  * vma_needs_reservation, vma_commit_reservation and vma_end_reservation
2103  * are used by the huge page allocation routines to manage reservations.
2104  *
2105  * vma_needs_reservation is called to determine if the huge page at addr
2106  * within the vma has an associated reservation.  If a reservation is
2107  * needed, the value 1 is returned.  The caller is then responsible for
2108  * managing the global reservation and subpool usage counts.  After
2109  * the huge page has been allocated, vma_commit_reservation is called
2110  * to add the page to the reservation map.  If the page allocation fails,
2111  * the reservation must be ended instead of committed.  vma_end_reservation
2112  * is called in such cases.
2113  *
2114  * In the normal case, vma_commit_reservation returns the same value
2115  * as the preceding vma_needs_reservation call.  The only time this
2116  * is not the case is if a reserve map was changed between calls.  It
2117  * is the responsibility of the caller to notice the difference and
2118  * take appropriate action.
2119  *
2120  * vma_add_reservation is used in error paths where a reservation must
2121  * be restored when a newly allocated huge page must be freed.  It is
2122  * to be called after calling vma_needs_reservation to determine if a
2123  * reservation exists.
2124  */
2125 enum vma_resv_mode {
2126         VMA_NEEDS_RESV,
2127         VMA_COMMIT_RESV,
2128         VMA_END_RESV,
2129         VMA_ADD_RESV,
2130 };
2131 static long __vma_reservation_common(struct hstate *h,
2132                                 struct vm_area_struct *vma, unsigned long addr,
2133                                 enum vma_resv_mode mode)
2134 {
2135         struct resv_map *resv;
2136         pgoff_t idx;
2137         long ret;
2138         long dummy_out_regions_needed;
2139
2140         resv = vma_resv_map(vma);
2141         if (!resv)
2142                 return 1;
2143
2144         idx = vma_hugecache_offset(h, vma, addr);
2145         switch (mode) {
2146         case VMA_NEEDS_RESV:
2147                 ret = region_chg(resv, idx, idx + 1, &dummy_out_regions_needed);
2148                 /* We assume that vma_reservation_* routines always operate on
2149                  * 1 page, and that adding to resv map a 1 page entry can only
2150                  * ever require 1 region.
2151                  */
2152                 VM_BUG_ON(dummy_out_regions_needed != 1);
2153                 break;
2154         case VMA_COMMIT_RESV:
2155                 ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2156                 /* region_add calls of range 1 should never fail. */
2157                 VM_BUG_ON(ret < 0);
2158                 break;
2159         case VMA_END_RESV:
2160                 region_abort(resv, idx, idx + 1, 1);
2161                 ret = 0;
2162                 break;
2163         case VMA_ADD_RESV:
2164                 if (vma->vm_flags & VM_MAYSHARE) {
2165                         ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2166                         /* region_add calls of range 1 should never fail. */
2167                         VM_BUG_ON(ret < 0);
2168                 } else {
2169                         region_abort(resv, idx, idx + 1, 1);
2170                         ret = region_del(resv, idx, idx + 1);
2171                 }
2172                 break;
2173         default:
2174                 BUG();
2175         }
2176
2177         if (vma->vm_flags & VM_MAYSHARE)
2178                 return ret;
2179         /*
2180          * We know private mapping must have HPAGE_RESV_OWNER set.
2181          *
2182          * In most cases, reserves always exist for private mappings.
2183          * However, a file associated with mapping could have been
2184          * hole punched or truncated after reserves were consumed.
2185          * As subsequent fault on such a range will not use reserves.
2186          * Subtle - The reserve map for private mappings has the
2187          * opposite meaning than that of shared mappings.  If NO
2188          * entry is in the reserve map, it means a reservation exists.
2189          * If an entry exists in the reserve map, it means the
2190          * reservation has already been consumed.  As a result, the
2191          * return value of this routine is the opposite of the
2192          * value returned from reserve map manipulation routines above.
2193          */
2194         if (ret > 0)
2195                 return 0;
2196         if (ret == 0)
2197                 return 1;
2198         return ret;
2199 }
2200
2201 static long vma_needs_reservation(struct hstate *h,
2202                         struct vm_area_struct *vma, unsigned long addr)
2203 {
2204         return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);
2205 }
2206
2207 static long vma_commit_reservation(struct hstate *h,
2208                         struct vm_area_struct *vma, unsigned long addr)
2209 {
2210         return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
2211 }
2212
2213 static void vma_end_reservation(struct hstate *h,
2214                         struct vm_area_struct *vma, unsigned long addr)
2215 {
2216         (void)__vma_reservation_common(h, vma, addr, VMA_END_RESV);
2217 }
2218
2219 static long vma_add_reservation(struct hstate *h,
2220                         struct vm_area_struct *vma, unsigned long addr)
2221 {
2222         return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV);
2223 }
2224
2225 /*
2226  * This routine is called to restore a reservation on error paths.  In the
2227  * specific error paths, a huge page was allocated (via alloc_huge_page)
2228  * and is about to be freed.  If a reservation for the page existed,
2229  * alloc_huge_page would have consumed the reservation and set
2230  * HPageRestoreReserve in the newly allocated page.  When the page is freed
2231  * via free_huge_page, the global reservation count will be incremented if
2232  * HPageRestoreReserve is set.  However, free_huge_page can not adjust the
2233  * reserve map.  Adjust the reserve map here to be consistent with global
2234  * reserve count adjustments to be made by free_huge_page.
2235  */
2236 static void restore_reserve_on_error(struct hstate *h,
2237                         struct vm_area_struct *vma, unsigned long address,
2238                         struct page *page)
2239 {
2240         if (unlikely(HPageRestoreReserve(page))) {
2241                 long rc = vma_needs_reservation(h, vma, address);
2242
2243                 if (unlikely(rc < 0)) {
2244                         /*
2245                          * Rare out of memory condition in reserve map
2246                          * manipulation.  Clear HPageRestoreReserve so that
2247                          * global reserve count will not be incremented
2248                          * by free_huge_page.  This will make it appear
2249                          * as though the reservation for this page was
2250                          * consumed.  This may prevent the task from
2251                          * faulting in the page at a later time.  This
2252                          * is better than inconsistent global huge page
2253                          * accounting of reserve counts.
2254                          */
2255                         ClearHPageRestoreReserve(page);
2256                 } else if (rc) {
2257                         rc = vma_add_reservation(h, vma, address);
2258                         if (unlikely(rc < 0))
2259                                 /*
2260                                  * See above comment about rare out of
2261                                  * memory condition.
2262                                  */
2263                                 ClearHPageRestoreReserve(page);
2264                 } else
2265                         vma_end_reservation(h, vma, address);
2266         }
2267 }
2268
2269 /*
2270  * alloc_and_dissolve_huge_page - Allocate a new page and dissolve the old one
2271  * @h: struct hstate old page belongs to
2272  * @old_page: Old page to dissolve
2273  * @list: List to isolate the page in case we need to
2274  * Returns 0 on success, otherwise negated error.
2275  */
2276 static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,
2277                                         struct list_head *list)
2278 {
2279         gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
2280         int nid = page_to_nid(old_page);
2281         struct page *new_page;
2282         int ret = 0;
2283
2284         /*
2285          * Before dissolving the page, we need to allocate a new one for the
2286          * pool to remain stable. Using alloc_buddy_huge_page() allows us to
2287          * not having to deal with prep_new_huge_page() and avoids dealing of any
2288          * counters. This simplifies and let us do the whole thing under the
2289          * lock.
2290          */
2291         new_page = alloc_buddy_huge_page(h, gfp_mask, nid, NULL, NULL);
2292         if (!new_page)
2293                 return -ENOMEM;
2294
2295 retry:
2296         spin_lock_irq(&hugetlb_lock);
2297         if (!PageHuge(old_page)) {
2298                 /*
2299                  * Freed from under us. Drop new_page too.
2300                  */
2301                 goto free_new;
2302         } else if (page_count(old_page)) {
2303                 /*
2304                  * Someone has grabbed the page, try to isolate it here.
2305                  * Fail with -EBUSY if not possible.
2306                  */
2307                 spin_unlock_irq(&hugetlb_lock);
2308                 if (!isolate_huge_page(old_page, list))
2309                         ret = -EBUSY;
2310                 spin_lock_irq(&hugetlb_lock);
2311                 goto free_new;
2312         } else if (!HPageFreed(old_page)) {
2313                 /*
2314                  * Page's refcount is 0 but it has not been enqueued in the
2315                  * freelist yet. Race window is small, so we can succeed here if
2316                  * we retry.
2317                  */
2318                 spin_unlock_irq(&hugetlb_lock);
2319                 cond_resched();
2320                 goto retry;
2321         } else {
2322                 /*
2323                  * Ok, old_page is still a genuine free hugepage. Remove it from
2324                  * the freelist and decrease the counters. These will be
2325                  * incremented again when calling __prep_account_new_huge_page()
2326                  * and enqueue_huge_page() for new_page. The counters will remain
2327                  * stable since this happens under the lock.
2328                  */
2329                 remove_hugetlb_page(h, old_page, false);
2330
2331                 /*
2332                  * new_page needs to be initialized with the standard hugetlb
2333                  * state. This is normally done by prep_new_huge_page() but
2334                  * that takes hugetlb_lock which is already held so we need to
2335                  * open code it here.
2336                  * Reference count trick is needed because allocator gives us
2337                  * referenced page but the pool requires pages with 0 refcount.
2338                  */
2339                 __prep_new_huge_page(new_page);
2340                 __prep_account_new_huge_page(h, nid);
2341                 page_ref_dec(new_page);
2342                 enqueue_huge_page(h, new_page);
2343
2344                 /*
2345                  * Pages have been replaced, we can safely free the old one.
2346                  */
2347                 spin_unlock_irq(&hugetlb_lock);
2348                 update_and_free_page(h, old_page);
2349         }
2350
2351         return ret;
2352
2353 free_new:
2354         spin_unlock_irq(&hugetlb_lock);
2355         __free_pages(new_page, huge_page_order(h));
2356
2357         return ret;
2358 }
2359
2360 int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list)
2361 {
2362         struct hstate *h;
2363         struct page *head;
2364         int ret = -EBUSY;
2365
2366         /*
2367          * The page might have been dissolved from under our feet, so make sure
2368          * to carefully check the state under the lock.
2369          * Return success when racing as if we dissolved the page ourselves.
2370          */
2371         spin_lock_irq(&hugetlb_lock);
2372         if (PageHuge(page)) {
2373                 head = compound_head(page);
2374                 h = page_hstate(head);
2375         } else {
2376                 spin_unlock_irq(&hugetlb_lock);
2377                 return 0;
2378         }
2379         spin_unlock_irq(&hugetlb_lock);
2380
2381         /*
2382          * Fence off gigantic pages as there is a cyclic dependency between
2383          * alloc_contig_range and them. Return -ENOMEM as this has the effect
2384          * of bailing out right away without further retrying.
2385          */
2386         if (hstate_is_gigantic(h))
2387                 return -ENOMEM;
2388
2389         if (page_count(head) && isolate_huge_page(head, list))
2390                 ret = 0;
2391         else if (!page_count(head))
2392                 ret = alloc_and_dissolve_huge_page(h, head, list);
2393
2394         return ret;
2395 }
2396
2397 struct page *alloc_huge_page(struct vm_area_struct *vma,
2398                                     unsigned long addr, int avoid_reserve)
2399 {
2400         struct hugepage_subpool *spool = subpool_vma(vma);
2401         struct hstate *h = hstate_vma(vma);
2402         struct page *page;
2403         long map_chg, map_commit;
2404         long gbl_chg;
2405         int ret, idx;
2406         struct hugetlb_cgroup *h_cg;
2407         bool deferred_reserve;
2408
2409         idx = hstate_index(h);
2410         /*
2411          * Examine the region/reserve map to determine if the process
2412          * has a reservation for the page to be allocated.  A return
2413          * code of zero indicates a reservation exists (no change).
2414          */
2415         map_chg = gbl_chg = vma_needs_reservation(h, vma, addr);
2416         if (map_chg < 0)
2417                 return ERR_PTR(-ENOMEM);
2418
2419         /*
2420          * Processes that did not create the mapping will have no
2421          * reserves as indicated by the region/reserve map. Check
2422          * that the allocation will not exceed the subpool limit.
2423          * Allocations for MAP_NORESERVE mappings also need to be
2424          * checked against any subpool limit.
2425          */
2426         if (map_chg || avoid_reserve) {
2427                 gbl_chg = hugepage_subpool_get_pages(spool, 1);
2428                 if (gbl_chg < 0) {
2429                         vma_end_reservation(h, vma, addr);
2430                         return ERR_PTR(-ENOSPC);
2431                 }
2432
2433                 /*
2434                  * Even though there was no reservation in the region/reserve
2435                  * map, there could be reservations associated with the
2436                  * subpool that can be used.  This would be indicated if the
2437                  * return value of hugepage_subpool_get_pages() is zero.
2438                  * However, if avoid_reserve is specified we still avoid even
2439                  * the subpool reservations.
2440                  */
2441                 if (avoid_reserve)
2442                         gbl_chg = 1;
2443         }
2444
2445         /* If this allocation is not consuming a reservation, charge it now.
2446          */
2447         deferred_reserve = map_chg || avoid_reserve;
2448         if (deferred_reserve) {
2449                 ret = hugetlb_cgroup_charge_cgroup_rsvd(
2450                         idx, pages_per_huge_page(h), &h_cg);
2451                 if (ret)
2452                         goto out_subpool_put;
2453         }
2454
2455         ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
2456         if (ret)
2457                 goto out_uncharge_cgroup_reservation;
2458
2459         spin_lock_irq(&hugetlb_lock);
2460         /*
2461          * glb_chg is passed to indicate whether or not a page must be taken
2462          * from the global free pool (global change).  gbl_chg == 0 indicates
2463          * a reservation exists for the allocation.
2464          */
2465         page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg);
2466         if (!page) {
2467                 spin_unlock_irq(&hugetlb_lock);
2468                 page = alloc_buddy_huge_page_with_mpol(h, vma, addr);
2469                 if (!page)
2470                         goto out_uncharge_cgroup;
2471                 if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) {
2472                         SetHPageRestoreReserve(page);
2473                         h->resv_huge_pages--;
2474                 }
2475                 spin_lock_irq(&hugetlb_lock);
2476                 list_add(&page->lru, &h->hugepage_activelist);
2477                 /* Fall through */
2478         }
2479         hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
2480         /* If allocation is not consuming a reservation, also store the
2481          * hugetlb_cgroup pointer on the page.
2482          */
2483         if (deferred_reserve) {
2484                 hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h),
2485                                                   h_cg, page);
2486         }
2487
2488         spin_unlock_irq(&hugetlb_lock);
2489
2490         hugetlb_set_page_subpool(page, spool);
2491
2492         map_commit = vma_commit_reservation(h, vma, addr);
2493         if (unlikely(map_chg > map_commit)) {
2494                 /*
2495                  * The page was added to the reservation map between
2496                  * vma_needs_reservation and vma_commit_reservation.
2497                  * This indicates a race with hugetlb_reserve_pages.
2498                  * Adjust for the subpool count incremented above AND
2499                  * in hugetlb_reserve_pages for the same page.  Also,
2500                  * the reservation count added in hugetlb_reserve_pages
2501                  * no longer applies.
2502                  */
2503                 long rsv_adjust;
2504
2505                 rsv_adjust = hugepage_subpool_put_pages(spool, 1);
2506                 hugetlb_acct_memory(h, -rsv_adjust);
2507                 if (deferred_reserve)
2508                         hugetlb_cgroup_uncharge_page_rsvd(hstate_index(h),
2509                                         pages_per_huge_page(h), page);
2510         }
2511         return page;
2512
2513 out_uncharge_cgroup:
2514         hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
2515 out_uncharge_cgroup_reservation:
2516         if (deferred_reserve)
2517                 hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h),
2518                                                     h_cg);
2519 out_subpool_put:
2520         if (map_chg || avoid_reserve)
2521                 hugepage_subpool_put_pages(spool, 1);
2522         vma_end_reservation(h, vma, addr);
2523         return ERR_PTR(-ENOSPC);
2524 }
2525
2526 int alloc_bootmem_huge_page(struct hstate *h)
2527         __attribute__ ((weak, alias("__alloc_bootmem_huge_page")));
2528 int __alloc_bootmem_huge_page(struct hstate *h)
2529 {
2530         struct huge_bootmem_page *m;
2531         int nr_nodes, node;
2532
2533         for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
2534                 void *addr;
2535
2536                 addr = memblock_alloc_try_nid_raw(
2537                                 huge_page_size(h), huge_page_size(h),
2538                                 0, MEMBLOCK_ALLOC_ACCESSIBLE, node);
2539                 if (addr) {
2540                         /*
2541                          * Use the beginning of the huge page to store the
2542                          * huge_bootmem_page struct (until gather_bootmem
2543                          * puts them into the mem_map).
2544                          */
2545                         m = addr;
2546                         goto found;
2547                 }
2548         }
2549         return 0;
2550
2551 found:
2552         BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h)));
2553         /* Put them into a private list first because mem_map is not up yet */
2554         INIT_LIST_HEAD(&m->list);
2555         list_add(&m->list, &huge_boot_pages);
2556         m->hstate = h;
2557         return 1;
2558 }
2559
2560 static void __init prep_compound_huge_page(struct page *page,
2561                 unsigned int order)
2562 {
2563         if (unlikely(order > (MAX_ORDER - 1)))
2564                 prep_compound_gigantic_page(page, order);
2565         else
2566                 prep_compound_page(page, order);
2567 }
2568
2569 /* Put bootmem huge pages into the standard lists after mem_map is up */
2570 static void __init gather_bootmem_prealloc(void)
2571 {
2572         struct huge_bootmem_page *m;
2573
2574         list_for_each_entry(m, &huge_boot_pages, list) {
2575                 struct page *page = virt_to_page(m);
2576                 struct hstate *h = m->hstate;
2577
2578                 WARN_ON(page_count(page) != 1);
2579                 prep_compound_huge_page(page, huge_page_order(h));
2580                 WARN_ON(PageReserved(page));
2581                 prep_new_huge_page(h, page, page_to_nid(page));
2582                 put_page(page); /* free it into the hugepage allocator */
2583
2584                 /*
2585                  * If we had gigantic hugepages allocated at boot time, we need
2586                  * to restore the 'stolen' pages to totalram_pages in order to
2587                  * fix confusing memory reports from free(1) and another
2588                  * side-effects, like CommitLimit going negative.
2589                  */
2590                 if (hstate_is_gigantic(h))
2591                         adjust_managed_page_count(page, pages_per_huge_page(h));
2592                 cond_resched();
2593         }
2594 }
2595
2596 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
2597 {
2598         unsigned long i;
2599         nodemask_t *node_alloc_noretry;
2600
2601         if (!hstate_is_gigantic(h)) {
2602                 /*
2603                  * Bit mask controlling how hard we retry per-node allocations.
2604                  * Ignore errors as lower level routines can deal with
2605                  * node_alloc_noretry == NULL.  If this kmalloc fails at boot
2606                  * time, we are likely in bigger trouble.
2607                  */
2608                 node_alloc_noretry = kmalloc(sizeof(*node_alloc_noretry),
2609                                                 GFP_KERNEL);
2610         } else {
2611                 /* allocations done at boot time */
2612                 node_alloc_noretry = NULL;
2613         }
2614
2615         /* bit mask controlling how hard we retry per-node allocations */
2616         if (node_alloc_noretry)
2617                 nodes_clear(*node_alloc_noretry);
2618
2619         for (i = 0; i < h->max_huge_pages; ++i) {
2620                 if (hstate_is_gigantic(h)) {
2621                         if (hugetlb_cma_size) {
2622                                 pr_warn_once("HugeTLB: hugetlb_cma is enabled, skip boot time allocation\n");
2623                                 goto free;
2624                         }
2625                         if (!alloc_bootmem_huge_page(h))
2626                                 break;
2627                 } else if (!alloc_pool_huge_page(h,
2628                                          &node_states[N_MEMORY],
2629                                          node_alloc_noretry))
2630                         break;
2631                 cond_resched();
2632         }
2633         if (i < h->max_huge_pages) {
2634                 char buf[32];
2635
2636                 string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
2637                 pr_warn("HugeTLB: allocating %lu of page size %s failed.  Only allocated %lu hugepages.\n",
2638                         h->max_huge_pages, buf, i);
2639                 h->max_huge_pages = i;
2640         }
2641 free:
2642         kfree(node_alloc_noretry);
2643 }
2644
2645 static void __init hugetlb_init_hstates(void)
2646 {
2647         struct hstate *h;
2648
2649         for_each_hstate(h) {
2650                 if (minimum_order > huge_page_order(h))
2651                         minimum_order = huge_page_order(h);
2652
2653                 /* oversize hugepages were init'ed in early boot */
2654                 if (!hstate_is_gigantic(h))
2655                         hugetlb_hstate_alloc_pages(h);
2656         }
2657         VM_BUG_ON(minimum_order == UINT_MAX);
2658 }
2659
2660 static void __init report_hugepages(void)
2661 {
2662         struct hstate *h;
2663
2664         for_each_hstate(h) {
2665                 char buf[32];
2666
2667                 string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
2668                 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
2669                         buf, h->free_huge_pages);
2670         }
2671 }
2672
2673 #ifdef CONFIG_HIGHMEM
2674 static void try_to_free_low(struct hstate *h, unsigned long count,
2675                                                 nodemask_t *nodes_allowed)
2676 {
2677         int i;
2678         LIST_HEAD(page_list);
2679
2680         lockdep_assert_held(&hugetlb_lock);
2681         if (hstate_is_gigantic(h))
2682                 return;
2683
2684         /*
2685          * Collect pages to be freed on a list, and free after dropping lock
2686          */
2687         for_each_node_mask(i, *nodes_allowed) {
2688                 struct page *page, *next;
2689                 struct list_head *freel = &h->hugepage_freelists[i];
2690                 list_for_each_entry_safe(page, next, freel, lru) {
2691                         if (count >= h->nr_huge_pages)
2692                                 goto out;
2693                         if (PageHighMem(page))
2694                                 continue;
2695                         remove_hugetlb_page(h, page, false);
2696                         list_add(&page->lru, &page_list);
2697                 }
2698         }
2699
2700 out:
2701         spin_unlock_irq(&hugetlb_lock);
2702         update_and_free_pages_bulk(h, &page_list);
2703         spin_lock_irq(&hugetlb_lock);
2704 }
2705 #else
2706 static inline void try_to_free_low(struct hstate *h, unsigned long count,
2707                                                 nodemask_t *nodes_allowed)
2708 {
2709 }
2710 #endif
2711
2712 /*
2713  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
2714  * balanced by operating on them in a round-robin fashion.
2715  * Returns 1 if an adjustment was made.
2716  */
2717 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
2718                                 int delta)
2719 {
2720         int nr_nodes, node;
2721
2722         lockdep_assert_held(&hugetlb_lock);
2723         VM_BUG_ON(delta != -1 && delta != 1);
2724
2725         if (delta < 0) {
2726                 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
2727                         if (h->surplus_huge_pages_node[node])
2728                                 goto found;
2729                 }
2730         } else {
2731                 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
2732                         if (h->surplus_huge_pages_node[node] <
2733                                         h->nr_huge_pages_node[node])
2734                                 goto found;
2735                 }
2736         }
2737         return 0;
2738
2739 found:
2740         h->surplus_huge_pages += delta;
2741         h->surplus_huge_pages_node[node] += delta;
2742         return 1;
2743 }
2744
2745 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
2746 static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid,
2747                               nodemask_t *nodes_allowed)
2748 {
2749         unsigned long min_count, ret;
2750         struct page *page;
2751         LIST_HEAD(page_list);
2752         NODEMASK_ALLOC(nodemask_t, node_alloc_noretry, GFP_KERNEL);
2753
2754         /*
2755          * Bit mask controlling how hard we retry per-node allocations.
2756          * If we can not allocate the bit mask, do not attempt to allocate
2757          * the requested huge pages.
2758          */
2759         if (node_alloc_noretry)
2760                 nodes_clear(*node_alloc_noretry);
2761         else
2762                 return -ENOMEM;
2763
2764         /*
2765          * resize_lock mutex prevents concurrent adjustments to number of
2766          * pages in hstate via the proc/sysfs interfaces.
2767          */
2768         mutex_lock(&h->resize_lock);
2769         spin_lock_irq(&hugetlb_lock);
2770
2771         /*
2772          * Check for a node specific request.
2773          * Changing node specific huge page count may require a corresponding
2774          * change to the global count.  In any case, the passed node mask
2775          * (nodes_allowed) will restrict alloc/free to the specified node.
2776          */
2777         if (nid != NUMA_NO_NODE) {
2778                 unsigned long old_count = count;
2779
2780                 count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
2781                 /*
2782                  * User may have specified a large count value which caused the
2783                  * above calculation to overflow.  In this case, they wanted
2784                  * to allocate as many huge pages as possible.  Set count to
2785                  * largest possible value to align with their intention.
2786                  */
2787                 if (count < old_count)
2788                         count = ULONG_MAX;
2789         }
2790
2791         /*
2792          * Gigantic pages runtime allocation depend on the capability for large
2793          * page range allocation.
2794          * If the system does not provide this feature, return an error when
2795          * the user tries to allocate gigantic pages but let the user free the
2796          * boottime allocated gigantic pages.
2797          */
2798         if (hstate_is_gigantic(h) && !IS_ENABLED(CONFIG_CONTIG_ALLOC)) {
2799                 if (count > persistent_huge_pages(h)) {
2800                         spin_unlock_irq(&hugetlb_lock);
2801                         mutex_unlock(&h->resize_lock);
2802                         NODEMASK_FREE(node_alloc_noretry);
2803                         return -EINVAL;
2804                 }
2805                 /* Fall through to decrease pool */
2806         }
2807
2808         /*
2809          * Increase the pool size
2810          * First take pages out of surplus state.  Then make up the
2811          * remaining difference by allocating fresh huge pages.
2812          *
2813          * We might race with alloc_surplus_huge_page() here and be unable
2814          * to convert a surplus huge page to a normal huge page. That is
2815          * not critical, though, it just means the overall size of the
2816          * pool might be one hugepage larger than it needs to be, but
2817          * within all the constraints specified by the sysctls.
2818          */
2819         while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
2820                 if (!adjust_pool_surplus(h, nodes_allowed, -1))
2821                         break;
2822         }
2823
2824         while (count > persistent_huge_pages(h)) {
2825                 /*
2826                  * If this allocation races such that we no longer need the
2827                  * page, free_huge_page will handle it by freeing the page
2828                  * and reducing the surplus.
2829                  */
2830                 spin_unlock_irq(&hugetlb_lock);
2831
2832                 /* yield cpu to avoid soft lockup */
2833                 cond_resched();
2834
2835                 ret = alloc_pool_huge_page(h, nodes_allowed,
2836                                                 node_alloc_noretry);
2837                 spin_lock_irq(&hugetlb_lock);
2838                 if (!ret)
2839                         goto out;
2840
2841                 /* Bail for signals. Probably ctrl-c from user */
2842                 if (signal_pending(current))
2843                         goto out;
2844         }
2845
2846         /*
2847          * Decrease the pool size
2848          * First return free pages to the buddy allocator (being careful
2849          * to keep enough around to satisfy reservations).  Then place
2850          * pages into surplus state as needed so the pool will shrink
2851          * to the desired size as pages become free.
2852          *
2853          * By placing pages into the surplus state independent of the
2854          * overcommit value, we are allowing the surplus pool size to
2855          * exceed overcommit. There are few sane options here. Since
2856          * alloc_surplus_huge_page() is checking the global counter,
2857          * though, we'll note that we're not allowed to exceed surplus
2858          * and won't grow the pool anywhere else. Not until one of the
2859          * sysctls are changed, or the surplus pages go out of use.
2860          */
2861         min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
2862         min_count = max(count, min_count);
2863         try_to_free_low(h, min_count, nodes_allowed);
2864
2865         /*
2866          * Collect pages to be removed on list without dropping lock
2867          */
2868         while (min_count < persistent_huge_pages(h)) {
2869                 page = remove_pool_huge_page(h, nodes_allowed, 0);
2870                 if (!page)
2871                         break;
2872
2873                 list_add(&page->lru, &page_list);
2874         }
2875         /* free the pages after dropping lock */
2876         spin_unlock_irq(&hugetlb_lock);
2877         update_and_free_pages_bulk(h, &page_list);
2878         spin_lock_irq(&hugetlb_lock);
2879
2880         while (count < persistent_huge_pages(h)) {
2881                 if (!adjust_pool_surplus(h, nodes_allowed, 1))
2882                         break;
2883         }
2884 out:
2885         h->max_huge_pages = persistent_huge_pages(h);
2886         spin_unlock_irq(&hugetlb_lock);
2887         mutex_unlock(&h->resize_lock);
2888
2889         NODEMASK_FREE(node_alloc_noretry);
2890
2891         return 0;
2892 }
2893
2894 #define HSTATE_ATTR_RO(_name) \
2895         static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
2896
2897 #define HSTATE_ATTR(_name) \
2898         static struct kobj_attribute _name##_attr = \
2899                 __ATTR(_name, 0644, _name##_show, _name##_store)
2900
2901 static struct kobject *hugepages_kobj;
2902 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
2903
2904 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
2905
2906 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
2907 {
2908         int i;
2909
2910         for (i = 0; i < HUGE_MAX_HSTATE; i++)
2911                 if (hstate_kobjs[i] == kobj) {
2912                         if (nidp)
2913                                 *nidp = NUMA_NO_NODE;
2914                         return &hstates[i];
2915                 }
2916
2917         return kobj_to_node_hstate(kobj, nidp);
2918 }
2919
2920 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
2921                                         struct kobj_attribute *attr, char *buf)
2922 {
2923         struct hstate *h;
2924         unsigned long nr_huge_pages;
2925         int nid;
2926
2927         h = kobj_to_hstate(kobj, &nid);
2928         if (nid == NUMA_NO_NODE)
2929                 nr_huge_pages = h->nr_huge_pages;
2930         else
2931                 nr_huge_pages = h->nr_huge_pages_node[nid];
2932
2933         return sysfs_emit(buf, "%lu\n", nr_huge_pages);
2934 }
2935
2936 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
2937                                            struct hstate *h, int nid,
2938                                            unsigned long count, size_t len)
2939 {
2940         int err;
2941         nodemask_t nodes_allowed, *n_mask;
2942
2943         if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
2944                 return -EINVAL;
2945
2946         if (nid == NUMA_NO_NODE) {
2947                 /*
2948                  * global hstate attribute
2949                  */
2950                 if (!(obey_mempolicy &&
2951                                 init_nodemask_of_mempolicy(&nodes_allowed)))
2952                         n_mask = &node_states[N_MEMORY];
2953                 else
2954                         n_mask = &nodes_allowed;
2955         } else {
2956                 /*
2957                  * Node specific request.  count adjustment happens in
2958                  * set_max_huge_pages() after acquiring hugetlb_lock.
2959                  */
2960                 init_nodemask_of_node(&nodes_allowed, nid);
2961                 n_mask = &nodes_allowed;
2962         }
2963
2964         err = set_max_huge_pages(h, count, nid, n_mask);
2965
2966         return err ? err : len;
2967 }
2968
2969 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
2970                                          struct kobject *kobj, const char *buf,
2971                                          size_t len)
2972 {
2973         struct hstate *h;
2974         unsigned long count;
2975         int nid;
2976         int err;
2977
2978         err = kstrtoul(buf, 10, &count);
2979         if (err)
2980                 return err;
2981
2982         h = kobj_to_hstate(kobj, &nid);
2983         return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
2984 }
2985
2986 static ssize_t nr_hugepages_show(struct kobject *kobj,
2987                                        struct kobj_attribute *attr, char *buf)
2988 {
2989         return nr_hugepages_show_common(kobj, attr, buf);
2990 }
2991
2992 static ssize_t nr_hugepages_store(struct kobject *kobj,
2993                struct kobj_attribute *attr, const char *buf, size_t len)
2994 {
2995         return nr_hugepages_store_common(false, kobj, buf, len);
2996 }
2997 HSTATE_ATTR(nr_hugepages);
2998
2999 #ifdef CONFIG_NUMA
3000
3001 /*
3002  * hstate attribute for optionally mempolicy-based constraint on persistent
3003  * huge page alloc/free.
3004  */
3005 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
3006                                            struct kobj_attribute *attr,
3007                                            char *buf)
3008 {
3009         return nr_hugepages_show_common(kobj, attr, buf);
3010 }
3011
3012 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
3013                struct kobj_attribute *attr, const char *buf, size_t len)
3014 {
3015         return nr_hugepages_store_common(true, kobj, buf, len);
3016 }
3017 HSTATE_ATTR(nr_hugepages_mempolicy);
3018 #endif
3019
3020
3021 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
3022                                         struct kobj_attribute *attr, char *buf)
3023 {
3024         struct hstate *h = kobj_to_hstate(kobj, NULL);
3025         return sysfs_emit(buf, "%lu\n", h->nr_overcommit_huge_pages);
3026 }
3027
3028 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
3029                 struct kobj_attribute *attr, const char *buf, size_t count)
3030 {
3031         int err;
3032         unsigned long input;
3033         struct hstate *h = kobj_to_hstate(kobj, NULL);
3034
3035         if (hstate_is_gigantic(h))
3036                 return -EINVAL;
3037
3038         err = kstrtoul(buf, 10, &input);
3039         if (err)
3040                 return err;
3041
3042         spin_lock_irq(&hugetlb_lock);
3043         h->nr_overcommit_huge_pages = input;
3044         spin_unlock_irq(&hugetlb_lock);
3045
3046         return count;
3047 }
3048 HSTATE_ATTR(nr_overcommit_hugepages);
3049
3050 static ssize_t free_hugepages_show(struct kobject *kobj,
3051                                         struct kobj_attribute *attr, char *buf)
3052 {
3053         struct hstate *h;
3054         unsigned long free_huge_pages;
3055         int nid;
3056
3057         h = kobj_to_hstate(kobj, &nid);
3058         if (nid == NUMA_NO_NODE)
3059                 free_huge_pages = h->free_huge_pages;
3060         else
3061                 free_huge_pages = h->free_huge_pages_node[nid];
3062
3063         return sysfs_emit(buf, "%lu\n", free_huge_pages);
3064 }
3065 HSTATE_ATTR_RO(free_hugepages);
3066
3067 static ssize_t resv_hugepages_show(struct kobject *kobj,
3068                                         struct kobj_attribute *attr, char *buf)
3069 {
3070         struct hstate *h = kobj_to_hstate(kobj, NULL);
3071         return sysfs_emit(buf, "%lu\n", h->resv_huge_pages);
3072 }
3073 HSTATE_ATTR_RO(resv_hugepages);
3074
3075 static ssize_t surplus_hugepages_show(struct kobject *kobj,
3076                                         struct kobj_attribute *attr, char *buf)
3077 {
3078         struct hstate *h;
3079         unsigned long surplus_huge_pages;
3080         int nid;
3081
3082         h = kobj_to_hstate(kobj, &nid);
3083         if (nid == NUMA_NO_NODE)
3084                 surplus_huge_pages = h->surplus_huge_pages;
3085         else
3086                 surplus_huge_pages = h->surplus_huge_pages_node[nid];
3087
3088         return sysfs_emit(buf, "%lu\n", surplus_huge_pages);
3089 }
3090 HSTATE_ATTR_RO(surplus_hugepages);
3091
3092 static struct attribute *hstate_attrs[] = {
3093         &nr_hugepages_attr.attr,
3094         &nr_overcommit_hugepages_attr.attr,
3095         &free_hugepages_attr.attr,
3096         &resv_hugepages_attr.attr,
3097         &surplus_hugepages_attr.attr,
3098 #ifdef CONFIG_NUMA
3099         &nr_hugepages_mempolicy_attr.attr,
3100 #endif
3101         NULL,
3102 };
3103
3104 static const struct attribute_group hstate_attr_group = {
3105         .attrs = hstate_attrs,
3106 };
3107
3108 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
3109                                     struct kobject **hstate_kobjs,
3110                                     const struct attribute_group *hstate_attr_group)
3111 {
3112         int retval;
3113         int hi = hstate_index(h);
3114
3115         hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
3116         if (!hstate_kobjs[hi])
3117                 return -ENOMEM;
3118
3119         retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
3120         if (retval) {
3121                 kobject_put(hstate_kobjs[hi]);
3122                 hstate_kobjs[hi] = NULL;
3123         }
3124
3125         return retval;
3126 }
3127
3128 static void __init hugetlb_sysfs_init(void)
3129 {
3130         struct hstate *h;
3131         int err;
3132
3133         hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
3134         if (!hugepages_kobj)
3135                 return;
3136
3137         for_each_hstate(h) {
3138                 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
3139                                          hstate_kobjs, &hstate_attr_group);
3140                 if (err)
3141                         pr_err("HugeTLB: Unable to add hstate %s", h->name);
3142         }
3143 }
3144
3145 #ifdef CONFIG_NUMA
3146
3147 /*
3148  * node_hstate/s - associate per node hstate attributes, via their kobjects,
3149  * with node devices in node_devices[] using a parallel array.  The array
3150  * index of a node device or _hstate == node id.
3151  * This is here to avoid any static dependency of the node device driver, in
3152  * the base kernel, on the hugetlb module.
3153  */
3154 struct node_hstate {
3155         struct kobject          *hugepages_kobj;
3156         struct kobject          *hstate_kobjs[HUGE_MAX_HSTATE];
3157 };
3158 static struct node_hstate node_hstates[MAX_NUMNODES];
3159
3160 /*
3161  * A subset of global hstate attributes for node devices
3162  */
3163 static struct attribute *per_node_hstate_attrs[] = {
3164         &nr_hugepages_attr.attr,
3165         &free_hugepages_attr.attr,
3166         &surplus_hugepages_attr.attr,
3167         NULL,
3168 };
3169
3170 static const struct attribute_group per_node_hstate_attr_group = {
3171         .attrs = per_node_hstate_attrs,
3172 };
3173
3174 /*
3175  * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
3176  * Returns node id via non-NULL nidp.
3177  */
3178 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
3179 {
3180         int nid;
3181
3182         for (nid = 0; nid < nr_node_ids; nid++) {
3183                 struct node_hstate *nhs = &node_hstates[nid];
3184                 int i;
3185                 for (i = 0; i < HUGE_MAX_HSTATE; i++)
3186                         if (nhs->hstate_kobjs[i] == kobj) {
3187                                 if (nidp)
3188                                         *nidp = nid;
3189                                 return &hstates[i];
3190                         }
3191         }
3192
3193         BUG();
3194         return NULL;
3195 }
3196
3197 /*
3198  * Unregister hstate attributes from a single node device.
3199  * No-op if no hstate attributes attached.
3200  */
3201 static void hugetlb_unregister_node(struct node *node)
3202 {
3203         struct hstate *h;
3204         struct node_hstate *nhs = &node_hstates[node->dev.id];
3205
3206         if (!nhs->hugepages_kobj)
3207                 return;         /* no hstate attributes */
3208
3209         for_each_hstate(h) {
3210                 int idx = hstate_index(h);
3211                 if (nhs->hstate_kobjs[idx]) {
3212                         kobject_put(nhs->hstate_kobjs[idx]);
3213                         nhs->hstate_kobjs[idx] = NULL;
3214                 }
3215         }
3216
3217         kobject_put(nhs->hugepages_kobj);
3218         nhs->hugepages_kobj = NULL;
3219 }
3220
3221
3222 /*
3223  * Register hstate attributes for a single node device.
3224  * No-op if attributes already registered.
3225  */
3226 static void hugetlb_register_node(struct node *node)
3227 {
3228         struct hstate *h;
3229         struct node_hstate *nhs = &node_hstates[node->dev.id];
3230         int err;
3231
3232         if (nhs->hugepages_kobj)
3233                 return;         /* already allocated */
3234
3235         nhs->hugepages_kobj = kobject_create_and_add("hugepages",
3236                                                         &node->dev.kobj);
3237         if (!nhs->hugepages_kobj)
3238                 return;
3239
3240         for_each_hstate(h) {
3241                 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
3242                                                 nhs->hstate_kobjs,
3243                                                 &per_node_hstate_attr_group);
3244                 if (err) {
3245                         pr_err("HugeTLB: Unable to add hstate %s for node %d\n",
3246                                 h->name, node->dev.id);
3247                         hugetlb_unregister_node(node);
3248                         break;
3249                 }
3250         }
3251 }
3252
3253 /*
3254  * hugetlb init time:  register hstate attributes for all registered node
3255  * devices of nodes that have memory.  All on-line nodes should have
3256  * registered their associated device by this time.
3257  */
3258 static void __init hugetlb_register_all_nodes(void)
3259 {
3260         int nid;
3261
3262         for_each_node_state(nid, N_MEMORY) {
3263                 struct node *node = node_devices[nid];
3264                 if (node->dev.id == nid)
3265                         hugetlb_register_node(node);
3266         }
3267
3268         /*
3269          * Let the node device driver know we're here so it can
3270          * [un]register hstate attributes on node hotplug.
3271          */
3272         register_hugetlbfs_with_node(hugetlb_register_node,
3273                                      hugetlb_unregister_node);
3274 }
3275 #else   /* !CONFIG_NUMA */
3276
3277 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
3278 {
3279         BUG();
3280         if (nidp)
3281                 *nidp = -1;
3282         return NULL;
3283 }
3284
3285 static void hugetlb_register_all_nodes(void) { }
3286
3287 #endif
3288
3289 static int __init hugetlb_init(void)
3290 {
3291         int i;
3292
3293         BUILD_BUG_ON(sizeof_field(struct page, private) * BITS_PER_BYTE <
3294                         __NR_HPAGEFLAGS);
3295
3296         if (!hugepages_supported()) {
3297                 if (hugetlb_max_hstate || default_hstate_max_huge_pages)
3298                         pr_warn("HugeTLB: huge pages not supported, ignoring associated command-line parameters\n");
3299                 return 0;
3300         }
3301
3302         /*
3303          * Make sure HPAGE_SIZE (HUGETLB_PAGE_ORDER) hstate exists.  Some
3304          * architectures depend on setup being done here.
3305          */
3306         hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
3307         if (!parsed_default_hugepagesz) {
3308                 /*
3309                  * If we did not parse a default huge page size, set
3310                  * default_hstate_idx to HPAGE_SIZE hstate. And, if the
3311                  * number of huge pages for this default size was implicitly
3312                  * specified, set that here as well.
3313                  * Note that the implicit setting will overwrite an explicit
3314                  * setting.  A warning will be printed in this case.
3315                  */
3316                 default_hstate_idx = hstate_index(size_to_hstate(HPAGE_SIZE));
3317                 if (default_hstate_max_huge_pages) {
3318                         if (default_hstate.max_huge_pages) {
3319                                 char buf[32];
3320
3321                                 string_get_size(huge_page_size(&default_hstate),
3322                                         1, STRING_UNITS_2, buf, 32);
3323                                 pr_warn("HugeTLB: Ignoring hugepages=%lu associated with %s page size\n",
3324                                         default_hstate.max_huge_pages, buf);
3325                                 pr_warn("HugeTLB: Using hugepages=%lu for number of default huge pages\n",
3326                                         default_hstate_max_huge_pages);
3327                         }
3328                         default_hstate.max_huge_pages =
3329                                 default_hstate_max_huge_pages;
3330                 }
3331         }
3332
3333         hugetlb_cma_check();
3334         hugetlb_init_hstates();
3335         gather_bootmem_prealloc();
3336         report_hugepages();
3337
3338         hugetlb_sysfs_init();
3339         hugetlb_register_all_nodes();
3340         hugetlb_cgroup_file_init();
3341
3342 #ifdef CONFIG_SMP
3343         num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
3344 #else
3345         num_fault_mutexes = 1;
3346 #endif
3347         hugetlb_fault_mutex_table =
3348                 kmalloc_array(num_fault_mutexes, sizeof(struct mutex),
3349                               GFP_KERNEL);
3350         BUG_ON(!hugetlb_fault_mutex_table);
3351
3352         for (i = 0; i < num_fault_mutexes; i++)
3353                 mutex_init(&hugetlb_fault_mutex_table[i]);
3354         return 0;
3355 }
3356 subsys_initcall(hugetlb_init);
3357
3358 /* Overwritten by architectures with more huge page sizes */
3359 bool __init __attribute((weak)) arch_hugetlb_valid_size(unsigned long size)
3360 {
3361         return size == HPAGE_SIZE;
3362 }
3363
3364 void __init hugetlb_add_hstate(unsigned int order)
3365 {
3366         struct hstate *h;
3367         unsigned long i;
3368
3369         if (size_to_hstate(PAGE_SIZE << order)) {
3370                 return;
3371         }
3372         BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
3373         BUG_ON(order == 0);
3374         h = &hstates[hugetlb_max_hstate++];
3375         mutex_init(&h->resize_lock);
3376         h->order = order;
3377         h->mask = ~(huge_page_size(h) - 1);
3378         for (i = 0; i < MAX_NUMNODES; ++i)
3379                 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
3380         INIT_LIST_HEAD(&h->hugepage_activelist);
3381         h->next_nid_to_alloc = first_memory_node;
3382         h->next_nid_to_free = first_memory_node;
3383         snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
3384                                         huge_page_size(h)/1024);
3385
3386         parsed_hstate = h;
3387 }
3388
3389 /*
3390  * hugepages command line processing
3391  * hugepages normally follows a valid hugepagsz or default_hugepagsz
3392  * specification.  If not, ignore the hugepages value.  hugepages can also
3393  * be the first huge page command line  option in which case it implicitly
3394  * specifies the number of huge pages for the default size.
3395  */
3396 static int __init hugepages_setup(char *s)
3397 {
3398         unsigned long *mhp;
3399         static unsigned long *last_mhp;
3400
3401         if (!parsed_valid_hugepagesz) {
3402                 pr_warn("HugeTLB: hugepages=%s does not follow a valid hugepagesz, ignoring\n", s);
3403                 parsed_valid_hugepagesz = true;
3404                 return 0;
3405         }
3406
3407         /*
3408          * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter
3409          * yet, so this hugepages= parameter goes to the "default hstate".
3410          * Otherwise, it goes with the previously parsed hugepagesz or
3411          * default_hugepagesz.
3412          */
3413         else if (!hugetlb_max_hstate)
3414                 mhp = &default_hstate_max_huge_pages;
3415         else
3416                 mhp = &parsed_hstate->max_huge_pages;
3417
3418         if (mhp == last_mhp) {
3419                 pr_warn("HugeTLB: hugepages= specified twice without interleaving hugepagesz=, ignoring hugepages=%s\n", s);
3420                 return 0;
3421         }
3422
3423         if (sscanf(s, "%lu", mhp) <= 0)
3424                 *mhp = 0;
3425
3426         /*
3427          * Global state is always initialized later in hugetlb_init.
3428          * But we need to allocate gigantic hstates here early to still
3429          * use the bootmem allocator.
3430          */
3431         if (hugetlb_max_hstate && hstate_is_gigantic(parsed_hstate))
3432                 hugetlb_hstate_alloc_pages(parsed_hstate);
3433
3434         last_mhp = mhp;
3435
3436         return 1;
3437 }
3438 __setup("hugepages=", hugepages_setup);
3439
3440 /*
3441  * hugepagesz command line processing
3442  * A specific huge page size can only be specified once with hugepagesz.
3443  * hugepagesz is followed by hugepages on the command line.  The global
3444  * variable 'parsed_valid_hugepagesz' is used to determine if prior
3445  * hugepagesz argument was valid.
3446  */
3447 static int __init hugepagesz_setup(char *s)
3448 {
3449         unsigned long size;
3450         struct hstate *h;
3451
3452         parsed_valid_hugepagesz = false;
3453         size = (unsigned long)memparse(s, NULL);
3454
3455         if (!arch_hugetlb_valid_size(size)) {
3456                 pr_err("HugeTLB: unsupported hugepagesz=%s\n", s);
3457                 return 0;
3458         }
3459
3460         h = size_to_hstate(size);
3461         if (h) {
3462                 /*
3463                  * hstate for this size already exists.  This is normally
3464                  * an error, but is allowed if the existing hstate is the
3465                  * default hstate.  More specifically, it is only allowed if
3466                  * the number of huge pages for the default hstate was not
3467                  * previously specified.
3468                  */
3469                 if (!parsed_default_hugepagesz ||  h != &default_hstate ||
3470                     default_hstate.max_huge_pages) {
3471                         pr_warn("HugeTLB: hugepagesz=%s specified twice, ignoring\n", s);
3472                         return 0;
3473                 }
3474
3475                 /*
3476                  * No need to call hugetlb_add_hstate() as hstate already
3477                  * exists.  But, do set parsed_hstate so that a following
3478                  * hugepages= parameter will be applied to this hstate.
3479                  */
3480                 parsed_hstate = h;
3481                 parsed_valid_hugepagesz = true;
3482                 return 1;
3483         }
3484
3485         hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
3486         parsed_valid_hugepagesz = true;
3487         return 1;
3488 }
3489 __setup("hugepagesz=", hugepagesz_setup);
3490
3491 /*
3492  * default_hugepagesz command line input
3493  * Only one instance of default_hugepagesz allowed on command line.
3494  */
3495 static int __init default_hugepagesz_setup(char *s)
3496 {
3497         unsigned long size;
3498
3499         parsed_valid_hugepagesz = false;
3500         if (parsed_default_hugepagesz) {
3501                 pr_err("HugeTLB: default_hugepagesz previously specified, ignoring %s\n", s);
3502                 return 0;
3503         }
3504
3505         size = (unsigned long)memparse(s, NULL);
3506
3507         if (!arch_hugetlb_valid_size(size)) {
3508                 pr_err("HugeTLB: unsupported default_hugepagesz=%s\n", s);
3509                 return 0;
3510         }
3511
3512         hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
3513         parsed_valid_hugepagesz = true;
3514         parsed_default_hugepagesz = true;
3515         default_hstate_idx = hstate_index(size_to_hstate(size));
3516
3517         /*
3518          * The number of default huge pages (for this size) could have been
3519          * specified as the first hugetlb parameter: hugepages=X.  If so,
3520          * then default_hstate_max_huge_pages is set.  If the default huge
3521          * page size is gigantic (>= MAX_ORDER), then the pages must be
3522          * allocated here from bootmem allocator.
3523          */
3524         if (default_hstate_max_huge_pages) {
3525                 default_hstate.max_huge_pages = default_hstate_max_huge_pages;
3526                 if (hstate_is_gigantic(&default_hstate))
3527                         hugetlb_hstate_alloc_pages(&default_hstate);
3528                 default_hstate_max_huge_pages = 0;
3529         }
3530
3531         return 1;
3532 }
3533 __setup("default_hugepagesz=", default_hugepagesz_setup);
3534
3535 static unsigned int allowed_mems_nr(struct hstate *h)
3536 {
3537         int node;
3538         unsigned int nr = 0;
3539         nodemask_t *mpol_allowed;
3540         unsigned int *array = h->free_huge_pages_node;
3541         gfp_t gfp_mask = htlb_alloc_mask(h);
3542
3543         mpol_allowed = policy_nodemask_current(gfp_mask);
3544
3545         for_each_node_mask(node, cpuset_current_mems_allowed) {
3546                 if (!mpol_allowed || node_isset(node, *mpol_allowed))
3547                         nr += array[node];
3548         }
3549
3550         return nr;
3551 }
3552
3553 #ifdef CONFIG_SYSCTL
3554 static int proc_hugetlb_doulongvec_minmax(struct ctl_table *table, int write,
3555                                           void *buffer, size_t *length,
3556                                           loff_t *ppos, unsigned long *out)
3557 {
3558         struct ctl_table dup_table;
3559
3560         /*
3561          * In order to avoid races with __do_proc_doulongvec_minmax(), we
3562          * can duplicate the @table and alter the duplicate of it.
3563          */
3564         dup_table = *table;
3565         dup_table.data = out;
3566
3567         return proc_doulongvec_minmax(&dup_table, write, buffer, length, ppos);
3568 }
3569
3570 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
3571                          struct ctl_table *table, int write,
3572                          void *buffer, size_t *length, loff_t *ppos)
3573 {
3574         struct hstate *h = &default_hstate;
3575         unsigned long tmp = h->max_huge_pages;
3576         int ret;
3577
3578         if (!hugepages_supported())
3579                 return -EOPNOTSUPP;
3580
3581         ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos,
3582                                              &tmp);
3583         if (ret)
3584                 goto out;
3585
3586         if (write)
3587                 ret = __nr_hugepages_store_common(obey_mempolicy, h,
3588                                                   NUMA_NO_NODE, tmp, *length);
3589 out:
3590         return ret;
3591 }
3592
3593 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
3594                           void *buffer, size_t *length, loff_t *ppos)
3595 {
3596
3597         return hugetlb_sysctl_handler_common(false, table, write,
3598                                                         buffer, length, ppos);
3599 }
3600
3601 #ifdef CONFIG_NUMA
3602 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
3603                           void *buffer, size_t *length, loff_t *ppos)
3604 {
3605         return hugetlb_sysctl_handler_common(true, table, write,
3606                                                         buffer, length, ppos);
3607 }
3608 #endif /* CONFIG_NUMA */
3609
3610 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
3611                 void *buffer, size_t *length, loff_t *ppos)
3612 {
3613         struct hstate *h = &default_hstate;
3614         unsigned long tmp;
3615         int ret;
3616
3617         if (!hugepages_supported())
3618                 return -EOPNOTSUPP;
3619
3620         tmp = h->nr_overcommit_huge_pages;
3621
3622         if (write && hstate_is_gigantic(h))
3623                 return -EINVAL;
3624
3625         ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos,
3626                                              &tmp);
3627         if (ret)
3628                 goto out;
3629
3630         if (write) {
3631                 spin_lock_irq(&hugetlb_lock);
3632                 h->nr_overcommit_huge_pages = tmp;
3633                 spin_unlock_irq(&hugetlb_lock);
3634         }
3635 out:
3636         return ret;
3637 }
3638
3639 #endif /* CONFIG_SYSCTL */
3640
3641 void hugetlb_report_meminfo(struct seq_file *m)
3642 {
3643         struct hstate *h;
3644         unsigned long total = 0;
3645
3646         if (!hugepages_supported())
3647                 return;
3648
3649         for_each_hstate(h) {
3650                 unsigned long count = h->nr_huge_pages;
3651
3652                 total += huge_page_size(h) * count;
3653
3654                 if (h == &default_hstate)
3655                         seq_printf(m,
3656                                    "HugePages_Total:   %5lu\n"
3657                                    "HugePages_Free:    %5lu\n"
3658                                    "HugePages_Rsvd:    %5lu\n"
3659                                    "HugePages_Surp:    %5lu\n"
3660                                    "Hugepagesize:   %8lu kB\n",
3661                                    count,
3662                                    h->free_huge_pages,
3663                                    h->resv_huge_pages,
3664                                    h->surplus_huge_pages,
3665                                    huge_page_size(h) / SZ_1K);
3666         }
3667
3668         seq_printf(m, "Hugetlb:        %8lu kB\n", total / SZ_1K);
3669 }
3670
3671 int hugetlb_report_node_meminfo(char *buf, int len, int nid)
3672 {
3673         struct hstate *h = &default_hstate;
3674
3675         if (!hugepages_supported())
3676                 return 0;
3677
3678         return sysfs_emit_at(buf, len,
3679                              "Node %d HugePages_Total: %5u\n"
3680                              "Node %d HugePages_Free:  %5u\n"
3681                              "Node %d HugePages_Surp:  %5u\n",
3682                              nid, h->nr_huge_pages_node[nid],
3683                              nid, h->free_huge_pages_node[nid],
3684                              nid, h->surplus_huge_pages_node[nid]);
3685 }
3686
3687 void hugetlb_show_meminfo(void)
3688 {
3689         struct hstate *h;
3690         int nid;
3691
3692         if (!hugepages_supported())
3693                 return;
3694
3695         for_each_node_state(nid, N_MEMORY)
3696                 for_each_hstate(h)
3697                         pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
3698                                 nid,
3699                                 h->nr_huge_pages_node[nid],
3700                                 h->free_huge_pages_node[nid],
3701                                 h->surplus_huge_pages_node[nid],
3702                                 huge_page_size(h) / SZ_1K);
3703 }
3704
3705 void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm)
3706 {
3707         seq_printf(m, "HugetlbPages:\t%8lu kB\n",
3708                    atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10));
3709 }
3710
3711 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
3712 unsigned long hugetlb_total_pages(void)
3713 {
3714         struct hstate *h;
3715         unsigned long nr_total_pages = 0;
3716
3717         for_each_hstate(h)
3718                 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
3719         return nr_total_pages;
3720 }
3721
3722 static int hugetlb_acct_memory(struct hstate *h, long delta)
3723 {
3724         int ret = -ENOMEM;
3725
3726         if (!delta)
3727                 return 0;
3728
3729         spin_lock_irq(&hugetlb_lock);
3730         /*
3731          * When cpuset is configured, it breaks the strict hugetlb page
3732          * reservation as the accounting is done on a global variable. Such
3733          * reservation is completely rubbish in the presence of cpuset because
3734          * the reservation is not checked against page availability for the
3735          * current cpuset. Application can still potentially OOM'ed by kernel
3736          * with lack of free htlb page in cpuset that the task is in.
3737          * Attempt to enforce strict accounting with cpuset is almost
3738          * impossible (or too ugly) because cpuset is too fluid that
3739          * task or memory node can be dynamically moved between cpusets.
3740          *
3741          * The change of semantics for shared hugetlb mapping with cpuset is
3742          * undesirable. However, in order to preserve some of the semantics,
3743          * we fall back to check against current free page availability as
3744          * a best attempt and hopefully to minimize the impact of changing
3745          * semantics that cpuset has.
3746          *
3747          * Apart from cpuset, we also have memory policy mechanism that
3748          * also determines from which node the kernel will allocate memory
3749          * in a NUMA system. So similar to cpuset, we also should consider
3750          * the memory policy of the current task. Similar to the description
3751          * above.
3752          */
3753         if (delta > 0) {
3754                 if (gather_surplus_pages(h, delta) < 0)
3755                         goto out;
3756
3757                 if (delta > allowed_mems_nr(h)) {
3758                         return_unused_surplus_pages(h, delta);
3759                         goto out;
3760                 }
3761         }
3762
3763         ret = 0;
3764         if (delta < 0)
3765                 return_unused_surplus_pages(h, (unsigned long) -delta);
3766
3767 out:
3768         spin_unlock_irq(&hugetlb_lock);
3769         return ret;
3770 }
3771
3772 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
3773 {
3774         struct resv_map *resv = vma_resv_map(vma);
3775
3776         /*
3777          * This new VMA should share its siblings reservation map if present.
3778          * The VMA will only ever have a valid reservation map pointer where
3779          * it is being copied for another still existing VMA.  As that VMA
3780          * has a reference to the reservation map it cannot disappear until
3781          * after this open call completes.  It is therefore safe to take a
3782          * new reference here without additional locking.
3783          */
3784         if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3785                 kref_get(&resv->refs);
3786 }
3787
3788 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
3789 {
3790         struct hstate *h = hstate_vma(vma);
3791         struct resv_map *resv = vma_resv_map(vma);
3792         struct hugepage_subpool *spool = subpool_vma(vma);
3793         unsigned long reserve, start, end;
3794         long gbl_reserve;
3795
3796         if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3797                 return;
3798
3799         start = vma_hugecache_offset(h, vma, vma->vm_start);
3800         end = vma_hugecache_offset(h, vma, vma->vm_end);
3801
3802         reserve = (end - start) - region_count(resv, start, end);
3803         hugetlb_cgroup_uncharge_counter(resv, start, end);
3804         if (reserve) {
3805                 /*
3806                  * Decrement reserve counts.  The global reserve count may be
3807                  * adjusted if the subpool has a minimum size.
3808                  */
3809                 gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
3810                 hugetlb_acct_memory(h, -gbl_reserve);
3811         }
3812
3813         kref_put(&resv->refs, resv_map_release);
3814 }
3815
3816 static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr)
3817 {
3818         if (addr & ~(huge_page_mask(hstate_vma(vma))))
3819                 return -EINVAL;
3820         return 0;
3821 }
3822
3823 static unsigned long hugetlb_vm_op_pagesize(struct vm_area_struct *vma)
3824 {
3825         return huge_page_size(hstate_vma(vma));
3826 }
3827
3828 /*
3829  * We cannot handle pagefaults against hugetlb pages at all.  They cause
3830  * handle_mm_fault() to try to instantiate regular-sized pages in the
3831  * hugepage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
3832  * this far.
3833  */
3834 static vm_fault_t hugetlb_vm_op_fault(struct vm_fault *vmf)
3835 {
3836         BUG();
3837         return 0;
3838 }
3839
3840 /*
3841  * When a new function is introduced to vm_operations_struct and added
3842  * to hugetlb_vm_ops, please consider adding the function to shm_vm_ops.
3843  * This is because under System V memory model, mappings created via
3844  * shmget/shmat with "huge page" specified are backed by hugetlbfs files,
3845  * their original vm_ops are overwritten with shm_vm_ops.
3846  */
3847 const struct vm_operations_struct hugetlb_vm_ops = {
3848         .fault = hugetlb_vm_op_fault,
3849         .open = hugetlb_vm_op_open,
3850         .close = hugetlb_vm_op_close,
3851         .may_split = hugetlb_vm_op_split,
3852         .pagesize = hugetlb_vm_op_pagesize,
3853 };
3854
3855 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
3856                                 int writable)
3857 {
3858         pte_t entry;
3859
3860         if (writable) {
3861                 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
3862                                          vma->vm_page_prot)));
3863         } else {
3864                 entry = huge_pte_wrprotect(mk_huge_pte(page,
3865                                            vma->vm_page_prot));
3866         }
3867         entry = pte_mkyoung(entry);
3868         entry = pte_mkhuge(entry);
3869         entry = arch_make_huge_pte(entry, vma, page, writable);
3870
3871         return entry;
3872 }
3873
3874 static void set_huge_ptep_writable(struct vm_area_struct *vma,
3875                                    unsigned long address, pte_t *ptep)
3876 {
3877         pte_t entry;
3878
3879         entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
3880         if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
3881                 update_mmu_cache(vma, address, ptep);
3882 }
3883
3884 bool is_hugetlb_entry_migration(pte_t pte)
3885 {
3886         swp_entry_t swp;
3887
3888         if (huge_pte_none(pte) || pte_present(pte))
3889                 return false;
3890         swp = pte_to_swp_entry(pte);
3891         if (is_migration_entry(swp))
3892                 return true;
3893         else
3894                 return false;
3895 }
3896
3897 static bool is_hugetlb_entry_hwpoisoned(pte_t pte)
3898 {
3899         swp_entry_t swp;
3900
3901         if (huge_pte_none(pte) || pte_present(pte))
3902                 return false;
3903         swp = pte_to_swp_entry(pte);
3904         if (is_hwpoison_entry(swp))
3905                 return true;
3906         else
3907                 return false;
3908 }
3909
3910 static void
3911 hugetlb_install_page(struct vm_area_struct *vma, pte_t *ptep, unsigned long addr,
3912                      struct page *new_page)
3913 {
3914         __SetPageUptodate(new_page);
3915         set_huge_pte_at(vma->vm_mm, addr, ptep, make_huge_pte(vma, new_page, 1));
3916         hugepage_add_new_anon_rmap(new_page, vma, addr);
3917         hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm);
3918         ClearHPageRestoreReserve(new_page);
3919         SetHPageMigratable(new_page);
3920 }
3921
3922 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
3923                             struct vm_area_struct *vma)
3924 {
3925         pte_t *src_pte, *dst_pte, entry, dst_entry;
3926         struct page *ptepage;
3927         unsigned long addr;
3928         bool cow = is_cow_mapping(vma->vm_flags);
3929         struct hstate *h = hstate_vma(vma);
3930         unsigned long sz = huge_page_size(h);
3931         unsigned long npages = pages_per_huge_page(h);
3932         struct address_space *mapping = vma->vm_file->f_mapping;
3933         struct mmu_notifier_range range;
3934         int ret = 0;
3935
3936         if (cow) {
3937                 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, src,
3938                                         vma->vm_start,
3939                                         vma->vm_end);
3940                 mmu_notifier_invalidate_range_start(&range);
3941         } else {
3942                 /*
3943                  * For shared mappings i_mmap_rwsem must be held to call
3944                  * huge_pte_alloc, otherwise the returned ptep could go
3945                  * away if part of a shared pmd and another thread calls
3946                  * huge_pmd_unshare.
3947                  */
3948                 i_mmap_lock_read(mapping);
3949         }
3950
3951         for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
3952                 spinlock_t *src_ptl, *dst_ptl;
3953                 src_pte = huge_pte_offset(src, addr, sz);
3954                 if (!src_pte)
3955                         continue;
3956                 dst_pte = huge_pte_alloc(dst, vma, addr, sz);
3957                 if (!dst_pte) {
3958                         ret = -ENOMEM;
3959                         break;
3960                 }
3961
3962                 /*
3963                  * If the pagetables are shared don't copy or take references.
3964                  * dst_pte == src_pte is the common case of src/dest sharing.
3965                  *
3966                  * However, src could have 'unshared' and dst shares with
3967                  * another vma.  If dst_pte !none, this implies sharing.
3968                  * Check here before taking page table lock, and once again
3969                  * after taking the lock below.
3970                  */
3971                 dst_entry = huge_ptep_get(dst_pte);
3972                 if ((dst_pte == src_pte) || !huge_pte_none(dst_entry))
3973                         continue;
3974
3975                 dst_ptl = huge_pte_lock(h, dst, dst_pte);
3976                 src_ptl = huge_pte_lockptr(h, src, src_pte);
3977                 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
3978                 entry = huge_ptep_get(src_pte);
3979                 dst_entry = huge_ptep_get(dst_pte);
3980 again:
3981                 if (huge_pte_none(entry) || !huge_pte_none(dst_entry)) {
3982                         /*
3983                          * Skip if src entry none.  Also, skip in the
3984                          * unlikely case dst entry !none as this implies
3985                          * sharing with another vma.
3986                          */
3987                         ;
3988                 } else if (unlikely(is_hugetlb_entry_migration(entry) ||
3989                                     is_hugetlb_entry_hwpoisoned(entry))) {
3990                         swp_entry_t swp_entry = pte_to_swp_entry(entry);
3991
3992                         if (is_write_migration_entry(swp_entry) && cow) {
3993                                 /*
3994                                  * COW mappings require pages in both
3995                                  * parent and child to be set to read.
3996                                  */
3997                                 make_migration_entry_read(&swp_entry);
3998                                 entry = swp_entry_to_pte(swp_entry);
3999                                 set_huge_swap_pte_at(src, addr, src_pte,
4000                                                      entry, sz);
4001                         }
4002                         set_huge_swap_pte_at(dst, addr, dst_pte, entry, sz);
4003                 } else {
4004                         entry = huge_ptep_get(src_pte);
4005                         ptepage = pte_page(entry);
4006                         get_page(ptepage);
4007
4008                         /*
4009                          * This is a rare case where we see pinned hugetlb
4010                          * pages while they're prone to COW.  We need to do the
4011                          * COW earlier during fork.
4012                          *
4013                          * When pre-allocating the page or copying data, we
4014                          * need to be without the pgtable locks since we could
4015                          * sleep during the process.
4016                          */
4017                         if (unlikely(page_needs_cow_for_dma(vma, ptepage))) {
4018                                 pte_t src_pte_old = entry;
4019                                 struct page *new;
4020
4021                                 spin_unlock(src_ptl);
4022                                 spin_unlock(dst_ptl);
4023                                 /* Do not use reserve as it's private owned */
4024                                 new = alloc_huge_page(vma, addr, 1);
4025                                 if (IS_ERR(new)) {
4026                                         put_page(ptepage);
4027                                         ret = PTR_ERR(new);
4028                                         break;
4029                                 }
4030                                 copy_user_huge_page(new, ptepage, addr, vma,
4031                                                     npages);
4032                                 put_page(ptepage);
4033
4034                                 /* Install the new huge page if src pte stable */
4035                                 dst_ptl = huge_pte_lock(h, dst, dst_pte);
4036                                 src_ptl = huge_pte_lockptr(h, src, src_pte);
4037                                 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
4038                                 entry = huge_ptep_get(src_pte);
4039                                 if (!pte_same(src_pte_old, entry)) {
4040                                         put_page(new);
4041                                         /* dst_entry won't change as in child */
4042                                         goto again;
4043                                 }
4044                                 hugetlb_install_page(vma, dst_pte, addr, new);
4045                                 spin_unlock(src_ptl);
4046                                 spin_unlock(dst_ptl);
4047                                 continue;
4048                         }
4049
4050                         if (cow) {
4051                                 /*
4052                                  * No need to notify as we are downgrading page
4053                                  * table protection not changing it to point
4054                                  * to a new page.
4055                                  *
4056                                  * See Documentation/vm/mmu_notifier.rst
4057                                  */
4058                                 huge_ptep_set_wrprotect(src, addr, src_pte);
4059                         }
4060
4061                         page_dup_rmap(ptepage, true);
4062                         set_huge_pte_at(dst, addr, dst_pte, entry);
4063                         hugetlb_count_add(npages, dst);
4064                 }
4065                 spin_unlock(src_ptl);
4066                 spin_unlock(dst_ptl);
4067         }
4068
4069         if (cow)
4070                 mmu_notifier_invalidate_range_end(&range);
4071         else
4072                 i_mmap_unlock_read(mapping);
4073
4074         return ret;
4075 }
4076
4077 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
4078                             unsigned long start, unsigned long end,
4079                             struct page *ref_page)
4080 {
4081         struct mm_struct *mm = vma->vm_mm;
4082         unsigned long address;
4083         pte_t *ptep;
4084         pte_t pte;
4085         spinlock_t *ptl;
4086         struct page *page;
4087         struct hstate *h = hstate_vma(vma);
4088         unsigned long sz = huge_page_size(h);
4089         struct mmu_notifier_range range;
4090
4091         WARN_ON(!is_vm_hugetlb_page(vma));
4092         BUG_ON(start & ~huge_page_mask(h));
4093         BUG_ON(end & ~huge_page_mask(h));
4094
4095         /*
4096          * This is a hugetlb vma, all the pte entries should point
4097          * to huge page.
4098          */
4099         tlb_change_page_size(tlb, sz);
4100         tlb_start_vma(tlb, vma);
4101
4102         /*
4103          * If sharing possible, alert mmu notifiers of worst case.
4104          */
4105         mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, mm, start,
4106                                 end);
4107         adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
4108         mmu_notifier_invalidate_range_start(&range);
4109         address = start;
4110         for (; address < end; address += sz) {
4111                 ptep = huge_pte_offset(mm, address, sz);
4112                 if (!ptep)
4113                         continue;
4114
4115                 ptl = huge_pte_lock(h, mm, ptep);
4116                 if (huge_pmd_unshare(mm, vma, &address, ptep)) {
4117                         spin_unlock(ptl);
4118                         /*
4119                          * We just unmapped a page of PMDs by clearing a PUD.
4120                          * The caller's TLB flush range should cover this area.
4121                          */
4122                         continue;
4123                 }
4124
4125                 pte = huge_ptep_get(ptep);
4126                 if (huge_pte_none(pte)) {
4127                         spin_unlock(ptl);
4128                         continue;
4129                 }
4130
4131                 /*
4132                  * Migrating hugepage or HWPoisoned hugepage is already
4133                  * unmapped and its refcount is dropped, so just clear pte here.
4134                  */
4135                 if (unlikely(!pte_present(pte))) {
4136                         huge_pte_clear(mm, address, ptep, sz);
4137                         spin_unlock(ptl);
4138                         continue;
4139                 }
4140
4141                 page = pte_page(pte);
4142                 /*
4143                  * If a reference page is supplied, it is because a specific
4144                  * page is being unmapped, not a range. Ensure the page we
4145                  * are about to unmap is the actual page of interest.
4146                  */
4147                 if (ref_page) {
4148                         if (page != ref_page) {
4149                                 spin_unlock(ptl);
4150                                 continue;
4151                         }
4152                         /*
4153                          * Mark the VMA as having unmapped its page so that
4154                          * future faults in this VMA will fail rather than
4155                          * looking like data was lost
4156                          */
4157                         set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
4158                 }
4159
4160                 pte = huge_ptep_get_and_clear(mm, address, ptep);
4161                 tlb_remove_huge_tlb_entry(h, tlb, ptep, address);
4162                 if (huge_pte_dirty(pte))
4163                         set_page_dirty(page);
4164
4165                 hugetlb_count_sub(pages_per_huge_page(h), mm);
4166                 page_remove_rmap(page, true);
4167
4168                 spin_unlock(ptl);
4169                 tlb_remove_page_size(tlb, page, huge_page_size(h));
4170                 /*
4171                  * Bail out after unmapping reference page if supplied
4172                  */
4173                 if (ref_page)
4174                         break;
4175         }
4176         mmu_notifier_invalidate_range_end(&range);
4177         tlb_end_vma(tlb, vma);
4178 }
4179
4180 void __unmap_hugepage_range_final(struct mmu_gather *tlb,
4181                           struct vm_area_struct *vma, unsigned long start,
4182                           unsigned long end, struct page *ref_page)
4183 {
4184         __unmap_hugepage_range(tlb, vma, start, end, ref_page);
4185
4186         /*
4187          * Clear this flag so that x86's huge_pmd_share page_table_shareable
4188          * test will fail on a vma being torn down, and not grab a page table
4189          * on its way out.  We're lucky that the flag has such an appropriate
4190          * name, and can in fact be safely cleared here. We could clear it
4191          * before the __unmap_hugepage_range above, but all that's necessary
4192          * is to clear it before releasing the i_mmap_rwsem. This works
4193          * because in the context this is called, the VMA is about to be
4194          * destroyed and the i_mmap_rwsem is held.
4195          */
4196         vma->vm_flags &= ~VM_MAYSHARE;
4197 }
4198
4199 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
4200                           unsigned long end, struct page *ref_page)
4201 {
4202         struct mmu_gather tlb;
4203
4204         tlb_gather_mmu(&tlb, vma->vm_mm);
4205         __unmap_hugepage_range(&tlb, vma, start, end, ref_page);
4206         tlb_finish_mmu(&tlb);
4207 }
4208
4209 /*
4210  * This is called when the original mapper is failing to COW a MAP_PRIVATE
4211  * mapping it owns the reserve page for. The intention is to unmap the page
4212  * from other VMAs and let the children be SIGKILLed if they are faulting the
4213  * same region.
4214  */
4215 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
4216                               struct page *page, unsigned long address)
4217 {
4218         struct hstate *h = hstate_vma(vma);
4219         struct vm_area_struct *iter_vma;
4220         struct address_space *mapping;
4221         pgoff_t pgoff;
4222
4223         /*
4224          * vm_pgoff is in PAGE_SIZE units, hence the different calculation
4225          * from page cache lookup which is in HPAGE_SIZE units.
4226          */
4227         address = address & huge_page_mask(h);
4228         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
4229                         vma->vm_pgoff;
4230         mapping = vma->vm_file->f_mapping;
4231
4232         /*
4233          * Take the mapping lock for the duration of the table walk. As
4234          * this mapping should be shared between all the VMAs,
4235          * __unmap_hugepage_range() is called as the lock is already held
4236          */
4237         i_mmap_lock_write(mapping);
4238         vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
4239                 /* Do not unmap the current VMA */
4240                 if (iter_vma == vma)
4241                         continue;
4242
4243                 /*
4244                  * Shared VMAs have their own reserves and do not affect
4245                  * MAP_PRIVATE accounting but it is possible that a shared
4246                  * VMA is using the same page so check and skip such VMAs.
4247                  */
4248                 if (iter_vma->vm_flags & VM_MAYSHARE)
4249                         continue;
4250
4251                 /*
4252                  * Unmap the page from other VMAs without their own reserves.
4253                  * They get marked to be SIGKILLed if they fault in these
4254                  * areas. This is because a future no-page fault on this VMA
4255                  * could insert a zeroed page instead of the data existing
4256                  * from the time of fork. This would look like data corruption
4257                  */
4258                 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
4259                         unmap_hugepage_range(iter_vma, address,
4260                                              address + huge_page_size(h), page);
4261         }
4262         i_mmap_unlock_write(mapping);
4263 }
4264
4265 /*
4266  * Hugetlb_cow() should be called with page lock of the original hugepage held.
4267  * Called with hugetlb_instantiation_mutex held and pte_page locked so we
4268  * cannot race with other handlers or page migration.
4269  * Keep the pte_same checks anyway to make transition from the mutex easier.
4270  */
4271 static vm_fault_t hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
4272                        unsigned long address, pte_t *ptep,
4273                        struct page *pagecache_page, spinlock_t *ptl)
4274 {
4275         pte_t pte;
4276         struct hstate *h = hstate_vma(vma);
4277         struct page *old_page, *new_page;
4278         int outside_reserve = 0;
4279         vm_fault_t ret = 0;
4280         unsigned long haddr = address & huge_page_mask(h);
4281         struct mmu_notifier_range range;
4282
4283         pte = huge_ptep_get(ptep);
4284         old_page = pte_page(pte);
4285
4286 retry_avoidcopy:
4287         /* If no-one else is actually using this page, avoid the copy
4288          * and just make the page writable */
4289         if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
4290                 page_move_anon_rmap(old_page, vma);
4291                 set_huge_ptep_writable(vma, haddr, ptep);
4292                 return 0;
4293         }
4294
4295         /*
4296          * If the process that created a MAP_PRIVATE mapping is about to
4297          * perform a COW due to a shared page count, attempt to satisfy
4298          * the allocation without using the existing reserves. The pagecache
4299          * page is used to determine if the reserve at this address was
4300          * consumed or not. If reserves were used, a partial faulted mapping
4301          * at the time of fork() could consume its reserves on COW instead
4302          * of the full address range.
4303          */
4304         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
4305                         old_page != pagecache_page)
4306                 outside_reserve = 1;
4307
4308         get_page(old_page);
4309
4310         /*
4311          * Drop page table lock as buddy allocator may be called. It will
4312          * be acquired again before returning to the caller, as expected.
4313          */
4314         spin_unlock(ptl);
4315         new_page = alloc_huge_page(vma, haddr, outside_reserve);
4316
4317         if (IS_ERR(new_page)) {
4318                 /*
4319                  * If a process owning a MAP_PRIVATE mapping fails to COW,
4320                  * it is due to references held by a child and an insufficient
4321                  * huge page pool. To guarantee the original mappers
4322                  * reliability, unmap the page from child processes. The child
4323                  * may get SIGKILLed if it later faults.
4324                  */
4325                 if (outside_reserve) {
4326                         struct address_space *mapping = vma->vm_file->f_mapping;
4327                         pgoff_t idx;
4328                         u32 hash;
4329
4330                         put_page(old_page);
4331                         BUG_ON(huge_pte_none(pte));
4332                         /*
4333                          * Drop hugetlb_fault_mutex and i_mmap_rwsem before
4334                          * unmapping.  unmapping needs to hold i_mmap_rwsem
4335                          * in write mode.  Dropping i_mmap_rwsem in read mode
4336                          * here is OK as COW mappings do not interact with
4337                          * PMD sharing.
4338                          *
4339                          * Reacquire both after unmap operation.
4340                          */
4341                         idx = vma_hugecache_offset(h, vma, haddr);
4342                         hash = hugetlb_fault_mutex_hash(mapping, idx);
4343                         mutex_unlock(&hugetlb_fault_mutex_table[hash]);
4344                         i_mmap_unlock_read(mapping);
4345
4346                         unmap_ref_private(mm, vma, old_page, haddr);
4347
4348                         i_mmap_lock_read(mapping);
4349                         mutex_lock(&hugetlb_fault_mutex_table[hash]);
4350                         spin_lock(ptl);
4351                         ptep = huge_pte_offset(mm, haddr, huge_page_size(h));
4352                         if (likely(ptep &&
4353                                    pte_same(huge_ptep_get(ptep), pte)))
4354                                 goto retry_avoidcopy;
4355                         /*
4356                          * race occurs while re-acquiring page table
4357                          * lock, and our job is done.
4358                          */
4359                         return 0;
4360                 }
4361
4362                 ret = vmf_error(PTR_ERR(new_page));
4363                 goto out_release_old;
4364         }
4365
4366         /*
4367          * When the original hugepage is shared one, it does not have
4368          * anon_vma prepared.
4369          */
4370         if (unlikely(anon_vma_prepare(vma))) {
4371                 ret = VM_FAULT_OOM;
4372                 goto out_release_all;
4373         }
4374
4375         copy_user_huge_page(new_page, old_page, address, vma,
4376                             pages_per_huge_page(h));
4377         __SetPageUptodate(new_page);
4378
4379         mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, haddr,
4380                                 haddr + huge_page_size(h));
4381         mmu_notifier_invalidate_range_start(&range);
4382
4383         /*
4384          * Retake the page table lock to check for racing updates
4385          * before the page tables are altered
4386          */
4387         spin_lock(ptl);
4388         ptep = huge_pte_offset(mm, haddr, huge_page_size(h));
4389         if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
4390                 ClearHPageRestoreReserve(new_page);
4391
4392                 /* Break COW */
4393                 huge_ptep_clear_flush(vma, haddr, ptep);
4394                 mmu_notifier_invalidate_range(mm, range.start, range.end);
4395                 set_huge_pte_at(mm, haddr, ptep,
4396                                 make_huge_pte(vma, new_page, 1));
4397                 page_remove_rmap(old_page, true);
4398                 hugepage_add_new_anon_rmap(new_page, vma, haddr);
4399                 SetHPageMigratable(new_page);
4400                 /* Make the old page be freed below */
4401                 new_page = old_page;
4402         }
4403         spin_unlock(ptl);
4404         mmu_notifier_invalidate_range_end(&range);
4405 out_release_all:
4406         restore_reserve_on_error(h, vma, haddr, new_page);
4407         put_page(new_page);
4408 out_release_old:
4409         put_page(old_page);
4410
4411         spin_lock(ptl); /* Caller expects lock to be held */
4412         return ret;
4413 }
4414
4415 /* Return the pagecache page at a given address within a VMA */
4416 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
4417                         struct vm_area_struct *vma, unsigned long address)
4418 {
4419         struct address_space *mapping;
4420         pgoff_t idx;
4421
4422         mapping = vma->vm_file->f_mapping;
4423         idx = vma_hugecache_offset(h, vma, address);
4424
4425         return find_lock_page(mapping, idx);
4426 }
4427
4428 /*
4429  * Return whether there is a pagecache page to back given address within VMA.
4430  * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
4431  */
4432 static bool hugetlbfs_pagecache_present(struct hstate *h,
4433                         struct vm_area_struct *vma, unsigned long address)
4434 {
4435         struct address_space *mapping;
4436         pgoff_t idx;
4437         struct page *page;
4438
4439         mapping = vma->vm_file->f_mapping;
4440         idx = vma_hugecache_offset(h, vma, address);
4441
4442         page = find_get_page(mapping, idx);
4443         if (page)
4444                 put_page(page);
4445         return page != NULL;
4446 }
4447
4448 int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
4449                            pgoff_t idx)
4450 {
4451         struct inode *inode = mapping->host;
4452         struct hstate *h = hstate_inode(inode);
4453         int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
4454
4455         if (err)
4456                 return err;
4457         ClearHPageRestoreReserve(page);
4458
4459         /*
4460          * set page dirty so that it will not be removed from cache/file
4461          * by non-hugetlbfs specific code paths.
4462          */
4463         set_page_dirty(page);
4464
4465         spin_lock(&inode->i_lock);
4466         inode->i_blocks += blocks_per_huge_page(h);
4467         spin_unlock(&inode->i_lock);
4468         return 0;
4469 }
4470
4471 static inline vm_fault_t hugetlb_handle_userfault(struct vm_area_struct *vma,
4472                                                   struct address_space *mapping,
4473                                                   pgoff_t idx,
4474                                                   unsigned int flags,
4475                                                   unsigned long haddr,
4476                                                   unsigned long reason)
4477 {
4478         vm_fault_t ret;
4479         u32 hash;
4480         struct vm_fault vmf = {
4481                 .vma = vma,
4482                 .address = haddr,
4483                 .flags = flags,
4484
4485                 /*
4486                  * Hard to debug if it ends up being
4487                  * used by a callee that assumes
4488                  * something about the other
4489                  * uninitialized fields... same as in
4490                  * memory.c
4491                  */
4492         };
4493
4494         /*
4495          * hugetlb_fault_mutex and i_mmap_rwsem must be
4496          * dropped before handling userfault.  Reacquire
4497          * after handling fault to make calling code simpler.
4498          */
4499         hash = hugetlb_fault_mutex_hash(mapping, idx);
4500         mutex_unlock(&hugetlb_fault_mutex_table[hash]);
4501         i_mmap_unlock_read(mapping);
4502         ret = handle_userfault(&vmf, reason);
4503         i_mmap_lock_read(mapping);
4504         mutex_lock(&hugetlb_fault_mutex_table[hash]);
4505
4506         return ret;
4507 }
4508
4509 static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
4510                         struct vm_area_struct *vma,
4511                         struct address_space *mapping, pgoff_t idx,
4512                         unsigned long address, pte_t *ptep, unsigned int flags)
4513 {
4514         struct hstate *h = hstate_vma(vma);
4515         vm_fault_t ret = VM_FAULT_SIGBUS;
4516         int anon_rmap = 0;
4517         unsigned long size;
4518         struct page *page;
4519         pte_t new_pte;
4520         spinlock_t *ptl;
4521         unsigned long haddr = address & huge_page_mask(h);
4522         bool new_page = false;
4523
4524         /*
4525          * Currently, we are forced to kill the process in the event the
4526          * original mapper has unmapped pages from the child due to a failed
4527          * COW. Warn that such a situation has occurred as it may not be obvious
4528          */
4529         if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
4530                 pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n",
4531                            current->pid);
4532                 return ret;
4533         }
4534
4535         /*
4536          * We can not race with truncation due to holding i_mmap_rwsem.
4537          * i_size is modified when holding i_mmap_rwsem, so check here
4538          * once for faults beyond end of file.
4539          */
4540         size = i_size_read(mapping->host) >> huge_page_shift(h);
4541         if (idx >= size)
4542                 goto out;
4543
4544 retry:
4545         page = find_lock_page(mapping, idx);
4546         if (!page) {
4547                 /* Check for page in userfault range */
4548                 if (userfaultfd_missing(vma)) {
4549                         ret = hugetlb_handle_userfault(vma, mapping, idx,
4550                                                        flags, haddr,
4551                                                        VM_UFFD_MISSING);
4552                         goto out;
4553                 }
4554
4555                 page = alloc_huge_page(vma, haddr, 0);
4556                 if (IS_ERR(page)) {
4557                         /*
4558                          * Returning error will result in faulting task being
4559                          * sent SIGBUS.  The hugetlb fault mutex prevents two
4560                          * tasks from racing to fault in the same page which
4561                          * could result in false unable to allocate errors.
4562                          * Page migration does not take the fault mutex, but
4563                          * does a clear then write of pte's under page table
4564                          * lock.  Page fault code could race with migration,
4565                          * notice the clear pte and try to allocate a page
4566                          * here.  Before returning error, get ptl and make
4567                          * sure there really is no pte entry.
4568                          */
4569                         ptl = huge_pte_lock(h, mm, ptep);
4570                         ret = 0;
4571                         if (huge_pte_none(huge_ptep_get(ptep)))
4572                                 ret = vmf_error(PTR_ERR(page));
4573                         spin_unlock(ptl);
4574                         goto out;
4575                 }
4576                 clear_huge_page(page, address, pages_per_huge_page(h));
4577                 __SetPageUptodate(page);
4578                 new_page = true;
4579
4580                 if (vma->vm_flags & VM_MAYSHARE) {
4581                         int err = huge_add_to_page_cache(page, mapping, idx);
4582                         if (err) {
4583                                 put_page(page);
4584                                 if (err == -EEXIST)
4585                                         goto retry;
4586                                 goto out;
4587                         }
4588                 } else {
4589                         lock_page(page);
4590                         if (unlikely(anon_vma_prepare(vma))) {
4591                                 ret = VM_FAULT_OOM;
4592                                 goto backout_unlocked;
4593                         }
4594                         anon_rmap = 1;
4595                 }
4596         } else {
4597                 /*
4598                  * If memory error occurs between mmap() and fault, some process
4599                  * don't have hwpoisoned swap entry for errored virtual address.
4600                  * So we need to block hugepage fault by PG_hwpoison bit check.
4601                  */
4602                 if (unlikely(PageHWPoison(page))) {
4603                         ret = VM_FAULT_HWPOISON_LARGE |
4604                                 VM_FAULT_SET_HINDEX(hstate_index(h));
4605                         goto backout_unlocked;
4606                 }
4607
4608                 /* Check for page in userfault range. */
4609                 if (userfaultfd_minor(vma)) {
4610                         unlock_page(page);
4611                         put_page(page);
4612                         ret = hugetlb_handle_userfault(vma, mapping, idx,
4613                                                        flags, haddr,
4614                                                        VM_UFFD_MINOR);
4615                         goto out;
4616                 }
4617         }
4618
4619         /*
4620          * If we are going to COW a private mapping later, we examine the
4621          * pending reservations for this page now. This will ensure that
4622          * any allocations necessary to record that reservation occur outside
4623          * the spinlock.
4624          */
4625         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
4626                 if (vma_needs_reservation(h, vma, haddr) < 0) {
4627                         ret = VM_FAULT_OOM;
4628                         goto backout_unlocked;
4629                 }
4630                 /* Just decrements count, does not deallocate */
4631                 vma_end_reservation(h, vma, haddr);
4632         }
4633
4634         ptl = huge_pte_lock(h, mm, ptep);
4635         ret = 0;
4636         if (!huge_pte_none(huge_ptep_get(ptep)))
4637                 goto backout;
4638
4639         if (anon_rmap) {
4640                 ClearHPageRestoreReserve(page);
4641                 hugepage_add_new_anon_rmap(page, vma, haddr);
4642         } else
4643                 page_dup_rmap(page, true);
4644         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
4645                                 && (vma->vm_flags & VM_SHARED)));
4646         set_huge_pte_at(mm, haddr, ptep, new_pte);
4647
4648         hugetlb_count_add(pages_per_huge_page(h), mm);
4649         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
4650                 /* Optimization, do the COW without a second fault */
4651                 ret = hugetlb_cow(mm, vma, address, ptep, page, ptl);
4652         }
4653
4654         spin_unlock(ptl);
4655
4656         /*
4657          * Only set HPageMigratable in newly allocated pages.  Existing pages
4658          * found in the pagecache may not have HPageMigratableset if they have
4659          * been isolated for migration.
4660          */
4661         if (new_page)
4662                 SetHPageMigratable(page);
4663
4664         unlock_page(page);
4665 out:
4666         return ret;
4667
4668 backout:
4669         spin_unlock(ptl);
4670 backout_unlocked:
4671         unlock_page(page);
4672         restore_reserve_on_error(h, vma, haddr, page);
4673         put_page(page);
4674         goto out;
4675 }
4676
4677 #ifdef CONFIG_SMP
4678 u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
4679 {
4680         unsigned long key[2];
4681         u32 hash;
4682
4683         key[0] = (unsigned long) mapping;
4684         key[1] = idx;
4685
4686         hash = jhash2((u32 *)&key, sizeof(key)/(sizeof(u32)), 0);
4687
4688         return hash & (num_fault_mutexes - 1);
4689 }
4690 #else
4691 /*
4692  * For uniprocessor systems we always use a single mutex, so just
4693  * return 0 and avoid the hashing overhead.
4694  */
4695 u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
4696 {
4697         return 0;
4698 }
4699 #endif
4700
4701 vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
4702                         unsigned long address, unsigned int flags)
4703 {
4704         pte_t *ptep, entry;
4705         spinlock_t *ptl;
4706         vm_fault_t ret;
4707         u32 hash;
4708         pgoff_t idx;
4709         struct page *page = NULL;
4710         struct page *pagecache_page = NULL;
4711         struct hstate *h = hstate_vma(vma);
4712         struct address_space *mapping;
4713         int need_wait_lock = 0;
4714         unsigned long haddr = address & huge_page_mask(h);
4715
4716         ptep = huge_pte_offset(mm, haddr, huge_page_size(h));
4717         if (ptep) {
4718                 /*
4719                  * Since we hold no locks, ptep could be stale.  That is
4720                  * OK as we are only making decisions based on content and
4721                  * not actually modifying content here.
4722                  */
4723                 entry = huge_ptep_get(ptep);
4724                 if (unlikely(is_hugetlb_entry_migration(entry))) {
4725                         migration_entry_wait_huge(vma, mm, ptep);
4726                         return 0;
4727                 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
4728                         return VM_FAULT_HWPOISON_LARGE |
4729                                 VM_FAULT_SET_HINDEX(hstate_index(h));
4730         }
4731
4732         /*
4733          * Acquire i_mmap_rwsem before calling huge_pte_alloc and hold
4734          * until finished with ptep.  This serves two purposes:
4735          * 1) It prevents huge_pmd_unshare from being called elsewhere
4736          *    and making the ptep no longer valid.
4737          * 2) It synchronizes us with i_size modifications during truncation.
4738          *
4739          * ptep could have already be assigned via huge_pte_offset.  That
4740          * is OK, as huge_pte_alloc will return the same value unless
4741          * something has changed.
4742          */
4743         mapping = vma->vm_file->f_mapping;
4744         i_mmap_lock_read(mapping);
4745         ptep = huge_pte_alloc(mm, vma, haddr, huge_page_size(h));
4746         if (!ptep) {
4747                 i_mmap_unlock_read(mapping);
4748                 return VM_FAULT_OOM;
4749         }
4750
4751         /*
4752          * Serialize hugepage allocation and instantiation, so that we don't
4753          * get spurious allocation failures if two CPUs race to instantiate
4754          * the same page in the page cache.
4755          */
4756         idx = vma_hugecache_offset(h, vma, haddr);
4757         hash = hugetlb_fault_mutex_hash(mapping, idx);
4758         mutex_lock(&hugetlb_fault_mutex_table[hash]);
4759
4760         entry = huge_ptep_get(ptep);
4761         if (huge_pte_none(entry)) {
4762                 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags);
4763                 goto out_mutex;
4764         }
4765
4766         ret = 0;
4767
4768         /*
4769          * entry could be a migration/hwpoison entry at this point, so this
4770          * check prevents the kernel from going below assuming that we have
4771          * an active hugepage in pagecache. This goto expects the 2nd page
4772          * fault, and is_hugetlb_entry_(migration|hwpoisoned) check will
4773          * properly handle it.
4774          */
4775         if (!pte_present(entry))
4776                 goto out_mutex;
4777
4778         /*
4779          * If we are going to COW the mapping later, we examine the pending
4780          * reservations for this page now. This will ensure that any
4781          * allocations necessary to record that reservation occur outside the
4782          * spinlock. For private mappings, we also lookup the pagecache
4783          * page now as it is used to determine if a reservation has been
4784          * consumed.
4785          */
4786         if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
4787                 if (vma_needs_reservation(h, vma, haddr) < 0) {
4788                         ret = VM_FAULT_OOM;
4789                         goto out_mutex;
4790                 }
4791                 /* Just decrements count, does not deallocate */
4792                 vma_end_reservation(h, vma, haddr);
4793
4794                 if (!(vma->vm_flags & VM_MAYSHARE))
4795                         pagecache_page = hugetlbfs_pagecache_page(h,
4796                                                                 vma, haddr);
4797         }
4798
4799         ptl = huge_pte_lock(h, mm, ptep);
4800
4801         /* Check for a racing update before calling hugetlb_cow */
4802         if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
4803                 goto out_ptl;
4804
4805         /*
4806          * hugetlb_cow() requires page locks of pte_page(entry) and
4807          * pagecache_page, so here we need take the former one
4808          * when page != pagecache_page or !pagecache_page.
4809          */
4810         page = pte_page(entry);
4811         if (page != pagecache_page)
4812                 if (!trylock_page(page)) {
4813                         need_wait_lock = 1;
4814                         goto out_ptl;
4815                 }
4816
4817         get_page(page);
4818
4819         if (flags & FAULT_FLAG_WRITE) {
4820                 if (!huge_pte_write(entry)) {
4821                         ret = hugetlb_cow(mm, vma, address, ptep,
4822                                           pagecache_page, ptl);
4823                         goto out_put_page;
4824                 }
4825                 entry = huge_pte_mkdirty(entry);
4826         }
4827         entry = pte_mkyoung(entry);
4828         if (huge_ptep_set_access_flags(vma, haddr, ptep, entry,
4829                                                 flags & FAULT_FLAG_WRITE))
4830                 update_mmu_cache(vma, haddr, ptep);
4831 out_put_page:
4832         if (page != pagecache_page)
4833                 unlock_page(page);
4834         put_page(page);
4835 out_ptl:
4836         spin_unlock(ptl);
4837
4838         if (pagecache_page) {
4839                 unlock_page(pagecache_page);
4840                 put_page(pagecache_page);
4841         }
4842 out_mutex:
4843         mutex_unlock(&hugetlb_fault_mutex_table[hash]);
4844         i_mmap_unlock_read(mapping);
4845         /*
4846          * Generally it's safe to hold refcount during waiting page lock. But
4847          * here we just wait to defer the next page fault to avoid busy loop and
4848          * the page is not used after unlocked before returning from the current
4849          * page fault. So we are safe from accessing freed page, even if we wait
4850          * here without taking refcount.
4851          */
4852         if (need_wait_lock)
4853                 wait_on_page_locked(page);
4854         return ret;
4855 }
4856
4857 #ifdef CONFIG_USERFAULTFD
4858 /*
4859  * Used by userfaultfd UFFDIO_COPY.  Based on mcopy_atomic_pte with
4860  * modifications for huge pages.
4861  */
4862 int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm,
4863                             pte_t *dst_pte,
4864                             struct vm_area_struct *dst_vma,
4865                             unsigned long dst_addr,
4866                             unsigned long src_addr,
4867                             enum mcopy_atomic_mode mode,
4868                             struct page **pagep)
4869 {
4870         bool is_continue = (mode == MCOPY_ATOMIC_CONTINUE);
4871         struct address_space *mapping;
4872         pgoff_t idx;
4873         unsigned long size;
4874         int vm_shared = dst_vma->vm_flags & VM_SHARED;
4875         struct hstate *h = hstate_vma(dst_vma);
4876         pte_t _dst_pte;
4877         spinlock_t *ptl;
4878         int ret;
4879         struct page *page;
4880         int writable;
4881
4882         mapping = dst_vma->vm_file->f_mapping;
4883         idx = vma_hugecache_offset(h, dst_vma, dst_addr);
4884
4885         if (is_continue) {
4886                 ret = -EFAULT;
4887                 page = find_lock_page(mapping, idx);
4888                 if (!page)
4889                         goto out;
4890         } else if (!*pagep) {
4891                 ret = -ENOMEM;
4892                 page = alloc_huge_page(dst_vma, dst_addr, 0);
4893                 if (IS_ERR(page))
4894                         goto out;
4895
4896                 ret = copy_huge_page_from_user(page,
4897                                                 (const void __user *) src_addr,
4898                                                 pages_per_huge_page(h), false);
4899
4900                 /* fallback to copy_from_user outside mmap_lock */
4901                 if (unlikely(ret)) {
4902                         ret = -ENOENT;
4903                         *pagep = page;
4904                         /* don't free the page */
4905                         goto out;
4906                 }
4907         } else {
4908                 page = *pagep;
4909                 *pagep = NULL;
4910         }
4911
4912         /*
4913          * The memory barrier inside __SetPageUptodate makes sure that
4914          * preceding stores to the page contents become visible before
4915          * the set_pte_at() write.
4916          */
4917         __SetPageUptodate(page);
4918
4919         /* Add shared, newly allocated pages to the page cache. */
4920         if (vm_shared && !is_continue) {
4921                 size = i_size_read(mapping->host) >> huge_page_shift(h);
4922                 ret = -EFAULT;
4923                 if (idx >= size)
4924                         goto out_release_nounlock;
4925
4926                 /*
4927                  * Serialization between remove_inode_hugepages() and
4928                  * huge_add_to_page_cache() below happens through the
4929                  * hugetlb_fault_mutex_table that here must be hold by
4930                  * the caller.
4931                  */
4932                 ret = huge_add_to_page_cache(page, mapping, idx);
4933                 if (ret)
4934                         goto out_release_nounlock;
4935         }
4936
4937         ptl = huge_pte_lockptr(h, dst_mm, dst_pte);
4938         spin_lock(ptl);
4939
4940         /*
4941          * Recheck the i_size after holding PT lock to make sure not
4942          * to leave any page mapped (as page_mapped()) beyond the end
4943          * of the i_size (remove_inode_hugepages() is strict about
4944          * enforcing that). If we bail out here, we'll also leave a
4945          * page in the radix tree in the vm_shared case beyond the end
4946          * of the i_size, but remove_inode_hugepages() will take care
4947          * of it as soon as we drop the hugetlb_fault_mutex_table.
4948          */
4949         size = i_size_read(mapping->host) >> huge_page_shift(h);
4950         ret = -EFAULT;
4951         if (idx >= size)
4952                 goto out_release_unlock;
4953
4954         ret = -EEXIST;
4955         if (!huge_pte_none(huge_ptep_get(dst_pte)))
4956                 goto out_release_unlock;
4957
4958         if (vm_shared) {
4959                 page_dup_rmap(page, true);
4960         } else {
4961                 ClearHPageRestoreReserve(page);
4962                 hugepage_add_new_anon_rmap(page, dst_vma, dst_addr);
4963         }
4964
4965         /* For CONTINUE on a non-shared VMA, don't set VM_WRITE for CoW. */
4966         if (is_continue && !vm_shared)
4967                 writable = 0;
4968         else
4969                 writable = dst_vma->vm_flags & VM_WRITE;
4970
4971         _dst_pte = make_huge_pte(dst_vma, page, writable);
4972         if (writable)
4973                 _dst_pte = huge_pte_mkdirty(_dst_pte);
4974         _dst_pte = pte_mkyoung(_dst_pte);
4975
4976         set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
4977
4978         (void)huge_ptep_set_access_flags(dst_vma, dst_addr, dst_pte, _dst_pte,
4979                                         dst_vma->vm_flags & VM_WRITE);
4980         hugetlb_count_add(pages_per_huge_page(h), dst_mm);
4981
4982         /* No need to invalidate - it was non-present before */
4983         update_mmu_cache(dst_vma, dst_addr, dst_pte);
4984
4985         spin_unlock(ptl);
4986         if (!is_continue)
4987                 SetHPageMigratable(page);
4988         if (vm_shared || is_continue)
4989                 unlock_page(page);
4990         ret = 0;
4991 out:
4992         return ret;
4993 out_release_unlock:
4994         spin_unlock(ptl);
4995         if (vm_shared || is_continue)
4996                 unlock_page(page);
4997 out_release_nounlock:
4998         put_page(page);
4999         goto out;
5000 }
5001 #endif /* CONFIG_USERFAULTFD */
5002
5003 static void record_subpages_vmas(struct page *page, struct vm_area_struct *vma,
5004                                  int refs, struct page **pages,
5005                                  struct vm_area_struct **vmas)
5006 {
5007         int nr;
5008
5009         for (nr = 0; nr < refs; nr++) {
5010                 if (likely(pages))
5011                         pages[nr] = mem_map_offset(page, nr);
5012                 if (vmas)
5013                         vmas[nr] = vma;
5014         }
5015 }
5016
5017 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
5018                          struct page **pages, struct vm_area_struct **vmas,
5019                          unsigned long *position, unsigned long *nr_pages,
5020                          long i, unsigned int flags, int *locked)
5021 {
5022         unsigned long pfn_offset;
5023         unsigned long vaddr = *position;
5024         unsigned long remainder = *nr_pages;
5025         struct hstate *h = hstate_vma(vma);
5026         int err = -EFAULT, refs;
5027
5028         while (vaddr < vma->vm_end && remainder) {
5029                 pte_t *pte;
5030                 spinlock_t *ptl = NULL;
5031                 int absent;
5032                 struct page *page;
5033
5034                 /*
5035                  * If we have a pending SIGKILL, don't keep faulting pages and
5036                  * potentially allocating memory.
5037                  */
5038                 if (fatal_signal_pending(current)) {
5039                         remainder = 0;
5040                         break;
5041                 }
5042
5043                 /*
5044                  * Some archs (sparc64, sh*) have multiple pte_ts to
5045                  * each hugepage.  We have to make sure we get the
5046                  * first, for the page indexing below to work.
5047                  *
5048                  * Note that page table lock is not held when pte is null.
5049                  */
5050                 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h),
5051                                       huge_page_size(h));
5052                 if (pte)
5053                         ptl = huge_pte_lock(h, mm, pte);
5054                 absent = !pte || huge_pte_none(huge_ptep_get(pte));
5055
5056                 /*
5057                  * When coredumping, it suits get_dump_page if we just return
5058                  * an error where there's an empty slot with no huge pagecache
5059                  * to back it.  This way, we avoid allocating a hugepage, and
5060                  * the sparse dumpfile avoids allocating disk blocks, but its
5061                  * huge holes still show up with zeroes where they need to be.
5062                  */
5063                 if (absent && (flags & FOLL_DUMP) &&
5064                     !hugetlbfs_pagecache_present(h, vma, vaddr)) {
5065                         if (pte)
5066                                 spin_unlock(ptl);
5067                         remainder = 0;
5068                         break;
5069                 }
5070
5071                 /*
5072                  * We need call hugetlb_fault for both hugepages under migration
5073                  * (in which case hugetlb_fault waits for the migration,) and
5074                  * hwpoisoned hugepages (in which case we need to prevent the
5075                  * caller from accessing to them.) In order to do this, we use
5076                  * here is_swap_pte instead of is_hugetlb_entry_migration and
5077                  * is_hugetlb_entry_hwpoisoned. This is because it simply covers
5078                  * both cases, and because we can't follow correct pages
5079                  * directly from any kind of swap entries.
5080                  */
5081                 if (absent || is_swap_pte(huge_ptep_get(pte)) ||
5082                     ((flags & FOLL_WRITE) &&
5083                       !huge_pte_write(huge_ptep_get(pte)))) {
5084                         vm_fault_t ret;
5085                         unsigned int fault_flags = 0;
5086
5087                         if (pte)
5088                                 spin_unlock(ptl);
5089                         if (flags & FOLL_WRITE)
5090                                 fault_flags |= FAULT_FLAG_WRITE;
5091                         if (locked)
5092                                 fault_flags |= FAULT_FLAG_ALLOW_RETRY |
5093                                         FAULT_FLAG_KILLABLE;
5094                         if (flags & FOLL_NOWAIT)
5095                                 fault_flags |= FAULT_FLAG_ALLOW_RETRY |
5096                                         FAULT_FLAG_RETRY_NOWAIT;
5097                         if (flags & FOLL_TRIED) {
5098                                 /*
5099                                  * Note: FAULT_FLAG_ALLOW_RETRY and
5100                                  * FAULT_FLAG_TRIED can co-exist
5101                                  */
5102                                 fault_flags |= FAULT_FLAG_TRIED;
5103                         }
5104                         ret = hugetlb_fault(mm, vma, vaddr, fault_flags);
5105                         if (ret & VM_FAULT_ERROR) {
5106                                 err = vm_fault_to_errno(ret, flags);
5107                                 remainder = 0;
5108                                 break;
5109                         }
5110                         if (ret & VM_FAULT_RETRY) {
5111                                 if (locked &&
5112                                     !(fault_flags & FAULT_FLAG_RETRY_NOWAIT))
5113                                         *locked = 0;
5114                                 *nr_pages = 0;
5115                                 /*
5116                                  * VM_FAULT_RETRY must not return an
5117                                  * error, it will return zero
5118                                  * instead.
5119                                  *
5120                                  * No need to update "position" as the
5121                                  * caller will not check it after
5122                                  * *nr_pages is set to 0.
5123                                  */
5124                                 return i;
5125                         }
5126                         continue;
5127                 }
5128
5129                 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
5130                 page = pte_page(huge_ptep_get(pte));
5131
5132                 /*
5133                  * If subpage information not requested, update counters
5134                  * and skip the same_page loop below.
5135                  */
5136                 if (!pages && !vmas && !pfn_offset &&
5137                     (vaddr + huge_page_size(h) < vma->vm_end) &&
5138                     (remainder >= pages_per_huge_page(h))) {
5139                         vaddr += huge_page_size(h);
5140                         remainder -= pages_per_huge_page(h);
5141                         i += pages_per_huge_page(h);
5142                         spin_unlock(ptl);
5143                         continue;
5144                 }
5145
5146                 refs = min3(pages_per_huge_page(h) - pfn_offset,
5147                             (vma->vm_end - vaddr) >> PAGE_SHIFT, remainder);
5148
5149                 if (pages || vmas)
5150                         record_subpages_vmas(mem_map_offset(page, pfn_offset),
5151                                              vma, refs,
5152                                              likely(pages) ? pages + i : NULL,
5153                                              vmas ? vmas + i : NULL);
5154
5155                 if (pages) {
5156                         /*
5157                          * try_grab_compound_head() should always succeed here,
5158                          * because: a) we hold the ptl lock, and b) we've just
5159                          * checked that the huge page is present in the page
5160                          * tables. If the huge page is present, then the tail
5161                          * pages must also be present. The ptl prevents the
5162                          * head page and tail pages from being rearranged in
5163                          * any way. So this page must be available at this
5164                          * point, unless the page refcount overflowed:
5165                          */
5166                         if (WARN_ON_ONCE(!try_grab_compound_head(pages[i],
5167                                                                  refs,
5168                                                                  flags))) {
5169                                 spin_unlock(ptl);
5170                                 remainder = 0;
5171                                 err = -ENOMEM;
5172                                 break;
5173                         }
5174                 }
5175
5176                 vaddr += (refs << PAGE_SHIFT);
5177                 remainder -= refs;
5178                 i += refs;
5179
5180                 spin_unlock(ptl);
5181         }
5182         *nr_pages = remainder;
5183         /*
5184          * setting position is actually required only if remainder is
5185          * not zero but it's faster not to add a "if (remainder)"
5186          * branch.
5187          */
5188         *position = vaddr;
5189
5190         return i ? i : err;
5191 }
5192
5193 unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
5194                 unsigned long address, unsigned long end, pgprot_t newprot)
5195 {
5196         struct mm_struct *mm = vma->vm_mm;
5197         unsigned long start = address;
5198         pte_t *ptep;
5199         pte_t pte;
5200         struct hstate *h = hstate_vma(vma);
5201         unsigned long pages = 0;
5202         bool shared_pmd = false;
5203         struct mmu_notifier_range range;
5204
5205         /*
5206          * In the case of shared PMDs, the area to flush could be beyond
5207          * start/end.  Set range.start/range.end to cover the maximum possible
5208          * range if PMD sharing is possible.
5209          */
5210         mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA,
5211                                 0, vma, mm, start, end);
5212         adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
5213
5214         BUG_ON(address >= end);
5215         flush_cache_range(vma, range.start, range.end);
5216
5217         mmu_notifier_invalidate_range_start(&range);
5218         i_mmap_lock_write(vma->vm_file->f_mapping);
5219         for (; address < end; address += huge_page_size(h)) {
5220                 spinlock_t *ptl;
5221                 ptep = huge_pte_offset(mm, address, huge_page_size(h));
5222                 if (!ptep)
5223                         continue;
5224                 ptl = huge_pte_lock(h, mm, ptep);
5225                 if (huge_pmd_unshare(mm, vma, &address, ptep)) {
5226                         pages++;
5227                         spin_unlock(ptl);
5228                         shared_pmd = true;
5229                         continue;
5230                 }
5231                 pte = huge_ptep_get(ptep);
5232                 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
5233                         spin_unlock(ptl);
5234                         continue;
5235                 }
5236                 if (unlikely(is_hugetlb_entry_migration(pte))) {
5237                         swp_entry_t entry = pte_to_swp_entry(pte);
5238
5239                         if (is_write_migration_entry(entry)) {
5240                                 pte_t newpte;
5241
5242                                 make_migration_entry_read(&entry);
5243                                 newpte = swp_entry_to_pte(entry);
5244                                 set_huge_swap_pte_at(mm, address, ptep,
5245                                                      newpte, huge_page_size(h));
5246                                 pages++;
5247                         }
5248                         spin_unlock(ptl);
5249                         continue;
5250                 }
5251                 if (!huge_pte_none(pte)) {
5252                         pte_t old_pte;
5253
5254                         old_pte = huge_ptep_modify_prot_start(vma, address, ptep);
5255                         pte = pte_mkhuge(huge_pte_modify(old_pte, newprot));
5256                         pte = arch_make_huge_pte(pte, vma, NULL, 0);
5257                         huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte);
5258                         pages++;
5259                 }
5260                 spin_unlock(ptl);
5261         }
5262         /*
5263          * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
5264          * may have cleared our pud entry and done put_page on the page table:
5265          * once we release i_mmap_rwsem, another task can do the final put_page
5266          * and that page table be reused and filled with junk.  If we actually
5267          * did unshare a page of pmds, flush the range corresponding to the pud.
5268          */
5269         if (shared_pmd)
5270                 flush_hugetlb_tlb_range(vma, range.start, range.end);
5271         else
5272                 flush_hugetlb_tlb_range(vma, start, end);
5273         /*
5274          * No need to call mmu_notifier_invalidate_range() we are downgrading
5275          * page table protection not changing it to point to a new page.
5276          *
5277          * See Documentation/vm/mmu_notifier.rst
5278          */
5279         i_mmap_unlock_write(vma->vm_file->f_mapping);
5280         mmu_notifier_invalidate_range_end(&range);
5281
5282         return pages << h->order;
5283 }
5284
5285 /* Return true if reservation was successful, false otherwise.  */
5286 bool hugetlb_reserve_pages(struct inode *inode,
5287                                         long from, long to,
5288                                         struct vm_area_struct *vma,
5289                                         vm_flags_t vm_flags)
5290 {
5291         long chg, add = -1;
5292         struct hstate *h = hstate_inode(inode);
5293         struct hugepage_subpool *spool = subpool_inode(inode);
5294         struct resv_map *resv_map;
5295         struct hugetlb_cgroup *h_cg = NULL;
5296         long gbl_reserve, regions_needed = 0;
5297
5298         /* This should never happen */
5299         if (from > to) {
5300                 VM_WARN(1, "%s called with a negative range\n", __func__);
5301                 return false;
5302         }
5303
5304         /*
5305          * Only apply hugepage reservation if asked. At fault time, an
5306          * attempt will be made for VM_NORESERVE to allocate a page
5307          * without using reserves
5308          */
5309         if (vm_flags & VM_NORESERVE)
5310                 return true;
5311
5312         /*
5313          * Shared mappings base their reservation on the number of pages that
5314          * are already allocated on behalf of the file. Private mappings need
5315          * to reserve the full area even if read-only as mprotect() may be
5316          * called to make the mapping read-write. Assume !vma is a shm mapping
5317          */
5318         if (!vma || vma->vm_flags & VM_MAYSHARE) {
5319                 /*
5320                  * resv_map can not be NULL as hugetlb_reserve_pages is only
5321                  * called for inodes for which resv_maps were created (see
5322                  * hugetlbfs_get_inode).
5323                  */
5324                 resv_map = inode_resv_map(inode);
5325
5326                 chg = region_chg(resv_map, from, to, &regions_needed);
5327
5328         } else {
5329                 /* Private mapping. */
5330                 resv_map = resv_map_alloc();
5331                 if (!resv_map)
5332                         return false;
5333
5334                 chg = to - from;
5335
5336                 set_vma_resv_map(vma, resv_map);
5337                 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
5338         }
5339
5340         if (chg < 0)
5341                 goto out_err;
5342
5343         if (hugetlb_cgroup_charge_cgroup_rsvd(hstate_index(h),
5344                                 chg * pages_per_huge_page(h), &h_cg) < 0)
5345                 goto out_err;
5346
5347         if (vma && !(vma->vm_flags & VM_MAYSHARE) && h_cg) {
5348                 /* For private mappings, the hugetlb_cgroup uncharge info hangs
5349                  * of the resv_map.
5350                  */
5351                 resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, h_cg, h);
5352         }
5353
5354         /*
5355          * There must be enough pages in the subpool for the mapping. If
5356          * the subpool has a minimum size, there may be some global
5357          * reservations already in place (gbl_reserve).
5358          */
5359         gbl_reserve = hugepage_subpool_get_pages(spool, chg);
5360         if (gbl_reserve < 0)
5361                 goto out_uncharge_cgroup;
5362
5363         /*
5364          * Check enough hugepages are available for the reservation.
5365          * Hand the pages back to the subpool if there are not
5366          */
5367         if (hugetlb_acct_memory(h, gbl_reserve) < 0)
5368                 goto out_put_pages;
5369
5370         /*
5371          * Account for the reservations made. Shared mappings record regions
5372          * that have reservations as they are shared by multiple VMAs.
5373          * When the last VMA disappears, the region map says how much
5374          * the reservation was and the page cache tells how much of
5375          * the reservation was consumed. Private mappings are per-VMA and
5376          * only the consumed reservations are tracked. When the VMA
5377          * disappears, the original reservation is the VMA size and the
5378          * consumed reservations are stored in the map. Hence, nothing
5379          * else has to be done for private mappings here
5380          */
5381         if (!vma || vma->vm_flags & VM_MAYSHARE) {
5382                 add = region_add(resv_map, from, to, regions_needed, h, h_cg);
5383
5384                 if (unlikely(add < 0)) {
5385                         hugetlb_acct_memory(h, -gbl_reserve);
5386                         goto out_put_pages;
5387                 } else if (unlikely(chg > add)) {
5388                         /*
5389                          * pages in this range were added to the reserve
5390                          * map between region_chg and region_add.  This
5391                          * indicates a race with alloc_huge_page.  Adjust
5392                          * the subpool and reserve counts modified above
5393                          * based on the difference.
5394                          */
5395                         long rsv_adjust;
5396
5397                         /*
5398                          * hugetlb_cgroup_uncharge_cgroup_rsvd() will put the
5399                          * reference to h_cg->css. See comment below for detail.
5400                          */
5401                         hugetlb_cgroup_uncharge_cgroup_rsvd(
5402                                 hstate_index(h),
5403                                 (chg - add) * pages_per_huge_page(h), h_cg);
5404
5405                         rsv_adjust = hugepage_subpool_put_pages(spool,
5406                                                                 chg - add);
5407                         hugetlb_acct_memory(h, -rsv_adjust);
5408                 } else if (h_cg) {
5409                         /*
5410                          * The file_regions will hold their own reference to
5411                          * h_cg->css. So we should release the reference held
5412                          * via hugetlb_cgroup_charge_cgroup_rsvd() when we are
5413                          * done.
5414                          */
5415                         hugetlb_cgroup_put_rsvd_cgroup(h_cg);
5416                 }
5417         }
5418         return true;
5419
5420 out_put_pages:
5421         /* put back original number of pages, chg */
5422         (void)hugepage_subpool_put_pages(spool, chg);
5423 out_uncharge_cgroup:
5424         hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h),
5425                                             chg * pages_per_huge_page(h), h_cg);
5426 out_err:
5427         if (!vma || vma->vm_flags & VM_MAYSHARE)
5428                 /* Only call region_abort if the region_chg succeeded but the
5429                  * region_add failed or didn't run.
5430                  */
5431                 if (chg >= 0 && add < 0)
5432                         region_abort(resv_map, from, to, regions_needed);
5433         if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
5434                 kref_put(&resv_map->refs, resv_map_release);
5435         return false;
5436 }
5437
5438 long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
5439                                                                 long freed)
5440 {
5441         struct hstate *h = hstate_inode(inode);
5442         struct resv_map *resv_map = inode_resv_map(inode);
5443         long chg = 0;
5444         struct hugepage_subpool *spool = subpool_inode(inode);
5445         long gbl_reserve;
5446
5447         /*
5448          * Since this routine can be called in the evict inode path for all
5449          * hugetlbfs inodes, resv_map could be NULL.
5450          */
5451         if (resv_map) {
5452                 chg = region_del(resv_map, start, end);
5453                 /*
5454                  * region_del() can fail in the rare case where a region
5455                  * must be split and another region descriptor can not be
5456                  * allocated.  If end == LONG_MAX, it will not fail.
5457                  */
5458                 if (chg < 0)
5459                         return chg;
5460         }
5461
5462         spin_lock(&inode->i_lock);
5463         inode->i_blocks -= (blocks_per_huge_page(h) * freed);
5464         spin_unlock(&inode->i_lock);
5465
5466         /*
5467          * If the subpool has a minimum size, the number of global
5468          * reservations to be released may be adjusted.
5469          *
5470          * Note that !resv_map implies freed == 0. So (chg - freed)
5471          * won't go negative.
5472          */
5473         gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
5474         hugetlb_acct_memory(h, -gbl_reserve);
5475
5476         return 0;
5477 }
5478
5479 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
5480 static unsigned long page_table_shareable(struct vm_area_struct *svma,
5481                                 struct vm_area_struct *vma,
5482                                 unsigned long addr, pgoff_t idx)
5483 {
5484         unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
5485                                 svma->vm_start;
5486         unsigned long sbase = saddr & PUD_MASK;
5487         unsigned long s_end = sbase + PUD_SIZE;
5488
5489         /* Allow segments to share if only one is marked locked */
5490         unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
5491         unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK;
5492
5493         /*
5494          * match the virtual addresses, permission and the alignment of the
5495          * page table page.
5496          */
5497         if (pmd_index(addr) != pmd_index(saddr) ||
5498             vm_flags != svm_flags ||
5499             !range_in_vma(svma, sbase, s_end))
5500                 return 0;
5501
5502         return saddr;
5503 }
5504
5505 static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr)
5506 {
5507         unsigned long base = addr & PUD_MASK;
5508         unsigned long end = base + PUD_SIZE;
5509
5510         /*
5511          * check on proper vm_flags and page table alignment
5512          */
5513         if (vma->vm_flags & VM_MAYSHARE && range_in_vma(vma, base, end))
5514                 return true;
5515         return false;
5516 }
5517
5518 bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
5519 {
5520 #ifdef CONFIG_USERFAULTFD
5521         if (uffd_disable_huge_pmd_share(vma))
5522                 return false;
5523 #endif
5524         return vma_shareable(vma, addr);
5525 }
5526
5527 /*
5528  * Determine if start,end range within vma could be mapped by shared pmd.
5529  * If yes, adjust start and end to cover range associated with possible
5530  * shared pmd mappings.
5531  */
5532 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
5533                                 unsigned long *start, unsigned long *end)
5534 {
5535         unsigned long v_start = ALIGN(vma->vm_start, PUD_SIZE),
5536                 v_end = ALIGN_DOWN(vma->vm_end, PUD_SIZE);
5537
5538         /*
5539          * vma need span at least one aligned PUD size and the start,end range
5540          * must at least partialy within it.
5541          */
5542         if (!(vma->vm_flags & VM_MAYSHARE) || !(v_end > v_start) ||
5543                 (*end <= v_start) || (*start >= v_end))
5544                 return;
5545
5546         /* Extend the range to be PUD aligned for a worst case scenario */
5547         if (*start > v_start)
5548                 *start = ALIGN_DOWN(*start, PUD_SIZE);
5549
5550         if (*end < v_end)
5551                 *end = ALIGN(*end, PUD_SIZE);
5552 }
5553
5554 /*
5555  * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
5556  * and returns the corresponding pte. While this is not necessary for the
5557  * !shared pmd case because we can allocate the pmd later as well, it makes the
5558  * code much cleaner.
5559  *
5560  * This routine must be called with i_mmap_rwsem held in at least read mode if
5561  * sharing is possible.  For hugetlbfs, this prevents removal of any page
5562  * table entries associated with the address space.  This is important as we
5563  * are setting up sharing based on existing page table entries (mappings).
5564  *
5565  * NOTE: This routine is only called from huge_pte_alloc.  Some callers of
5566  * huge_pte_alloc know that sharing is not possible and do not take
5567  * i_mmap_rwsem as a performance optimization.  This is handled by the
5568  * if !vma_shareable check at the beginning of the routine. i_mmap_rwsem is
5569  * only required for subsequent processing.
5570  */
5571 pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
5572                       unsigned long addr, pud_t *pud)
5573 {
5574         struct address_space *mapping = vma->vm_file->f_mapping;
5575         pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
5576                         vma->vm_pgoff;
5577         struct vm_area_struct *svma;
5578         unsigned long saddr;
5579         pte_t *spte = NULL;
5580         pte_t *pte;
5581         spinlock_t *ptl;
5582
5583         i_mmap_assert_locked(mapping);
5584         vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
5585                 if (svma == vma)
5586                         continue;
5587
5588                 saddr = page_table_shareable(svma, vma, addr, idx);
5589                 if (saddr) {
5590                         spte = huge_pte_offset(svma->vm_mm, saddr,
5591                                                vma_mmu_pagesize(svma));
5592                         if (spte) {
5593                                 get_page(virt_to_page(spte));
5594                                 break;
5595                         }
5596                 }
5597         }
5598
5599         if (!spte)
5600                 goto out;
5601
5602         ptl = huge_pte_lock(hstate_vma(vma), mm, spte);
5603         if (pud_none(*pud)) {
5604                 pud_populate(mm, pud,
5605                                 (pmd_t *)((unsigned long)spte & PAGE_MASK));
5606                 mm_inc_nr_pmds(mm);
5607         } else {
5608                 put_page(virt_to_page(spte));
5609         }
5610         spin_unlock(ptl);
5611 out:
5612         pte = (pte_t *)pmd_alloc(mm, pud, addr);
5613         return pte;
5614 }
5615
5616 /*
5617  * unmap huge page backed by shared pte.
5618  *
5619  * Hugetlb pte page is ref counted at the time of mapping.  If pte is shared
5620  * indicated by page_count > 1, unmap is achieved by clearing pud and
5621  * decrementing the ref count. If count == 1, the pte page is not shared.
5622  *
5623  * Called with page table lock held and i_mmap_rwsem held in write mode.
5624  *
5625  * returns: 1 successfully unmapped a shared pte page
5626  *          0 the underlying pte page is not shared, or it is the last user
5627  */
5628 int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
5629                                         unsigned long *addr, pte_t *ptep)
5630 {
5631         pgd_t *pgd = pgd_offset(mm, *addr);
5632         p4d_t *p4d = p4d_offset(pgd, *addr);
5633         pud_t *pud = pud_offset(p4d, *addr);
5634
5635         i_mmap_assert_write_locked(vma->vm_file->f_mapping);
5636         BUG_ON(page_count(virt_to_page(ptep)) == 0);
5637         if (page_count(virt_to_page(ptep)) == 1)
5638                 return 0;
5639
5640         pud_clear(pud);
5641         put_page(virt_to_page(ptep));
5642         mm_dec_nr_pmds(mm);
5643         *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
5644         return 1;
5645 }
5646
5647 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
5648 pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
5649                       unsigned long addr, pud_t *pud)
5650 {
5651         return NULL;
5652 }
5653
5654 int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
5655                                 unsigned long *addr, pte_t *ptep)
5656 {
5657         return 0;
5658 }
5659
5660 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
5661                                 unsigned long *start, unsigned long *end)
5662 {
5663 }
5664
5665 bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
5666 {
5667         return false;
5668 }
5669 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
5670
5671 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
5672 pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
5673                         unsigned long addr, unsigned long sz)
5674 {
5675         pgd_t *pgd;
5676         p4d_t *p4d;
5677         pud_t *pud;
5678         pte_t *pte = NULL;
5679
5680         pgd = pgd_offset(mm, addr);
5681         p4d = p4d_alloc(mm, pgd, addr);
5682         if (!p4d)
5683                 return NULL;
5684         pud = pud_alloc(mm, p4d, addr);
5685         if (pud) {
5686                 if (sz == PUD_SIZE) {
5687                         pte = (pte_t *)pud;
5688                 } else {
5689                         BUG_ON(sz != PMD_SIZE);
5690                         if (want_pmd_share(vma, addr) && pud_none(*pud))
5691                                 pte = huge_pmd_share(mm, vma, addr, pud);
5692                         else
5693                                 pte = (pte_t *)pmd_alloc(mm, pud, addr);
5694                 }
5695         }
5696         BUG_ON(pte && pte_present(*pte) && !pte_huge(*pte));
5697
5698         return pte;
5699 }
5700
5701 /*
5702  * huge_pte_offset() - Walk the page table to resolve the hugepage
5703  * entry at address @addr
5704  *
5705  * Return: Pointer to page table entry (PUD or PMD) for
5706  * address @addr, or NULL if a !p*d_present() entry is encountered and the
5707  * size @sz doesn't match the hugepage size at this level of the page
5708  * table.
5709  */
5710 pte_t *huge_pte_offset(struct mm_struct *mm,
5711                        unsigned long addr, unsigned long sz)
5712 {
5713         pgd_t *pgd;
5714         p4d_t *p4d;
5715         pud_t *pud;
5716         pmd_t *pmd;
5717
5718         pgd = pgd_offset(mm, addr);
5719         if (!pgd_present(*pgd))
5720                 return NULL;
5721         p4d = p4d_offset(pgd, addr);
5722         if (!p4d_present(*p4d))
5723                 return NULL;
5724
5725         pud = pud_offset(p4d, addr);
5726         if (sz == PUD_SIZE)
5727                 /* must be pud huge, non-present or none */
5728                 return (pte_t *)pud;
5729         if (!pud_present(*pud))
5730                 return NULL;
5731         /* must have a valid entry and size to go further */
5732
5733         pmd = pmd_offset(pud, addr);
5734         /* must be pmd huge, non-present or none */
5735         return (pte_t *)pmd;
5736 }
5737
5738 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
5739
5740 /*
5741  * These functions are overwritable if your architecture needs its own
5742  * behavior.
5743  */
5744 struct page * __weak
5745 follow_huge_addr(struct mm_struct *mm, unsigned long address,
5746                               int write)
5747 {
5748         return ERR_PTR(-EINVAL);
5749 }
5750
5751 struct page * __weak
5752 follow_huge_pd(struct vm_area_struct *vma,
5753                unsigned long address, hugepd_t hpd, int flags, int pdshift)
5754 {
5755         WARN(1, "hugepd follow called with no support for hugepage directory format\n");
5756         return NULL;
5757 }
5758
5759 struct page * __weak
5760 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
5761                 pmd_t *pmd, int flags)
5762 {
5763         struct page *page = NULL;
5764         spinlock_t *ptl;
5765         pte_t pte;
5766
5767         /* FOLL_GET and FOLL_PIN are mutually exclusive. */
5768         if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) ==
5769                          (FOLL_PIN | FOLL_GET)))
5770                 return NULL;
5771
5772 retry:
5773         ptl = pmd_lockptr(mm, pmd);
5774         spin_lock(ptl);
5775         /*
5776          * make sure that the address range covered by this pmd is not
5777          * unmapped from other threads.
5778          */
5779         if (!pmd_huge(*pmd))
5780                 goto out;
5781         pte = huge_ptep_get((pte_t *)pmd);
5782         if (pte_present(pte)) {
5783                 page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT);
5784                 /*
5785                  * try_grab_page() should always succeed here, because: a) we
5786                  * hold the pmd (ptl) lock, and b) we've just checked that the
5787                  * huge pmd (head) page is present in the page tables. The ptl
5788                  * prevents the head page and tail pages from being rearranged
5789                  * in any way. So this page must be available at this point,
5790                  * unless the page refcount overflowed:
5791                  */
5792                 if (WARN_ON_ONCE(!try_grab_page(page, flags))) {
5793                         page = NULL;
5794                         goto out;
5795                 }
5796         } else {
5797                 if (is_hugetlb_entry_migration(pte)) {
5798                         spin_unlock(ptl);
5799                         __migration_entry_wait(mm, (pte_t *)pmd, ptl);
5800                         goto retry;
5801                 }
5802                 /*
5803                  * hwpoisoned entry is treated as no_page_table in
5804                  * follow_page_mask().
5805                  */
5806         }
5807 out:
5808         spin_unlock(ptl);
5809         return page;
5810 }
5811
5812 struct page * __weak
5813 follow_huge_pud(struct mm_struct *mm, unsigned long address,
5814                 pud_t *pud, int flags)
5815 {
5816         if (flags & (FOLL_GET | FOLL_PIN))
5817                 return NULL;
5818
5819         return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
5820 }
5821
5822 struct page * __weak
5823 follow_huge_pgd(struct mm_struct *mm, unsigned long address, pgd_t *pgd, int flags)
5824 {
5825         if (flags & (FOLL_GET | FOLL_PIN))
5826                 return NULL;
5827
5828         return pte_page(*(pte_t *)pgd) + ((address & ~PGDIR_MASK) >> PAGE_SHIFT);
5829 }
5830
5831 bool isolate_huge_page(struct page *page, struct list_head *list)
5832 {
5833         bool ret = true;
5834
5835         spin_lock_irq(&hugetlb_lock);
5836         if (!PageHeadHuge(page) ||
5837             !HPageMigratable(page) ||
5838             !get_page_unless_zero(page)) {
5839                 ret = false;
5840                 goto unlock;
5841         }
5842         ClearHPageMigratable(page);
5843         list_move_tail(&page->lru, list);
5844 unlock:
5845         spin_unlock_irq(&hugetlb_lock);
5846         return ret;
5847 }
5848
5849 void putback_active_hugepage(struct page *page)
5850 {
5851         spin_lock_irq(&hugetlb_lock);
5852         SetHPageMigratable(page);
5853         list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
5854         spin_unlock_irq(&hugetlb_lock);
5855         put_page(page);
5856 }
5857
5858 void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason)
5859 {
5860         struct hstate *h = page_hstate(oldpage);
5861
5862         hugetlb_cgroup_migrate(oldpage, newpage);
5863         set_page_owner_migrate_reason(newpage, reason);
5864
5865         /*
5866          * transfer temporary state of the new huge page. This is
5867          * reverse to other transitions because the newpage is going to
5868          * be final while the old one will be freed so it takes over
5869          * the temporary status.
5870          *
5871          * Also note that we have to transfer the per-node surplus state
5872          * here as well otherwise the global surplus count will not match
5873          * the per-node's.
5874          */
5875         if (HPageTemporary(newpage)) {
5876                 int old_nid = page_to_nid(oldpage);
5877                 int new_nid = page_to_nid(newpage);
5878
5879                 SetHPageTemporary(oldpage);
5880                 ClearHPageTemporary(newpage);
5881
5882                 /*
5883                  * There is no need to transfer the per-node surplus state
5884                  * when we do not cross the node.
5885                  */
5886                 if (new_nid == old_nid)
5887                         return;
5888                 spin_lock_irq(&hugetlb_lock);
5889                 if (h->surplus_huge_pages_node[old_nid]) {
5890                         h->surplus_huge_pages_node[old_nid]--;
5891                         h->surplus_huge_pages_node[new_nid]++;
5892                 }
5893                 spin_unlock_irq(&hugetlb_lock);
5894         }
5895 }
5896
5897 /*
5898  * This function will unconditionally remove all the shared pmd pgtable entries
5899  * within the specific vma for a hugetlbfs memory range.
5900  */
5901 void hugetlb_unshare_all_pmds(struct vm_area_struct *vma)
5902 {
5903         struct hstate *h = hstate_vma(vma);
5904         unsigned long sz = huge_page_size(h);
5905         struct mm_struct *mm = vma->vm_mm;
5906         struct mmu_notifier_range range;
5907         unsigned long address, start, end;
5908         spinlock_t *ptl;
5909         pte_t *ptep;
5910
5911         if (!(vma->vm_flags & VM_MAYSHARE))
5912                 return;
5913
5914         start = ALIGN(vma->vm_start, PUD_SIZE);
5915         end = ALIGN_DOWN(vma->vm_end, PUD_SIZE);
5916
5917         if (start >= end)
5918                 return;
5919
5920         /*
5921          * No need to call adjust_range_if_pmd_sharing_possible(), because
5922          * we have already done the PUD_SIZE alignment.
5923          */
5924         mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm,
5925                                 start, end);
5926         mmu_notifier_invalidate_range_start(&range);
5927         i_mmap_lock_write(vma->vm_file->f_mapping);
5928         for (address = start; address < end; address += PUD_SIZE) {
5929                 unsigned long tmp = address;
5930
5931                 ptep = huge_pte_offset(mm, address, sz);
5932                 if (!ptep)
5933                         continue;
5934                 ptl = huge_pte_lock(h, mm, ptep);
5935                 /* We don't want 'address' to be changed */
5936                 huge_pmd_unshare(mm, vma, &tmp, ptep);
5937                 spin_unlock(ptl);
5938         }
5939         flush_hugetlb_tlb_range(vma, start, end);
5940         i_mmap_unlock_write(vma->vm_file->f_mapping);
5941         /*
5942          * No need to call mmu_notifier_invalidate_range(), see
5943          * Documentation/vm/mmu_notifier.rst.
5944          */
5945         mmu_notifier_invalidate_range_end(&range);
5946 }
5947
5948 #ifdef CONFIG_CMA
5949 static bool cma_reserve_called __initdata;
5950
5951 static int __init cmdline_parse_hugetlb_cma(char *p)
5952 {
5953         hugetlb_cma_size = memparse(p, &p);
5954         return 0;
5955 }
5956
5957 early_param("hugetlb_cma", cmdline_parse_hugetlb_cma);
5958
5959 void __init hugetlb_cma_reserve(int order)
5960 {
5961         unsigned long size, reserved, per_node;
5962         int nid;
5963
5964         cma_reserve_called = true;
5965
5966         if (!hugetlb_cma_size)
5967                 return;
5968
5969         if (hugetlb_cma_size < (PAGE_SIZE << order)) {
5970                 pr_warn("hugetlb_cma: cma area should be at least %lu MiB\n",
5971                         (PAGE_SIZE << order) / SZ_1M);
5972                 return;
5973         }
5974
5975         /*
5976          * If 3 GB area is requested on a machine with 4 numa nodes,
5977          * let's allocate 1 GB on first three nodes and ignore the last one.
5978          */
5979         per_node = DIV_ROUND_UP(hugetlb_cma_size, nr_online_nodes);
5980         pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n",
5981                 hugetlb_cma_size / SZ_1M, per_node / SZ_1M);
5982
5983         reserved = 0;
5984         for_each_node_state(nid, N_ONLINE) {
5985                 int res;
5986                 char name[CMA_MAX_NAME];
5987
5988                 size = min(per_node, hugetlb_cma_size - reserved);
5989                 size = round_up(size, PAGE_SIZE << order);
5990
5991                 snprintf(name, sizeof(name), "hugetlb%d", nid);
5992                 res = cma_declare_contiguous_nid(0, size, 0, PAGE_SIZE << order,
5993                                                  0, false, name,
5994                                                  &hugetlb_cma[nid], nid);
5995                 if (res) {
5996                         pr_warn("hugetlb_cma: reservation failed: err %d, node %d",
5997                                 res, nid);
5998                         continue;
5999                 }
6000
6001                 reserved += size;
6002                 pr_info("hugetlb_cma: reserved %lu MiB on node %d\n",
6003                         size / SZ_1M, nid);
6004
6005                 if (reserved >= hugetlb_cma_size)
6006                         break;
6007         }
6008 }
6009
6010 void __init hugetlb_cma_check(void)
6011 {
6012         if (!hugetlb_cma_size || cma_reserve_called)
6013                 return;
6014
6015         pr_warn("hugetlb_cma: the option isn't supported by current arch\n");
6016 }
6017
6018 #endif /* CONFIG_CMA */