Merge tag 'parisc-for-6.0-2' of git://git.kernel.org/pub/scm/linux/kernel/git/deller...
[linux-2.6-microblaze.git] / mm / hugetlb_vmemmap.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * HugeTLB Vmemmap Optimization (HVO)
4  *
5  * Copyright (c) 2020, ByteDance. All rights reserved.
6  *
7  *     Author: Muchun Song <songmuchun@bytedance.com>
8  *
9  * See Documentation/mm/vmemmap_dedup.rst
10  */
11 #define pr_fmt(fmt)     "HugeTLB: " fmt
12
13 #include <linux/pgtable.h>
14 #include <linux/bootmem_info.h>
15 #include <asm/pgalloc.h>
16 #include <asm/tlbflush.h>
17 #include "hugetlb_vmemmap.h"
18
19 /**
20  * struct vmemmap_remap_walk - walk vmemmap page table
21  *
22  * @remap_pte:          called for each lowest-level entry (PTE).
23  * @nr_walked:          the number of walked pte.
24  * @reuse_page:         the page which is reused for the tail vmemmap pages.
25  * @reuse_addr:         the virtual address of the @reuse_page page.
26  * @vmemmap_pages:      the list head of the vmemmap pages that can be freed
27  *                      or is mapped from.
28  */
29 struct vmemmap_remap_walk {
30         void                    (*remap_pte)(pte_t *pte, unsigned long addr,
31                                              struct vmemmap_remap_walk *walk);
32         unsigned long           nr_walked;
33         struct page             *reuse_page;
34         unsigned long           reuse_addr;
35         struct list_head        *vmemmap_pages;
36 };
37
38 static int __split_vmemmap_huge_pmd(pmd_t *pmd, unsigned long start)
39 {
40         pmd_t __pmd;
41         int i;
42         unsigned long addr = start;
43         struct page *page = pmd_page(*pmd);
44         pte_t *pgtable = pte_alloc_one_kernel(&init_mm);
45
46         if (!pgtable)
47                 return -ENOMEM;
48
49         pmd_populate_kernel(&init_mm, &__pmd, pgtable);
50
51         for (i = 0; i < PTRS_PER_PTE; i++, addr += PAGE_SIZE) {
52                 pte_t entry, *pte;
53                 pgprot_t pgprot = PAGE_KERNEL;
54
55                 entry = mk_pte(page + i, pgprot);
56                 pte = pte_offset_kernel(&__pmd, addr);
57                 set_pte_at(&init_mm, addr, pte, entry);
58         }
59
60         spin_lock(&init_mm.page_table_lock);
61         if (likely(pmd_leaf(*pmd))) {
62                 /*
63                  * Higher order allocations from buddy allocator must be able to
64                  * be treated as indepdenent small pages (as they can be freed
65                  * individually).
66                  */
67                 if (!PageReserved(page))
68                         split_page(page, get_order(PMD_SIZE));
69
70                 /* Make pte visible before pmd. See comment in pmd_install(). */
71                 smp_wmb();
72                 pmd_populate_kernel(&init_mm, pmd, pgtable);
73                 flush_tlb_kernel_range(start, start + PMD_SIZE);
74         } else {
75                 pte_free_kernel(&init_mm, pgtable);
76         }
77         spin_unlock(&init_mm.page_table_lock);
78
79         return 0;
80 }
81
82 static int split_vmemmap_huge_pmd(pmd_t *pmd, unsigned long start)
83 {
84         int leaf;
85
86         spin_lock(&init_mm.page_table_lock);
87         leaf = pmd_leaf(*pmd);
88         spin_unlock(&init_mm.page_table_lock);
89
90         if (!leaf)
91                 return 0;
92
93         return __split_vmemmap_huge_pmd(pmd, start);
94 }
95
96 static void vmemmap_pte_range(pmd_t *pmd, unsigned long addr,
97                               unsigned long end,
98                               struct vmemmap_remap_walk *walk)
99 {
100         pte_t *pte = pte_offset_kernel(pmd, addr);
101
102         /*
103          * The reuse_page is found 'first' in table walk before we start
104          * remapping (which is calling @walk->remap_pte).
105          */
106         if (!walk->reuse_page) {
107                 walk->reuse_page = pte_page(*pte);
108                 /*
109                  * Because the reuse address is part of the range that we are
110                  * walking, skip the reuse address range.
111                  */
112                 addr += PAGE_SIZE;
113                 pte++;
114                 walk->nr_walked++;
115         }
116
117         for (; addr != end; addr += PAGE_SIZE, pte++) {
118                 walk->remap_pte(pte, addr, walk);
119                 walk->nr_walked++;
120         }
121 }
122
123 static int vmemmap_pmd_range(pud_t *pud, unsigned long addr,
124                              unsigned long end,
125                              struct vmemmap_remap_walk *walk)
126 {
127         pmd_t *pmd;
128         unsigned long next;
129
130         pmd = pmd_offset(pud, addr);
131         do {
132                 int ret;
133
134                 ret = split_vmemmap_huge_pmd(pmd, addr & PMD_MASK);
135                 if (ret)
136                         return ret;
137
138                 next = pmd_addr_end(addr, end);
139                 vmemmap_pte_range(pmd, addr, next, walk);
140         } while (pmd++, addr = next, addr != end);
141
142         return 0;
143 }
144
145 static int vmemmap_pud_range(p4d_t *p4d, unsigned long addr,
146                              unsigned long end,
147                              struct vmemmap_remap_walk *walk)
148 {
149         pud_t *pud;
150         unsigned long next;
151
152         pud = pud_offset(p4d, addr);
153         do {
154                 int ret;
155
156                 next = pud_addr_end(addr, end);
157                 ret = vmemmap_pmd_range(pud, addr, next, walk);
158                 if (ret)
159                         return ret;
160         } while (pud++, addr = next, addr != end);
161
162         return 0;
163 }
164
165 static int vmemmap_p4d_range(pgd_t *pgd, unsigned long addr,
166                              unsigned long end,
167                              struct vmemmap_remap_walk *walk)
168 {
169         p4d_t *p4d;
170         unsigned long next;
171
172         p4d = p4d_offset(pgd, addr);
173         do {
174                 int ret;
175
176                 next = p4d_addr_end(addr, end);
177                 ret = vmemmap_pud_range(p4d, addr, next, walk);
178                 if (ret)
179                         return ret;
180         } while (p4d++, addr = next, addr != end);
181
182         return 0;
183 }
184
185 static int vmemmap_remap_range(unsigned long start, unsigned long end,
186                                struct vmemmap_remap_walk *walk)
187 {
188         unsigned long addr = start;
189         unsigned long next;
190         pgd_t *pgd;
191
192         VM_BUG_ON(!PAGE_ALIGNED(start));
193         VM_BUG_ON(!PAGE_ALIGNED(end));
194
195         pgd = pgd_offset_k(addr);
196         do {
197                 int ret;
198
199                 next = pgd_addr_end(addr, end);
200                 ret = vmemmap_p4d_range(pgd, addr, next, walk);
201                 if (ret)
202                         return ret;
203         } while (pgd++, addr = next, addr != end);
204
205         /*
206          * We only change the mapping of the vmemmap virtual address range
207          * [@start + PAGE_SIZE, end), so we only need to flush the TLB which
208          * belongs to the range.
209          */
210         flush_tlb_kernel_range(start + PAGE_SIZE, end);
211
212         return 0;
213 }
214
215 /*
216  * Free a vmemmap page. A vmemmap page can be allocated from the memblock
217  * allocator or buddy allocator. If the PG_reserved flag is set, it means
218  * that it allocated from the memblock allocator, just free it via the
219  * free_bootmem_page(). Otherwise, use __free_page().
220  */
221 static inline void free_vmemmap_page(struct page *page)
222 {
223         if (PageReserved(page))
224                 free_bootmem_page(page);
225         else
226                 __free_page(page);
227 }
228
229 /* Free a list of the vmemmap pages */
230 static void free_vmemmap_page_list(struct list_head *list)
231 {
232         struct page *page, *next;
233
234         list_for_each_entry_safe(page, next, list, lru) {
235                 list_del(&page->lru);
236                 free_vmemmap_page(page);
237         }
238 }
239
240 static void vmemmap_remap_pte(pte_t *pte, unsigned long addr,
241                               struct vmemmap_remap_walk *walk)
242 {
243         /*
244          * Remap the tail pages as read-only to catch illegal write operation
245          * to the tail pages.
246          */
247         pgprot_t pgprot = PAGE_KERNEL_RO;
248         pte_t entry = mk_pte(walk->reuse_page, pgprot);
249         struct page *page = pte_page(*pte);
250
251         list_add_tail(&page->lru, walk->vmemmap_pages);
252         set_pte_at(&init_mm, addr, pte, entry);
253 }
254
255 /*
256  * How many struct page structs need to be reset. When we reuse the head
257  * struct page, the special metadata (e.g. page->flags or page->mapping)
258  * cannot copy to the tail struct page structs. The invalid value will be
259  * checked in the free_tail_pages_check(). In order to avoid the message
260  * of "corrupted mapping in tail page". We need to reset at least 3 (one
261  * head struct page struct and two tail struct page structs) struct page
262  * structs.
263  */
264 #define NR_RESET_STRUCT_PAGE            3
265
266 static inline void reset_struct_pages(struct page *start)
267 {
268         int i;
269         struct page *from = start + NR_RESET_STRUCT_PAGE;
270
271         for (i = 0; i < NR_RESET_STRUCT_PAGE; i++)
272                 memcpy(start + i, from, sizeof(*from));
273 }
274
275 static void vmemmap_restore_pte(pte_t *pte, unsigned long addr,
276                                 struct vmemmap_remap_walk *walk)
277 {
278         pgprot_t pgprot = PAGE_KERNEL;
279         struct page *page;
280         void *to;
281
282         BUG_ON(pte_page(*pte) != walk->reuse_page);
283
284         page = list_first_entry(walk->vmemmap_pages, struct page, lru);
285         list_del(&page->lru);
286         to = page_to_virt(page);
287         copy_page(to, (void *)walk->reuse_addr);
288         reset_struct_pages(to);
289
290         set_pte_at(&init_mm, addr, pte, mk_pte(page, pgprot));
291 }
292
293 /**
294  * vmemmap_remap_free - remap the vmemmap virtual address range [@start, @end)
295  *                      to the page which @reuse is mapped to, then free vmemmap
296  *                      which the range are mapped to.
297  * @start:      start address of the vmemmap virtual address range that we want
298  *              to remap.
299  * @end:        end address of the vmemmap virtual address range that we want to
300  *              remap.
301  * @reuse:      reuse address.
302  *
303  * Return: %0 on success, negative error code otherwise.
304  */
305 static int vmemmap_remap_free(unsigned long start, unsigned long end,
306                               unsigned long reuse)
307 {
308         int ret;
309         LIST_HEAD(vmemmap_pages);
310         struct vmemmap_remap_walk walk = {
311                 .remap_pte      = vmemmap_remap_pte,
312                 .reuse_addr     = reuse,
313                 .vmemmap_pages  = &vmemmap_pages,
314         };
315
316         /*
317          * In order to make remapping routine most efficient for the huge pages,
318          * the routine of vmemmap page table walking has the following rules
319          * (see more details from the vmemmap_pte_range()):
320          *
321          * - The range [@start, @end) and the range [@reuse, @reuse + PAGE_SIZE)
322          *   should be continuous.
323          * - The @reuse address is part of the range [@reuse, @end) that we are
324          *   walking which is passed to vmemmap_remap_range().
325          * - The @reuse address is the first in the complete range.
326          *
327          * So we need to make sure that @start and @reuse meet the above rules.
328          */
329         BUG_ON(start - reuse != PAGE_SIZE);
330
331         mmap_read_lock(&init_mm);
332         ret = vmemmap_remap_range(reuse, end, &walk);
333         if (ret && walk.nr_walked) {
334                 end = reuse + walk.nr_walked * PAGE_SIZE;
335                 /*
336                  * vmemmap_pages contains pages from the previous
337                  * vmemmap_remap_range call which failed.  These
338                  * are pages which were removed from the vmemmap.
339                  * They will be restored in the following call.
340                  */
341                 walk = (struct vmemmap_remap_walk) {
342                         .remap_pte      = vmemmap_restore_pte,
343                         .reuse_addr     = reuse,
344                         .vmemmap_pages  = &vmemmap_pages,
345                 };
346
347                 vmemmap_remap_range(reuse, end, &walk);
348         }
349         mmap_read_unlock(&init_mm);
350
351         free_vmemmap_page_list(&vmemmap_pages);
352
353         return ret;
354 }
355
356 static int alloc_vmemmap_page_list(unsigned long start, unsigned long end,
357                                    gfp_t gfp_mask, struct list_head *list)
358 {
359         unsigned long nr_pages = (end - start) >> PAGE_SHIFT;
360         int nid = page_to_nid((struct page *)start);
361         struct page *page, *next;
362
363         while (nr_pages--) {
364                 page = alloc_pages_node(nid, gfp_mask, 0);
365                 if (!page)
366                         goto out;
367                 list_add_tail(&page->lru, list);
368         }
369
370         return 0;
371 out:
372         list_for_each_entry_safe(page, next, list, lru)
373                 __free_pages(page, 0);
374         return -ENOMEM;
375 }
376
377 /**
378  * vmemmap_remap_alloc - remap the vmemmap virtual address range [@start, end)
379  *                       to the page which is from the @vmemmap_pages
380  *                       respectively.
381  * @start:      start address of the vmemmap virtual address range that we want
382  *              to remap.
383  * @end:        end address of the vmemmap virtual address range that we want to
384  *              remap.
385  * @reuse:      reuse address.
386  * @gfp_mask:   GFP flag for allocating vmemmap pages.
387  *
388  * Return: %0 on success, negative error code otherwise.
389  */
390 static int vmemmap_remap_alloc(unsigned long start, unsigned long end,
391                                unsigned long reuse, gfp_t gfp_mask)
392 {
393         LIST_HEAD(vmemmap_pages);
394         struct vmemmap_remap_walk walk = {
395                 .remap_pte      = vmemmap_restore_pte,
396                 .reuse_addr     = reuse,
397                 .vmemmap_pages  = &vmemmap_pages,
398         };
399
400         /* See the comment in the vmemmap_remap_free(). */
401         BUG_ON(start - reuse != PAGE_SIZE);
402
403         if (alloc_vmemmap_page_list(start, end, gfp_mask, &vmemmap_pages))
404                 return -ENOMEM;
405
406         mmap_read_lock(&init_mm);
407         vmemmap_remap_range(reuse, end, &walk);
408         mmap_read_unlock(&init_mm);
409
410         return 0;
411 }
412
413 DEFINE_STATIC_KEY_FALSE(hugetlb_optimize_vmemmap_key);
414 EXPORT_SYMBOL(hugetlb_optimize_vmemmap_key);
415
416 static bool vmemmap_optimize_enabled = IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP_DEFAULT_ON);
417 core_param(hugetlb_free_vmemmap, vmemmap_optimize_enabled, bool, 0);
418
419 /**
420  * hugetlb_vmemmap_restore - restore previously optimized (by
421  *                           hugetlb_vmemmap_optimize()) vmemmap pages which
422  *                           will be reallocated and remapped.
423  * @h:          struct hstate.
424  * @head:       the head page whose vmemmap pages will be restored.
425  *
426  * Return: %0 if @head's vmemmap pages have been reallocated and remapped,
427  * negative error code otherwise.
428  */
429 int hugetlb_vmemmap_restore(const struct hstate *h, struct page *head)
430 {
431         int ret;
432         unsigned long vmemmap_start = (unsigned long)head, vmemmap_end;
433         unsigned long vmemmap_reuse;
434
435         if (!HPageVmemmapOptimized(head))
436                 return 0;
437
438         vmemmap_end     = vmemmap_start + hugetlb_vmemmap_size(h);
439         vmemmap_reuse   = vmemmap_start;
440         vmemmap_start   += HUGETLB_VMEMMAP_RESERVE_SIZE;
441
442         /*
443          * The pages which the vmemmap virtual address range [@vmemmap_start,
444          * @vmemmap_end) are mapped to are freed to the buddy allocator, and
445          * the range is mapped to the page which @vmemmap_reuse is mapped to.
446          * When a HugeTLB page is freed to the buddy allocator, previously
447          * discarded vmemmap pages must be allocated and remapping.
448          */
449         ret = vmemmap_remap_alloc(vmemmap_start, vmemmap_end, vmemmap_reuse,
450                                   GFP_KERNEL | __GFP_NORETRY | __GFP_THISNODE);
451         if (!ret) {
452                 ClearHPageVmemmapOptimized(head);
453                 static_branch_dec(&hugetlb_optimize_vmemmap_key);
454         }
455
456         return ret;
457 }
458
459 /* Return true iff a HugeTLB whose vmemmap should and can be optimized. */
460 static bool vmemmap_should_optimize(const struct hstate *h, const struct page *head)
461 {
462         if (!READ_ONCE(vmemmap_optimize_enabled))
463                 return false;
464
465         if (!hugetlb_vmemmap_optimizable(h))
466                 return false;
467
468         if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG)) {
469                 pmd_t *pmdp, pmd;
470                 struct page *vmemmap_page;
471                 unsigned long vaddr = (unsigned long)head;
472
473                 /*
474                  * Only the vmemmap page's vmemmap page can be self-hosted.
475                  * Walking the page tables to find the backing page of the
476                  * vmemmap page.
477                  */
478                 pmdp = pmd_off_k(vaddr);
479                 /*
480                  * The READ_ONCE() is used to stabilize *pmdp in a register or
481                  * on the stack so that it will stop changing under the code.
482                  * The only concurrent operation where it can be changed is
483                  * split_vmemmap_huge_pmd() (*pmdp will be stable after this
484                  * operation).
485                  */
486                 pmd = READ_ONCE(*pmdp);
487                 if (pmd_leaf(pmd))
488                         vmemmap_page = pmd_page(pmd) + pte_index(vaddr);
489                 else
490                         vmemmap_page = pte_page(*pte_offset_kernel(pmdp, vaddr));
491                 /*
492                  * Due to HugeTLB alignment requirements and the vmemmap pages
493                  * being at the start of the hotplugged memory region in
494                  * memory_hotplug.memmap_on_memory case. Checking any vmemmap
495                  * page's vmemmap page if it is marked as VmemmapSelfHosted is
496                  * sufficient.
497                  *
498                  * [                  hotplugged memory                  ]
499                  * [        section        ][...][        section        ]
500                  * [ vmemmap ][              usable memory               ]
501                  *   ^   |     |                                        |
502                  *   +---+     |                                        |
503                  *     ^       |                                        |
504                  *     +-------+                                        |
505                  *          ^                                           |
506                  *          +-------------------------------------------+
507                  */
508                 if (PageVmemmapSelfHosted(vmemmap_page))
509                         return false;
510         }
511
512         return true;
513 }
514
515 /**
516  * hugetlb_vmemmap_optimize - optimize @head page's vmemmap pages.
517  * @h:          struct hstate.
518  * @head:       the head page whose vmemmap pages will be optimized.
519  *
520  * This function only tries to optimize @head's vmemmap pages and does not
521  * guarantee that the optimization will succeed after it returns. The caller
522  * can use HPageVmemmapOptimized(@head) to detect if @head's vmemmap pages
523  * have been optimized.
524  */
525 void hugetlb_vmemmap_optimize(const struct hstate *h, struct page *head)
526 {
527         unsigned long vmemmap_start = (unsigned long)head, vmemmap_end;
528         unsigned long vmemmap_reuse;
529
530         if (!vmemmap_should_optimize(h, head))
531                 return;
532
533         static_branch_inc(&hugetlb_optimize_vmemmap_key);
534
535         vmemmap_end     = vmemmap_start + hugetlb_vmemmap_size(h);
536         vmemmap_reuse   = vmemmap_start;
537         vmemmap_start   += HUGETLB_VMEMMAP_RESERVE_SIZE;
538
539         /*
540          * Remap the vmemmap virtual address range [@vmemmap_start, @vmemmap_end)
541          * to the page which @vmemmap_reuse is mapped to, then free the pages
542          * which the range [@vmemmap_start, @vmemmap_end] is mapped to.
543          */
544         if (vmemmap_remap_free(vmemmap_start, vmemmap_end, vmemmap_reuse))
545                 static_branch_dec(&hugetlb_optimize_vmemmap_key);
546         else
547                 SetHPageVmemmapOptimized(head);
548 }
549
550 static struct ctl_table hugetlb_vmemmap_sysctls[] = {
551         {
552                 .procname       = "hugetlb_optimize_vmemmap",
553                 .data           = &vmemmap_optimize_enabled,
554                 .maxlen         = sizeof(int),
555                 .mode           = 0644,
556                 .proc_handler   = proc_dobool,
557         },
558         { }
559 };
560
561 static int __init hugetlb_vmemmap_init(void)
562 {
563         /* HUGETLB_VMEMMAP_RESERVE_SIZE should cover all used struct pages */
564         BUILD_BUG_ON(__NR_USED_SUBPAGE * sizeof(struct page) > HUGETLB_VMEMMAP_RESERVE_SIZE);
565
566         if (IS_ENABLED(CONFIG_PROC_SYSCTL)) {
567                 const struct hstate *h;
568
569                 for_each_hstate(h) {
570                         if (hugetlb_vmemmap_optimizable(h)) {
571                                 register_sysctl_init("vm", hugetlb_vmemmap_sysctls);
572                                 break;
573                         }
574                 }
575         }
576         return 0;
577 }
578 late_initcall(hugetlb_vmemmap_init);