Merge tag 'sched-urgent-2022-08-06' of git://git.kernel.org/pub/scm/linux/kernel...

[linux-2.6-microblaze.git] / mm / hugetlb_vmemmap.c

diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index 1089ea8..1362feb 100644 (file)
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -6,11 +6,11 @@
  *
  *     Author: Muchun Song <songmuchun@bytedance.com>
  *
- * See Documentation/vm/vmemmap_dedup.rst
+ * See Documentation/mm/vmemmap_dedup.rst
  */
 #define pr_fmt(fmt)    "HugeTLB: " fmt
 
-#include <linux/memory_hotplug.h>
+#include <linux/memory.h>
 #include "hugetlb_vmemmap.h"
 
 /*
@@ -97,18 +97,68 @@ int hugetlb_vmemmap_alloc(struct hstate *h, struct page *head)
        return ret;
 }
 
+static unsigned int vmemmap_optimizable_pages(struct hstate *h,
+                                             struct page *head)
+{
+       if (READ_ONCE(vmemmap_optimize_mode) == VMEMMAP_OPTIMIZE_OFF)
+               return 0;
+
+       if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG)) {
+               pmd_t *pmdp, pmd;
+               struct page *vmemmap_page;
+               unsigned long vaddr = (unsigned long)head;
+
+               /*
+                * Only the vmemmap page's vmemmap page can be self-hosted.
+                * Walking the page tables to find the backing page of the
+                * vmemmap page.
+                */
+               pmdp = pmd_off_k(vaddr);
+               /*
+                * The READ_ONCE() is used to stabilize *pmdp in a register or
+                * on the stack so that it will stop changing under the code.
+                * The only concurrent operation where it can be changed is
+                * split_vmemmap_huge_pmd() (*pmdp will be stable after this
+                * operation).
+                */
+               pmd = READ_ONCE(*pmdp);
+               if (pmd_leaf(pmd))
+                       vmemmap_page = pmd_page(pmd) + pte_index(vaddr);
+               else
+                       vmemmap_page = pte_page(*pte_offset_kernel(pmdp, vaddr));
+               /*
+                * Due to HugeTLB alignment requirements and the vmemmap pages
+                * being at the start of the hotplugged memory region in
+                * memory_hotplug.memmap_on_memory case. Checking any vmemmap
+                * page's vmemmap page if it is marked as VmemmapSelfHosted is
+                * sufficient.
+                *
+                * [                  hotplugged memory                  ]
+                * [        section        ][...][        section        ]
+                * [ vmemmap ][              usable memory               ]
+                *   ^   |     |                                        |
+                *   +---+     |                                        |
+                *     ^       |                                        |
+                *     +-------+                                        |
+                *          ^                                           |
+                *          +-------------------------------------------+
+                */
+               if (PageVmemmapSelfHosted(vmemmap_page))
+                       return 0;
+       }
+
+       return hugetlb_optimize_vmemmap_pages(h);
+}
+
 void hugetlb_vmemmap_free(struct hstate *h, struct page *head)
 {
        unsigned long vmemmap_addr = (unsigned long)head;
        unsigned long vmemmap_end, vmemmap_reuse, vmemmap_pages;
 
-       vmemmap_pages = hugetlb_optimize_vmemmap_pages(h);
+       vmemmap_pages = vmemmap_optimizable_pages(h, head);
        if (!vmemmap_pages)
                return;
 
-       if (READ_ONCE(vmemmap_optimize_mode) == VMEMMAP_OPTIMIZE_OFF)
-               return;
-
        static_branch_inc(&hugetlb_optimize_vmemmap_key);
 
        vmemmap_addr    += RESERVE_VMEMMAP_SIZE;
@@ -199,10 +249,10 @@ static struct ctl_table hugetlb_vmemmap_sysctls[] = {
 static __init int hugetlb_vmemmap_sysctls_init(void)
 {
        /*
-        * If "memory_hotplug.memmap_on_memory" is enabled or "struct page"
-        * crosses page boundaries, the vmemmap pages cannot be optimized.
+        * If "struct page" crosses page boundaries, the vmemmap pages cannot
+        * be optimized.
         */
-       if (!mhp_memmap_on_memory() && is_power_of_2(sizeof(struct page)))
+       if (is_power_of_2(sizeof(struct page)))
                register_sysctl_init("vm", hugetlb_vmemmap_sysctls);
 
        return 0;