mm/memory_hotplug: avoid calling zone_intersects() for ZONE_NORMAL

[linux-2.6-microblaze.git] / mm / sparse-vmemmap.c

diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index db6df27..8aecd6b 100644 (file)
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -34,6 +34,7 @@
 #include <asm/pgalloc.h>
 #include <asm/tlbflush.h>
 
+#ifdef CONFIG_HUGETLB_PAGE_FREE_VMEMMAP
 /**
  * struct vmemmap_remap_walk - walk vmemmap page table
  *
@@ -53,8 +54,7 @@ struct vmemmap_remap_walk {
        struct list_head *vmemmap_pages;
 };
 
-static int split_vmemmap_huge_pmd(pmd_t *pmd, unsigned long start,
-                                 struct vmemmap_remap_walk *walk)
+static int __split_vmemmap_huge_pmd(pmd_t *pmd, unsigned long start)
 {
        pmd_t __pmd;
        int i;
@@ -76,15 +76,34 @@ static int split_vmemmap_huge_pmd(pmd_t *pmd, unsigned long start,
                set_pte_at(&init_mm, addr, pte, entry);
        }
 
-       /* Make pte visible before pmd. See comment in pmd_install(). */
-       smp_wmb();
-       pmd_populate_kernel(&init_mm, pmd, pgtable);
-
-       flush_tlb_kernel_range(start, start + PMD_SIZE);
+       spin_lock(&init_mm.page_table_lock);
+       if (likely(pmd_leaf(*pmd))) {
+               /* Make pte visible before pmd. See comment in pmd_install(). */
+               smp_wmb();
+               pmd_populate_kernel(&init_mm, pmd, pgtable);
+               flush_tlb_kernel_range(start, start + PMD_SIZE);
+       } else {
+               pte_free_kernel(&init_mm, pgtable);
+       }
+       spin_unlock(&init_mm.page_table_lock);
 
        return 0;
 }
 
+static int split_vmemmap_huge_pmd(pmd_t *pmd, unsigned long start)
+{
+       int leaf;
+
+       spin_lock(&init_mm.page_table_lock);
+       leaf = pmd_leaf(*pmd);
+       spin_unlock(&init_mm.page_table_lock);
+
+       if (!leaf)
+               return 0;
+
+       return __split_vmemmap_huge_pmd(pmd, start);
+}
+
 static void vmemmap_pte_range(pmd_t *pmd, unsigned long addr,
                              unsigned long end,
                              struct vmemmap_remap_walk *walk)
@@ -121,13 +140,12 @@ static int vmemmap_pmd_range(pud_t *pud, unsigned long addr,
 
        pmd = pmd_offset(pud, addr);
        do {
-               if (pmd_leaf(*pmd)) {
-                       int ret;
+               int ret;
+
+               ret = split_vmemmap_huge_pmd(pmd, addr & PMD_MASK);
+               if (ret)
+                       return ret;
 
-                       ret = split_vmemmap_huge_pmd(pmd, addr & PMD_MASK, walk);
-                       if (ret)
-                               return ret;
-               }
                next = pmd_addr_end(addr, end);
                vmemmap_pte_range(pmd, addr, next, walk);
        } while (pmd++, addr = next, addr != end);
@@ -245,6 +263,26 @@ static void vmemmap_remap_pte(pte_t *pte, unsigned long addr,
        set_pte_at(&init_mm, addr, pte, entry);
 }
 
+/*
+ * How many struct page structs need to be reset. When we reuse the head
+ * struct page, the special metadata (e.g. page->flags or page->mapping)
+ * cannot copy to the tail struct page structs. The invalid value will be
+ * checked in the free_tail_pages_check(). In order to avoid the message
+ * of "corrupted mapping in tail page". We need to reset at least 3 (one
+ * head struct page struct and two tail struct page structs) struct page
+ * structs.
+ */
+#define NR_RESET_STRUCT_PAGE           3
+
+static inline void reset_struct_pages(struct page *start)
+{
+       int i;
+       struct page *from = start + NR_RESET_STRUCT_PAGE;
+
+       for (i = 0; i < NR_RESET_STRUCT_PAGE; i++)
+               memcpy(start + i, from, sizeof(*from));
+}
+
 static void vmemmap_restore_pte(pte_t *pte, unsigned long addr,
                                struct vmemmap_remap_walk *walk)
 {
@@ -258,6 +296,7 @@ static void vmemmap_restore_pte(pte_t *pte, unsigned long addr,
        list_del(&page->lru);
        to = page_to_virt(page);
        copy_page(to, (void *)walk->reuse_addr);
+       reset_struct_pages(to);
 
        set_pte_at(&init_mm, addr, pte, mk_pte(page, pgprot));
 }
@@ -300,10 +339,8 @@ int vmemmap_remap_free(unsigned long start, unsigned long end,
         */
        BUG_ON(start - reuse != PAGE_SIZE);
 
-       mmap_write_lock(&init_mm);
+       mmap_read_lock(&init_mm);
        ret = vmemmap_remap_range(reuse, end, &walk);
-       mmap_write_downgrade(&init_mm);
-
        if (ret && walk.nr_walked) {
                end = reuse + walk.nr_walked * PAGE_SIZE;
                /*
@@ -383,6 +420,7 @@ int vmemmap_remap_alloc(unsigned long start, unsigned long end,
 
        return 0;
 }
+#endif /* CONFIG_HUGETLB_PAGE_FREE_VMEMMAP */
 
 /*
  * Allocate a block of memory to be used to back the virtual memory map