hugetlb: skip to end of PT page mapping when pte not present

[linux-2.6-microblaze.git] / mm / hugetlb.c

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index ffdf3fc..95fd1c3 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -4727,6 +4727,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
        unsigned long npages = pages_per_huge_page(h);
        struct address_space *mapping = src_vma->vm_file->f_mapping;
        struct mmu_notifier_range range;
+       unsigned long last_addr_mask;
        int ret = 0;
 
        if (cow) {
@@ -4746,11 +4747,14 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
                i_mmap_lock_read(mapping);
        }
 
+       last_addr_mask = hugetlb_mask_last_page(h);
        for (addr = src_vma->vm_start; addr < src_vma->vm_end; addr += sz) {
                spinlock_t *src_ptl, *dst_ptl;
                src_pte = huge_pte_offset(src, addr, sz);
-               if (!src_pte)
+               if (!src_pte) {
+                       addr |= last_addr_mask;
                        continue;
+               }
                dst_pte = huge_pte_alloc(dst, dst_vma, addr, sz);
                if (!dst_pte) {
                        ret = -ENOMEM;
@@ -4767,8 +4771,10 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
                 * after taking the lock below.
                 */
                dst_entry = huge_ptep_get(dst_pte);
-               if ((dst_pte == src_pte) || !huge_pte_none(dst_entry))
+               if ((dst_pte == src_pte) || !huge_pte_none(dst_entry)) {
+                       addr |= last_addr_mask;
                        continue;
+               }
 
                dst_ptl = huge_pte_lock(h, dst, dst_pte);
                src_ptl = huge_pte_lockptr(h, src, src_pte);
@@ -4928,6 +4934,7 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma,
        unsigned long sz = huge_page_size(h);
        struct mm_struct *mm = vma->vm_mm;
        unsigned long old_end = old_addr + len;
+       unsigned long last_addr_mask;
        unsigned long old_addr_copy;
        pte_t *src_pte, *dst_pte;
        struct mmu_notifier_range range;
@@ -4943,12 +4950,16 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma,
        flush_cache_range(vma, range.start, range.end);
 
        mmu_notifier_invalidate_range_start(&range);
+       last_addr_mask = hugetlb_mask_last_page(h);
        /* Prevent race with file truncation */
        i_mmap_lock_write(mapping);
        for (; old_addr < old_end; old_addr += sz, new_addr += sz) {
                src_pte = huge_pte_offset(mm, old_addr, sz);
-               if (!src_pte)
+               if (!src_pte) {
+                       old_addr |= last_addr_mask;
+                       new_addr |= last_addr_mask;
                        continue;
+               }
                if (huge_pte_none(huge_ptep_get(src_pte)))
                        continue;
 
@@ -4993,6 +5004,7 @@ static void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct
        struct hstate *h = hstate_vma(vma);
        unsigned long sz = huge_page_size(h);
        struct mmu_notifier_range range;
+       unsigned long last_addr_mask;
        bool force_flush = false;
 
        WARN_ON(!is_vm_hugetlb_page(vma));
@@ -5013,11 +5025,14 @@ static void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct
                                end);
        adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
        mmu_notifier_invalidate_range_start(&range);
+       last_addr_mask = hugetlb_mask_last_page(h);
        address = start;
        for (; address < end; address += sz) {
                ptep = huge_pte_offset(mm, address, sz);
-               if (!ptep)
+               if (!ptep) {
+                       address |= last_addr_mask;
                        continue;
+               }
 
                ptl = huge_pte_lock(h, mm, ptep);
                if (huge_pmd_unshare(mm, vma, &address, ptep)) {
@@ -6285,6 +6300,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
        unsigned long pages = 0, psize = huge_page_size(h);
        bool shared_pmd = false;
        struct mmu_notifier_range range;
+       unsigned long last_addr_mask;
        bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
        bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
 
@@ -6301,12 +6317,15 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
        flush_cache_range(vma, range.start, range.end);
 
        mmu_notifier_invalidate_range_start(&range);
+       last_addr_mask = hugetlb_mask_last_page(h);
        i_mmap_lock_write(vma->vm_file->f_mapping);
        for (; address < end; address += psize) {
                spinlock_t *ptl;
                ptep = huge_pte_offset(mm, address, psize);
-               if (!ptep)
+               if (!ptep) {
+                       address |= last_addr_mask;
                        continue;
+               }
                ptl = huge_pte_lock(h, mm, ptep);
                if (huge_pmd_unshare(mm, vma, &address, ptep)) {
                        /*
@@ -6856,6 +6875,33 @@ pte_t *huge_pte_offset(struct mm_struct *mm,
        return (pte_t *)pmd;
 }
 
+/*
+ * Return a mask that can be used to update an address to the last huge
+ * page in a page table page mapping size.  Used to skip non-present
+ * page table entries when linearly scanning address ranges.  Architectures
+ * with unique huge page to page table relationships can define their own
+ * version of this routine.
+ */
+unsigned long hugetlb_mask_last_page(struct hstate *h)
+{
+       unsigned long hp_size = huge_page_size(h);
+
+       if (hp_size == PUD_SIZE)
+               return P4D_SIZE - PUD_SIZE;
+       else if (hp_size == PMD_SIZE)
+               return PUD_SIZE - PMD_SIZE;
+       else
+               return 0UL;
+}
+
+#else
+
+/* See description above.  Architectures can provide their own version. */
+__weak unsigned long hugetlb_mask_last_page(struct hstate *h)
+{
+       return 0UL;
+}
+
 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
 
 /*