Merge tag 'libnvdimm-for-4.18' of git://git.kernel.org/pub/scm/linux/kernel/git/nvdim...

[linux-2.6-microblaze.git] / mm / gup.c

diff --git a/mm/gup.c b/mm/gup.c
index 3d8472d..b70d7ba 100644 (file)
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -212,53 +212,69 @@ static struct page *follow_pmd_mask(struct vm_area_struct *vma,
                                    unsigned long address, pud_t *pudp,
                                    unsigned int flags, unsigned int *page_mask)
 {
-       pmd_t *pmd;
+       pmd_t *pmd, pmdval;
        spinlock_t *ptl;
        struct page *page;
        struct mm_struct *mm = vma->vm_mm;
 
        pmd = pmd_offset(pudp, address);
-       if (pmd_none(*pmd))
+       /*
+        * The READ_ONCE() will stabilize the pmdval in a register or
+        * on the stack so that it will stop changing under the code.
+        */
+       pmdval = READ_ONCE(*pmd);
+       if (pmd_none(pmdval))
                return no_page_table(vma, flags);
-       if (pmd_huge(*pmd) && vma->vm_flags & VM_HUGETLB) {
+       if (pmd_huge(pmdval) && vma->vm_flags & VM_HUGETLB) {
                page = follow_huge_pmd(mm, address, pmd, flags);
                if (page)
                        return page;
                return no_page_table(vma, flags);
        }
-       if (is_hugepd(__hugepd(pmd_val(*pmd)))) {
+       if (is_hugepd(__hugepd(pmd_val(pmdval)))) {
                page = follow_huge_pd(vma, address,
-                                     __hugepd(pmd_val(*pmd)), flags,
+                                     __hugepd(pmd_val(pmdval)), flags,
                                      PMD_SHIFT);
                if (page)
                        return page;
                return no_page_table(vma, flags);
        }
 retry:
-       if (!pmd_present(*pmd)) {
+       if (!pmd_present(pmdval)) {
                if (likely(!(flags & FOLL_MIGRATION)))
                        return no_page_table(vma, flags);
                VM_BUG_ON(thp_migration_supported() &&
-                                 !is_pmd_migration_entry(*pmd));
-               if (is_pmd_migration_entry(*pmd))
+                                 !is_pmd_migration_entry(pmdval));
+               if (is_pmd_migration_entry(pmdval))
                        pmd_migration_entry_wait(mm, pmd);
+               pmdval = READ_ONCE(*pmd);
+               /*
+                * MADV_DONTNEED may convert the pmd to null because
+                * mmap_sem is held in read mode
+                */
+               if (pmd_none(pmdval))
+                       return no_page_table(vma, flags);
                goto retry;
        }
-       if (pmd_devmap(*pmd)) {
+       if (pmd_devmap(pmdval)) {
                ptl = pmd_lock(mm, pmd);
                page = follow_devmap_pmd(vma, address, pmd, flags);
                spin_unlock(ptl);
                if (page)
                        return page;
        }
-       if (likely(!pmd_trans_huge(*pmd)))
+       if (likely(!pmd_trans_huge(pmdval)))
                return follow_page_pte(vma, address, pmd, flags);
 
-       if ((flags & FOLL_NUMA) && pmd_protnone(*pmd))
+       if ((flags & FOLL_NUMA) && pmd_protnone(pmdval))
                return no_page_table(vma, flags);
 
 retry_locked:
        ptl = pmd_lock(mm, pmd);
+       if (unlikely(pmd_none(*pmd))) {
+               spin_unlock(ptl);
+               return no_page_table(vma, flags);
+       }
        if (unlikely(!pmd_present(*pmd))) {
                spin_unlock(ptl);
                if (likely(!(flags & FOLL_MIGRATION)))
@@ -1354,7 +1370,7 @@ static void undo_dev_pagemap(int *nr, int nr_start, struct page **pages)
        }
 }
 
-#ifdef __HAVE_ARCH_PTE_SPECIAL
+#ifdef CONFIG_ARCH_HAS_PTE_SPECIAL
 static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
                         int write, struct page **pages, int *nr)
 {
@@ -1430,7 +1446,7 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
 {
        return 0;
 }
-#endif /* __HAVE_ARCH_PTE_SPECIAL */
+#endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */
 
 #if defined(__HAVE_ARCH_PTE_DEVMAP) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
 static int __gup_device_huge(unsigned long pfn, unsigned long addr,