mm/gup: add FOLL_LONGTERM capability to GUP fast

DAX pages were previously unprotected from longterm pins when users called
get_user_pages_fast().

Use the new FOLL_LONGTERM flag to check for DEVMAP pages and fall back to
regular GUP processing if a DEVMAP page is encountered.

[ira.weiny@intel.com: v3]
Link: http://lkml.kernel.org/r/20190328084422.29911-5-ira.weiny@intel.com
Link: http://lkml.kernel.org/r/20190328084422.29911-5-ira.weiny@intel.com
Link: http://lkml.kernel.org/r/20190317183438.2057-5-ira.weiny@intel.com
Signed-off-by: Ira Weiny <ira.weiny@intel.com>
Reviewed-by: Andrew Morton <akpm@linux-foundation.org>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: James Hogan <jhogan@kernel.org>
Cc: Jason Gunthorpe <jgg@ziepe.ca>
Cc: John Hubbard <jhubbard@nvidia.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: Rich Felker <dalias@libc.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Yoshinori Sato <ysato@users.sourceforge.jp>
Cc: Mike Marshall <hubcap@omnibond.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/mm/gup.c b/mm/gup.c
index 3dde6a8..8e0a0a3 100644 (file)
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -1637,6 +1637,9 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
                        goto pte_unmap;
 
                if (pte_devmap(pte)) {
+                       if (unlikely(flags & FOLL_LONGTERM))
+                               goto pte_unmap;
+
                        pgmap = get_dev_pagemap(pte_pfn(pte), pgmap);
                        if (unlikely(!pgmap)) {
                                undo_dev_pagemap(nr, nr_start, pages);
@@ -1776,8 +1779,11 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
        if (!pmd_access_permitted(orig, flags & FOLL_WRITE))
                return 0;
 
-       if (pmd_devmap(orig))
+       if (pmd_devmap(orig)) {
+               if (unlikely(flags & FOLL_LONGTERM))
+                       return 0;
                return __gup_device_huge_pmd(orig, pmdp, addr, end, pages, nr);
+       }
 
        refs = 0;
        page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
@@ -1814,8 +1820,11 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
        if (!pud_access_permitted(orig, flags & FOLL_WRITE))
                return 0;
 
-       if (pud_devmap(orig))
+       if (pud_devmap(orig)) {
+               if (unlikely(flags & FOLL_LONGTERM))
+                       return 0;
                return __gup_device_huge_pud(orig, pudp, addr, end, pages, nr);
+       }
 
        refs = 0;
        page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
@@ -2058,6 +2067,29 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
        return nr;
 }
 
+static int __gup_longterm_unlocked(unsigned long start, int nr_pages,
+                                  unsigned int gup_flags, struct page **pages)
+{
+       int ret;
+
+       /*
+        * FIXME: FOLL_LONGTERM does not work with
+        * get_user_pages_unlocked() (see comments in that function)
+        */
+       if (gup_flags & FOLL_LONGTERM) {
+               down_read(&current->mm->mmap_sem);
+               ret = __gup_longterm_locked(current, current->mm,
+                                           start, nr_pages,
+                                           pages, NULL, gup_flags);
+               up_read(&current->mm->mmap_sem);
+       } else {
+               ret = get_user_pages_unlocked(start, nr_pages,
+                                             pages, gup_flags);
+       }
+
+       return ret;
+}
+
 /**
  * get_user_pages_fast() - pin user pages in memory
  * @start:     starting user address
@@ -2103,8 +2135,8 @@ int get_user_pages_fast(unsigned long start, int nr_pages,
                start += nr << PAGE_SHIFT;
                pages += nr;
 
-               ret = get_user_pages_unlocked(start, nr_pages - nr, pages,
-                                             gup_flags);
+               ret = __gup_longterm_unlocked(start, nr_pages - nr,
+                                             gup_flags, pages);
 
                /* Have to be a bit careful with return values */
                if (nr > 0) {