mm/hmm: add new helper to hotplug CDM memory region
[linux-2.6-microblaze.git] / mm / userfaultfd.c
1 /*
2  *  mm/userfaultfd.c
3  *
4  *  Copyright (C) 2015  Red Hat, Inc.
5  *
6  *  This work is licensed under the terms of the GNU GPL, version 2. See
7  *  the COPYING file in the top-level directory.
8  */
9
10 #include <linux/mm.h>
11 #include <linux/sched/signal.h>
12 #include <linux/pagemap.h>
13 #include <linux/rmap.h>
14 #include <linux/swap.h>
15 #include <linux/swapops.h>
16 #include <linux/userfaultfd_k.h>
17 #include <linux/mmu_notifier.h>
18 #include <linux/hugetlb.h>
19 #include <linux/pagemap.h>
20 #include <linux/shmem_fs.h>
21 #include <asm/tlbflush.h>
22 #include "internal.h"
23
24 static int mcopy_atomic_pte(struct mm_struct *dst_mm,
25                             pmd_t *dst_pmd,
26                             struct vm_area_struct *dst_vma,
27                             unsigned long dst_addr,
28                             unsigned long src_addr,
29                             struct page **pagep)
30 {
31         struct mem_cgroup *memcg;
32         pte_t _dst_pte, *dst_pte;
33         spinlock_t *ptl;
34         void *page_kaddr;
35         int ret;
36         struct page *page;
37
38         if (!*pagep) {
39                 ret = -ENOMEM;
40                 page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, dst_vma, dst_addr);
41                 if (!page)
42                         goto out;
43
44                 page_kaddr = kmap_atomic(page);
45                 ret = copy_from_user(page_kaddr,
46                                      (const void __user *) src_addr,
47                                      PAGE_SIZE);
48                 kunmap_atomic(page_kaddr);
49
50                 /* fallback to copy_from_user outside mmap_sem */
51                 if (unlikely(ret)) {
52                         ret = -EFAULT;
53                         *pagep = page;
54                         /* don't free the page */
55                         goto out;
56                 }
57         } else {
58                 page = *pagep;
59                 *pagep = NULL;
60         }
61
62         /*
63          * The memory barrier inside __SetPageUptodate makes sure that
64          * preceeding stores to the page contents become visible before
65          * the set_pte_at() write.
66          */
67         __SetPageUptodate(page);
68
69         ret = -ENOMEM;
70         if (mem_cgroup_try_charge(page, dst_mm, GFP_KERNEL, &memcg, false))
71                 goto out_release;
72
73         _dst_pte = mk_pte(page, dst_vma->vm_page_prot);
74         if (dst_vma->vm_flags & VM_WRITE)
75                 _dst_pte = pte_mkwrite(pte_mkdirty(_dst_pte));
76
77         ret = -EEXIST;
78         dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
79         if (!pte_none(*dst_pte))
80                 goto out_release_uncharge_unlock;
81
82         inc_mm_counter(dst_mm, MM_ANONPAGES);
83         page_add_new_anon_rmap(page, dst_vma, dst_addr, false);
84         mem_cgroup_commit_charge(page, memcg, false, false);
85         lru_cache_add_active_or_unevictable(page, dst_vma);
86
87         set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
88
89         /* No need to invalidate - it was non-present before */
90         update_mmu_cache(dst_vma, dst_addr, dst_pte);
91
92         pte_unmap_unlock(dst_pte, ptl);
93         ret = 0;
94 out:
95         return ret;
96 out_release_uncharge_unlock:
97         pte_unmap_unlock(dst_pte, ptl);
98         mem_cgroup_cancel_charge(page, memcg, false);
99 out_release:
100         put_page(page);
101         goto out;
102 }
103
104 static int mfill_zeropage_pte(struct mm_struct *dst_mm,
105                               pmd_t *dst_pmd,
106                               struct vm_area_struct *dst_vma,
107                               unsigned long dst_addr)
108 {
109         pte_t _dst_pte, *dst_pte;
110         spinlock_t *ptl;
111         int ret;
112
113         _dst_pte = pte_mkspecial(pfn_pte(my_zero_pfn(dst_addr),
114                                          dst_vma->vm_page_prot));
115         ret = -EEXIST;
116         dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
117         if (!pte_none(*dst_pte))
118                 goto out_unlock;
119         set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
120         /* No need to invalidate - it was non-present before */
121         update_mmu_cache(dst_vma, dst_addr, dst_pte);
122         ret = 0;
123 out_unlock:
124         pte_unmap_unlock(dst_pte, ptl);
125         return ret;
126 }
127
128 static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address)
129 {
130         pgd_t *pgd;
131         p4d_t *p4d;
132         pud_t *pud;
133
134         pgd = pgd_offset(mm, address);
135         p4d = p4d_alloc(mm, pgd, address);
136         if (!p4d)
137                 return NULL;
138         pud = pud_alloc(mm, p4d, address);
139         if (!pud)
140                 return NULL;
141         /*
142          * Note that we didn't run this because the pmd was
143          * missing, the *pmd may be already established and in
144          * turn it may also be a trans_huge_pmd.
145          */
146         return pmd_alloc(mm, pud, address);
147 }
148
149 #ifdef CONFIG_HUGETLB_PAGE
150 /*
151  * __mcopy_atomic processing for HUGETLB vmas.  Note that this routine is
152  * called with mmap_sem held, it will release mmap_sem before returning.
153  */
154 static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm,
155                                               struct vm_area_struct *dst_vma,
156                                               unsigned long dst_start,
157                                               unsigned long src_start,
158                                               unsigned long len,
159                                               bool zeropage)
160 {
161         int vm_alloc_shared = dst_vma->vm_flags & VM_SHARED;
162         int vm_shared = dst_vma->vm_flags & VM_SHARED;
163         ssize_t err;
164         pte_t *dst_pte;
165         unsigned long src_addr, dst_addr;
166         long copied;
167         struct page *page;
168         struct hstate *h;
169         unsigned long vma_hpagesize;
170         pgoff_t idx;
171         u32 hash;
172         struct address_space *mapping;
173
174         /*
175          * There is no default zero huge page for all huge page sizes as
176          * supported by hugetlb.  A PMD_SIZE huge pages may exist as used
177          * by THP.  Since we can not reliably insert a zero page, this
178          * feature is not supported.
179          */
180         if (zeropage) {
181                 up_read(&dst_mm->mmap_sem);
182                 return -EINVAL;
183         }
184
185         src_addr = src_start;
186         dst_addr = dst_start;
187         copied = 0;
188         page = NULL;
189         vma_hpagesize = vma_kernel_pagesize(dst_vma);
190
191         /*
192          * Validate alignment based on huge page size
193          */
194         err = -EINVAL;
195         if (dst_start & (vma_hpagesize - 1) || len & (vma_hpagesize - 1))
196                 goto out_unlock;
197
198 retry:
199         /*
200          * On routine entry dst_vma is set.  If we had to drop mmap_sem and
201          * retry, dst_vma will be set to NULL and we must lookup again.
202          */
203         if (!dst_vma) {
204                 err = -ENOENT;
205                 dst_vma = find_vma(dst_mm, dst_start);
206                 if (!dst_vma || !is_vm_hugetlb_page(dst_vma))
207                         goto out_unlock;
208                 /*
209                  * Only allow __mcopy_atomic_hugetlb on userfaultfd
210                  * registered ranges.
211                  */
212                 if (!dst_vma->vm_userfaultfd_ctx.ctx)
213                         goto out_unlock;
214
215                 if (dst_start < dst_vma->vm_start ||
216                     dst_start + len > dst_vma->vm_end)
217                         goto out_unlock;
218
219                 err = -EINVAL;
220                 if (vma_hpagesize != vma_kernel_pagesize(dst_vma))
221                         goto out_unlock;
222
223                 vm_shared = dst_vma->vm_flags & VM_SHARED;
224         }
225
226         if (WARN_ON(dst_addr & (vma_hpagesize - 1) ||
227                     (len - copied) & (vma_hpagesize - 1)))
228                 goto out_unlock;
229
230         /*
231          * If not shared, ensure the dst_vma has a anon_vma.
232          */
233         err = -ENOMEM;
234         if (!vm_shared) {
235                 if (unlikely(anon_vma_prepare(dst_vma)))
236                         goto out_unlock;
237         }
238
239         h = hstate_vma(dst_vma);
240
241         while (src_addr < src_start + len) {
242                 pte_t dst_pteval;
243
244                 BUG_ON(dst_addr >= dst_start + len);
245                 VM_BUG_ON(dst_addr & ~huge_page_mask(h));
246
247                 /*
248                  * Serialize via hugetlb_fault_mutex
249                  */
250                 idx = linear_page_index(dst_vma, dst_addr);
251                 mapping = dst_vma->vm_file->f_mapping;
252                 hash = hugetlb_fault_mutex_hash(h, dst_mm, dst_vma, mapping,
253                                                                 idx, dst_addr);
254                 mutex_lock(&hugetlb_fault_mutex_table[hash]);
255
256                 err = -ENOMEM;
257                 dst_pte = huge_pte_alloc(dst_mm, dst_addr, huge_page_size(h));
258                 if (!dst_pte) {
259                         mutex_unlock(&hugetlb_fault_mutex_table[hash]);
260                         goto out_unlock;
261                 }
262
263                 err = -EEXIST;
264                 dst_pteval = huge_ptep_get(dst_pte);
265                 if (!huge_pte_none(dst_pteval)) {
266                         mutex_unlock(&hugetlb_fault_mutex_table[hash]);
267                         goto out_unlock;
268                 }
269
270                 err = hugetlb_mcopy_atomic_pte(dst_mm, dst_pte, dst_vma,
271                                                 dst_addr, src_addr, &page);
272
273                 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
274                 vm_alloc_shared = vm_shared;
275
276                 cond_resched();
277
278                 if (unlikely(err == -EFAULT)) {
279                         up_read(&dst_mm->mmap_sem);
280                         BUG_ON(!page);
281
282                         err = copy_huge_page_from_user(page,
283                                                 (const void __user *)src_addr,
284                                                 pages_per_huge_page(h), true);
285                         if (unlikely(err)) {
286                                 err = -EFAULT;
287                                 goto out;
288                         }
289                         down_read(&dst_mm->mmap_sem);
290
291                         dst_vma = NULL;
292                         goto retry;
293                 } else
294                         BUG_ON(page);
295
296                 if (!err) {
297                         dst_addr += vma_hpagesize;
298                         src_addr += vma_hpagesize;
299                         copied += vma_hpagesize;
300
301                         if (fatal_signal_pending(current))
302                                 err = -EINTR;
303                 }
304                 if (err)
305                         break;
306         }
307
308 out_unlock:
309         up_read(&dst_mm->mmap_sem);
310 out:
311         if (page) {
312                 /*
313                  * We encountered an error and are about to free a newly
314                  * allocated huge page.
315                  *
316                  * Reservation handling is very subtle, and is different for
317                  * private and shared mappings.  See the routine
318                  * restore_reserve_on_error for details.  Unfortunately, we
319                  * can not call restore_reserve_on_error now as it would
320                  * require holding mmap_sem.
321                  *
322                  * If a reservation for the page existed in the reservation
323                  * map of a private mapping, the map was modified to indicate
324                  * the reservation was consumed when the page was allocated.
325                  * We clear the PagePrivate flag now so that the global
326                  * reserve count will not be incremented in free_huge_page.
327                  * The reservation map will still indicate the reservation
328                  * was consumed and possibly prevent later page allocation.
329                  * This is better than leaking a global reservation.  If no
330                  * reservation existed, it is still safe to clear PagePrivate
331                  * as no adjustments to reservation counts were made during
332                  * allocation.
333                  *
334                  * The reservation map for shared mappings indicates which
335                  * pages have reservations.  When a huge page is allocated
336                  * for an address with a reservation, no change is made to
337                  * the reserve map.  In this case PagePrivate will be set
338                  * to indicate that the global reservation count should be
339                  * incremented when the page is freed.  This is the desired
340                  * behavior.  However, when a huge page is allocated for an
341                  * address without a reservation a reservation entry is added
342                  * to the reservation map, and PagePrivate will not be set.
343                  * When the page is freed, the global reserve count will NOT
344                  * be incremented and it will appear as though we have leaked
345                  * reserved page.  In this case, set PagePrivate so that the
346                  * global reserve count will be incremented to match the
347                  * reservation map entry which was created.
348                  *
349                  * Note that vm_alloc_shared is based on the flags of the vma
350                  * for which the page was originally allocated.  dst_vma could
351                  * be different or NULL on error.
352                  */
353                 if (vm_alloc_shared)
354                         SetPagePrivate(page);
355                 else
356                         ClearPagePrivate(page);
357                 put_page(page);
358         }
359         BUG_ON(copied < 0);
360         BUG_ON(err > 0);
361         BUG_ON(!copied && !err);
362         return copied ? copied : err;
363 }
364 #else /* !CONFIG_HUGETLB_PAGE */
365 /* fail at build time if gcc attempts to use this */
366 extern ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm,
367                                       struct vm_area_struct *dst_vma,
368                                       unsigned long dst_start,
369                                       unsigned long src_start,
370                                       unsigned long len,
371                                       bool zeropage);
372 #endif /* CONFIG_HUGETLB_PAGE */
373
374 static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm,
375                                                 pmd_t *dst_pmd,
376                                                 struct vm_area_struct *dst_vma,
377                                                 unsigned long dst_addr,
378                                                 unsigned long src_addr,
379                                                 struct page **page,
380                                                 bool zeropage)
381 {
382         ssize_t err;
383
384         if (vma_is_anonymous(dst_vma)) {
385                 if (!zeropage)
386                         err = mcopy_atomic_pte(dst_mm, dst_pmd, dst_vma,
387                                                dst_addr, src_addr, page);
388                 else
389                         err = mfill_zeropage_pte(dst_mm, dst_pmd,
390                                                  dst_vma, dst_addr);
391         } else {
392                 if (!zeropage)
393                         err = shmem_mcopy_atomic_pte(dst_mm, dst_pmd,
394                                                      dst_vma, dst_addr,
395                                                      src_addr, page);
396                 else
397                         err = shmem_mfill_zeropage_pte(dst_mm, dst_pmd,
398                                                        dst_vma, dst_addr);
399         }
400
401         return err;
402 }
403
404 static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm,
405                                               unsigned long dst_start,
406                                               unsigned long src_start,
407                                               unsigned long len,
408                                               bool zeropage)
409 {
410         struct vm_area_struct *dst_vma;
411         ssize_t err;
412         pmd_t *dst_pmd;
413         unsigned long src_addr, dst_addr;
414         long copied;
415         struct page *page;
416
417         /*
418          * Sanitize the command parameters:
419          */
420         BUG_ON(dst_start & ~PAGE_MASK);
421         BUG_ON(len & ~PAGE_MASK);
422
423         /* Does the address range wrap, or is the span zero-sized? */
424         BUG_ON(src_start + len <= src_start);
425         BUG_ON(dst_start + len <= dst_start);
426
427         src_addr = src_start;
428         dst_addr = dst_start;
429         copied = 0;
430         page = NULL;
431 retry:
432         down_read(&dst_mm->mmap_sem);
433
434         /*
435          * Make sure the vma is not shared, that the dst range is
436          * both valid and fully within a single existing vma.
437          */
438         err = -ENOENT;
439         dst_vma = find_vma(dst_mm, dst_start);
440         if (!dst_vma)
441                 goto out_unlock;
442         /*
443          * Be strict and only allow __mcopy_atomic on userfaultfd
444          * registered ranges to prevent userland errors going
445          * unnoticed. As far as the VM consistency is concerned, it
446          * would be perfectly safe to remove this check, but there's
447          * no useful usage for __mcopy_atomic ouside of userfaultfd
448          * registered ranges. This is after all why these are ioctls
449          * belonging to the userfaultfd and not syscalls.
450          */
451         if (!dst_vma->vm_userfaultfd_ctx.ctx)
452                 goto out_unlock;
453
454         if (dst_start < dst_vma->vm_start ||
455             dst_start + len > dst_vma->vm_end)
456                 goto out_unlock;
457
458         err = -EINVAL;
459         /*
460          * shmem_zero_setup is invoked in mmap for MAP_ANONYMOUS|MAP_SHARED but
461          * it will overwrite vm_ops, so vma_is_anonymous must return false.
462          */
463         if (WARN_ON_ONCE(vma_is_anonymous(dst_vma) &&
464             dst_vma->vm_flags & VM_SHARED))
465                 goto out_unlock;
466
467         /*
468          * If this is a HUGETLB vma, pass off to appropriate routine
469          */
470         if (is_vm_hugetlb_page(dst_vma))
471                 return  __mcopy_atomic_hugetlb(dst_mm, dst_vma, dst_start,
472                                                 src_start, len, zeropage);
473
474         if (!vma_is_anonymous(dst_vma) && !vma_is_shmem(dst_vma))
475                 goto out_unlock;
476
477         /*
478          * Ensure the dst_vma has a anon_vma or this page
479          * would get a NULL anon_vma when moved in the
480          * dst_vma.
481          */
482         err = -ENOMEM;
483         if (vma_is_anonymous(dst_vma) && unlikely(anon_vma_prepare(dst_vma)))
484                 goto out_unlock;
485
486         while (src_addr < src_start + len) {
487                 pmd_t dst_pmdval;
488
489                 BUG_ON(dst_addr >= dst_start + len);
490
491                 dst_pmd = mm_alloc_pmd(dst_mm, dst_addr);
492                 if (unlikely(!dst_pmd)) {
493                         err = -ENOMEM;
494                         break;
495                 }
496
497                 dst_pmdval = pmd_read_atomic(dst_pmd);
498                 /*
499                  * If the dst_pmd is mapped as THP don't
500                  * override it and just be strict.
501                  */
502                 if (unlikely(pmd_trans_huge(dst_pmdval))) {
503                         err = -EEXIST;
504                         break;
505                 }
506                 if (unlikely(pmd_none(dst_pmdval)) &&
507                     unlikely(__pte_alloc(dst_mm, dst_pmd, dst_addr))) {
508                         err = -ENOMEM;
509                         break;
510                 }
511                 /* If an huge pmd materialized from under us fail */
512                 if (unlikely(pmd_trans_huge(*dst_pmd))) {
513                         err = -EFAULT;
514                         break;
515                 }
516
517                 BUG_ON(pmd_none(*dst_pmd));
518                 BUG_ON(pmd_trans_huge(*dst_pmd));
519
520                 err = mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
521                                        src_addr, &page, zeropage);
522                 cond_resched();
523
524                 if (unlikely(err == -EFAULT)) {
525                         void *page_kaddr;
526
527                         up_read(&dst_mm->mmap_sem);
528                         BUG_ON(!page);
529
530                         page_kaddr = kmap(page);
531                         err = copy_from_user(page_kaddr,
532                                              (const void __user *) src_addr,
533                                              PAGE_SIZE);
534                         kunmap(page);
535                         if (unlikely(err)) {
536                                 err = -EFAULT;
537                                 goto out;
538                         }
539                         goto retry;
540                 } else
541                         BUG_ON(page);
542
543                 if (!err) {
544                         dst_addr += PAGE_SIZE;
545                         src_addr += PAGE_SIZE;
546                         copied += PAGE_SIZE;
547
548                         if (fatal_signal_pending(current))
549                                 err = -EINTR;
550                 }
551                 if (err)
552                         break;
553         }
554
555 out_unlock:
556         up_read(&dst_mm->mmap_sem);
557 out:
558         if (page)
559                 put_page(page);
560         BUG_ON(copied < 0);
561         BUG_ON(err > 0);
562         BUG_ON(!copied && !err);
563         return copied ? copied : err;
564 }
565
566 ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start,
567                      unsigned long src_start, unsigned long len)
568 {
569         return __mcopy_atomic(dst_mm, dst_start, src_start, len, false);
570 }
571
572 ssize_t mfill_zeropage(struct mm_struct *dst_mm, unsigned long start,
573                        unsigned long len)
574 {
575         return __mcopy_atomic(dst_mm, start, 0, len, true);
576 }