include/linux/userfaultfd_k.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  *  include/linux/userfaultfd_k.h
   4  *
   5  *  Copyright (C) 2015  Red Hat, Inc.
   6  *
   7  */
   8
   9 #ifndef _LINUX_USERFAULTFD_K_H
  10 #define _LINUX_USERFAULTFD_K_H
  11
  12 #ifdef CONFIG_USERFAULTFD
  13
  14 #include <linux/userfaultfd.h> /* linux/include/uapi/linux/userfaultfd.h */
  15
  16 #include <linux/fcntl.h>
  17 #include <linux/mm.h>
  18 #include <asm-generic/pgtable_uffd.h>
  19
  20 /* The set of all possible UFFD-related VM flags. */
  21 #define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR)
  22
  23 /*
  24  * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining
  25  * new flags, since they might collide with O_* ones. We want
  26  * to re-use O_* flags that couldn't possibly have a meaning
  27  * from userfaultfd, in order to leave a free define-space for
  28  * shared O_* flags.
  29  */
  30 #define UFFD_CLOEXEC O_CLOEXEC
  31 #define UFFD_NONBLOCK O_NONBLOCK
  32
  33 #define UFFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK)
  34 #define UFFD_FLAGS_SET (EFD_SHARED_FCNTL_FLAGS)
  35
  36 extern int sysctl_unprivileged_userfaultfd;
  37
  38 extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason);
  39
  40 /*
  41  * The mode of operation for __mcopy_atomic and its helpers.
  42  *
  43  * This is almost an implementation detail (mcopy_atomic below doesn't take this
  44  * as a parameter), but it's exposed here because memory-kind-specific
  45  * implementations (e.g. hugetlbfs) need to know the mode of operation.
  46  */
  47 enum mcopy_atomic_mode {
  48         /* A normal copy_from_user into the destination range. */
  49         MCOPY_ATOMIC_NORMAL,
  50         /* Don't copy; map the destination range to the zero page. */
  51         MCOPY_ATOMIC_ZEROPAGE,
  52         /* Just install pte(s) with the existing page(s) in the page cache. */
  53         MCOPY_ATOMIC_CONTINUE,
  54 };
  55
  56 extern int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd,
  57                                     struct vm_area_struct *dst_vma,
  58                                     unsigned long dst_addr, struct page *page,
  59                                     bool newly_allocated, bool wp_copy);
  60
  61 extern ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start,
  62                             unsigned long src_start, unsigned long len,
  63                             bool *mmap_changing, __u64 mode);
  64 extern ssize_t mfill_zeropage(struct mm_struct *dst_mm,
  65                               unsigned long dst_start,
  66                               unsigned long len,
  67                               bool *mmap_changing);
  68 extern ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long dst_start,
  69                               unsigned long len, bool *mmap_changing);
  70 extern int mwriteprotect_range(struct mm_struct *dst_mm,
  71                                unsigned long start, unsigned long len,
  72                                bool enable_wp, bool *mmap_changing);
  73
  74 /* mm helpers */
  75 static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma,
  76                                         struct vm_userfaultfd_ctx vm_ctx)
  77 {
  78         return vma->vm_userfaultfd_ctx.ctx == vm_ctx.ctx;
  79 }
  80
  81 /*
  82  * Never enable huge pmd sharing on some uffd registered vmas:
  83  *
  84  * - VM_UFFD_WP VMAs, because write protect information is per pgtable entry.
  85  *
  86  * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for
  87  *   VMAs which share huge pmds. (If you have two mappings to the same
  88  *   underlying pages, and fault in the non-UFFD-registered one with a write,
  89  *   with huge pmd sharing this would *also* setup the second UFFD-registered
  90  *   mapping, and we'd not get minor faults.)
  91  */
  92 static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma)
  93 {
  94         return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR);
  95 }
  96
  97 static inline bool userfaultfd_missing(struct vm_area_struct *vma)
  98 {
  99         return vma->vm_flags & VM_UFFD_MISSING;
 100 }
 101
 102 static inline bool userfaultfd_wp(struct vm_area_struct *vma)
 103 {
 104         return vma->vm_flags & VM_UFFD_WP;
 105 }
 106
 107 static inline bool userfaultfd_minor(struct vm_area_struct *vma)
 108 {
 109         return vma->vm_flags & VM_UFFD_MINOR;
 110 }
 111
 112 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma,
 113                                       pte_t pte)
 114 {
 115         return userfaultfd_wp(vma) && pte_uffd_wp(pte);
 116 }
 117
 118 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma,
 119                                            pmd_t pmd)
 120 {
 121         return userfaultfd_wp(vma) && pmd_uffd_wp(pmd);
 122 }
 123
 124 static inline bool userfaultfd_armed(struct vm_area_struct *vma)
 125 {
 126         return vma->vm_flags & __VM_UFFD_FLAGS;
 127 }
 128
 129 extern int dup_userfaultfd(struct vm_area_struct *, struct list_head *);
 130 extern void dup_userfaultfd_complete(struct list_head *);
 131
 132 extern void mremap_userfaultfd_prep(struct vm_area_struct *,
 133                                     struct vm_userfaultfd_ctx *);
 134 extern void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *,
 135                                         unsigned long from, unsigned long to,
 136                                         unsigned long len);
 137
 138 extern bool userfaultfd_remove(struct vm_area_struct *vma,
 139                                unsigned long start,
 140                                unsigned long end);
 141
 142 extern int userfaultfd_unmap_prep(struct vm_area_struct *vma,
 143                                   unsigned long start, unsigned long end,
 144                                   struct list_head *uf);
 145 extern void userfaultfd_unmap_complete(struct mm_struct *mm,
 146                                        struct list_head *uf);
 147
 148 #else /* CONFIG_USERFAULTFD */
 149
 150 /* mm helpers */
 151 static inline vm_fault_t handle_userfault(struct vm_fault *vmf,
 152                                 unsigned long reason)
 153 {
 154         return VM_FAULT_SIGBUS;
 155 }
 156
 157 static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma,
 158                                         struct vm_userfaultfd_ctx vm_ctx)
 159 {
 160         return true;
 161 }
 162
 163 static inline bool userfaultfd_missing(struct vm_area_struct *vma)
 164 {
 165         return false;
 166 }
 167
 168 static inline bool userfaultfd_wp(struct vm_area_struct *vma)
 169 {
 170         return false;
 171 }
 172
 173 static inline bool userfaultfd_minor(struct vm_area_struct *vma)
 174 {
 175         return false;
 176 }
 177
 178 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma,
 179                                       pte_t pte)
 180 {
 181         return false;
 182 }
 183
 184 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma,
 185                                            pmd_t pmd)
 186 {
 187         return false;
 188 }
 189
 190
 191 static inline bool userfaultfd_armed(struct vm_area_struct *vma)
 192 {
 193         return false;
 194 }
 195
 196 static inline int dup_userfaultfd(struct vm_area_struct *vma,
 197                                   struct list_head *l)
 198 {
 199         return 0;
 200 }
 201
 202 static inline void dup_userfaultfd_complete(struct list_head *l)
 203 {
 204 }
 205
 206 static inline void mremap_userfaultfd_prep(struct vm_area_struct *vma,
 207                                            struct vm_userfaultfd_ctx *ctx)
 208 {
 209 }
 210
 211 static inline void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *ctx,
 212                                                unsigned long from,
 213                                                unsigned long to,
 214                                                unsigned long len)
 215 {
 216 }
 217
 218 static inline bool userfaultfd_remove(struct vm_area_struct *vma,
 219                                       unsigned long start,
 220                                       unsigned long end)
 221 {
 222         return true;
 223 }
 224
 225 static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma,
 226                                          unsigned long start, unsigned long end,
 227                                          struct list_head *uf)
 228 {
 229         return 0;
 230 }
 231
 232 static inline void userfaultfd_unmap_complete(struct mm_struct *mm,
 233                                               struct list_head *uf)
 234 {
 235 }
 236
 237 #endif /* CONFIG_USERFAULTFD */
 238
 239 #endif /* _LINUX_USERFAULTFD_K_H */