include/linux/userfaultfd_k.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  *  include/linux/userfaultfd_k.h
   4  *
   5  *  Copyright (C) 2015  Red Hat, Inc.
   6  *
   7  */
   8
   9 #ifndef _LINUX_USERFAULTFD_K_H
  10 #define _LINUX_USERFAULTFD_K_H
  11
  12 #ifdef CONFIG_USERFAULTFD
  13
  14 #include <linux/userfaultfd.h> /* linux/include/uapi/linux/userfaultfd.h */
  15
  16 #include <linux/fcntl.h>
  17 #include <linux/mm.h>
  18 #include <asm-generic/pgtable_uffd.h>
  19
  20 /* The set of all possible UFFD-related VM flags. */
  21 #define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR)
  22
  23 /*
  24  * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining
  25  * new flags, since they might collide with O_* ones. We want
  26  * to re-use O_* flags that couldn't possibly have a meaning
  27  * from userfaultfd, in order to leave a free define-space for
  28  * shared O_* flags.
  29  */
  30 #define UFFD_CLOEXEC O_CLOEXEC
  31 #define UFFD_NONBLOCK O_NONBLOCK
  32
  33 #define UFFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK)
  34 #define UFFD_FLAGS_SET (EFD_SHARED_FCNTL_FLAGS)
  35
  36 extern int sysctl_unprivileged_userfaultfd;
  37
  38 extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason);
  39
  40 /*
  41  * The mode of operation for __mcopy_atomic and its helpers.
  42  *
  43  * This is almost an implementation detail (mcopy_atomic below doesn't take this
  44  * as a parameter), but it's exposed here because memory-kind-specific
  45  * implementations (e.g. hugetlbfs) need to know the mode of operation.
  46  */
  47 enum mcopy_atomic_mode {
  48         /* A normal copy_from_user into the destination range. */
  49         MCOPY_ATOMIC_NORMAL,
  50         /* Don't copy; map the destination range to the zero page. */
  51         MCOPY_ATOMIC_ZEROPAGE,
  52         /* Just install pte(s) with the existing page(s) in the page cache. */
  53         MCOPY_ATOMIC_CONTINUE,
  54 };
  55
  56 extern ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start,
  57                             unsigned long src_start, unsigned long len,
  58                             bool *mmap_changing, __u64 mode);
  59 extern ssize_t mfill_zeropage(struct mm_struct *dst_mm,
  60                               unsigned long dst_start,
  61                               unsigned long len,
  62                               bool *mmap_changing);
  63 extern ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long dst_start,
  64                               unsigned long len, bool *mmap_changing);
  65 extern int mwriteprotect_range(struct mm_struct *dst_mm,
  66                                unsigned long start, unsigned long len,
  67                                bool enable_wp, bool *mmap_changing);
  68
  69 /* mm helpers */
  70 static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma,
  71                                         struct vm_userfaultfd_ctx vm_ctx)
  72 {
  73         return vma->vm_userfaultfd_ctx.ctx == vm_ctx.ctx;
  74 }
  75
  76 /*
  77  * Never enable huge pmd sharing on some uffd registered vmas:
  78  *
  79  * - VM_UFFD_WP VMAs, because write protect information is per pgtable entry.
  80  *
  81  * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for
  82  *   VMAs which share huge pmds. (If you have two mappings to the same
  83  *   underlying pages, and fault in the non-UFFD-registered one with a write,
  84  *   with huge pmd sharing this would *also* setup the second UFFD-registered
  85  *   mapping, and we'd not get minor faults.)
  86  */
  87 static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma)
  88 {
  89         return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR);
  90 }
  91
  92 static inline bool userfaultfd_missing(struct vm_area_struct *vma)
  93 {
  94         return vma->vm_flags & VM_UFFD_MISSING;
  95 }
  96
  97 static inline bool userfaultfd_wp(struct vm_area_struct *vma)
  98 {
  99         return vma->vm_flags & VM_UFFD_WP;
 100 }
 101
 102 static inline bool userfaultfd_minor(struct vm_area_struct *vma)
 103 {
 104         return vma->vm_flags & VM_UFFD_MINOR;
 105 }
 106
 107 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma,
 108                                       pte_t pte)
 109 {
 110         return userfaultfd_wp(vma) && pte_uffd_wp(pte);
 111 }
 112
 113 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma,
 114                                            pmd_t pmd)
 115 {
 116         return userfaultfd_wp(vma) && pmd_uffd_wp(pmd);
 117 }
 118
 119 static inline bool userfaultfd_armed(struct vm_area_struct *vma)
 120 {
 121         return vma->vm_flags & __VM_UFFD_FLAGS;
 122 }
 123
 124 extern int dup_userfaultfd(struct vm_area_struct *, struct list_head *);
 125 extern void dup_userfaultfd_complete(struct list_head *);
 126
 127 extern void mremap_userfaultfd_prep(struct vm_area_struct *,
 128                                     struct vm_userfaultfd_ctx *);
 129 extern void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *,
 130                                         unsigned long from, unsigned long to,
 131                                         unsigned long len);
 132
 133 extern bool userfaultfd_remove(struct vm_area_struct *vma,
 134                                unsigned long start,
 135                                unsigned long end);
 136
 137 extern int userfaultfd_unmap_prep(struct vm_area_struct *vma,
 138                                   unsigned long start, unsigned long end,
 139                                   struct list_head *uf);
 140 extern void userfaultfd_unmap_complete(struct mm_struct *mm,
 141                                        struct list_head *uf);
 142
 143 #else /* CONFIG_USERFAULTFD */
 144
 145 /* mm helpers */
 146 static inline vm_fault_t handle_userfault(struct vm_fault *vmf,
 147                                 unsigned long reason)
 148 {
 149         return VM_FAULT_SIGBUS;
 150 }
 151
 152 static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma,
 153                                         struct vm_userfaultfd_ctx vm_ctx)
 154 {
 155         return true;
 156 }
 157
 158 static inline bool userfaultfd_missing(struct vm_area_struct *vma)
 159 {
 160         return false;
 161 }
 162
 163 static inline bool userfaultfd_wp(struct vm_area_struct *vma)
 164 {
 165         return false;
 166 }
 167
 168 static inline bool userfaultfd_minor(struct vm_area_struct *vma)
 169 {
 170         return false;
 171 }
 172
 173 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma,
 174                                       pte_t pte)
 175 {
 176         return false;
 177 }
 178
 179 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma,
 180                                            pmd_t pmd)
 181 {
 182         return false;
 183 }
 184
 185
 186 static inline bool userfaultfd_armed(struct vm_area_struct *vma)
 187 {
 188         return false;
 189 }
 190
 191 static inline int dup_userfaultfd(struct vm_area_struct *vma,
 192                                   struct list_head *l)
 193 {
 194         return 0;
 195 }
 196
 197 static inline void dup_userfaultfd_complete(struct list_head *l)
 198 {
 199 }
 200
 201 static inline void mremap_userfaultfd_prep(struct vm_area_struct *vma,
 202                                            struct vm_userfaultfd_ctx *ctx)
 203 {
 204 }
 205
 206 static inline void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *ctx,
 207                                                unsigned long from,
 208                                                unsigned long to,
 209                                                unsigned long len)
 210 {
 211 }
 212
 213 static inline bool userfaultfd_remove(struct vm_area_struct *vma,
 214                                       unsigned long start,
 215                                       unsigned long end)
 216 {
 217         return true;
 218 }
 219
 220 static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma,
 221                                          unsigned long start, unsigned long end,
 222                                          struct list_head *uf)
 223 {
 224         return 0;
 225 }
 226
 227 static inline void userfaultfd_unmap_complete(struct mm_struct *mm,
 228                                               struct list_head *uf)
 229 {
 230 }
 231
 232 #endif /* CONFIG_USERFAULTFD */
 233
 234 #endif /* _LINUX_USERFAULTFD_K_H */