1 // SPDX-License-Identifier: GPL-2.0-only
3 * Access kernel or user memory without faulting.
5 #include <linux/export.h>
7 #include <linux/uaccess.h>
9 bool __weak probe_kernel_read_allowed(const void *unsafe_src, size_t size)
14 #ifdef HAVE_GET_KERNEL_NOFAULT
16 #define probe_kernel_read_loop(dst, src, len, type, err_label) \
17 while (len >= sizeof(type)) { \
18 __get_kernel_nofault(dst, src, type, err_label); \
19 dst += sizeof(type); \
20 src += sizeof(type); \
21 len -= sizeof(type); \
24 long probe_kernel_read(void *dst, const void *src, size_t size)
26 if (!probe_kernel_read_allowed(src, size))
30 probe_kernel_read_loop(dst, src, size, u64, Efault);
31 probe_kernel_read_loop(dst, src, size, u32, Efault);
32 probe_kernel_read_loop(dst, src, size, u16, Efault);
33 probe_kernel_read_loop(dst, src, size, u8, Efault);
40 EXPORT_SYMBOL_GPL(probe_kernel_read);
42 #define probe_kernel_write_loop(dst, src, len, type, err_label) \
43 while (len >= sizeof(type)) { \
44 __put_kernel_nofault(dst, src, type, err_label); \
45 dst += sizeof(type); \
46 src += sizeof(type); \
47 len -= sizeof(type); \
50 long probe_kernel_write(void *dst, const void *src, size_t size)
53 probe_kernel_write_loop(dst, src, size, u64, Efault);
54 probe_kernel_write_loop(dst, src, size, u32, Efault);
55 probe_kernel_write_loop(dst, src, size, u16, Efault);
56 probe_kernel_write_loop(dst, src, size, u8, Efault);
64 long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
66 const void *src = unsafe_addr;
68 if (unlikely(count <= 0))
70 if (!probe_kernel_read_allowed(unsafe_addr, count))
75 __get_kernel_nofault(dst, src, u8, Efault);
78 } while (dst[-1] && src - unsafe_addr < count);
82 return src - unsafe_addr;
88 #else /* HAVE_GET_KERNEL_NOFAULT */
90 * probe_kernel_read(): safely attempt to read from kernel-space
91 * @dst: pointer to the buffer that shall take the data
92 * @src: address to read from
93 * @size: size of the data chunk
95 * Safely read from kernel address @src to the buffer at @dst. If a kernel
96 * fault happens, handle that and return -EFAULT. If @src is not a valid kernel
97 * address, return -ERANGE.
99 * We ensure that the copy_from_user is executed in atomic context so that
100 * do_page_fault() doesn't attempt to take mmap_lock. This makes
101 * probe_kernel_read() suitable for use within regions where the caller
102 * already holds mmap_lock, or other locks which nest inside mmap_lock.
104 long probe_kernel_read(void *dst, const void *src, size_t size)
107 mm_segment_t old_fs = get_fs();
109 if (!probe_kernel_read_allowed(src, size))
114 ret = __copy_from_user_inatomic(dst, (__force const void __user *)src,
123 EXPORT_SYMBOL_GPL(probe_kernel_read);
126 * probe_kernel_write(): safely attempt to write to a location
127 * @dst: address to write to
128 * @src: pointer to the data that shall be written
129 * @size: size of the data chunk
131 * Safely write to address @dst from the buffer at @src. If a kernel fault
132 * happens, handle that and return -EFAULT.
134 long probe_kernel_write(void *dst, const void *src, size_t size)
137 mm_segment_t old_fs = get_fs();
141 ret = __copy_to_user_inatomic((__force void __user *)dst, src, size);
151 * strncpy_from_kernel_nofault: - Copy a NUL terminated string from unsafe
153 * @dst: Destination address, in kernel space. This buffer must be at
154 * least @count bytes long.
155 * @unsafe_addr: Unsafe address.
156 * @count: Maximum number of bytes to copy, including the trailing NUL.
158 * Copies a NUL-terminated string from unsafe address to kernel buffer.
160 * On success, returns the length of the string INCLUDING the trailing NUL.
162 * If access fails, returns -EFAULT (some data may have been copied and the
163 * trailing NUL added). If @unsafe_addr is not a valid kernel address, return
166 * If @count is smaller than the length of the string, copies @count-1 bytes,
167 * sets the last byte of @dst buffer to NUL and returns @count.
169 long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
171 mm_segment_t old_fs = get_fs();
172 const void *src = unsafe_addr;
175 if (unlikely(count <= 0))
177 if (!probe_kernel_read_allowed(unsafe_addr, count))
184 ret = __get_user(*dst++, (const char __user __force *)src++);
185 } while (dst[-1] && ret == 0 && src - unsafe_addr < count);
191 return ret ? -EFAULT : src - unsafe_addr;
193 #endif /* HAVE_GET_KERNEL_NOFAULT */
196 * probe_user_read(): safely attempt to read from a user-space location
197 * @dst: pointer to the buffer that shall take the data
198 * @src: address to read from. This must be a user address.
199 * @size: size of the data chunk
201 * Safely read from user address @src to the buffer at @dst. If a kernel fault
202 * happens, handle that and return -EFAULT.
204 long probe_user_read(void *dst, const void __user *src, size_t size)
207 mm_segment_t old_fs = get_fs();
210 if (access_ok(src, size)) {
212 ret = __copy_from_user_inatomic(dst, src, size);
221 EXPORT_SYMBOL_GPL(probe_user_read);
224 * probe_user_write(): safely attempt to write to a user-space location
225 * @dst: address to write to
226 * @src: pointer to the data that shall be written
227 * @size: size of the data chunk
229 * Safely write to address @dst from the buffer at @src. If a kernel fault
230 * happens, handle that and return -EFAULT.
232 long probe_user_write(void __user *dst, const void *src, size_t size)
235 mm_segment_t old_fs = get_fs();
238 if (access_ok(dst, size)) {
240 ret = __copy_to_user_inatomic(dst, src, size);
249 EXPORT_SYMBOL_GPL(probe_user_write);
252 * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user
254 * @dst: Destination address, in kernel space. This buffer must be at
255 * least @count bytes long.
256 * @unsafe_addr: Unsafe user address.
257 * @count: Maximum number of bytes to copy, including the trailing NUL.
259 * Copies a NUL-terminated string from unsafe user address to kernel buffer.
261 * On success, returns the length of the string INCLUDING the trailing NUL.
263 * If access fails, returns -EFAULT (some data may have been copied
264 * and the trailing NUL added).
266 * If @count is smaller than the length of the string, copies @count-1 bytes,
267 * sets the last byte of @dst buffer to NUL and returns @count.
269 long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr,
272 mm_segment_t old_fs = get_fs();
275 if (unlikely(count <= 0))
280 ret = strncpy_from_user(dst, unsafe_addr, count);
287 } else if (ret > 0) {
295 * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL.
296 * @unsafe_addr: The string to measure.
297 * @count: Maximum count (including NUL)
299 * Get the size of a NUL-terminated string in user space without pagefault.
301 * Returns the size of the string INCLUDING the terminating NUL.
303 * If the string is too long, returns a number larger than @count. User
304 * has to check the return value against "> count".
305 * On exception (or invalid count), returns 0.
307 * Unlike strnlen_user, this can be used from IRQ handler etc. because
308 * it disables pagefaults.
310 long strnlen_user_nofault(const void __user *unsafe_addr, long count)
312 mm_segment_t old_fs = get_fs();
317 ret = strnlen_user(unsafe_addr, count);