1 /* SPDX-License-Identifier: GPL-2.0-only */
3 * Based on arch/arm/include/asm/uaccess.h
5 * Copyright (C) 2012 ARM Ltd.
7 #ifndef __ASM_UACCESS_H
8 #define __ASM_UACCESS_H
10 #include <asm/alternative.h>
11 #include <asm/kernel-pgtable.h>
12 #include <asm/sysreg.h>
15 * User space memory access functions
17 #include <linux/bitops.h>
18 #include <linux/kasan-checks.h>
19 #include <linux/string.h>
21 #include <asm/cpufeature.h>
24 #include <asm/ptrace.h>
25 #include <asm/memory.h>
26 #include <asm/extable.h>
28 #define HAVE_GET_KERNEL_NOFAULT
31 * Test whether a block of memory is a valid user space address.
32 * Returns 1 if the range is valid, 0 otherwise.
34 * This is equivalent to the following test:
35 * (u65)addr + (u65)size <= (u65)TASK_SIZE_MAX
37 static inline unsigned long __range_ok(const void __user *addr, unsigned long size)
39 unsigned long ret, limit = TASK_SIZE_MAX - 1;
42 * Asynchronous I/O running in a kernel thread does not have the
43 * TIF_TAGGED_ADDR flag of the process owning the mm, so always untag
44 * the user address before checking.
46 if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI) &&
47 (current->flags & PF_KTHREAD || test_thread_flag(TIF_TAGGED_ADDR)))
48 addr = untagged_addr(addr);
52 // A + B <= C + 1 for all A,B,C, in four easy steps:
53 // 1: X = A + B; X' = X % 2^64
55 // 2: Set C = 0 if X > 2^64, to guarantee X' > C in step 4
56 " csel %1, xzr, %1, hi\n"
57 // 3: Set X' = ~0 if X >= 2^64. For X == 2^64, this decrements X'
58 // to compensate for the carry flag being set in step 4. For
59 // X > 2^64, X' merely has to remain nonzero, which it does.
60 " csinv %0, %0, xzr, cc\n"
61 // 4: For X < 2^64, this gives us X' - C - 1 <= 0, where the -1
62 // comes from the carry in being clear. Otherwise, we are
63 // testing X' - C == 0, subject to the previous adjustments.
66 : "=&r" (ret), "+r" (limit) : "Ir" (size), "0" (addr) : "cc");
71 #define access_ok(addr, size) __range_ok(addr, size)
73 #define _ASM_EXTABLE(from, to) \
74 " .pushsection __ex_table, \"a\"\n" \
76 " .long (" #from " - .), (" #to " - .)\n" \
80 * User access enabling/disabling.
82 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
83 static inline void __uaccess_ttbr0_disable(void)
85 unsigned long flags, ttbr;
87 local_irq_save(flags);
88 ttbr = read_sysreg(ttbr1_el1);
89 ttbr &= ~TTBR_ASID_MASK;
90 /* reserved_pg_dir placed before swapper_pg_dir */
91 write_sysreg(ttbr - RESERVED_SWAPPER_OFFSET, ttbr0_el1);
93 /* Set reserved ASID */
94 write_sysreg(ttbr, ttbr1_el1);
96 local_irq_restore(flags);
99 static inline void __uaccess_ttbr0_enable(void)
101 unsigned long flags, ttbr0, ttbr1;
104 * Disable interrupts to avoid preemption between reading the 'ttbr0'
105 * variable and the MSR. A context switch could trigger an ASID
106 * roll-over and an update of 'ttbr0'.
108 local_irq_save(flags);
109 ttbr0 = READ_ONCE(current_thread_info()->ttbr0);
111 /* Restore active ASID */
112 ttbr1 = read_sysreg(ttbr1_el1);
113 ttbr1 &= ~TTBR_ASID_MASK; /* safety measure */
114 ttbr1 |= ttbr0 & TTBR_ASID_MASK;
115 write_sysreg(ttbr1, ttbr1_el1);
118 /* Restore user page table */
119 write_sysreg(ttbr0, ttbr0_el1);
121 local_irq_restore(flags);
124 static inline bool uaccess_ttbr0_disable(void)
126 if (!system_uses_ttbr0_pan())
128 __uaccess_ttbr0_disable();
132 static inline bool uaccess_ttbr0_enable(void)
134 if (!system_uses_ttbr0_pan())
136 __uaccess_ttbr0_enable();
140 static inline bool uaccess_ttbr0_disable(void)
145 static inline bool uaccess_ttbr0_enable(void)
151 static inline void __uaccess_disable_hw_pan(void)
153 asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN,
157 static inline void __uaccess_enable_hw_pan(void)
159 asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN,
164 * The Tag Check Flag (TCF) mode for MTE is per EL, hence TCF0
165 * affects EL0 and TCF affects EL1 irrespective of which TTBR is
167 * The kernel accesses TTBR0 usually with LDTR/STTR instructions
168 * when UAO is available, so these would act as EL0 accesses using
170 * However futex.h code uses exclusives which would be executed as
171 * EL1, this can potentially cause a tag check fault even if the
172 * user disables TCF0.
174 * To address the problem we set the PSTATE.TCO bit in uaccess_enable()
175 * and reset it in uaccess_disable().
177 * The Tag check override (TCO) bit disables temporarily the tag checking
178 * preventing the issue.
180 static inline void __uaccess_disable_tco(void)
182 asm volatile(ALTERNATIVE("nop", SET_PSTATE_TCO(0),
183 ARM64_MTE, CONFIG_KASAN_HW_TAGS));
186 static inline void __uaccess_enable_tco(void)
188 asm volatile(ALTERNATIVE("nop", SET_PSTATE_TCO(1),
189 ARM64_MTE, CONFIG_KASAN_HW_TAGS));
193 * These functions disable tag checking only if in MTE async mode
194 * since the sync mode generates exceptions synchronously and the
195 * nofault or load_unaligned_zeropad can handle them.
197 static inline void __uaccess_disable_tco_async(void)
199 if (system_uses_mte_async_mode())
200 __uaccess_disable_tco();
203 static inline void __uaccess_enable_tco_async(void)
205 if (system_uses_mte_async_mode())
206 __uaccess_enable_tco();
209 static inline void uaccess_disable_privileged(void)
211 __uaccess_disable_tco();
213 if (uaccess_ttbr0_disable())
216 __uaccess_enable_hw_pan();
219 static inline void uaccess_enable_privileged(void)
221 __uaccess_enable_tco();
223 if (uaccess_ttbr0_enable())
226 __uaccess_disable_hw_pan();
230 * Sanitise a uaccess pointer such that it becomes NULL if above the maximum
231 * user address. In case the pointer is tagged (has the top byte set), untag
232 * the pointer before checking.
234 #define uaccess_mask_ptr(ptr) (__typeof__(ptr))__uaccess_mask_ptr(ptr)
235 static inline void __user *__uaccess_mask_ptr(const void __user *ptr)
237 void __user *safe_ptr;
240 " bics xzr, %3, %2\n"
241 " csel %0, %1, xzr, eq\n"
243 : "r" (ptr), "r" (TASK_SIZE_MAX - 1),
244 "r" (untagged_addr(ptr))
252 * The "__xxx" versions of the user access functions do not verify the address
253 * space - it must have been done previously with a separate "access_ok()"
256 * The "__xxx_error" versions set the third argument to -EFAULT if an error
257 * occurs, and leave it unchanged on success.
259 #define __get_mem_asm(load, reg, x, addr, err) \
261 "1: " load " " reg "1, [%2]\n" \
263 " .section .fixup, \"ax\"\n" \
269 _ASM_EXTABLE(1b, 3b) \
270 : "+r" (err), "=&r" (x) \
271 : "r" (addr), "i" (-EFAULT))
273 #define __raw_get_mem(ldr, x, ptr, err) \
275 unsigned long __gu_val; \
276 switch (sizeof(*(ptr))) { \
278 __get_mem_asm(ldr "b", "%w", __gu_val, (ptr), (err)); \
281 __get_mem_asm(ldr "h", "%w", __gu_val, (ptr), (err)); \
284 __get_mem_asm(ldr, "%w", __gu_val, (ptr), (err)); \
287 __get_mem_asm(ldr, "%x", __gu_val, (ptr), (err)); \
292 (x) = (__force __typeof__(*(ptr)))__gu_val; \
295 #define __raw_get_user(x, ptr, err) \
297 __chk_user_ptr(ptr); \
298 uaccess_ttbr0_enable(); \
299 __raw_get_mem("ldtr", x, ptr, err); \
300 uaccess_ttbr0_disable(); \
303 #define __get_user_error(x, ptr, err) \
305 __typeof__(*(ptr)) __user *__p = (ptr); \
307 if (access_ok(__p, sizeof(*__p))) { \
308 __p = uaccess_mask_ptr(__p); \
309 __raw_get_user((x), __p, (err)); \
311 (x) = (__force __typeof__(x))0; (err) = -EFAULT; \
315 #define __get_user(x, ptr) \
318 __get_user_error((x), (ptr), __gu_err); \
322 #define get_user __get_user
324 #define __get_kernel_nofault(dst, src, type, err_label) \
328 __uaccess_enable_tco_async(); \
329 __raw_get_mem("ldr", *((type *)(dst)), \
330 (__force type *)(src), __gkn_err); \
331 __uaccess_disable_tco_async(); \
332 if (unlikely(__gkn_err)) \
336 #define __put_mem_asm(store, reg, x, addr, err) \
338 "1: " store " " reg "1, [%2]\n" \
340 " .section .fixup,\"ax\"\n" \
345 _ASM_EXTABLE(1b, 3b) \
347 : "r" (x), "r" (addr), "i" (-EFAULT))
349 #define __raw_put_mem(str, x, ptr, err) \
351 __typeof__(*(ptr)) __pu_val = (x); \
352 switch (sizeof(*(ptr))) { \
354 __put_mem_asm(str "b", "%w", __pu_val, (ptr), (err)); \
357 __put_mem_asm(str "h", "%w", __pu_val, (ptr), (err)); \
360 __put_mem_asm(str, "%w", __pu_val, (ptr), (err)); \
363 __put_mem_asm(str, "%x", __pu_val, (ptr), (err)); \
370 #define __raw_put_user(x, ptr, err) \
372 __chk_user_ptr(ptr); \
373 uaccess_ttbr0_enable(); \
374 __raw_put_mem("sttr", x, ptr, err); \
375 uaccess_ttbr0_disable(); \
378 #define __put_user_error(x, ptr, err) \
380 __typeof__(*(ptr)) __user *__p = (ptr); \
382 if (access_ok(__p, sizeof(*__p))) { \
383 __p = uaccess_mask_ptr(__p); \
384 __raw_put_user((x), __p, (err)); \
390 #define __put_user(x, ptr) \
393 __put_user_error((x), (ptr), __pu_err); \
397 #define put_user __put_user
399 #define __put_kernel_nofault(dst, src, type, err_label) \
403 __uaccess_enable_tco_async(); \
404 __raw_put_mem("str", *((type *)(src)), \
405 (__force type *)(dst), __pkn_err); \
406 __uaccess_disable_tco_async(); \
407 if (unlikely(__pkn_err)) \
411 extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n);
412 #define raw_copy_from_user(to, from, n) \
414 unsigned long __acfu_ret; \
415 uaccess_ttbr0_enable(); \
416 __acfu_ret = __arch_copy_from_user((to), \
417 __uaccess_mask_ptr(from), (n)); \
418 uaccess_ttbr0_disable(); \
422 extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n);
423 #define raw_copy_to_user(to, from, n) \
425 unsigned long __actu_ret; \
426 uaccess_ttbr0_enable(); \
427 __actu_ret = __arch_copy_to_user(__uaccess_mask_ptr(to), \
429 uaccess_ttbr0_disable(); \
433 extern unsigned long __must_check __arch_copy_in_user(void __user *to, const void __user *from, unsigned long n);
434 #define raw_copy_in_user(to, from, n) \
436 unsigned long __aciu_ret; \
437 uaccess_ttbr0_enable(); \
438 __aciu_ret = __arch_copy_in_user(__uaccess_mask_ptr(to), \
439 __uaccess_mask_ptr(from), (n)); \
440 uaccess_ttbr0_disable(); \
444 #define INLINE_COPY_TO_USER
445 #define INLINE_COPY_FROM_USER
447 extern unsigned long __must_check __arch_clear_user(void __user *to, unsigned long n);
448 static inline unsigned long __must_check __clear_user(void __user *to, unsigned long n)
450 if (access_ok(to, n)) {
451 uaccess_ttbr0_enable();
452 n = __arch_clear_user(__uaccess_mask_ptr(to), n);
453 uaccess_ttbr0_disable();
457 #define clear_user __clear_user
459 extern long strncpy_from_user(char *dest, const char __user *src, long count);
461 extern __must_check long strnlen_user(const char __user *str, long n);
463 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
465 void memcpy_page_flushcache(char *to, struct page *page, size_t offset, size_t len);
466 extern unsigned long __must_check __copy_user_flushcache(void *to, const void __user *from, unsigned long n);
468 static inline int __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size)
470 kasan_check_write(dst, size);
471 return __copy_user_flushcache(dst, __uaccess_mask_ptr(src), size);
475 #endif /* __ASM_UACCESS_H */