1 #ifndef _ASM_X86_UACCESS_H
2 #define _ASM_X86_UACCESS_H
4 * User space memory access functions
6 #include <linux/errno.h>
7 #include <linux/compiler.h>
8 #include <linux/thread_info.h>
9 #include <linux/string.h>
14 #define VERIFY_WRITE 1
17 * The fs value determines whether argument validity checking should be
18 * performed or not. If get_fs() == USER_DS, checking is performed, with
19 * get_fs() == KERNEL_DS, checking is bypassed.
21 * For historical reasons, these macros are grossly misnamed.
24 #define MAKE_MM_SEG(s) ((mm_segment_t) { (s) })
26 #define KERNEL_DS MAKE_MM_SEG(-1UL)
27 #define USER_DS MAKE_MM_SEG(TASK_SIZE_MAX)
29 #define get_ds() (KERNEL_DS)
30 #define get_fs() (current_thread_info()->addr_limit)
31 #define set_fs(x) (current_thread_info()->addr_limit = (x))
33 #define segment_eq(a, b) ((a).seg == (b).seg)
35 #define __addr_ok(addr) \
36 ((unsigned long __force)(addr) < \
37 (current_thread_info()->addr_limit.seg))
40 * Test whether a block of memory is a valid user space address.
41 * Returns 0 if the range is valid, nonzero otherwise.
43 * This is equivalent to the following test:
44 * (u33)addr + (u33)size > (u33)current->addr_limit.seg (u65 for x86_64)
46 * This needs 33-bit (65-bit for x86_64) arithmetic. We have a carry...
49 #define __range_not_ok(addr, size) \
51 unsigned long flag, roksum; \
52 __chk_user_ptr(addr); \
53 asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" \
54 : "=&r" (flag), "=r" (roksum) \
55 : "1" (addr), "g" ((long)(size)), \
56 "rm" (current_thread_info()->addr_limit.seg)); \
61 * access_ok: - Checks if a user space pointer is valid
62 * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that
63 * %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe
64 * to write to a block, it is always safe to read from it.
65 * @addr: User space pointer to start of block to check
66 * @size: Size of block to check
68 * Context: User context only. This function may sleep.
70 * Checks if a pointer to a block of memory in user space is valid.
72 * Returns true (nonzero) if the memory block may be valid, false (zero)
73 * if it is definitely invalid.
75 * Note that, depending on architecture, this function probably just
76 * checks that the pointer is in the user space range - after calling
77 * this function, memory access functions may still return -EFAULT.
79 #define access_ok(type, addr, size) (likely(__range_not_ok(addr, size) == 0))
82 * The exception table consists of pairs of addresses relative to the
83 * exception table enty itself: the first is the address of an
84 * instruction that is allowed to fault, and the second is the address
85 * at which the program should continue. No registers are modified,
86 * so it is entirely up to the continuation code to figure out what to
89 * All the routines below use bits of fixup code that are out of line
90 * with the main instruction path. This means when everything is well,
91 * we don't even have to jump over them. Further, they do not intrude
92 * on our cache or tlb entries.
95 struct exception_table_entry {
98 /* This is not the generic standard exception_table_entry format */
99 #define ARCH_HAS_SORT_EXTABLE
100 #define ARCH_HAS_SEARCH_EXTABLE
102 extern int fixup_exception(struct pt_regs *regs);
103 extern int early_fixup_exception(unsigned long *ip);
106 * These are the main single-value transfer routines. They automatically
107 * use the right size if we just have the right pointer type.
109 * This gets kind of ugly. We want to return _two_ values in "get_user()"
110 * and yet we don't want to do any pointers, because that is too much
111 * of a performance impact. Thus we have a few rather ugly macros here,
112 * and hide all the ugliness from the user.
114 * The "__xxx" versions of the user access functions are versions that
115 * do not verify the address space, that must have been done previously
116 * with a separate "access_ok()" call (this is used when we do multiple
117 * accesses to the same area of user memory).
120 extern int __get_user_1(void);
121 extern int __get_user_2(void);
122 extern int __get_user_4(void);
123 extern int __get_user_8(void);
124 extern int __get_user_bad(void);
126 #define __get_user_x(size, ret, x, ptr) \
127 asm volatile("call __get_user_" #size \
128 : "=a" (ret), "=d" (x) \
131 /* Careful: we have to cast the result to the type of the pointer
132 * for sign reasons */
135 * get_user: - Get a simple variable from user space.
136 * @x: Variable to store result.
137 * @ptr: Source address, in user space.
139 * Context: User context only. This function may sleep.
141 * This macro copies a single simple variable from user space to kernel
142 * space. It supports simple types like char and int, but not larger
143 * data types like structures or arrays.
145 * @ptr must have pointer-to-simple-variable type, and the result of
146 * dereferencing @ptr must be assignable to @x without a cast.
148 * Returns zero on success, or -EFAULT on error.
149 * On error, the variable @x is set to zero.
152 #define __get_user_8(__ret_gu, __val_gu, ptr) \
153 __get_user_x(X, __ret_gu, __val_gu, ptr)
155 #define __get_user_8(__ret_gu, __val_gu, ptr) \
156 __get_user_x(8, __ret_gu, __val_gu, ptr)
159 #define get_user(x, ptr) \
162 unsigned long __val_gu; \
163 __chk_user_ptr(ptr); \
165 switch (sizeof(*(ptr))) { \
167 __get_user_x(1, __ret_gu, __val_gu, ptr); \
170 __get_user_x(2, __ret_gu, __val_gu, ptr); \
173 __get_user_x(4, __ret_gu, __val_gu, ptr); \
176 __get_user_8(__ret_gu, __val_gu, ptr); \
179 __get_user_x(X, __ret_gu, __val_gu, ptr); \
182 (x) = (__typeof__(*(ptr)))__val_gu; \
186 #define __put_user_x(size, x, ptr, __ret_pu) \
187 asm volatile("call __put_user_" #size : "=a" (__ret_pu) \
188 : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
193 #define __put_user_asm_u64(x, addr, err, errret) \
194 asm volatile("1: movl %%eax,0(%2)\n" \
195 "2: movl %%edx,4(%2)\n" \
197 ".section .fixup,\"ax\"\n" \
201 _ASM_EXTABLE(1b, 4b) \
202 _ASM_EXTABLE(2b, 4b) \
204 : "A" (x), "r" (addr), "i" (errret), "0" (err))
206 #define __put_user_asm_ex_u64(x, addr) \
207 asm volatile("1: movl %%eax,0(%1)\n" \
208 "2: movl %%edx,4(%1)\n" \
210 _ASM_EXTABLE_EX(1b, 2b) \
211 _ASM_EXTABLE_EX(2b, 3b) \
212 : : "A" (x), "r" (addr))
214 #define __put_user_x8(x, ptr, __ret_pu) \
215 asm volatile("call __put_user_8" : "=a" (__ret_pu) \
216 : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
218 #define __put_user_asm_u64(x, ptr, retval, errret) \
219 __put_user_asm(x, ptr, retval, "q", "", "er", errret)
220 #define __put_user_asm_ex_u64(x, addr) \
221 __put_user_asm_ex(x, addr, "q", "", "er")
222 #define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu)
225 extern void __put_user_bad(void);
228 * Strange magic calling convention: pointer in %ecx,
229 * value in %eax(:%edx), return value in %eax. clobbers %rbx
231 extern void __put_user_1(void);
232 extern void __put_user_2(void);
233 extern void __put_user_4(void);
234 extern void __put_user_8(void);
236 #ifdef CONFIG_X86_WP_WORKS_OK
239 * put_user: - Write a simple value into user space.
240 * @x: Value to copy to user space.
241 * @ptr: Destination address, in user space.
243 * Context: User context only. This function may sleep.
245 * This macro copies a single simple value from kernel space to user
246 * space. It supports simple types like char and int, but not larger
247 * data types like structures or arrays.
249 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
250 * to the result of dereferencing @ptr.
252 * Returns zero on success, or -EFAULT on error.
254 #define put_user(x, ptr) \
257 __typeof__(*(ptr)) __pu_val; \
258 __chk_user_ptr(ptr); \
261 switch (sizeof(*(ptr))) { \
263 __put_user_x(1, __pu_val, ptr, __ret_pu); \
266 __put_user_x(2, __pu_val, ptr, __ret_pu); \
269 __put_user_x(4, __pu_val, ptr, __ret_pu); \
272 __put_user_x8(__pu_val, ptr, __ret_pu); \
275 __put_user_x(X, __pu_val, ptr, __ret_pu); \
281 #define __put_user_size(x, ptr, size, retval, errret) \
284 __chk_user_ptr(ptr); \
287 __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \
290 __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \
293 __put_user_asm(x, ptr, retval, "l", "k", "ir", errret); \
296 __put_user_asm_u64((__typeof__(*ptr))(x), ptr, retval, \
304 #define __put_user_size_ex(x, ptr, size) \
306 __chk_user_ptr(ptr); \
309 __put_user_asm_ex(x, ptr, "b", "b", "iq"); \
312 __put_user_asm_ex(x, ptr, "w", "w", "ir"); \
315 __put_user_asm_ex(x, ptr, "l", "k", "ir"); \
318 __put_user_asm_ex_u64((__typeof__(*ptr))(x), ptr); \
327 #define __put_user_size(x, ptr, size, retval, errret) \
329 __typeof__(*(ptr))__pus_tmp = x; \
332 if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, size) != 0)) \
336 #define put_user(x, ptr) \
339 __typeof__(*(ptr))__pus_tmp = x; \
341 if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, \
342 sizeof(*(ptr))) != 0)) \
343 __ret_pu = -EFAULT; \
349 #define __get_user_asm_u64(x, ptr, retval, errret) (x) = __get_user_bad()
350 #define __get_user_asm_ex_u64(x, ptr) (x) = __get_user_bad()
352 #define __get_user_asm_u64(x, ptr, retval, errret) \
353 __get_user_asm(x, ptr, retval, "q", "", "=r", errret)
354 #define __get_user_asm_ex_u64(x, ptr) \
355 __get_user_asm_ex(x, ptr, "q", "", "=r")
358 #define __get_user_size(x, ptr, size, retval, errret) \
361 __chk_user_ptr(ptr); \
364 __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \
367 __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \
370 __get_user_asm(x, ptr, retval, "l", "k", "=r", errret); \
373 __get_user_asm_u64(x, ptr, retval, errret); \
376 (x) = __get_user_bad(); \
380 #define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \
381 asm volatile("1: mov"itype" %2,%"rtype"1\n" \
383 ".section .fixup,\"ax\"\n" \
385 " xor"itype" %"rtype"1,%"rtype"1\n" \
388 _ASM_EXTABLE(1b, 3b) \
389 : "=r" (err), ltype(x) \
390 : "m" (__m(addr)), "i" (errret), "0" (err))
392 #define __get_user_size_ex(x, ptr, size) \
394 __chk_user_ptr(ptr); \
397 __get_user_asm_ex(x, ptr, "b", "b", "=q"); \
400 __get_user_asm_ex(x, ptr, "w", "w", "=r"); \
403 __get_user_asm_ex(x, ptr, "l", "k", "=r"); \
406 __get_user_asm_ex_u64(x, ptr); \
409 (x) = __get_user_bad(); \
413 #define __get_user_asm_ex(x, addr, itype, rtype, ltype) \
414 asm volatile("1: mov"itype" %1,%"rtype"0\n" \
416 _ASM_EXTABLE_EX(1b, 2b) \
417 : ltype(x) : "m" (__m(addr)))
419 #define __put_user_nocheck(x, ptr, size) \
422 __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \
426 #define __get_user_nocheck(x, ptr, size) \
429 unsigned long __gu_val; \
430 __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \
431 (x) = (__force __typeof__(*(ptr)))__gu_val; \
435 /* FIXME: this hack is definitely wrong -AK */
436 struct __large_struct { unsigned long buf[100]; };
437 #define __m(x) (*(struct __large_struct __user *)(x))
440 * Tell gcc we read from memory instead of writing: this is because
441 * we do not write to any memory gcc knows about, so there are no
444 #define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \
445 asm volatile("1: mov"itype" %"rtype"1,%2\n" \
447 ".section .fixup,\"ax\"\n" \
451 _ASM_EXTABLE(1b, 3b) \
453 : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err))
455 #define __put_user_asm_ex(x, addr, itype, rtype, ltype) \
456 asm volatile("1: mov"itype" %"rtype"0,%1\n" \
458 _ASM_EXTABLE_EX(1b, 2b) \
459 : : ltype(x), "m" (__m(addr)))
462 * uaccess_try and catch
464 #define uaccess_try do { \
465 int prev_err = current_thread_info()->uaccess_err; \
466 current_thread_info()->uaccess_err = 0; \
469 #define uaccess_catch(err) \
470 (err) |= (current_thread_info()->uaccess_err ? -EFAULT : 0); \
471 current_thread_info()->uaccess_err = prev_err; \
475 * __get_user: - Get a simple variable from user space, with less checking.
476 * @x: Variable to store result.
477 * @ptr: Source address, in user space.
479 * Context: User context only. This function may sleep.
481 * This macro copies a single simple variable from user space to kernel
482 * space. It supports simple types like char and int, but not larger
483 * data types like structures or arrays.
485 * @ptr must have pointer-to-simple-variable type, and the result of
486 * dereferencing @ptr must be assignable to @x without a cast.
488 * Caller must check the pointer with access_ok() before calling this
491 * Returns zero on success, or -EFAULT on error.
492 * On error, the variable @x is set to zero.
495 #define __get_user(x, ptr) \
496 __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
499 * __put_user: - Write a simple value into user space, with less checking.
500 * @x: Value to copy to user space.
501 * @ptr: Destination address, in user space.
503 * Context: User context only. This function may sleep.
505 * This macro copies a single simple value from kernel space to user
506 * space. It supports simple types like char and int, but not larger
507 * data types like structures or arrays.
509 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
510 * to the result of dereferencing @ptr.
512 * Caller must check the pointer with access_ok() before calling this
515 * Returns zero on success, or -EFAULT on error.
518 #define __put_user(x, ptr) \
519 __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
521 #define __get_user_unaligned __get_user
522 #define __put_user_unaligned __put_user
525 * {get|put}_user_try and catch
529 * } get_user_catch(err)
531 #define get_user_try uaccess_try
532 #define get_user_catch(err) uaccess_catch(err)
534 #define get_user_ex(x, ptr) do { \
535 unsigned long __gue_val; \
536 __get_user_size_ex((__gue_val), (ptr), (sizeof(*(ptr)))); \
537 (x) = (__force __typeof__(*(ptr)))__gue_val; \
540 #ifdef CONFIG_X86_WP_WORKS_OK
542 #define put_user_try uaccess_try
543 #define put_user_catch(err) uaccess_catch(err)
545 #define put_user_ex(x, ptr) \
546 __put_user_size_ex((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
548 #else /* !CONFIG_X86_WP_WORKS_OK */
550 #define put_user_try do { \
551 int __uaccess_err = 0;
553 #define put_user_catch(err) \
554 (err) |= __uaccess_err; \
557 #define put_user_ex(x, ptr) do { \
558 __uaccess_err |= __put_user(x, ptr); \
561 #endif /* CONFIG_X86_WP_WORKS_OK */
564 copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
565 extern __must_check long
566 strncpy_from_user(char *dst, const char __user *src, long count);
569 * movsl can be slow when source and dest are not both 8-byte aligned
571 #ifdef CONFIG_X86_INTEL_USERCOPY
572 extern struct movsl_mask {
574 } ____cacheline_aligned_in_smp movsl_mask;
577 #define ARCH_HAS_NOCACHE_UACCESS 1
580 # include "uaccess_32.h"
582 # include "uaccess_64.h"
585 #endif /* _ASM_X86_UACCESS_H */