1 /* SPDX-License-Identifier: GPL-2.0 */
3 * __get_user functions.
5 * (C) Copyright 1998 Linus Torvalds
6 * (C) Copyright 2005 Andi Kleen
7 * (C) Copyright 2008 Glauber Costa
9 * These functions have a non-standard call interface
10 * to make them more efficient, especially as they
11 * return an error value in addition to the "real"
18 * Inputs: %[r|e]ax contains the address.
20 * Outputs: %[r|e]ax is error code (0 or -EFAULT)
21 * %[r|e]dx contains zero-extended value
22 * %ecx contains the high half for 32-bit __get_user_8
25 * These functions should not modify any other registers,
26 * as they get called from within inline assembly.
29 #include <linux/linkage.h>
30 #include <asm/page_types.h>
31 #include <asm/errno.h>
32 #include <asm/asm-offsets.h>
33 #include <asm/thread_info.h>
36 #include <asm/export.h>
38 #define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC
41 SYM_FUNC_START(__get_user_1)
42 mov PER_CPU_VAR(current_task), %_ASM_DX
43 cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
45 sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */
46 and %_ASM_DX, %_ASM_AX
48 1: movzbl (%_ASM_AX),%edx
52 SYM_FUNC_END(__get_user_1)
53 EXPORT_SYMBOL(__get_user_1)
55 SYM_FUNC_START(__get_user_2)
58 mov PER_CPU_VAR(current_task), %_ASM_DX
59 cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
61 sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */
62 and %_ASM_DX, %_ASM_AX
64 2: movzwl -1(%_ASM_AX),%edx
68 SYM_FUNC_END(__get_user_2)
69 EXPORT_SYMBOL(__get_user_2)
71 SYM_FUNC_START(__get_user_4)
74 mov PER_CPU_VAR(current_task), %_ASM_DX
75 cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
77 sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */
78 and %_ASM_DX, %_ASM_AX
80 3: movl -3(%_ASM_AX),%edx
84 SYM_FUNC_END(__get_user_4)
85 EXPORT_SYMBOL(__get_user_4)
87 SYM_FUNC_START(__get_user_8)
91 mov PER_CPU_VAR(current_task), %_ASM_DX
92 cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
94 sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */
95 and %_ASM_DX, %_ASM_AX
97 4: movq -7(%_ASM_AX),%rdx
104 mov PER_CPU_VAR(current_task), %_ASM_DX
105 cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
107 sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */
108 and %_ASM_DX, %_ASM_AX
110 4: movl -7(%_ASM_AX),%edx
111 5: movl -3(%_ASM_AX),%ecx
116 SYM_FUNC_END(__get_user_8)
117 EXPORT_SYMBOL(__get_user_8)
119 /* .. and the same for __get_user, just without the range checks */
120 SYM_FUNC_START(__get_user_nocheck_1)
123 6: movzbl (%_ASM_AX),%edx
127 SYM_FUNC_END(__get_user_nocheck_1)
128 EXPORT_SYMBOL(__get_user_nocheck_1)
130 SYM_FUNC_START(__get_user_nocheck_2)
133 7: movzwl (%_ASM_AX),%edx
137 SYM_FUNC_END(__get_user_nocheck_2)
138 EXPORT_SYMBOL(__get_user_nocheck_2)
140 SYM_FUNC_START(__get_user_nocheck_4)
143 8: movl (%_ASM_AX),%edx
147 SYM_FUNC_END(__get_user_nocheck_4)
148 EXPORT_SYMBOL(__get_user_nocheck_4)
150 SYM_FUNC_START(__get_user_nocheck_8)
154 9: movq (%_ASM_AX),%rdx
156 9: movl (%_ASM_AX),%edx
157 10: movl 4(%_ASM_AX),%ecx
162 SYM_FUNC_END(__get_user_nocheck_8)
163 EXPORT_SYMBOL(__get_user_nocheck_8)
166 SYM_CODE_START_LOCAL(.Lbad_get_user_clac)
170 mov $(-EFAULT),%_ASM_AX
172 SYM_CODE_END(.Lbad_get_user_clac)
175 SYM_CODE_START_LOCAL(.Lbad_get_user_8_clac)
180 mov $(-EFAULT),%_ASM_AX
182 SYM_CODE_END(.Lbad_get_user_8_clac)
186 _ASM_EXTABLE_UA(1b, .Lbad_get_user_clac)
187 _ASM_EXTABLE_UA(2b, .Lbad_get_user_clac)
188 _ASM_EXTABLE_UA(3b, .Lbad_get_user_clac)
190 _ASM_EXTABLE_UA(4b, .Lbad_get_user_clac)
192 _ASM_EXTABLE_UA(4b, .Lbad_get_user_8_clac)
193 _ASM_EXTABLE_UA(5b, .Lbad_get_user_8_clac)
197 _ASM_EXTABLE_UA(6b, .Lbad_get_user_clac)
198 _ASM_EXTABLE_UA(7b, .Lbad_get_user_clac)
199 _ASM_EXTABLE_UA(8b, .Lbad_get_user_clac)
201 _ASM_EXTABLE_UA(9b, .Lbad_get_user_clac)
203 _ASM_EXTABLE_UA(9b, .Lbad_get_user_8_clac)
204 _ASM_EXTABLE_UA(10b, .Lbad_get_user_8_clac)