Merge tag 'for-linus-5.11-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/rw/uml
[linux-2.6-microblaze.git] / arch / x86 / lib / usercopy_32.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * User address space access functions.
4  * The non inlined parts of asm-i386/uaccess.h are here.
5  *
6  * Copyright 1997 Andi Kleen <ak@muc.de>
7  * Copyright 1997 Linus Torvalds
8  */
9 #include <linux/export.h>
10 #include <linux/uaccess.h>
11 #include <asm/mmx.h>
12 #include <asm/asm.h>
13
14 #ifdef CONFIG_X86_INTEL_USERCOPY
15 /*
16  * Alignment at which movsl is preferred for bulk memory copies.
17  */
18 struct movsl_mask movsl_mask __read_mostly;
19 #endif
20
21 static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned long n)
22 {
23 #ifdef CONFIG_X86_INTEL_USERCOPY
24         if (n >= 64 && ((a1 ^ a2) & movsl_mask.mask))
25                 return 0;
26 #endif
27         return 1;
28 }
29 #define movsl_is_ok(a1, a2, n) \
30         __movsl_is_ok((unsigned long)(a1), (unsigned long)(a2), (n))
31
32 /*
33  * Zero Userspace
34  */
35
36 #define __do_clear_user(addr,size)                                      \
37 do {                                                                    \
38         int __d0;                                                       \
39         might_fault();                                                  \
40         __asm__ __volatile__(                                           \
41                 ASM_STAC "\n"                                           \
42                 "0:     rep; stosl\n"                                   \
43                 "       movl %2,%0\n"                                   \
44                 "1:     rep; stosb\n"                                   \
45                 "2: " ASM_CLAC "\n"                                     \
46                 ".section .fixup,\"ax\"\n"                              \
47                 "3:     lea 0(%2,%0,4),%0\n"                            \
48                 "       jmp 2b\n"                                       \
49                 ".previous\n"                                           \
50                 _ASM_EXTABLE_UA(0b, 3b)                                 \
51                 _ASM_EXTABLE_UA(1b, 2b)                                 \
52                 : "=&c"(size), "=&D" (__d0)                             \
53                 : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0));     \
54 } while (0)
55
56 /**
57  * clear_user - Zero a block of memory in user space.
58  * @to:   Destination address, in user space.
59  * @n:    Number of bytes to zero.
60  *
61  * Zero a block of memory in user space.
62  *
63  * Return: number of bytes that could not be cleared.
64  * On success, this will be zero.
65  */
66 unsigned long
67 clear_user(void __user *to, unsigned long n)
68 {
69         might_fault();
70         if (access_ok(to, n))
71                 __do_clear_user(to, n);
72         return n;
73 }
74 EXPORT_SYMBOL(clear_user);
75
76 /**
77  * __clear_user - Zero a block of memory in user space, with less checking.
78  * @to:   Destination address, in user space.
79  * @n:    Number of bytes to zero.
80  *
81  * Zero a block of memory in user space.  Caller must check
82  * the specified block with access_ok() before calling this function.
83  *
84  * Return: number of bytes that could not be cleared.
85  * On success, this will be zero.
86  */
87 unsigned long
88 __clear_user(void __user *to, unsigned long n)
89 {
90         __do_clear_user(to, n);
91         return n;
92 }
93 EXPORT_SYMBOL(__clear_user);
94
95 #ifdef CONFIG_X86_INTEL_USERCOPY
96 static unsigned long
97 __copy_user_intel(void __user *to, const void *from, unsigned long size)
98 {
99         int d0, d1;
100         __asm__ __volatile__(
101                        "       .align 2,0x90\n"
102                        "1:     movl 32(%4), %%eax\n"
103                        "       cmpl $67, %0\n"
104                        "       jbe 3f\n"
105                        "2:     movl 64(%4), %%eax\n"
106                        "       .align 2,0x90\n"
107                        "3:     movl 0(%4), %%eax\n"
108                        "4:     movl 4(%4), %%edx\n"
109                        "5:     movl %%eax, 0(%3)\n"
110                        "6:     movl %%edx, 4(%3)\n"
111                        "7:     movl 8(%4), %%eax\n"
112                        "8:     movl 12(%4),%%edx\n"
113                        "9:     movl %%eax, 8(%3)\n"
114                        "10:    movl %%edx, 12(%3)\n"
115                        "11:    movl 16(%4), %%eax\n"
116                        "12:    movl 20(%4), %%edx\n"
117                        "13:    movl %%eax, 16(%3)\n"
118                        "14:    movl %%edx, 20(%3)\n"
119                        "15:    movl 24(%4), %%eax\n"
120                        "16:    movl 28(%4), %%edx\n"
121                        "17:    movl %%eax, 24(%3)\n"
122                        "18:    movl %%edx, 28(%3)\n"
123                        "19:    movl 32(%4), %%eax\n"
124                        "20:    movl 36(%4), %%edx\n"
125                        "21:    movl %%eax, 32(%3)\n"
126                        "22:    movl %%edx, 36(%3)\n"
127                        "23:    movl 40(%4), %%eax\n"
128                        "24:    movl 44(%4), %%edx\n"
129                        "25:    movl %%eax, 40(%3)\n"
130                        "26:    movl %%edx, 44(%3)\n"
131                        "27:    movl 48(%4), %%eax\n"
132                        "28:    movl 52(%4), %%edx\n"
133                        "29:    movl %%eax, 48(%3)\n"
134                        "30:    movl %%edx, 52(%3)\n"
135                        "31:    movl 56(%4), %%eax\n"
136                        "32:    movl 60(%4), %%edx\n"
137                        "33:    movl %%eax, 56(%3)\n"
138                        "34:    movl %%edx, 60(%3)\n"
139                        "       addl $-64, %0\n"
140                        "       addl $64, %4\n"
141                        "       addl $64, %3\n"
142                        "       cmpl $63, %0\n"
143                        "       ja  1b\n"
144                        "35:    movl  %0, %%eax\n"
145                        "       shrl  $2, %0\n"
146                        "       andl  $3, %%eax\n"
147                        "       cld\n"
148                        "99:    rep; movsl\n"
149                        "36:    movl %%eax, %0\n"
150                        "37:    rep; movsb\n"
151                        "100:\n"
152                        ".section .fixup,\"ax\"\n"
153                        "101:   lea 0(%%eax,%0,4),%0\n"
154                        "       jmp 100b\n"
155                        ".previous\n"
156                        _ASM_EXTABLE_UA(1b, 100b)
157                        _ASM_EXTABLE_UA(2b, 100b)
158                        _ASM_EXTABLE_UA(3b, 100b)
159                        _ASM_EXTABLE_UA(4b, 100b)
160                        _ASM_EXTABLE_UA(5b, 100b)
161                        _ASM_EXTABLE_UA(6b, 100b)
162                        _ASM_EXTABLE_UA(7b, 100b)
163                        _ASM_EXTABLE_UA(8b, 100b)
164                        _ASM_EXTABLE_UA(9b, 100b)
165                        _ASM_EXTABLE_UA(10b, 100b)
166                        _ASM_EXTABLE_UA(11b, 100b)
167                        _ASM_EXTABLE_UA(12b, 100b)
168                        _ASM_EXTABLE_UA(13b, 100b)
169                        _ASM_EXTABLE_UA(14b, 100b)
170                        _ASM_EXTABLE_UA(15b, 100b)
171                        _ASM_EXTABLE_UA(16b, 100b)
172                        _ASM_EXTABLE_UA(17b, 100b)
173                        _ASM_EXTABLE_UA(18b, 100b)
174                        _ASM_EXTABLE_UA(19b, 100b)
175                        _ASM_EXTABLE_UA(20b, 100b)
176                        _ASM_EXTABLE_UA(21b, 100b)
177                        _ASM_EXTABLE_UA(22b, 100b)
178                        _ASM_EXTABLE_UA(23b, 100b)
179                        _ASM_EXTABLE_UA(24b, 100b)
180                        _ASM_EXTABLE_UA(25b, 100b)
181                        _ASM_EXTABLE_UA(26b, 100b)
182                        _ASM_EXTABLE_UA(27b, 100b)
183                        _ASM_EXTABLE_UA(28b, 100b)
184                        _ASM_EXTABLE_UA(29b, 100b)
185                        _ASM_EXTABLE_UA(30b, 100b)
186                        _ASM_EXTABLE_UA(31b, 100b)
187                        _ASM_EXTABLE_UA(32b, 100b)
188                        _ASM_EXTABLE_UA(33b, 100b)
189                        _ASM_EXTABLE_UA(34b, 100b)
190                        _ASM_EXTABLE_UA(35b, 100b)
191                        _ASM_EXTABLE_UA(36b, 100b)
192                        _ASM_EXTABLE_UA(37b, 100b)
193                        _ASM_EXTABLE_UA(99b, 101b)
194                        : "=&c"(size), "=&D" (d0), "=&S" (d1)
195                        :  "1"(to), "2"(from), "0"(size)
196                        : "eax", "edx", "memory");
197         return size;
198 }
199
200 static unsigned long __copy_user_intel_nocache(void *to,
201                                 const void __user *from, unsigned long size)
202 {
203         int d0, d1;
204
205         __asm__ __volatile__(
206                "        .align 2,0x90\n"
207                "0:      movl 32(%4), %%eax\n"
208                "        cmpl $67, %0\n"
209                "        jbe 2f\n"
210                "1:      movl 64(%4), %%eax\n"
211                "        .align 2,0x90\n"
212                "2:      movl 0(%4), %%eax\n"
213                "21:     movl 4(%4), %%edx\n"
214                "        movnti %%eax, 0(%3)\n"
215                "        movnti %%edx, 4(%3)\n"
216                "3:      movl 8(%4), %%eax\n"
217                "31:     movl 12(%4),%%edx\n"
218                "        movnti %%eax, 8(%3)\n"
219                "        movnti %%edx, 12(%3)\n"
220                "4:      movl 16(%4), %%eax\n"
221                "41:     movl 20(%4), %%edx\n"
222                "        movnti %%eax, 16(%3)\n"
223                "        movnti %%edx, 20(%3)\n"
224                "10:     movl 24(%4), %%eax\n"
225                "51:     movl 28(%4), %%edx\n"
226                "        movnti %%eax, 24(%3)\n"
227                "        movnti %%edx, 28(%3)\n"
228                "11:     movl 32(%4), %%eax\n"
229                "61:     movl 36(%4), %%edx\n"
230                "        movnti %%eax, 32(%3)\n"
231                "        movnti %%edx, 36(%3)\n"
232                "12:     movl 40(%4), %%eax\n"
233                "71:     movl 44(%4), %%edx\n"
234                "        movnti %%eax, 40(%3)\n"
235                "        movnti %%edx, 44(%3)\n"
236                "13:     movl 48(%4), %%eax\n"
237                "81:     movl 52(%4), %%edx\n"
238                "        movnti %%eax, 48(%3)\n"
239                "        movnti %%edx, 52(%3)\n"
240                "14:     movl 56(%4), %%eax\n"
241                "91:     movl 60(%4), %%edx\n"
242                "        movnti %%eax, 56(%3)\n"
243                "        movnti %%edx, 60(%3)\n"
244                "        addl $-64, %0\n"
245                "        addl $64, %4\n"
246                "        addl $64, %3\n"
247                "        cmpl $63, %0\n"
248                "        ja  0b\n"
249                "        sfence \n"
250                "5:      movl  %0, %%eax\n"
251                "        shrl  $2, %0\n"
252                "        andl $3, %%eax\n"
253                "        cld\n"
254                "6:      rep; movsl\n"
255                "        movl %%eax,%0\n"
256                "7:      rep; movsb\n"
257                "8:\n"
258                ".section .fixup,\"ax\"\n"
259                "9:      lea 0(%%eax,%0,4),%0\n"
260                "16:     jmp 8b\n"
261                ".previous\n"
262                _ASM_EXTABLE_UA(0b, 16b)
263                _ASM_EXTABLE_UA(1b, 16b)
264                _ASM_EXTABLE_UA(2b, 16b)
265                _ASM_EXTABLE_UA(21b, 16b)
266                _ASM_EXTABLE_UA(3b, 16b)
267                _ASM_EXTABLE_UA(31b, 16b)
268                _ASM_EXTABLE_UA(4b, 16b)
269                _ASM_EXTABLE_UA(41b, 16b)
270                _ASM_EXTABLE_UA(10b, 16b)
271                _ASM_EXTABLE_UA(51b, 16b)
272                _ASM_EXTABLE_UA(11b, 16b)
273                _ASM_EXTABLE_UA(61b, 16b)
274                _ASM_EXTABLE_UA(12b, 16b)
275                _ASM_EXTABLE_UA(71b, 16b)
276                _ASM_EXTABLE_UA(13b, 16b)
277                _ASM_EXTABLE_UA(81b, 16b)
278                _ASM_EXTABLE_UA(14b, 16b)
279                _ASM_EXTABLE_UA(91b, 16b)
280                _ASM_EXTABLE_UA(6b, 9b)
281                _ASM_EXTABLE_UA(7b, 16b)
282                : "=&c"(size), "=&D" (d0), "=&S" (d1)
283                :  "1"(to), "2"(from), "0"(size)
284                : "eax", "edx", "memory");
285         return size;
286 }
287
288 #else
289
290 /*
291  * Leave these declared but undefined.  They should not be any references to
292  * them
293  */
294 unsigned long __copy_user_intel(void __user *to, const void *from,
295                                         unsigned long size);
296 #endif /* CONFIG_X86_INTEL_USERCOPY */
297
298 /* Generic arbitrary sized copy.  */
299 #define __copy_user(to, from, size)                                     \
300 do {                                                                    \
301         int __d0, __d1, __d2;                                           \
302         __asm__ __volatile__(                                           \
303                 "       cmp  $7,%0\n"                                   \
304                 "       jbe  1f\n"                                      \
305                 "       movl %1,%0\n"                                   \
306                 "       negl %0\n"                                      \
307                 "       andl $7,%0\n"                                   \
308                 "       subl %0,%3\n"                                   \
309                 "4:     rep; movsb\n"                                   \
310                 "       movl %3,%0\n"                                   \
311                 "       shrl $2,%0\n"                                   \
312                 "       andl $3,%3\n"                                   \
313                 "       .align 2,0x90\n"                                \
314                 "0:     rep; movsl\n"                                   \
315                 "       movl %3,%0\n"                                   \
316                 "1:     rep; movsb\n"                                   \
317                 "2:\n"                                                  \
318                 ".section .fixup,\"ax\"\n"                              \
319                 "5:     addl %3,%0\n"                                   \
320                 "       jmp 2b\n"                                       \
321                 "3:     lea 0(%3,%0,4),%0\n"                            \
322                 "       jmp 2b\n"                                       \
323                 ".previous\n"                                           \
324                 _ASM_EXTABLE_UA(4b, 5b)                                 \
325                 _ASM_EXTABLE_UA(0b, 3b)                                 \
326                 _ASM_EXTABLE_UA(1b, 2b)                                 \
327                 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2)   \
328                 : "3"(size), "0"(size), "1"(to), "2"(from)              \
329                 : "memory");                                            \
330 } while (0)
331
332 unsigned long __copy_user_ll(void *to, const void *from, unsigned long n)
333 {
334         __uaccess_begin_nospec();
335         if (movsl_is_ok(to, from, n))
336                 __copy_user(to, from, n);
337         else
338                 n = __copy_user_intel(to, from, n);
339         __uaccess_end();
340         return n;
341 }
342 EXPORT_SYMBOL(__copy_user_ll);
343
344 unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from,
345                                         unsigned long n)
346 {
347         __uaccess_begin_nospec();
348 #ifdef CONFIG_X86_INTEL_USERCOPY
349         if (n > 64 && static_cpu_has(X86_FEATURE_XMM2))
350                 n = __copy_user_intel_nocache(to, from, n);
351         else
352                 __copy_user(to, from, n);
353 #else
354         __copy_user(to, from, n);
355 #endif
356         __uaccess_end();
357         return n;
358 }
359 EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero);