1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/string.h>
3 #include <linux/export.h>
8 __visible void *memcpy(void *to, const void *from, size_t n)
10 return __memcpy(to, from, n);
12 EXPORT_SYMBOL(memcpy);
14 __visible void *memset(void *s, int c, size_t count)
16 return __memset(s, c, count);
18 EXPORT_SYMBOL(memset);
20 __visible void *memmove(void *dest, const void *src, size_t n)
22 int d0,d1,d2,d3,d4,d5;
26 /* Handle more 16 bytes in loop */
30 /* Decide forward/backward copy mode */
35 * movs instruction have many startup latency
36 * so we handle small size by general register.
41 * movs instruction is only good for aligned case.
51 * We gobble 16 bytes forward in each loop.
63 "lea 0x10(%1), %1\n\t"
64 "lea 0x10(%2), %2\n\t"
70 * Handle data forward by movs.
74 "mov -4(%1, %0), %3\n\t"
75 "lea -4(%2, %0), %4\n\t"
81 * Handle data backward by movs.
87 "lea -4(%1, %0), %1\n\t"
88 "lea -4(%2, %0), %2\n\t"
97 * Start to prepare for backward copy.
109 * Calculate copy position to tail.
117 * We gobble 16 bytes backward in each loop.
122 "mov -1*4(%1), %3\n\t"
123 "mov -2*4(%1), %4\n\t"
124 "mov %3, -1*4(%2)\n\t"
125 "mov %4, -2*4(%2)\n\t"
126 "mov -3*4(%1), %3\n\t"
127 "mov -4*4(%1), %4\n\t"
128 "mov %3, -3*4(%2)\n\t"
129 "mov %4, -4*4(%2)\n\t"
130 "lea -0x10(%1), %1\n\t"
131 "lea -0x10(%2), %2\n\t"
134 * Calculate copy position to head.
141 * Move data from 8 bytes to 15 bytes.
147 "mov 0*4(%1), %3\n\t"
148 "mov 1*4(%1), %4\n\t"
149 "mov -2*4(%1, %0), %5\n\t"
150 "mov -1*4(%1, %0), %1\n\t"
152 "mov %3, 0*4(%2)\n\t"
153 "mov %4, 1*4(%2)\n\t"
154 "mov %5, -2*4(%2, %0)\n\t"
155 "mov %1, -1*4(%2, %0)\n\t"
159 * Move data from 4 bytes to 7 bytes.
165 "mov 0*4(%1), %3\n\t"
166 "mov -1*4(%1, %0), %4\n\t"
167 "mov %3, 0*4(%2)\n\t"
168 "mov %4, -1*4(%2, %0)\n\t"
172 * Move data from 2 bytes to 3 bytes.
178 "movw 0*2(%1), %%dx\n\t"
179 "movw -1*2(%1, %0), %%bx\n\t"
180 "movw %%dx, 0*2(%2)\n\t"
181 "movw %%bx, -1*2(%2, %0)\n\t"
185 * Move data for 1 byte.
191 "movb (%1), %%cl\n\t"
192 "movb %%cl, (%2)\n\t"
195 : "=&c" (d0), "=&S" (d1), "=&D" (d2),
196 "=r" (d3),"=r" (d4), "=r"(d5)
205 EXPORT_SYMBOL(memmove);