1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* Copyright 2002 Andi Kleen */
4 #include <linux/linkage.h>
6 #include <asm/cpufeatures.h>
7 #include <asm/alternative-asm.h>
8 #include <asm/export.h>
10 .pushsection .noinstr.text, "ax"
13 * We build a jump to memcpy_orig by default which gets NOPped out on
14 * the majority of x86 CPUs which set REP_GOOD. In addition, CPUs which
15 * have the enhanced REP MOVSB/STOSB feature (ERMS), change those NOPs
16 * to a jmp to memcpy_erms which does the REP; MOVSB mem copy.
22 * memcpy - Copy a memory block.
30 * rax original destination
32 SYM_FUNC_START_ALIAS(__memcpy)
33 SYM_FUNC_START_LOCAL(memcpy)
34 ALTERNATIVE_2 "jmp memcpy_orig", "", X86_FEATURE_REP_GOOD, \
35 "jmp memcpy_erms", X86_FEATURE_ERMS
46 SYM_FUNC_END_ALIAS(__memcpy)
48 EXPORT_SYMBOL(__memcpy)
51 * memcpy_erms() - enhanced fast string memcpy. This is faster and
52 * simpler than memcpy. Use memcpy_erms when possible.
54 SYM_FUNC_START_LOCAL(memcpy_erms)
59 SYM_FUNC_END(memcpy_erms)
61 SYM_FUNC_START_LOCAL(memcpy_orig)
68 * We check whether memory false dependence could occur,
69 * then jump to corresponding copy mode.
78 * Move in blocks of 4x8 bytes:
91 jae .Lcopy_forward_loop
97 * Calculate copy position to tail.
103 * At most 3 ALU operations in one cycle,
104 * so append NOPS in the same 16 bytes trunk.
107 .Lcopy_backward_loop:
111 movq -3*8(%rsi), %r10
112 movq -4*8(%rsi), %r11
113 leaq -4*8(%rsi), %rsi
116 movq %r10, -3*8(%rdi)
117 movq %r11, -4*8(%rdi)
118 leaq -4*8(%rdi), %rdi
119 jae .Lcopy_backward_loop
122 * Calculate copy position to head.
132 * Move data from 16 bytes to 31 bytes.
136 movq -2*8(%rsi, %rdx), %r10
137 movq -1*8(%rsi, %rdx), %r11
140 movq %r10, -2*8(%rdi, %rdx)
141 movq %r11, -1*8(%rdi, %rdx)
148 * Move data from 8 bytes to 15 bytes.
151 movq -1*8(%rsi, %rdx), %r9
153 movq %r9, -1*8(%rdi, %rdx)
161 * Move data from 4 bytes to 7 bytes.
164 movl -4(%rsi, %rdx), %r8d
166 movl %r8d, -4(%rdi, %rdx)
173 * Move data from 1 bytes to 3 bytes.
178 movzbq (%rsi, %rdx), %r9
180 movb %r9b, (%rdi, %rdx)
186 SYM_FUNC_END(memcpy_orig)