1 /* SPDX-License-Identifier: GPL-2.0 */
3 #ifndef __NOSPEC_BRANCH_H__
4 #define __NOSPEC_BRANCH_H__
6 #include <asm/alternative.h>
7 #include <asm/alternative-asm.h>
8 #include <asm/cpufeatures.h>
11 * Fill the CPU return stack buffer.
13 * Each entry in the RSB, if used for a speculative 'ret', contains an
14 * infinite 'pause; lfence; jmp' loop to capture speculative execution.
16 * This is required in various cases for retpoline and IBRS-based
17 * mitigations for the Spectre variant 2 vulnerability. Sometimes to
18 * eliminate potentially bogus entries from the RSB, and sometimes
19 * purely to ensure that it doesn't get empty, which on some CPUs would
20 * allow predictions from other (unwanted!) sources to be used.
22 * We define a CPP macro such that it can be used from both .S files and
23 * inline assembly. It's possible to do a .macro and then include that
24 * from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
27 #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */
28 #define RSB_FILL_LOOPS 16 /* To avoid underflow */
31 * Google experimented with loop-unrolling and this turned out to be
32 * the optimal version — two calls, each with their own speculation
33 * trap should their return address end up getting used, in a loop.
35 #define __FILL_RETURN_BUFFER(reg, nr, sp) \
39 773: /* speculation trap */ \
45 775: /* speculation trap */ \
52 add $(BITS_PER_LONG/8) * nr, sp;
57 * This should be used immediately before a retpoline alternative. It tells
58 * objtool where the retpolines are so that it can make sense of the control
59 * flow by just reading the original instruction(s) and ignoring the
62 .macro ANNOTATE_NOSPEC_ALTERNATIVE
64 .pushsection .discard.nospec
65 .long .Lannotate_\@ - .
70 * These are the bare retpoline primitives for indirect jmp and call.
71 * Do not use these directly; they only exist to make the ALTERNATIVE
72 * invocation below less ugly.
74 .macro RETPOLINE_JMP reg:req
86 * This is a wrapper around RETPOLINE_JMP so the called function in reg
87 * returns to the instruction after the macro.
89 .macro RETPOLINE_CALL reg:req
91 .Ldo_retpoline_jmp_\@:
94 call .Ldo_retpoline_jmp_\@
98 * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
99 * indirect jmp/call which may be susceptible to the Spectre variant 2
102 .macro JMP_NOSPEC reg:req
103 #ifdef CONFIG_RETPOLINE
104 ANNOTATE_NOSPEC_ALTERNATIVE
105 ALTERNATIVE_2 __stringify(jmp *\reg), \
106 __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \
107 __stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
113 .macro CALL_NOSPEC reg:req
114 #ifdef CONFIG_RETPOLINE
115 ANNOTATE_NOSPEC_ALTERNATIVE
116 ALTERNATIVE_2 __stringify(call *\reg), \
117 __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
118 __stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD
125 * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
126 * monstrosity above, manually.
128 .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
129 #ifdef CONFIG_RETPOLINE
130 ANNOTATE_NOSPEC_ALTERNATIVE
131 ALTERNATIVE "jmp .Lskip_rsb_\@", \
132 __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \
138 #else /* __ASSEMBLY__ */
140 #define ANNOTATE_NOSPEC_ALTERNATIVE \
142 ".pushsection .discard.nospec\n\t" \
143 ".long 999b - .\n\t" \
146 #if defined(CONFIG_X86_64) && defined(RETPOLINE)
149 * Since the inline asm uses the %V modifier which is only in newer GCC,
150 * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE.
152 # define CALL_NOSPEC \
153 ANNOTATE_NOSPEC_ALTERNATIVE \
155 "call *%[thunk_target]\n", \
156 "call __x86_indirect_thunk_%V[thunk_target]\n", \
157 X86_FEATURE_RETPOLINE)
158 # define THUNK_TARGET(addr) [thunk_target] "r" (addr)
160 #elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE)
162 * For i386 we use the original ret-equivalent retpoline, because
163 * otherwise we'll run out of registers. We don't care about CET
166 # define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n", \
169 "901: call 903f;\n" \
174 "903: addl $4, %%esp;\n" \
175 " pushl %[thunk_target];\n" \
178 "904: call 901b;\n", \
179 X86_FEATURE_RETPOLINE)
181 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
182 #else /* No retpoline for C / inline asm */
183 # define CALL_NOSPEC "call *%[thunk_target]\n"
184 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
187 /* The Spectre V2 mitigation variants */
188 enum spectre_v2_mitigation {
190 SPECTRE_V2_RETPOLINE_MINIMAL,
191 SPECTRE_V2_RETPOLINE_MINIMAL_AMD,
192 SPECTRE_V2_RETPOLINE_GENERIC,
193 SPECTRE_V2_RETPOLINE_AMD,
197 extern char __indirect_thunk_start[];
198 extern char __indirect_thunk_end[];
201 * On VMEXIT we must ensure that no RSB predictions learned in the guest
202 * can be followed in the host, by overwriting the RSB completely. Both
203 * retpoline and IBRS mitigations for Spectre v2 need this; only on future
204 * CPUs with IBRS_ATT *might* it be avoided.
206 static inline void vmexit_fill_RSB(void)
208 #ifdef CONFIG_RETPOLINE
211 asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
212 ALTERNATIVE("jmp 910f",
213 __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
214 X86_FEATURE_RETPOLINE)
216 : "=r" (loops), ASM_CALL_CONSTRAINT
221 #endif /* __ASSEMBLY__ */
222 #endif /* __NOSPEC_BRANCH_H__ */