1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_STATIC_CALL_H
3 #define _LINUX_STATIC_CALL_H
8 * Static calls use code patching to hard-code function pointers into direct
9 * branch instructions. They give the flexibility of function pointers, but
10 * with improved performance. This is especially important for cases where
11 * retpolines would otherwise be used, as retpolines can significantly impact
17 * DECLARE_STATIC_CALL(name, func);
18 * DEFINE_STATIC_CALL(name, func);
19 * DEFINE_STATIC_CALL_NULL(name, typename);
20 * static_call(name)(args...);
21 * static_call_cond(name)(args...);
22 * static_call_update(name, func);
26 * # Start with the following functions (with identical prototypes):
27 * int func_a(int arg1, int arg2);
28 * int func_b(int arg1, int arg2);
30 * # Define a 'my_name' reference, associated with func_a() by default
31 * DEFINE_STATIC_CALL(my_name, func_a);
34 * static_call(my_name)(arg1, arg2);
36 * # Update 'my_name' to point to func_b()
37 * static_call_update(my_name, &func_b);
40 * static_call(my_name)(arg1, arg2);
43 * Implementation details:
45 * This requires some arch-specific code (CONFIG_HAVE_STATIC_CALL).
46 * Otherwise basic indirect calls are used (with function pointers).
48 * Each static_call() site calls into a trampoline associated with the name.
49 * The trampoline has a direct branch to the default function. Updates to a
50 * name will modify the trampoline's branch destination.
52 * If the arch has CONFIG_HAVE_STATIC_CALL_INLINE, then the call sites
53 * themselves will be patched at runtime to call the functions directly,
54 * rather than calling through the trampoline. This requires objtool or a
55 * compiler plugin to detect all the static_call() sites and annotate them
56 * in the .static_call_sites section.
59 * Notes on NULL function pointers:
61 * Static_call()s support NULL functions, with many of the caveats that
62 * regular function pointers have.
64 * Clearly calling a NULL function pointer is 'BAD', so too for
65 * static_call()s (although when HAVE_STATIC_CALL it might not be immediately
66 * fatal). A NULL static_call can be the result of:
68 * DECLARE_STATIC_CALL_NULL(my_static_call, void (*)(int));
70 * which is equivalent to declaring a NULL function pointer with just a
73 * void (*my_func_ptr)(int arg1) = NULL;
75 * or using static_call_update() with a NULL function. In both cases the
76 * HAVE_STATIC_CALL implementation will patch the trampoline with a RET
77 * instruction, instead of an immediate tail-call JMP. HAVE_STATIC_CALL_INLINE
78 * architectures can patch the trampoline call to a NOP.
80 * In all cases, any argument evaluation is unconditional. Unlike a regular
81 * conditional function pointer call:
86 * where the argument evaludation also depends on the pointer value.
88 * When calling a static_call that can be NULL, use:
90 * static_call_cond(name)(arg1);
92 * which will include the required value tests to avoid NULL-pointer
96 #include <linux/types.h>
97 #include <linux/cpu.h>
98 #include <linux/static_call_types.h>
100 #ifdef CONFIG_HAVE_STATIC_CALL
101 #include <asm/static_call.h>
104 * Either @site or @tramp can be NULL.
106 extern void arch_static_call_transform(void *site, void *tramp, void *func, bool tail);
108 #define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name)
111 #define STATIC_CALL_TRAMP_ADDR(name) NULL
114 #define static_call_update(name, func) \
116 BUILD_BUG_ON(!__same_type(*(func), STATIC_CALL_TRAMP(name))); \
117 __static_call_update(&STATIC_CALL_KEY(name), \
118 STATIC_CALL_TRAMP_ADDR(name), func); \
121 #ifdef CONFIG_HAVE_STATIC_CALL_INLINE
123 extern int __init static_call_init(void);
125 struct static_call_mod {
126 struct static_call_mod *next;
127 struct module *mod; /* for vmlinux, mod == NULL */
128 struct static_call_site *sites;
131 struct static_call_key {
134 /* bit 0: 0 = mods, 1 = sites */
136 struct static_call_mod *mods;
137 struct static_call_site *sites;
141 /* For finding the key associated with a trampoline */
142 struct static_call_tramp_key {
147 extern void __static_call_update(struct static_call_key *key, void *tramp, void *func);
148 extern int static_call_mod_init(struct module *mod);
149 extern int static_call_text_reserved(void *start, void *end);
151 extern long __static_call_return0(void);
153 #define __DEFINE_STATIC_CALL(name, _func, _func_init) \
154 DECLARE_STATIC_CALL(name, _func); \
155 struct static_call_key STATIC_CALL_KEY(name) = { \
156 .func = _func_init, \
159 ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func_init)
161 #define DEFINE_STATIC_CALL_NULL(name, _func) \
162 DECLARE_STATIC_CALL(name, _func); \
163 struct static_call_key STATIC_CALL_KEY(name) = { \
167 ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)
169 #define static_call_cond(name) (void)__static_call(name)
171 #define EXPORT_STATIC_CALL(name) \
172 EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \
173 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
174 #define EXPORT_STATIC_CALL_GPL(name) \
175 EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \
176 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
178 /* Leave the key unexported, so modules can't change static call targets: */
179 #define EXPORT_STATIC_CALL_TRAMP(name) \
180 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)); \
181 ARCH_ADD_TRAMP_KEY(name)
182 #define EXPORT_STATIC_CALL_TRAMP_GPL(name) \
183 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)); \
184 ARCH_ADD_TRAMP_KEY(name)
186 #elif defined(CONFIG_HAVE_STATIC_CALL)
188 static inline int static_call_init(void) { return 0; }
190 struct static_call_key {
194 #define __DEFINE_STATIC_CALL(name, _func, _func_init) \
195 DECLARE_STATIC_CALL(name, _func); \
196 struct static_call_key STATIC_CALL_KEY(name) = { \
197 .func = _func_init, \
199 ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func_init)
201 #define DEFINE_STATIC_CALL_NULL(name, _func) \
202 DECLARE_STATIC_CALL(name, _func); \
203 struct static_call_key STATIC_CALL_KEY(name) = { \
206 ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)
208 #define static_call_cond(name) (void)__static_call(name)
211 void __static_call_update(struct static_call_key *key, void *tramp, void *func)
214 WRITE_ONCE(key->func, func);
215 arch_static_call_transform(NULL, tramp, func, false);
219 static inline int static_call_text_reserved(void *start, void *end)
224 static inline long __static_call_return0(void)
229 #define EXPORT_STATIC_CALL(name) \
230 EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \
231 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
232 #define EXPORT_STATIC_CALL_GPL(name) \
233 EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \
234 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
236 /* Leave the key unexported, so modules can't change static call targets: */
237 #define EXPORT_STATIC_CALL_TRAMP(name) \
238 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
239 #define EXPORT_STATIC_CALL_TRAMP_GPL(name) \
240 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
242 #else /* Generic implementation */
244 static inline int static_call_init(void) { return 0; }
246 struct static_call_key {
250 static inline long __static_call_return0(void)
255 #define __DEFINE_STATIC_CALL(name, _func, _func_init) \
256 DECLARE_STATIC_CALL(name, _func); \
257 struct static_call_key STATIC_CALL_KEY(name) = { \
258 .func = _func_init, \
261 #define DEFINE_STATIC_CALL_NULL(name, _func) \
262 DECLARE_STATIC_CALL(name, _func); \
263 struct static_call_key STATIC_CALL_KEY(name) = { \
267 static inline void __static_call_nop(void) { }
270 * This horrific hack takes care of two things:
272 * - it ensures the compiler will only load the function pointer ONCE,
273 * which avoids a reload race.
275 * - it ensures the argument evaluation is unconditional, similar
276 * to the HAVE_STATIC_CALL variant.
278 * Sadly current GCC/Clang (10 for both) do not optimize this properly
279 * and will emit an indirect call for the NULL case :-(
281 #define __static_call_cond(name) \
283 void *func = READ_ONCE(STATIC_CALL_KEY(name).func); \
285 func = &__static_call_nop; \
286 (typeof(STATIC_CALL_TRAMP(name))*)func; \
289 #define static_call_cond(name) (void)__static_call_cond(name)
292 void __static_call_update(struct static_call_key *key, void *tramp, void *func)
294 WRITE_ONCE(key->func, func);
297 static inline int static_call_text_reserved(void *start, void *end)
302 #define EXPORT_STATIC_CALL(name) EXPORT_SYMBOL(STATIC_CALL_KEY(name))
303 #define EXPORT_STATIC_CALL_GPL(name) EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name))
305 #endif /* CONFIG_HAVE_STATIC_CALL */
307 #define DEFINE_STATIC_CALL(name, _func) \
308 __DEFINE_STATIC_CALL(name, _func, _func)
310 #define DEFINE_STATIC_CALL_RET0(name, _func) \
311 __DEFINE_STATIC_CALL(name, _func, __static_call_return0)
313 #endif /* _LINUX_STATIC_CALL_H */