Merge tag 'gvt-next-2020-11-23' of https://github.com/intel/gvt-linux into drm-intel...
[linux-2.6-microblaze.git] / include / linux / static_call.h
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_STATIC_CALL_H
3 #define _LINUX_STATIC_CALL_H
4
5 /*
6  * Static call support
7  *
8  * Static calls use code patching to hard-code function pointers into direct
9  * branch instructions. They give the flexibility of function pointers, but
10  * with improved performance. This is especially important for cases where
11  * retpolines would otherwise be used, as retpolines can significantly impact
12  * performance.
13  *
14  *
15  * API overview:
16  *
17  *   DECLARE_STATIC_CALL(name, func);
18  *   DEFINE_STATIC_CALL(name, func);
19  *   DEFINE_STATIC_CALL_NULL(name, typename);
20  *   static_call(name)(args...);
21  *   static_call_cond(name)(args...);
22  *   static_call_update(name, func);
23  *
24  * Usage example:
25  *
26  *   # Start with the following functions (with identical prototypes):
27  *   int func_a(int arg1, int arg2);
28  *   int func_b(int arg1, int arg2);
29  *
30  *   # Define a 'my_name' reference, associated with func_a() by default
31  *   DEFINE_STATIC_CALL(my_name, func_a);
32  *
33  *   # Call func_a()
34  *   static_call(my_name)(arg1, arg2);
35  *
36  *   # Update 'my_name' to point to func_b()
37  *   static_call_update(my_name, &func_b);
38  *
39  *   # Call func_b()
40  *   static_call(my_name)(arg1, arg2);
41  *
42  *
43  * Implementation details:
44  *
45  *   This requires some arch-specific code (CONFIG_HAVE_STATIC_CALL).
46  *   Otherwise basic indirect calls are used (with function pointers).
47  *
48  *   Each static_call() site calls into a trampoline associated with the name.
49  *   The trampoline has a direct branch to the default function.  Updates to a
50  *   name will modify the trampoline's branch destination.
51  *
52  *   If the arch has CONFIG_HAVE_STATIC_CALL_INLINE, then the call sites
53  *   themselves will be patched at runtime to call the functions directly,
54  *   rather than calling through the trampoline.  This requires objtool or a
55  *   compiler plugin to detect all the static_call() sites and annotate them
56  *   in the .static_call_sites section.
57  *
58  *
59  * Notes on NULL function pointers:
60  *
61  *   Static_call()s support NULL functions, with many of the caveats that
62  *   regular function pointers have.
63  *
64  *   Clearly calling a NULL function pointer is 'BAD', so too for
65  *   static_call()s (although when HAVE_STATIC_CALL it might not be immediately
66  *   fatal). A NULL static_call can be the result of:
67  *
68  *     DECLARE_STATIC_CALL_NULL(my_static_call, void (*)(int));
69  *
70  *   which is equivalent to declaring a NULL function pointer with just a
71  *   typename:
72  *
73  *     void (*my_func_ptr)(int arg1) = NULL;
74  *
75  *   or using static_call_update() with a NULL function. In both cases the
76  *   HAVE_STATIC_CALL implementation will patch the trampoline with a RET
77  *   instruction, instead of an immediate tail-call JMP. HAVE_STATIC_CALL_INLINE
78  *   architectures can patch the trampoline call to a NOP.
79  *
80  *   In all cases, any argument evaluation is unconditional. Unlike a regular
81  *   conditional function pointer call:
82  *
83  *     if (my_func_ptr)
84  *         my_func_ptr(arg1)
85  *
86  *   where the argument evaludation also depends on the pointer value.
87  *
88  *   When calling a static_call that can be NULL, use:
89  *
90  *     static_call_cond(name)(arg1);
91  *
92  *   which will include the required value tests to avoid NULL-pointer
93  *   dereferences.
94  */
95
96 #include <linux/types.h>
97 #include <linux/cpu.h>
98 #include <linux/static_call_types.h>
99
100 #ifdef CONFIG_HAVE_STATIC_CALL
101 #include <asm/static_call.h>
102
103 /*
104  * Either @site or @tramp can be NULL.
105  */
106 extern void arch_static_call_transform(void *site, void *tramp, void *func, bool tail);
107
108 #define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name)
109
110 /*
111  * __ADDRESSABLE() is used to ensure the key symbol doesn't get stripped from
112  * the symbol table so that objtool can reference it when it generates the
113  * .static_call_sites section.
114  */
115 #define __static_call(name)                                             \
116 ({                                                                      \
117         __ADDRESSABLE(STATIC_CALL_KEY(name));                           \
118         &STATIC_CALL_TRAMP(name);                                       \
119 })
120
121 #else
122 #define STATIC_CALL_TRAMP_ADDR(name) NULL
123 #endif
124
125
126 #define DECLARE_STATIC_CALL(name, func)                                 \
127         extern struct static_call_key STATIC_CALL_KEY(name);            \
128         extern typeof(func) STATIC_CALL_TRAMP(name);
129
130 #define static_call_update(name, func)                                  \
131 ({                                                                      \
132         BUILD_BUG_ON(!__same_type(*(func), STATIC_CALL_TRAMP(name)));   \
133         __static_call_update(&STATIC_CALL_KEY(name),                    \
134                              STATIC_CALL_TRAMP_ADDR(name), func);       \
135 })
136
137 #ifdef CONFIG_HAVE_STATIC_CALL_INLINE
138
139 extern int __init static_call_init(void);
140
141 struct static_call_mod {
142         struct static_call_mod *next;
143         struct module *mod; /* for vmlinux, mod == NULL */
144         struct static_call_site *sites;
145 };
146
147 struct static_call_key {
148         void *func;
149         union {
150                 /* bit 0: 0 = mods, 1 = sites */
151                 unsigned long type;
152                 struct static_call_mod *mods;
153                 struct static_call_site *sites;
154         };
155 };
156
157 extern void __static_call_update(struct static_call_key *key, void *tramp, void *func);
158 extern int static_call_mod_init(struct module *mod);
159 extern int static_call_text_reserved(void *start, void *end);
160
161 #define DEFINE_STATIC_CALL(name, _func)                                 \
162         DECLARE_STATIC_CALL(name, _func);                               \
163         struct static_call_key STATIC_CALL_KEY(name) = {                \
164                 .func = _func,                                          \
165                 .type = 1,                                              \
166         };                                                              \
167         ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
168
169 #define DEFINE_STATIC_CALL_NULL(name, _func)                            \
170         DECLARE_STATIC_CALL(name, _func);                               \
171         struct static_call_key STATIC_CALL_KEY(name) = {                \
172                 .func = NULL,                                           \
173                 .type = 1,                                              \
174         };                                                              \
175         ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)
176
177 #define static_call(name)       __static_call(name)
178 #define static_call_cond(name)  (void)__static_call(name)
179
180 #define EXPORT_STATIC_CALL(name)                                        \
181         EXPORT_SYMBOL(STATIC_CALL_KEY(name));                           \
182         EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
183
184 #define EXPORT_STATIC_CALL_GPL(name)                                    \
185         EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name));                       \
186         EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
187
188 #elif defined(CONFIG_HAVE_STATIC_CALL)
189
190 static inline int static_call_init(void) { return 0; }
191
192 struct static_call_key {
193         void *func;
194 };
195
196 #define DEFINE_STATIC_CALL(name, _func)                                 \
197         DECLARE_STATIC_CALL(name, _func);                               \
198         struct static_call_key STATIC_CALL_KEY(name) = {                \
199                 .func = _func,                                          \
200         };                                                              \
201         ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
202
203 #define DEFINE_STATIC_CALL_NULL(name, _func)                            \
204         DECLARE_STATIC_CALL(name, _func);                               \
205         struct static_call_key STATIC_CALL_KEY(name) = {                \
206                 .func = NULL,                                           \
207         };                                                              \
208         ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)
209
210 #define static_call(name)       __static_call(name)
211 #define static_call_cond(name)  (void)__static_call(name)
212
213 static inline
214 void __static_call_update(struct static_call_key *key, void *tramp, void *func)
215 {
216         cpus_read_lock();
217         WRITE_ONCE(key->func, func);
218         arch_static_call_transform(NULL, tramp, func, false);
219         cpus_read_unlock();
220 }
221
222 static inline int static_call_text_reserved(void *start, void *end)
223 {
224         return 0;
225 }
226
227 #define EXPORT_STATIC_CALL(name)                                        \
228         EXPORT_SYMBOL(STATIC_CALL_KEY(name));                           \
229         EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
230
231 #define EXPORT_STATIC_CALL_GPL(name)                                    \
232         EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name));                       \
233         EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
234
235 #else /* Generic implementation */
236
237 static inline int static_call_init(void) { return 0; }
238
239 struct static_call_key {
240         void *func;
241 };
242
243 #define DEFINE_STATIC_CALL(name, _func)                                 \
244         DECLARE_STATIC_CALL(name, _func);                               \
245         struct static_call_key STATIC_CALL_KEY(name) = {                \
246                 .func = _func,                                          \
247         }
248
249 #define DEFINE_STATIC_CALL_NULL(name, _func)                            \
250         DECLARE_STATIC_CALL(name, _func);                               \
251         struct static_call_key STATIC_CALL_KEY(name) = {                \
252                 .func = NULL,                                           \
253         }
254
255 #define static_call(name)                                               \
256         ((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_KEY(name).func))
257
258 static inline void __static_call_nop(void) { }
259
260 /*
261  * This horrific hack takes care of two things:
262  *
263  *  - it ensures the compiler will only load the function pointer ONCE,
264  *    which avoids a reload race.
265  *
266  *  - it ensures the argument evaluation is unconditional, similar
267  *    to the HAVE_STATIC_CALL variant.
268  *
269  * Sadly current GCC/Clang (10 for both) do not optimize this properly
270  * and will emit an indirect call for the NULL case :-(
271  */
272 #define __static_call_cond(name)                                        \
273 ({                                                                      \
274         void *func = READ_ONCE(STATIC_CALL_KEY(name).func);             \
275         if (!func)                                                      \
276                 func = &__static_call_nop;                              \
277         (typeof(STATIC_CALL_TRAMP(name))*)func;                         \
278 })
279
280 #define static_call_cond(name)  (void)__static_call_cond(name)
281
282 static inline
283 void __static_call_update(struct static_call_key *key, void *tramp, void *func)
284 {
285         WRITE_ONCE(key->func, func);
286 }
287
288 static inline int static_call_text_reserved(void *start, void *end)
289 {
290         return 0;
291 }
292
293 #define EXPORT_STATIC_CALL(name)        EXPORT_SYMBOL(STATIC_CALL_KEY(name))
294 #define EXPORT_STATIC_CALL_GPL(name)    EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name))
295
296 #endif /* CONFIG_HAVE_STATIC_CALL */
297
298 #endif /* _LINUX_STATIC_CALL_H */