include/linux/static_call.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _LINUX_STATIC_CALL_H
   3 #define _LINUX_STATIC_CALL_H
   4
   5 /*
   6  * Static call support
   7  *
   8  * Static calls use code patching to hard-code function pointers into direct
   9  * branch instructions. They give the flexibility of function pointers, but
  10  * with improved performance. This is especially important for cases where
  11  * retpolines would otherwise be used, as retpolines can significantly impact
  12  * performance.
  13  *
  14  *
  15  * API overview:
  16  *
  17  *   DECLARE_STATIC_CALL(name, func);
  18  *   DEFINE_STATIC_CALL(name, func);
  19  *   DEFINE_STATIC_CALL_NULL(name, typename);
  20  *   DEFINE_STATIC_CALL_RET0(name, typename);
  21  *
  22  *   __static_call_return0;
  23  *
  24  *   static_call(name)(args...);
  25  *   static_call_cond(name)(args...);
  26  *   static_call_update(name, func);
  27  *   static_call_query(name);
  28  *
  29  *   EXPORT_STATIC_CALL{,_TRAMP}{,_GPL}()
  30  *
  31  * Usage example:
  32  *
  33  *   # Start with the following functions (with identical prototypes):
  34  *   int func_a(int arg1, int arg2);
  35  *   int func_b(int arg1, int arg2);
  36  *
  37  *   # Define a 'my_name' reference, associated with func_a() by default
  38  *   DEFINE_STATIC_CALL(my_name, func_a);
  39  *
  40  *   # Call func_a()
  41  *   static_call(my_name)(arg1, arg2);
  42  *
  43  *   # Update 'my_name' to point to func_b()
  44  *   static_call_update(my_name, &func_b);
  45  *
  46  *   # Call func_b()
  47  *   static_call(my_name)(arg1, arg2);
  48  *
  49  *
  50  * Implementation details:
  51  *
  52  *   This requires some arch-specific code (CONFIG_HAVE_STATIC_CALL).
  53  *   Otherwise basic indirect calls are used (with function pointers).
  54  *
  55  *   Each static_call() site calls into a trampoline associated with the name.
  56  *   The trampoline has a direct branch to the default function.  Updates to a
  57  *   name will modify the trampoline's branch destination.
  58  *
  59  *   If the arch has CONFIG_HAVE_STATIC_CALL_INLINE, then the call sites
  60  *   themselves will be patched at runtime to call the functions directly,
  61  *   rather than calling through the trampoline.  This requires objtool or a
  62  *   compiler plugin to detect all the static_call() sites and annotate them
  63  *   in the .static_call_sites section.
  64  *
  65  *
  66  * Notes on NULL function pointers:
  67  *
  68  *   Static_call()s support NULL functions, with many of the caveats that
  69  *   regular function pointers have.
  70  *
  71  *   Clearly calling a NULL function pointer is 'BAD', so too for
  72  *   static_call()s (although when HAVE_STATIC_CALL it might not be immediately
  73  *   fatal). A NULL static_call can be the result of:
  74  *
  75  *     DECLARE_STATIC_CALL_NULL(my_static_call, void (*)(int));
  76  *
  77  *   which is equivalent to declaring a NULL function pointer with just a
  78  *   typename:
  79  *
  80  *     void (*my_func_ptr)(int arg1) = NULL;
  81  *
  82  *   or using static_call_update() with a NULL function. In both cases the
  83  *   HAVE_STATIC_CALL implementation will patch the trampoline with a RET
  84  *   instruction, instead of an immediate tail-call JMP. HAVE_STATIC_CALL_INLINE
  85  *   architectures can patch the trampoline call to a NOP.
  86  *
  87  *   In all cases, any argument evaluation is unconditional. Unlike a regular
  88  *   conditional function pointer call:
  89  *
  90  *     if (my_func_ptr)
  91  *         my_func_ptr(arg1)
  92  *
  93  *   where the argument evaludation also depends on the pointer value.
  94  *
  95  *   When calling a static_call that can be NULL, use:
  96  *
  97  *     static_call_cond(name)(arg1);
  98  *
  99  *   which will include the required value tests to avoid NULL-pointer
 100  *   dereferences.
 101  *
 102  *   To query which function is currently set to be called, use:
 103  *
 104  *   func = static_call_query(name);
 105  *
 106  *
 107  * DEFINE_STATIC_CALL_RET0 / __static_call_return0:
 108  *
 109  *   Just like how DEFINE_STATIC_CALL_NULL() / static_call_cond() optimize the
 110  *   conditional void function call, DEFINE_STATIC_CALL_RET0 /
 111  *   __static_call_return0 optimize the do nothing return 0 function.
 112  *
 113  *   This feature is strictly UB per the C standard (since it casts a function
 114  *   pointer to a different signature) and relies on the architecture ABI to
 115  *   make things work. In particular it relies on Caller Stack-cleanup and the
 116  *   whole return register being clobbered for short return values. All normal
 117  *   CDECL style ABIs conform.
 118  *
 119  *   In particular the x86_64 implementation replaces the 5 byte CALL
 120  *   instruction at the callsite with a 5 byte clear of the RAX register,
 121  *   completely eliding any function call overhead.
 122  *
 123  *   Notably argument setup is unconditional.
 124  *
 125  *
 126  * EXPORT_STATIC_CALL() vs EXPORT_STATIC_CALL_TRAMP():
 127  *
 128  *   The difference is that the _TRAMP variant tries to only export the
 129  *   trampoline with the result that a module can use static_call{,_cond}() but
 130  *   not static_call_update().
 131  *
 132  */
 133
 134 #include <linux/types.h>
 135 #include <linux/cpu.h>
 136 #include <linux/static_call_types.h>
 137
 138 #ifdef CONFIG_HAVE_STATIC_CALL
 139 #include <asm/static_call.h>
 140
 141 /*
 142  * Either @site or @tramp can be NULL.
 143  */
 144 extern void arch_static_call_transform(void *site, void *tramp, void *func, bool tail);
 145
 146 #define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name)
 147
 148 #else
 149 #define STATIC_CALL_TRAMP_ADDR(name) NULL
 150 #endif
 151
 152 #define static_call_update(name, func)                                  \
 153 ({                                                                      \
 154         typeof(&STATIC_CALL_TRAMP(name)) __F = (func);                  \
 155         __static_call_update(&STATIC_CALL_KEY(name),                    \
 156                              STATIC_CALL_TRAMP_ADDR(name), __F);        \
 157 })
 158
 159 #define static_call_query(name) (READ_ONCE(STATIC_CALL_KEY(name).func))
 160
 161 #ifdef CONFIG_HAVE_STATIC_CALL_INLINE
 162
 163 extern int __init static_call_init(void);
 164
 165 struct static_call_mod {
 166         struct static_call_mod *next;
 167         struct module *mod; /* for vmlinux, mod == NULL */
 168         struct static_call_site *sites;
 169 };
 170
 171 /* For finding the key associated with a trampoline */
 172 struct static_call_tramp_key {
 173         s32 tramp;
 174         s32 key;
 175 };
 176
 177 extern void __static_call_update(struct static_call_key *key, void *tramp, void *func);
 178 extern int static_call_mod_init(struct module *mod);
 179 extern int static_call_text_reserved(void *start, void *end);
 180
 181 extern long __static_call_return0(void);
 182
 183 #define DEFINE_STATIC_CALL(name, _func)                                 \
 184         DECLARE_STATIC_CALL(name, _func);                               \
 185         struct static_call_key STATIC_CALL_KEY(name) = {                \
 186                 .func = _func,                                          \
 187                 .type = 1,                                              \
 188         };                                                              \
 189         ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
 190
 191 #define DEFINE_STATIC_CALL_NULL(name, _func)                            \
 192         DECLARE_STATIC_CALL(name, _func);                               \
 193         struct static_call_key STATIC_CALL_KEY(name) = {                \
 194                 .func = NULL,                                           \
 195                 .type = 1,                                              \
 196         };                                                              \
 197         ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)
 198
 199 #define DEFINE_STATIC_CALL_RET0(name, _func)                            \
 200         DECLARE_STATIC_CALL(name, _func);                               \
 201         struct static_call_key STATIC_CALL_KEY(name) = {                \
 202                 .func = __static_call_return0,                          \
 203                 .type = 1,                                              \
 204         };                                                              \
 205         ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name)
 206
 207 #define static_call_cond(name)  (void)__static_call(name)
 208
 209 #define EXPORT_STATIC_CALL(name)                                        \
 210         EXPORT_SYMBOL(STATIC_CALL_KEY(name));                           \
 211         EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
 212 #define EXPORT_STATIC_CALL_GPL(name)                                    \
 213         EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name));                       \
 214         EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
 215
 216 /* Leave the key unexported, so modules can't change static call targets: */
 217 #define EXPORT_STATIC_CALL_TRAMP(name)                                  \
 218         EXPORT_SYMBOL(STATIC_CALL_TRAMP(name));                         \
 219         ARCH_ADD_TRAMP_KEY(name)
 220 #define EXPORT_STATIC_CALL_TRAMP_GPL(name)                              \
 221         EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name));                     \
 222         ARCH_ADD_TRAMP_KEY(name)
 223
 224 #elif defined(CONFIG_HAVE_STATIC_CALL)
 225
 226 static inline int static_call_init(void) { return 0; }
 227
 228 #define DEFINE_STATIC_CALL(name, _func)                                 \
 229         DECLARE_STATIC_CALL(name, _func);                               \
 230         struct static_call_key STATIC_CALL_KEY(name) = {                \
 231                 .func = _func,                                          \
 232         };                                                              \
 233         ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
 234
 235 #define DEFINE_STATIC_CALL_NULL(name, _func)                            \
 236         DECLARE_STATIC_CALL(name, _func);                               \
 237         struct static_call_key STATIC_CALL_KEY(name) = {                \
 238                 .func = NULL,                                           \
 239         };                                                              \
 240         ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)
 241
 242 #define DEFINE_STATIC_CALL_RET0(name, _func)                            \
 243         DECLARE_STATIC_CALL(name, _func);                               \
 244         struct static_call_key STATIC_CALL_KEY(name) = {                \
 245                 .func = __static_call_return0,                          \
 246         };                                                              \
 247         ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name)
 248
 249 #define static_call_cond(name)  (void)__static_call(name)
 250
 251 static inline
 252 void __static_call_update(struct static_call_key *key, void *tramp, void *func)
 253 {
 254         cpus_read_lock();
 255         WRITE_ONCE(key->func, func);
 256         arch_static_call_transform(NULL, tramp, func, false);
 257         cpus_read_unlock();
 258 }
 259
 260 static inline int static_call_text_reserved(void *start, void *end)
 261 {
 262         return 0;
 263 }
 264
 265 extern long __static_call_return0(void);
 266
 267 #define EXPORT_STATIC_CALL(name)                                        \
 268         EXPORT_SYMBOL(STATIC_CALL_KEY(name));                           \
 269         EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
 270 #define EXPORT_STATIC_CALL_GPL(name)                                    \
 271         EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name));                       \
 272         EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
 273
 274 /* Leave the key unexported, so modules can't change static call targets: */
 275 #define EXPORT_STATIC_CALL_TRAMP(name)                                  \
 276         EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
 277 #define EXPORT_STATIC_CALL_TRAMP_GPL(name)                              \
 278         EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
 279
 280 #else /* Generic implementation */
 281
 282 static inline int static_call_init(void) { return 0; }
 283
 284 static inline long __static_call_return0(void)
 285 {
 286         return 0;
 287 }
 288
 289 #define __DEFINE_STATIC_CALL(name, _func, _func_init)                   \
 290         DECLARE_STATIC_CALL(name, _func);                               \
 291         struct static_call_key STATIC_CALL_KEY(name) = {                \
 292                 .func = _func_init,                                     \
 293         }
 294
 295 #define DEFINE_STATIC_CALL(name, _func)                                 \
 296         __DEFINE_STATIC_CALL(name, _func, _func)
 297
 298 #define DEFINE_STATIC_CALL_NULL(name, _func)                            \
 299         __DEFINE_STATIC_CALL(name, _func, NULL)
 300
 301 #define DEFINE_STATIC_CALL_RET0(name, _func)                            \
 302         __DEFINE_STATIC_CALL(name, _func, __static_call_return0)
 303
 304 static inline void __static_call_nop(void) { }
 305
 306 /*
 307  * This horrific hack takes care of two things:
 308  *
 309  *  - it ensures the compiler will only load the function pointer ONCE,
 310  *    which avoids a reload race.
 311  *
 312  *  - it ensures the argument evaluation is unconditional, similar
 313  *    to the HAVE_STATIC_CALL variant.
 314  *
 315  * Sadly current GCC/Clang (10 for both) do not optimize this properly
 316  * and will emit an indirect call for the NULL case :-(
 317  */
 318 #define __static_call_cond(name)                                        \
 319 ({                                                                      \
 320         void *func = READ_ONCE(STATIC_CALL_KEY(name).func);             \
 321         if (!func)                                                      \
 322                 func = &__static_call_nop;                              \
 323         (typeof(STATIC_CALL_TRAMP(name))*)func;                         \
 324 })
 325
 326 #define static_call_cond(name)  (void)__static_call_cond(name)
 327
 328 static inline
 329 void __static_call_update(struct static_call_key *key, void *tramp, void *func)
 330 {
 331         WRITE_ONCE(key->func, func);
 332 }
 333
 334 static inline int static_call_text_reserved(void *start, void *end)
 335 {
 336         return 0;
 337 }
 338
 339 #define EXPORT_STATIC_CALL(name)        EXPORT_SYMBOL(STATIC_CALL_KEY(name))
 340 #define EXPORT_STATIC_CALL_GPL(name)    EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name))
 341
 342 #endif /* CONFIG_HAVE_STATIC_CALL */
 343
 344 #endif /* _LINUX_STATIC_CALL_H */