1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/init.h>
3 #include <linux/static_call.h>
6 #include <linux/sort.h>
7 #include <linux/slab.h>
8 #include <linux/module.h>
10 #include <linux/processor.h>
11 #include <asm/sections.h>
13 extern struct static_call_site __start_static_call_sites[],
14 __stop_static_call_sites[];
15 extern struct static_call_tramp_key __start_static_call_tramp_key[],
16 __stop_static_call_tramp_key[];
18 static bool static_call_initialized;
20 /* mutex to protect key modules/sites */
21 static DEFINE_MUTEX(static_call_mutex);
23 static void static_call_lock(void)
25 mutex_lock(&static_call_mutex);
28 static void static_call_unlock(void)
30 mutex_unlock(&static_call_mutex);
33 static inline void *static_call_addr(struct static_call_site *site)
35 return (void *)((long)site->addr + (long)&site->addr);
38 static inline unsigned long __static_call_key(const struct static_call_site *site)
40 return (long)site->key + (long)&site->key;
43 static inline struct static_call_key *static_call_key(const struct static_call_site *site)
45 return (void *)(__static_call_key(site) & ~STATIC_CALL_SITE_FLAGS);
48 /* These assume the key is word-aligned. */
49 static inline bool static_call_is_init(struct static_call_site *site)
51 return __static_call_key(site) & STATIC_CALL_SITE_INIT;
54 static inline bool static_call_is_tail(struct static_call_site *site)
56 return __static_call_key(site) & STATIC_CALL_SITE_TAIL;
59 static inline void static_call_set_init(struct static_call_site *site)
61 site->key = (__static_call_key(site) | STATIC_CALL_SITE_INIT) -
65 static int static_call_site_cmp(const void *_a, const void *_b)
67 const struct static_call_site *a = _a;
68 const struct static_call_site *b = _b;
69 const struct static_call_key *key_a = static_call_key(a);
70 const struct static_call_key *key_b = static_call_key(b);
81 static void static_call_site_swap(void *_a, void *_b, int size)
83 long delta = (unsigned long)_a - (unsigned long)_b;
84 struct static_call_site *a = _a;
85 struct static_call_site *b = _b;
86 struct static_call_site tmp = *a;
88 a->addr = b->addr - delta;
89 a->key = b->key - delta;
91 b->addr = tmp.addr + delta;
92 b->key = tmp.key + delta;
95 static inline void static_call_sort_entries(struct static_call_site *start,
96 struct static_call_site *stop)
98 sort(start, stop - start, sizeof(struct static_call_site),
99 static_call_site_cmp, static_call_site_swap);
102 static inline bool static_call_key_has_mods(struct static_call_key *key)
104 return !(key->type & 1);
107 static inline struct static_call_mod *static_call_key_next(struct static_call_key *key)
109 if (!static_call_key_has_mods(key))
115 static inline struct static_call_site *static_call_key_sites(struct static_call_key *key)
117 if (static_call_key_has_mods(key))
120 return (struct static_call_site *)(key->type & ~1);
123 void __static_call_update(struct static_call_key *key, void *tramp, void *func)
125 struct static_call_site *site, *stop;
126 struct static_call_mod *site_mod, first;
131 if (key->func == func)
136 arch_static_call_transform(NULL, tramp, func, false);
139 * If uninitialized, we'll not update the callsites, but they still
140 * point to the trampoline and we just patched that.
142 if (WARN_ON_ONCE(!static_call_initialized))
145 first = (struct static_call_mod){
146 .next = static_call_key_next(key),
148 .sites = static_call_key_sites(key),
151 for (site_mod = &first; site_mod; site_mod = site_mod->next) {
152 struct module *mod = site_mod->mod;
154 if (!site_mod->sites) {
156 * This can happen if the static call key is defined in
157 * a module which doesn't use it.
159 * It also happens in the has_mods case, where the
160 * 'first' entry has no sites associated with it.
165 stop = __stop_static_call_sites;
167 #ifdef CONFIG_MODULES
169 stop = mod->static_call_sites +
170 mod->num_static_call_sites;
174 for (site = site_mod->sites;
175 site < stop && static_call_key(site) == key; site++) {
176 void *site_addr = static_call_addr(site);
178 if (static_call_is_init(site)) {
180 * Don't write to call sites which were in
181 * initmem and have since been freed.
183 if (!mod && system_state >= SYSTEM_RUNNING)
185 if (mod && !within_module_init((unsigned long)site_addr, mod))
189 if (!kernel_text_address((unsigned long)site_addr)) {
190 WARN_ONCE(1, "can't patch static call site at %pS",
195 arch_static_call_transform(site_addr, NULL, func,
196 static_call_is_tail(site));
201 static_call_unlock();
204 EXPORT_SYMBOL_GPL(__static_call_update);
206 static int __static_call_init(struct module *mod,
207 struct static_call_site *start,
208 struct static_call_site *stop)
210 struct static_call_site *site;
211 struct static_call_key *key, *prev_key = NULL;
212 struct static_call_mod *site_mod;
217 static_call_sort_entries(start, stop);
219 for (site = start; site < stop; site++) {
220 void *site_addr = static_call_addr(site);
222 if ((mod && within_module_init((unsigned long)site_addr, mod)) ||
223 (!mod && init_section_contains(site_addr, 1)))
224 static_call_set_init(site);
226 key = static_call_key(site);
227 if (key != prev_key) {
231 * For vmlinux (!mod) avoid the allocation by storing
232 * the sites pointer in the key itself. Also see
233 * __static_call_update()'s @first.
235 * This allows architectures (eg. x86) to call
236 * static_call_init() before memory allocation works.
244 site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
249 * When the key has a direct sites pointer, extract
250 * that into an explicit struct static_call_mod, so we
251 * can have a list of modules.
253 if (static_call_key_sites(key)) {
254 site_mod->mod = NULL;
255 site_mod->next = NULL;
256 site_mod->sites = static_call_key_sites(key);
258 key->mods = site_mod;
260 site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
266 site_mod->sites = site;
267 site_mod->next = static_call_key_next(key);
268 key->mods = site_mod;
272 arch_static_call_transform(site_addr, NULL, key->func,
273 static_call_is_tail(site));
279 static int addr_conflict(struct static_call_site *site, void *start, void *end)
281 unsigned long addr = (unsigned long)static_call_addr(site);
283 if (addr <= (unsigned long)end &&
284 addr + CALL_INSN_SIZE > (unsigned long)start)
290 static int __static_call_text_reserved(struct static_call_site *iter_start,
291 struct static_call_site *iter_stop,
292 void *start, void *end)
294 struct static_call_site *iter = iter_start;
296 while (iter < iter_stop) {
297 if (addr_conflict(iter, start, end))
305 #ifdef CONFIG_MODULES
307 static int __static_call_mod_text_reserved(void *start, void *end)
313 mod = __module_text_address((unsigned long)start);
314 WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
315 if (!try_module_get(mod))
322 ret = __static_call_text_reserved(mod->static_call_sites,
323 mod->static_call_sites + mod->num_static_call_sites,
331 static unsigned long tramp_key_lookup(unsigned long addr)
333 struct static_call_tramp_key *start = __start_static_call_tramp_key;
334 struct static_call_tramp_key *stop = __stop_static_call_tramp_key;
335 struct static_call_tramp_key *tramp_key;
337 for (tramp_key = start; tramp_key != stop; tramp_key++) {
340 tramp = (long)tramp_key->tramp + (long)&tramp_key->tramp;
342 return (long)tramp_key->key + (long)&tramp_key->key;
348 static int static_call_add_module(struct module *mod)
350 struct static_call_site *start = mod->static_call_sites;
351 struct static_call_site *stop = start + mod->num_static_call_sites;
352 struct static_call_site *site;
354 for (site = start; site != stop; site++) {
355 unsigned long s_key = __static_call_key(site);
356 unsigned long addr = s_key & ~STATIC_CALL_SITE_FLAGS;
360 * Is the key is exported, 'addr' points to the key, which
361 * means modules are allowed to call static_call_update() on
364 * Otherwise, the key isn't exported, and 'addr' points to the
365 * trampoline so we need to lookup the key.
367 * We go through this dance to prevent crazy modules from
368 * abusing sensitive static calls.
370 if (!kernel_text_address(addr))
373 key = tramp_key_lookup(addr);
375 pr_warn("Failed to fixup __raw_static_call() usage at: %ps\n",
376 static_call_addr(site));
380 key |= s_key & STATIC_CALL_SITE_FLAGS;
381 site->key = key - (long)&site->key;
384 return __static_call_init(mod, start, stop);
387 static void static_call_del_module(struct module *mod)
389 struct static_call_site *start = mod->static_call_sites;
390 struct static_call_site *stop = mod->static_call_sites +
391 mod->num_static_call_sites;
392 struct static_call_key *key, *prev_key = NULL;
393 struct static_call_mod *site_mod, **prev;
394 struct static_call_site *site;
396 for (site = start; site < stop; site++) {
397 key = static_call_key(site);
403 for (prev = &key->mods, site_mod = key->mods;
404 site_mod && site_mod->mod != mod;
405 prev = &site_mod->next, site_mod = site_mod->next)
411 *prev = site_mod->next;
416 static int static_call_module_notify(struct notifier_block *nb,
417 unsigned long val, void *data)
419 struct module *mod = data;
426 case MODULE_STATE_COMING:
427 ret = static_call_add_module(mod);
429 WARN(1, "Failed to allocate memory for static calls");
430 static_call_del_module(mod);
433 case MODULE_STATE_GOING:
434 static_call_del_module(mod);
438 static_call_unlock();
441 return notifier_from_errno(ret);
444 static struct notifier_block static_call_module_nb = {
445 .notifier_call = static_call_module_notify,
450 static inline int __static_call_mod_text_reserved(void *start, void *end)
455 #endif /* CONFIG_MODULES */
457 int static_call_text_reserved(void *start, void *end)
459 int ret = __static_call_text_reserved(__start_static_call_sites,
460 __stop_static_call_sites, start, end);
465 return __static_call_mod_text_reserved(start, end);
468 int __init static_call_init(void)
472 if (static_call_initialized)
477 ret = __static_call_init(NULL, __start_static_call_sites,
478 __stop_static_call_sites);
479 static_call_unlock();
483 pr_err("Failed to allocate memory for static_call!\n");
487 static_call_initialized = true;
489 #ifdef CONFIG_MODULES
490 register_module_notifier(&static_call_module_nb);
494 early_initcall(static_call_init);
496 long __static_call_return0(void)
501 #ifdef CONFIG_STATIC_CALL_SELFTEST
503 static int func_a(int x)
508 static int func_b(int x)
513 DEFINE_STATIC_CALL(sc_selftest, func_a);
515 static struct static_call_data {
519 } static_call_data [] __initdata = {
525 static int __init test_static_call_init(void)
529 for (i = 0; i < ARRAY_SIZE(static_call_data); i++ ) {
530 struct static_call_data *scd = &static_call_data[i];
533 static_call_update(sc_selftest, scd->func);
535 WARN_ON(static_call(sc_selftest)(scd->val) != scd->expect);
540 early_initcall(test_static_call_init);
542 #endif /* CONFIG_STATIC_CALL_SELFTEST */