1 // SPDX-License-Identifier: GPL-2.0
3 * fprobe - Simple ftrace probe wrapper for function entry.
5 #define pr_fmt(fmt) "fprobe: " fmt
8 #include <linux/fprobe.h>
9 #include <linux/kallsyms.h>
10 #include <linux/kprobes.h>
11 #include <linux/rethook.h>
12 #include <linux/slab.h>
13 #include <linux/sort.h>
17 struct fprobe_rethook_node {
18 struct rethook_node node;
19 unsigned long entry_ip;
22 static void fprobe_handler(unsigned long ip, unsigned long parent_ip,
23 struct ftrace_ops *ops, struct ftrace_regs *fregs)
25 struct fprobe_rethook_node *fpr;
26 struct rethook_node *rh;
30 fp = container_of(ops, struct fprobe, ops);
31 if (fprobe_disabled(fp))
34 bit = ftrace_test_recursion_trylock(ip, parent_ip);
40 if (fp->entry_handler)
41 fp->entry_handler(fp, ip, ftrace_get_regs(fregs));
43 if (fp->exit_handler) {
44 rh = rethook_try_get(fp->rethook);
49 fpr = container_of(rh, struct fprobe_rethook_node, node);
51 rethook_hook(rh, ftrace_get_regs(fregs), true);
55 ftrace_test_recursion_unlock(bit);
57 NOKPROBE_SYMBOL(fprobe_handler);
59 static void fprobe_kprobe_handler(unsigned long ip, unsigned long parent_ip,
60 struct ftrace_ops *ops, struct ftrace_regs *fregs)
62 struct fprobe *fp = container_of(ops, struct fprobe, ops);
64 if (unlikely(kprobe_running())) {
69 fprobe_handler(ip, parent_ip, ops, fregs);
73 static void fprobe_exit_handler(struct rethook_node *rh, void *data,
76 struct fprobe *fp = (struct fprobe *)data;
77 struct fprobe_rethook_node *fpr;
79 if (!fp || fprobe_disabled(fp))
82 fpr = container_of(rh, struct fprobe_rethook_node, node);
84 fp->exit_handler(fp, fpr->entry_ip, regs);
86 NOKPROBE_SYMBOL(fprobe_exit_handler);
88 /* Convert ftrace location address from symbols */
89 static unsigned long *get_ftrace_locations(const char **syms, int num)
91 unsigned long addr, size;
95 /* Convert symbols to symbol address */
96 addrs = kcalloc(num, sizeof(*addrs), GFP_KERNEL);
98 return ERR_PTR(-ENOMEM);
100 for (i = 0; i < num; i++) {
101 addr = kallsyms_lookup_name(syms[i]);
102 if (!addr) /* Maybe wrong symbol */
105 /* Convert symbol address to ftrace location. */
106 if (!kallsyms_lookup_size_offset(addr, &size, NULL) || !size)
109 addr = ftrace_location_range(addr, addr + size - 1);
110 if (!addr) /* No dynamic ftrace there. */
121 return ERR_PTR(-ENOENT);
124 static void fprobe_init(struct fprobe *fp)
127 if (fprobe_shared_with_kprobes(fp))
128 fp->ops.func = fprobe_kprobe_handler;
130 fp->ops.func = fprobe_handler;
131 fp->ops.flags |= FTRACE_OPS_FL_SAVE_REGS;
134 static int fprobe_init_rethook(struct fprobe *fp, int num)
141 if (!fp->exit_handler) {
146 /* Initialize rethook if needed */
147 size = num * num_possible_cpus() * 2;
151 fp->rethook = rethook_alloc((void *)fp, fprobe_exit_handler);
152 for (i = 0; i < size; i++) {
153 struct fprobe_rethook_node *node;
155 node = kzalloc(sizeof(*node), GFP_KERNEL);
157 rethook_free(fp->rethook);
161 rethook_add_node(fp->rethook, &node->node);
166 static void fprobe_fail_cleanup(struct fprobe *fp)
169 /* Don't need to cleanup rethook->handler because this is not used. */
170 rethook_free(fp->rethook);
173 ftrace_free_filter(&fp->ops);
177 * register_fprobe() - Register fprobe to ftrace by pattern.
178 * @fp: A fprobe data structure to be registered.
179 * @filter: A wildcard pattern of probed symbols.
180 * @notfilter: A wildcard pattern of NOT probed symbols.
182 * Register @fp to ftrace for enabling the probe on the symbols matched to @filter.
183 * If @notfilter is not NULL, the symbols matched the @notfilter are not probed.
185 * Return 0 if @fp is registered successfully, -errno if not.
187 int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter)
189 struct ftrace_hash *hash;
198 len = strlen(filter);
199 str = kstrdup(filter, GFP_KERNEL);
200 ret = ftrace_set_filter(&fp->ops, str, len, 0);
206 len = strlen(notfilter);
207 str = kstrdup(notfilter, GFP_KERNEL);
208 ret = ftrace_set_notrace(&fp->ops, str, len, 0);
215 * correctly calculate the total number of filtered symbols
216 * from both filter and notfilter.
218 hash = rcu_access_pointer(fp->ops.local_hash.filter_hash);
219 if (WARN_ON_ONCE(!hash))
222 ret = fprobe_init_rethook(fp, (int)hash->count);
224 ret = register_ftrace_function(&fp->ops);
228 fprobe_fail_cleanup(fp);
231 EXPORT_SYMBOL_GPL(register_fprobe);
234 * register_fprobe_ips() - Register fprobe to ftrace by address.
235 * @fp: A fprobe data structure to be registered.
236 * @addrs: An array of target ftrace location addresses.
237 * @num: The number of entries of @addrs.
239 * Register @fp to ftrace for enabling the probe on the address given by @addrs.
240 * The @addrs must be the addresses of ftrace location address, which may be
241 * the symbol address + arch-dependent offset.
242 * If you unsure what this mean, please use other registration functions.
244 * Return 0 if @fp is registered successfully, -errno if not.
246 int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num)
250 if (!fp || !addrs || num <= 0)
255 ret = ftrace_set_filter_ips(&fp->ops, addrs, num, 0, 0);
259 ret = fprobe_init_rethook(fp, num);
261 ret = register_ftrace_function(&fp->ops);
264 fprobe_fail_cleanup(fp);
267 EXPORT_SYMBOL_GPL(register_fprobe_ips);
270 * register_fprobe_syms() - Register fprobe to ftrace by symbols.
271 * @fp: A fprobe data structure to be registered.
272 * @syms: An array of target symbols.
273 * @num: The number of entries of @syms.
275 * Register @fp to the symbols given by @syms array. This will be useful if
276 * you are sure the symbols exist in the kernel.
278 * Return 0 if @fp is registered successfully, -errno if not.
280 int register_fprobe_syms(struct fprobe *fp, const char **syms, int num)
282 unsigned long *addrs;
285 if (!fp || !syms || num <= 0)
288 addrs = get_ftrace_locations(syms, num);
290 return PTR_ERR(addrs);
292 ret = register_fprobe_ips(fp, addrs, num);
298 EXPORT_SYMBOL_GPL(register_fprobe_syms);
301 * unregister_fprobe() - Unregister fprobe from ftrace
302 * @fp: A fprobe data structure to be unregistered.
304 * Unregister fprobe (and remove ftrace hooks from the function entries).
306 * Return 0 if @fp is unregistered successfully, -errno if not.
308 int unregister_fprobe(struct fprobe *fp)
312 if (!fp || fp->ops.func != fprobe_handler)
316 * rethook_free() starts disabling the rethook, but the rethook handlers
317 * may be running on other processors at this point. To make sure that all
318 * current running handlers are finished, call unregister_ftrace_function()
322 rethook_free(fp->rethook);
324 ret = unregister_ftrace_function(&fp->ops);
328 ftrace_free_filter(&fp->ops);
332 EXPORT_SYMBOL_GPL(unregister_fprobe);