1 // SPDX-License-Identifier: GPL-2.0
2 #include <test_progs.h>
3 #include "kprobe_multi.skel.h"
4 #include "trace_helpers.h"
5 #include "kprobe_multi_empty.skel.h"
6 #include "bpf/libbpf_internal.h"
7 #include "bpf/hashmap.h"
9 static void kprobe_multi_test_run(struct kprobe_multi *skel, bool test_return)
11 LIBBPF_OPTS(bpf_test_run_opts, topts);
14 prog_fd = bpf_program__fd(skel->progs.trigger);
15 err = bpf_prog_test_run_opts(prog_fd, &topts);
16 ASSERT_OK(err, "test_run");
17 ASSERT_EQ(topts.retval, 0, "test_run");
19 ASSERT_EQ(skel->bss->kprobe_test1_result, 1, "kprobe_test1_result");
20 ASSERT_EQ(skel->bss->kprobe_test2_result, 1, "kprobe_test2_result");
21 ASSERT_EQ(skel->bss->kprobe_test3_result, 1, "kprobe_test3_result");
22 ASSERT_EQ(skel->bss->kprobe_test4_result, 1, "kprobe_test4_result");
23 ASSERT_EQ(skel->bss->kprobe_test5_result, 1, "kprobe_test5_result");
24 ASSERT_EQ(skel->bss->kprobe_test6_result, 1, "kprobe_test6_result");
25 ASSERT_EQ(skel->bss->kprobe_test7_result, 1, "kprobe_test7_result");
26 ASSERT_EQ(skel->bss->kprobe_test8_result, 1, "kprobe_test8_result");
29 ASSERT_EQ(skel->bss->kretprobe_test1_result, 1, "kretprobe_test1_result");
30 ASSERT_EQ(skel->bss->kretprobe_test2_result, 1, "kretprobe_test2_result");
31 ASSERT_EQ(skel->bss->kretprobe_test3_result, 1, "kretprobe_test3_result");
32 ASSERT_EQ(skel->bss->kretprobe_test4_result, 1, "kretprobe_test4_result");
33 ASSERT_EQ(skel->bss->kretprobe_test5_result, 1, "kretprobe_test5_result");
34 ASSERT_EQ(skel->bss->kretprobe_test6_result, 1, "kretprobe_test6_result");
35 ASSERT_EQ(skel->bss->kretprobe_test7_result, 1, "kretprobe_test7_result");
36 ASSERT_EQ(skel->bss->kretprobe_test8_result, 1, "kretprobe_test8_result");
40 static void test_skel_api(void)
42 struct kprobe_multi *skel = NULL;
45 skel = kprobe_multi__open_and_load();
46 if (!ASSERT_OK_PTR(skel, "kprobe_multi__open_and_load"))
49 skel->bss->pid = getpid();
50 err = kprobe_multi__attach(skel);
51 if (!ASSERT_OK(err, "kprobe_multi__attach"))
54 kprobe_multi_test_run(skel, true);
57 kprobe_multi__destroy(skel);
60 static void test_link_api(struct bpf_link_create_opts *opts)
62 int prog_fd, link1_fd = -1, link2_fd = -1;
63 struct kprobe_multi *skel = NULL;
65 skel = kprobe_multi__open_and_load();
66 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
69 skel->bss->pid = getpid();
70 prog_fd = bpf_program__fd(skel->progs.test_kprobe);
71 link1_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts);
72 if (!ASSERT_GE(link1_fd, 0, "link_fd"))
75 opts->kprobe_multi.flags = BPF_F_KPROBE_MULTI_RETURN;
76 prog_fd = bpf_program__fd(skel->progs.test_kretprobe);
77 link2_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts);
78 if (!ASSERT_GE(link2_fd, 0, "link_fd"))
81 kprobe_multi_test_run(skel, true);
88 kprobe_multi__destroy(skel);
91 #define GET_ADDR(__sym, __addr) ({ \
92 __addr = ksym_get_addr(__sym); \
93 if (!ASSERT_NEQ(__addr, 0, "kallsyms load failed for " #__sym)) \
97 static void test_link_api_addrs(void)
99 LIBBPF_OPTS(bpf_link_create_opts, opts);
100 unsigned long long addrs[8];
102 GET_ADDR("bpf_fentry_test1", addrs[0]);
103 GET_ADDR("bpf_fentry_test2", addrs[1]);
104 GET_ADDR("bpf_fentry_test3", addrs[2]);
105 GET_ADDR("bpf_fentry_test4", addrs[3]);
106 GET_ADDR("bpf_fentry_test5", addrs[4]);
107 GET_ADDR("bpf_fentry_test6", addrs[5]);
108 GET_ADDR("bpf_fentry_test7", addrs[6]);
109 GET_ADDR("bpf_fentry_test8", addrs[7]);
111 opts.kprobe_multi.addrs = (const unsigned long*) addrs;
112 opts.kprobe_multi.cnt = ARRAY_SIZE(addrs);
113 test_link_api(&opts);
116 static void test_link_api_syms(void)
118 LIBBPF_OPTS(bpf_link_create_opts, opts);
119 const char *syms[8] = {
130 opts.kprobe_multi.syms = syms;
131 opts.kprobe_multi.cnt = ARRAY_SIZE(syms);
132 test_link_api(&opts);
136 test_attach_api(const char *pattern, struct bpf_kprobe_multi_opts *opts)
138 struct bpf_link *link1 = NULL, *link2 = NULL;
139 struct kprobe_multi *skel = NULL;
141 skel = kprobe_multi__open_and_load();
142 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
145 skel->bss->pid = getpid();
146 link1 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
148 if (!ASSERT_OK_PTR(link1, "bpf_program__attach_kprobe_multi_opts"))
152 opts->retprobe = true;
153 link2 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kretprobe_manual,
155 if (!ASSERT_OK_PTR(link2, "bpf_program__attach_kprobe_multi_opts"))
159 kprobe_multi_test_run(skel, !!opts);
162 bpf_link__destroy(link2);
163 bpf_link__destroy(link1);
164 kprobe_multi__destroy(skel);
167 static void test_attach_api_pattern(void)
169 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
171 test_attach_api("bpf_fentry_test*", &opts);
172 test_attach_api("bpf_fentry_test?", NULL);
175 static void test_attach_api_addrs(void)
177 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
178 unsigned long long addrs[8];
180 GET_ADDR("bpf_fentry_test1", addrs[0]);
181 GET_ADDR("bpf_fentry_test2", addrs[1]);
182 GET_ADDR("bpf_fentry_test3", addrs[2]);
183 GET_ADDR("bpf_fentry_test4", addrs[3]);
184 GET_ADDR("bpf_fentry_test5", addrs[4]);
185 GET_ADDR("bpf_fentry_test6", addrs[5]);
186 GET_ADDR("bpf_fentry_test7", addrs[6]);
187 GET_ADDR("bpf_fentry_test8", addrs[7]);
189 opts.addrs = (const unsigned long *) addrs;
190 opts.cnt = ARRAY_SIZE(addrs);
191 test_attach_api(NULL, &opts);
194 static void test_attach_api_syms(void)
196 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
197 const char *syms[8] = {
209 opts.cnt = ARRAY_SIZE(syms);
210 test_attach_api(NULL, &opts);
213 static void test_attach_api_fails(void)
215 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
216 struct kprobe_multi *skel = NULL;
217 struct bpf_link *link = NULL;
218 unsigned long long addrs[2];
219 const char *syms[2] = {
225 addrs[0] = ksym_get_addr("bpf_fentry_test1");
226 addrs[1] = ksym_get_addr("bpf_fentry_test2");
228 if (!ASSERT_FALSE(!addrs[0] || !addrs[1], "ksym_get_addr"))
231 skel = kprobe_multi__open_and_load();
232 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
235 skel->bss->pid = getpid();
237 /* fail_1 - pattern and opts NULL */
238 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
240 if (!ASSERT_ERR_PTR(link, "fail_1"))
243 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_1_error"))
246 /* fail_2 - both addrs and syms set */
247 opts.addrs = (const unsigned long *) addrs;
249 opts.cnt = ARRAY_SIZE(syms);
252 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
254 if (!ASSERT_ERR_PTR(link, "fail_2"))
257 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_2_error"))
260 /* fail_3 - pattern and addrs set */
261 opts.addrs = (const unsigned long *) addrs;
263 opts.cnt = ARRAY_SIZE(syms);
266 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
268 if (!ASSERT_ERR_PTR(link, "fail_3"))
271 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_3_error"))
274 /* fail_4 - pattern and cnt set */
277 opts.cnt = ARRAY_SIZE(syms);
280 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
282 if (!ASSERT_ERR_PTR(link, "fail_4"))
285 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_4_error"))
288 /* fail_5 - pattern and cookies */
292 opts.cookies = cookies;
294 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
296 if (!ASSERT_ERR_PTR(link, "fail_5"))
299 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_5_error"))
303 bpf_link__destroy(link);
304 kprobe_multi__destroy(skel);
307 static inline __u64 get_time_ns(void)
311 clock_gettime(CLOCK_MONOTONIC, &t);
312 return (__u64) t.tv_sec * 1000000000 + t.tv_nsec;
315 static size_t symbol_hash(const void *key, void *ctx __maybe_unused)
317 return str_hash((const char *) key);
320 static bool symbol_equal(const void *key1, const void *key2, void *ctx __maybe_unused)
322 return strcmp((const char *) key1, (const char *) key2) == 0;
325 static int get_syms(char ***symsp, size_t *cntp)
327 size_t cap = 0, cnt = 0, i;
328 char *name, **syms = NULL;
335 * The available_filter_functions contains many duplicates,
336 * but other than that all symbols are usable in kprobe multi
338 * Filtering out duplicates by using hashmap__add, which won't
339 * add existing entry.
341 f = fopen("/sys/kernel/debug/tracing/available_filter_functions", "r");
345 map = hashmap__new(symbol_hash, symbol_equal, NULL);
347 err = libbpf_get_error(map);
351 while (fgets(buf, sizeof(buf), f)) {
353 if (strchr(buf, '['))
355 if (sscanf(buf, "%ms$*[^\n]\n", &name) != 1)
358 * We attach to almost all kernel functions and some of them
359 * will cause 'suspicious RCU usage' when fprobe is attached
360 * to them. Filter out the current culprits - arch_cpu_idle
361 * and rcu_* functions.
363 if (!strcmp(name, "arch_cpu_idle"))
365 if (!strncmp(name, "rcu_", 4))
367 if (!strcmp(name, "bpf_dispatcher_xdp_func"))
369 if (!strncmp(name, "__ftrace_invalid_address__",
370 sizeof("__ftrace_invalid_address__") - 1))
372 err = hashmap__add(map, name, NULL);
379 err = libbpf_ensure_mem((void **) &syms, &cap,
380 sizeof(*syms), cnt + 1);
396 for (i = 0; i < cnt; i++)
403 static void test_bench_attach(void)
405 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
406 struct kprobe_multi_empty *skel = NULL;
407 long attach_start_ns, attach_end_ns;
408 long detach_start_ns, detach_end_ns;
409 double attach_delta, detach_delta;
410 struct bpf_link *link = NULL;
414 if (!ASSERT_OK(get_syms(&syms, &cnt), "get_syms"))
417 skel = kprobe_multi_empty__open_and_load();
418 if (!ASSERT_OK_PTR(skel, "kprobe_multi_empty__open_and_load"))
421 opts.syms = (const char **) syms;
424 attach_start_ns = get_time_ns();
425 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_empty,
427 attach_end_ns = get_time_ns();
429 if (!ASSERT_OK_PTR(link, "bpf_program__attach_kprobe_multi_opts"))
432 detach_start_ns = get_time_ns();
433 bpf_link__destroy(link);
434 detach_end_ns = get_time_ns();
436 attach_delta = (attach_end_ns - attach_start_ns) / 1000000000.0;
437 detach_delta = (detach_end_ns - detach_start_ns) / 1000000000.0;
439 printf("%s: found %lu functions\n", __func__, cnt);
440 printf("%s: attached in %7.3lfs\n", __func__, attach_delta);
441 printf("%s: detached in %7.3lfs\n", __func__, detach_delta);
444 kprobe_multi_empty__destroy(skel);
446 for (i = 0; i < cnt; i++)
452 void test_kprobe_multi_test(void)
454 if (!ASSERT_OK(load_kallsyms(), "load_kallsyms"))
457 if (test__start_subtest("skel_api"))
459 if (test__start_subtest("link_api_addrs"))
460 test_link_api_syms();
461 if (test__start_subtest("link_api_syms"))
462 test_link_api_addrs();
463 if (test__start_subtest("attach_api_pattern"))
464 test_attach_api_pattern();
465 if (test__start_subtest("attach_api_addrs"))
466 test_attach_api_addrs();
467 if (test__start_subtest("attach_api_syms"))
468 test_attach_api_syms();
469 if (test__start_subtest("attach_api_fails"))
470 test_attach_api_fails();
471 if (test__start_subtest("bench_attach"))