tools/testing/selftests/bpf/prog_tests/bpf_iter.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2020 Facebook */
   3 #include <test_progs.h>
   4 #include <unistd.h>
   5 #include <sys/syscall.h>
   6 #include <task_local_storage_helpers.h>
   7 #include "bpf_iter_ipv6_route.skel.h"
   8 #include "bpf_iter_netlink.skel.h"
   9 #include "bpf_iter_bpf_map.skel.h"
  10 #include "bpf_iter_tasks.skel.h"
  11 #include "bpf_iter_task_stack.skel.h"
  12 #include "bpf_iter_task_file.skel.h"
  13 #include "bpf_iter_task_vmas.skel.h"
  14 #include "bpf_iter_task_btf.skel.h"
  15 #include "bpf_iter_tcp4.skel.h"
  16 #include "bpf_iter_tcp6.skel.h"
  17 #include "bpf_iter_udp4.skel.h"
  18 #include "bpf_iter_udp6.skel.h"
  19 #include "bpf_iter_unix.skel.h"
  20 #include "bpf_iter_vma_offset.skel.h"
  21 #include "bpf_iter_test_kern1.skel.h"
  22 #include "bpf_iter_test_kern2.skel.h"
  23 #include "bpf_iter_test_kern3.skel.h"
  24 #include "bpf_iter_test_kern4.skel.h"
  25 #include "bpf_iter_bpf_hash_map.skel.h"
  26 #include "bpf_iter_bpf_percpu_hash_map.skel.h"
  27 #include "bpf_iter_bpf_array_map.skel.h"
  28 #include "bpf_iter_bpf_percpu_array_map.skel.h"
  29 #include "bpf_iter_bpf_sk_storage_helpers.skel.h"
  30 #include "bpf_iter_bpf_sk_storage_map.skel.h"
  31 #include "bpf_iter_test_kern5.skel.h"
  32 #include "bpf_iter_test_kern6.skel.h"
  33 #include "bpf_iter_bpf_link.skel.h"
  34 #include "bpf_iter_ksym.skel.h"
  35 #include "bpf_iter_sockmap.skel.h"
  36
  37 static void test_btf_id_or_null(void)
  38 {
  39         struct bpf_iter_test_kern3 *skel;
  40
  41         skel = bpf_iter_test_kern3__open_and_load();
  42         if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern3__open_and_load")) {
  43                 bpf_iter_test_kern3__destroy(skel);
  44                 return;
  45         }
  46 }
  47
  48 static void do_dummy_read_opts(struct bpf_program *prog, struct bpf_iter_attach_opts *opts)
  49 {
  50         struct bpf_link *link;
  51         char buf[16] = {};
  52         int iter_fd, len;
  53
  54         link = bpf_program__attach_iter(prog, opts);
  55         if (!ASSERT_OK_PTR(link, "attach_iter"))
  56                 return;
  57
  58         iter_fd = bpf_iter_create(bpf_link__fd(link));
  59         if (!ASSERT_GE(iter_fd, 0, "create_iter"))
  60                 goto free_link;
  61
  62         /* not check contents, but ensure read() ends without error */
  63         while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
  64                 ;
  65         ASSERT_GE(len, 0, "read");
  66
  67         close(iter_fd);
  68
  69 free_link:
  70         bpf_link__destroy(link);
  71 }
  72
  73 static void do_dummy_read(struct bpf_program *prog)
  74 {
  75         do_dummy_read_opts(prog, NULL);
  76 }
  77
  78 static void do_read_map_iter_fd(struct bpf_object_skeleton **skel, struct bpf_program *prog,
  79                                 struct bpf_map *map)
  80 {
  81         DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
  82         union bpf_iter_link_info linfo;
  83         struct bpf_link *link;
  84         char buf[16] = {};
  85         int iter_fd, len;
  86
  87         memset(&linfo, 0, sizeof(linfo));
  88         linfo.map.map_fd = bpf_map__fd(map);
  89         opts.link_info = &linfo;
  90         opts.link_info_len = sizeof(linfo);
  91         link = bpf_program__attach_iter(prog, &opts);
  92         if (!ASSERT_OK_PTR(link, "attach_map_iter"))
  93                 return;
  94
  95         iter_fd = bpf_iter_create(bpf_link__fd(link));
  96         if (!ASSERT_GE(iter_fd, 0, "create_map_iter")) {
  97                 bpf_link__destroy(link);
  98                 return;
  99         }
 100
 101         /* Close link and map fd prematurely */
 102         bpf_link__destroy(link);
 103         bpf_object__destroy_skeleton(*skel);
 104         *skel = NULL;
 105
 106         /* Try to let map free work to run first if map is freed */
 107         usleep(100);
 108         /* Memory used by both sock map and sock local storage map are
 109          * freed after two synchronize_rcu() calls, so wait for it
 110          */
 111         kern_sync_rcu();
 112         kern_sync_rcu();
 113
 114         /* Read after both map fd and link fd are closed */
 115         while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
 116                 ;
 117         ASSERT_GE(len, 0, "read_iterator");
 118
 119         close(iter_fd);
 120 }
 121
 122 static int read_fd_into_buffer(int fd, char *buf, int size)
 123 {
 124         int bufleft = size;
 125         int len;
 126
 127         do {
 128                 len = read(fd, buf, bufleft);
 129                 if (len > 0) {
 130                         buf += len;
 131                         bufleft -= len;
 132                 }
 133         } while (len > 0);
 134
 135         return len < 0 ? len : size - bufleft;
 136 }
 137
 138 static void test_ipv6_route(void)
 139 {
 140         struct bpf_iter_ipv6_route *skel;
 141
 142         skel = bpf_iter_ipv6_route__open_and_load();
 143         if (!ASSERT_OK_PTR(skel, "bpf_iter_ipv6_route__open_and_load"))
 144                 return;
 145
 146         do_dummy_read(skel->progs.dump_ipv6_route);
 147
 148         bpf_iter_ipv6_route__destroy(skel);
 149 }
 150
 151 static void test_netlink(void)
 152 {
 153         struct bpf_iter_netlink *skel;
 154
 155         skel = bpf_iter_netlink__open_and_load();
 156         if (!ASSERT_OK_PTR(skel, "bpf_iter_netlink__open_and_load"))
 157                 return;
 158
 159         do_dummy_read(skel->progs.dump_netlink);
 160
 161         bpf_iter_netlink__destroy(skel);
 162 }
 163
 164 static void test_bpf_map(void)
 165 {
 166         struct bpf_iter_bpf_map *skel;
 167
 168         skel = bpf_iter_bpf_map__open_and_load();
 169         if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_map__open_and_load"))
 170                 return;
 171
 172         do_dummy_read(skel->progs.dump_bpf_map);
 173
 174         bpf_iter_bpf_map__destroy(skel);
 175 }
 176
 177 static void check_bpf_link_info(const struct bpf_program *prog)
 178 {
 179         LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 180         union bpf_iter_link_info linfo;
 181         struct bpf_link_info info = {};
 182         struct bpf_link *link;
 183         __u32 info_len;
 184         int err;
 185
 186         memset(&linfo, 0, sizeof(linfo));
 187         linfo.task.tid = getpid();
 188         opts.link_info = &linfo;
 189         opts.link_info_len = sizeof(linfo);
 190
 191         link = bpf_program__attach_iter(prog, &opts);
 192         if (!ASSERT_OK_PTR(link, "attach_iter"))
 193                 return;
 194
 195         info_len = sizeof(info);
 196         err = bpf_link_get_info_by_fd(bpf_link__fd(link), &info, &info_len);
 197         ASSERT_OK(err, "bpf_link_get_info_by_fd");
 198         ASSERT_EQ(info.iter.task.tid, getpid(), "check_task_tid");
 199
 200         bpf_link__destroy(link);
 201 }
 202
 203 static pthread_mutex_t do_nothing_mutex;
 204
 205 static void *do_nothing_wait(void *arg)
 206 {
 207         pthread_mutex_lock(&do_nothing_mutex);
 208         pthread_mutex_unlock(&do_nothing_mutex);
 209
 210         pthread_exit(arg);
 211 }
 212
 213 static void test_task_common_nocheck(struct bpf_iter_attach_opts *opts,
 214                                      int *num_unknown, int *num_known)
 215 {
 216         struct bpf_iter_tasks *skel;
 217         pthread_t thread_id;
 218         void *ret;
 219
 220         skel = bpf_iter_tasks__open_and_load();
 221         if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load"))
 222                 return;
 223
 224         ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock");
 225
 226         ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),
 227                   "pthread_create");
 228
 229         skel->bss->tid = getpid();
 230
 231         do_dummy_read_opts(skel->progs.dump_task, opts);
 232
 233         *num_unknown = skel->bss->num_unknown_tid;
 234         *num_known = skel->bss->num_known_tid;
 235
 236         ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock");
 237         ASSERT_FALSE(pthread_join(thread_id, &ret) || ret != NULL,
 238                      "pthread_join");
 239
 240         bpf_iter_tasks__destroy(skel);
 241 }
 242
 243 static void test_task_common(struct bpf_iter_attach_opts *opts, int num_unknown, int num_known)
 244 {
 245         int num_unknown_tid, num_known_tid;
 246
 247         test_task_common_nocheck(opts, &num_unknown_tid, &num_known_tid);
 248         ASSERT_EQ(num_unknown_tid, num_unknown, "check_num_unknown_tid");
 249         ASSERT_EQ(num_known_tid, num_known, "check_num_known_tid");
 250 }
 251
 252 static void test_task_tid(void)
 253 {
 254         LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 255         union bpf_iter_link_info linfo;
 256         int num_unknown_tid, num_known_tid;
 257
 258         memset(&linfo, 0, sizeof(linfo));
 259         linfo.task.tid = getpid();
 260         opts.link_info = &linfo;
 261         opts.link_info_len = sizeof(linfo);
 262         test_task_common(&opts, 0, 1);
 263
 264         linfo.task.tid = 0;
 265         linfo.task.pid = getpid();
 266         test_task_common(&opts, 1, 1);
 267
 268         test_task_common_nocheck(NULL, &num_unknown_tid, &num_known_tid);
 269         ASSERT_GT(num_unknown_tid, 1, "check_num_unknown_tid");
 270         ASSERT_EQ(num_known_tid, 1, "check_num_known_tid");
 271 }
 272
 273 static void test_task_pid(void)
 274 {
 275         LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 276         union bpf_iter_link_info linfo;
 277
 278         memset(&linfo, 0, sizeof(linfo));
 279         linfo.task.pid = getpid();
 280         opts.link_info = &linfo;
 281         opts.link_info_len = sizeof(linfo);
 282
 283         test_task_common(&opts, 1, 1);
 284 }
 285
 286 static void test_task_pidfd(void)
 287 {
 288         LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 289         union bpf_iter_link_info linfo;
 290         int pidfd;
 291
 292         pidfd = sys_pidfd_open(getpid(), 0);
 293         if (!ASSERT_GT(pidfd, 0, "sys_pidfd_open"))
 294                 return;
 295
 296         memset(&linfo, 0, sizeof(linfo));
 297         linfo.task.pid_fd = pidfd;
 298         opts.link_info = &linfo;
 299         opts.link_info_len = sizeof(linfo);
 300
 301         test_task_common(&opts, 1, 1);
 302
 303         close(pidfd);
 304 }
 305
 306 static void test_task_sleepable(void)
 307 {
 308         struct bpf_iter_tasks *skel;
 309
 310         skel = bpf_iter_tasks__open_and_load();
 311         if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load"))
 312                 return;
 313
 314         do_dummy_read(skel->progs.dump_task_sleepable);
 315
 316         ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task, 0,
 317                   "num_expected_failure_copy_from_user_task");
 318         ASSERT_GT(skel->bss->num_success_copy_from_user_task, 0,
 319                   "num_success_copy_from_user_task");
 320
 321         bpf_iter_tasks__destroy(skel);
 322 }
 323
 324 static void test_task_stack(void)
 325 {
 326         struct bpf_iter_task_stack *skel;
 327
 328         skel = bpf_iter_task_stack__open_and_load();
 329         if (!ASSERT_OK_PTR(skel, "bpf_iter_task_stack__open_and_load"))
 330                 return;
 331
 332         do_dummy_read(skel->progs.dump_task_stack);
 333         do_dummy_read(skel->progs.get_task_user_stacks);
 334
 335         ASSERT_EQ(skel->bss->num_user_stacks, 1, "num_user_stacks");
 336
 337         bpf_iter_task_stack__destroy(skel);
 338 }
 339
 340 static void test_task_file(void)
 341 {
 342         LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 343         struct bpf_iter_task_file *skel;
 344         union bpf_iter_link_info linfo;
 345         pthread_t thread_id;
 346         void *ret;
 347
 348         skel = bpf_iter_task_file__open_and_load();
 349         if (!ASSERT_OK_PTR(skel, "bpf_iter_task_file__open_and_load"))
 350                 return;
 351
 352         skel->bss->tgid = getpid();
 353
 354         ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock");
 355
 356         ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),
 357                   "pthread_create");
 358
 359         memset(&linfo, 0, sizeof(linfo));
 360         linfo.task.tid = getpid();
 361         opts.link_info = &linfo;
 362         opts.link_info_len = sizeof(linfo);
 363
 364         do_dummy_read_opts(skel->progs.dump_task_file, &opts);
 365
 366         ASSERT_EQ(skel->bss->count, 0, "check_count");
 367         ASSERT_EQ(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count");
 368
 369         skel->bss->last_tgid = 0;
 370         skel->bss->count = 0;
 371         skel->bss->unique_tgid_count = 0;
 372
 373         do_dummy_read(skel->progs.dump_task_file);
 374
 375         ASSERT_EQ(skel->bss->count, 0, "check_count");
 376         ASSERT_GT(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count");
 377
 378         check_bpf_link_info(skel->progs.dump_task_file);
 379
 380         ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock");
 381         ASSERT_OK(pthread_join(thread_id, &ret), "pthread_join");
 382         ASSERT_NULL(ret, "pthread_join");
 383
 384         bpf_iter_task_file__destroy(skel);
 385 }
 386
 387 #define TASKBUFSZ               32768
 388
 389 static char taskbuf[TASKBUFSZ];
 390
 391 static int do_btf_read(struct bpf_iter_task_btf *skel)
 392 {
 393         struct bpf_program *prog = skel->progs.dump_task_struct;
 394         struct bpf_iter_task_btf__bss *bss = skel->bss;
 395         int iter_fd = -1, err;
 396         struct bpf_link *link;
 397         char *buf = taskbuf;
 398         int ret = 0;
 399
 400         link = bpf_program__attach_iter(prog, NULL);
 401         if (!ASSERT_OK_PTR(link, "attach_iter"))
 402                 return ret;
 403
 404         iter_fd = bpf_iter_create(bpf_link__fd(link));
 405         if (!ASSERT_GE(iter_fd, 0, "create_iter"))
 406                 goto free_link;
 407
 408         err = read_fd_into_buffer(iter_fd, buf, TASKBUFSZ);
 409         if (bss->skip) {
 410                 printf("%s:SKIP:no __builtin_btf_type_id\n", __func__);
 411                 ret = 1;
 412                 test__skip();
 413                 goto free_link;
 414         }
 415
 416         if (!ASSERT_GE(err, 0, "read"))
 417                 goto free_link;
 418
 419         ASSERT_HAS_SUBSTR(taskbuf, "(struct task_struct)",
 420               "check for btf representation of task_struct in iter data");
 421 free_link:
 422         if (iter_fd > 0)
 423                 close(iter_fd);
 424         bpf_link__destroy(link);
 425         return ret;
 426 }
 427
 428 static void test_task_btf(void)
 429 {
 430         struct bpf_iter_task_btf__bss *bss;
 431         struct bpf_iter_task_btf *skel;
 432         int ret;
 433
 434         skel = bpf_iter_task_btf__open_and_load();
 435         if (!ASSERT_OK_PTR(skel, "bpf_iter_task_btf__open_and_load"))
 436                 return;
 437
 438         bss = skel->bss;
 439
 440         ret = do_btf_read(skel);
 441         if (ret)
 442                 goto cleanup;
 443
 444         if (!ASSERT_NEQ(bss->tasks, 0, "no task iteration, did BPF program run?"))
 445                 goto cleanup;
 446
 447         ASSERT_EQ(bss->seq_err, 0, "check for unexpected err");
 448
 449 cleanup:
 450         bpf_iter_task_btf__destroy(skel);
 451 }
 452
 453 static void test_tcp4(void)
 454 {
 455         struct bpf_iter_tcp4 *skel;
 456
 457         skel = bpf_iter_tcp4__open_and_load();
 458         if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp4__open_and_load"))
 459                 return;
 460
 461         do_dummy_read(skel->progs.dump_tcp4);
 462
 463         bpf_iter_tcp4__destroy(skel);
 464 }
 465
 466 static void test_tcp6(void)
 467 {
 468         struct bpf_iter_tcp6 *skel;
 469
 470         skel = bpf_iter_tcp6__open_and_load();
 471         if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp6__open_and_load"))
 472                 return;
 473
 474         do_dummy_read(skel->progs.dump_tcp6);
 475
 476         bpf_iter_tcp6__destroy(skel);
 477 }
 478
 479 static void test_udp4(void)
 480 {
 481         struct bpf_iter_udp4 *skel;
 482
 483         skel = bpf_iter_udp4__open_and_load();
 484         if (!ASSERT_OK_PTR(skel, "bpf_iter_udp4__open_and_load"))
 485                 return;
 486
 487         do_dummy_read(skel->progs.dump_udp4);
 488
 489         bpf_iter_udp4__destroy(skel);
 490 }
 491
 492 static void test_udp6(void)
 493 {
 494         struct bpf_iter_udp6 *skel;
 495
 496         skel = bpf_iter_udp6__open_and_load();
 497         if (!ASSERT_OK_PTR(skel, "bpf_iter_udp6__open_and_load"))
 498                 return;
 499
 500         do_dummy_read(skel->progs.dump_udp6);
 501
 502         bpf_iter_udp6__destroy(skel);
 503 }
 504
 505 static void test_unix(void)
 506 {
 507         struct bpf_iter_unix *skel;
 508
 509         skel = bpf_iter_unix__open_and_load();
 510         if (!ASSERT_OK_PTR(skel, "bpf_iter_unix__open_and_load"))
 511                 return;
 512
 513         do_dummy_read(skel->progs.dump_unix);
 514
 515         bpf_iter_unix__destroy(skel);
 516 }
 517
 518 /* The expected string is less than 16 bytes */
 519 static int do_read_with_fd(int iter_fd, const char *expected,
 520                            bool read_one_char)
 521 {
 522         int len, read_buf_len, start;
 523         char buf[16] = {};
 524
 525         read_buf_len = read_one_char ? 1 : 16;
 526         start = 0;
 527         while ((len = read(iter_fd, buf + start, read_buf_len)) > 0) {
 528                 start += len;
 529                 if (!ASSERT_LT(start, 16, "read"))
 530                         return -1;
 531                 read_buf_len = read_one_char ? 1 : 16 - start;
 532         }
 533         if (!ASSERT_GE(len, 0, "read"))
 534                 return -1;
 535
 536         if (!ASSERT_STREQ(buf, expected, "read"))
 537                 return -1;
 538
 539         return 0;
 540 }
 541
 542 static void test_anon_iter(bool read_one_char)
 543 {
 544         struct bpf_iter_test_kern1 *skel;
 545         struct bpf_link *link;
 546         int iter_fd, err;
 547
 548         skel = bpf_iter_test_kern1__open_and_load();
 549         if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern1__open_and_load"))
 550                 return;
 551
 552         err = bpf_iter_test_kern1__attach(skel);
 553         if (!ASSERT_OK(err, "bpf_iter_test_kern1__attach")) {
 554                 goto out;
 555         }
 556
 557         link = skel->links.dump_task;
 558         iter_fd = bpf_iter_create(bpf_link__fd(link));
 559         if (!ASSERT_GE(iter_fd, 0, "create_iter"))
 560                 goto out;
 561
 562         do_read_with_fd(iter_fd, "abcd", read_one_char);
 563         close(iter_fd);
 564
 565 out:
 566         bpf_iter_test_kern1__destroy(skel);
 567 }
 568
 569 static int do_read(const char *path, const char *expected)
 570 {
 571         int err, iter_fd;
 572
 573         iter_fd = open(path, O_RDONLY);
 574         if (!ASSERT_GE(iter_fd, 0, "open"))
 575                 return -1;
 576
 577         err = do_read_with_fd(iter_fd, expected, false);
 578         close(iter_fd);
 579         return err;
 580 }
 581
 582 static void test_file_iter(void)
 583 {
 584         const char *path = "/sys/fs/bpf/bpf_iter_test1";
 585         struct bpf_iter_test_kern1 *skel1;
 586         struct bpf_iter_test_kern2 *skel2;
 587         struct bpf_link *link;
 588         int err;
 589
 590         skel1 = bpf_iter_test_kern1__open_and_load();
 591         if (!ASSERT_OK_PTR(skel1, "bpf_iter_test_kern1__open_and_load"))
 592                 return;
 593
 594         link = bpf_program__attach_iter(skel1->progs.dump_task, NULL);
 595         if (!ASSERT_OK_PTR(link, "attach_iter"))
 596                 goto out;
 597
 598         /* unlink this path if it exists. */
 599         unlink(path);
 600
 601         err = bpf_link__pin(link, path);
 602         if (!ASSERT_OK(err, "pin_iter"))
 603                 goto free_link;
 604
 605         err = do_read(path, "abcd");
 606         if (err)
 607                 goto unlink_path;
 608
 609         /* file based iterator seems working fine. Let us a link update
 610          * of the underlying link and `cat` the iterator again, its content
 611          * should change.
 612          */
 613         skel2 = bpf_iter_test_kern2__open_and_load();
 614         if (!ASSERT_OK_PTR(skel2, "bpf_iter_test_kern2__open_and_load"))
 615                 goto unlink_path;
 616
 617         err = bpf_link__update_program(link, skel2->progs.dump_task);
 618         if (!ASSERT_OK(err, "update_prog"))
 619                 goto destroy_skel2;
 620
 621         do_read(path, "ABCD");
 622
 623 destroy_skel2:
 624         bpf_iter_test_kern2__destroy(skel2);
 625 unlink_path:
 626         unlink(path);
 627 free_link:
 628         bpf_link__destroy(link);
 629 out:
 630         bpf_iter_test_kern1__destroy(skel1);
 631 }
 632
 633 static void test_overflow(bool test_e2big_overflow, bool ret1)
 634 {
 635         __u32 map_info_len, total_read_len, expected_read_len;
 636         int err, iter_fd, map1_fd, map2_fd, len;
 637         struct bpf_map_info map_info = {};
 638         struct bpf_iter_test_kern4 *skel;
 639         struct bpf_link *link;
 640         __u32 iter_size;
 641         char *buf;
 642
 643         skel = bpf_iter_test_kern4__open();
 644         if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern4__open"))
 645                 return;
 646
 647         /* create two maps: bpf program will only do bpf_seq_write
 648          * for these two maps. The goal is one map output almost
 649          * fills seq_file buffer and then the other will trigger
 650          * overflow and needs restart.
 651          */
 652         map1_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL);
 653         if (!ASSERT_GE(map1_fd, 0, "bpf_map_create"))
 654                 goto out;
 655         map2_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL);
 656         if (!ASSERT_GE(map2_fd, 0, "bpf_map_create"))
 657                 goto free_map1;
 658
 659         /* bpf_seq_printf kernel buffer is 8 pages, so one map
 660          * bpf_seq_write will mostly fill it, and the other map
 661          * will partially fill and then trigger overflow and need
 662          * bpf_seq_read restart.
 663          */
 664         iter_size = sysconf(_SC_PAGE_SIZE) << 3;
 665
 666         if (test_e2big_overflow) {
 667                 skel->rodata->print_len = (iter_size + 8) / 8;
 668                 expected_read_len = 2 * (iter_size + 8);
 669         } else if (!ret1) {
 670                 skel->rodata->print_len = (iter_size - 8) / 8;
 671                 expected_read_len = 2 * (iter_size - 8);
 672         } else {
 673                 skel->rodata->print_len = 1;
 674                 expected_read_len = 2 * 8;
 675         }
 676         skel->rodata->ret1 = ret1;
 677
 678         if (!ASSERT_OK(bpf_iter_test_kern4__load(skel),
 679                   "bpf_iter_test_kern4__load"))
 680                 goto free_map2;
 681
 682         /* setup filtering map_id in bpf program */
 683         map_info_len = sizeof(map_info);
 684         err = bpf_map_get_info_by_fd(map1_fd, &map_info, &map_info_len);
 685         if (!ASSERT_OK(err, "get_map_info"))
 686                 goto free_map2;
 687         skel->bss->map1_id = map_info.id;
 688
 689         err = bpf_map_get_info_by_fd(map2_fd, &map_info, &map_info_len);
 690         if (!ASSERT_OK(err, "get_map_info"))
 691                 goto free_map2;
 692         skel->bss->map2_id = map_info.id;
 693
 694         link = bpf_program__attach_iter(skel->progs.dump_bpf_map, NULL);
 695         if (!ASSERT_OK_PTR(link, "attach_iter"))
 696                 goto free_map2;
 697
 698         iter_fd = bpf_iter_create(bpf_link__fd(link));
 699         if (!ASSERT_GE(iter_fd, 0, "create_iter"))
 700                 goto free_link;
 701
 702         buf = malloc(expected_read_len);
 703         if (!ASSERT_OK_PTR(buf, "malloc"))
 704                 goto close_iter;
 705
 706         /* do read */
 707         total_read_len = 0;
 708         if (test_e2big_overflow) {
 709                 while ((len = read(iter_fd, buf, expected_read_len)) > 0)
 710                         total_read_len += len;
 711
 712                 ASSERT_EQ(len, -1, "read");
 713                 ASSERT_EQ(errno, E2BIG, "read");
 714                 goto free_buf;
 715         } else if (!ret1) {
 716                 while ((len = read(iter_fd, buf, expected_read_len)) > 0)
 717                         total_read_len += len;
 718
 719                 if (!ASSERT_GE(len, 0, "read"))
 720                         goto free_buf;
 721         } else {
 722                 do {
 723                         len = read(iter_fd, buf, expected_read_len);
 724                         if (len > 0)
 725                                 total_read_len += len;
 726                 } while (len > 0 || len == -EAGAIN);
 727
 728                 if (!ASSERT_GE(len, 0, "read"))
 729                         goto free_buf;
 730         }
 731
 732         if (!ASSERT_EQ(total_read_len, expected_read_len, "read"))
 733                 goto free_buf;
 734
 735         if (!ASSERT_EQ(skel->bss->map1_accessed, 1, "map1_accessed"))
 736                 goto free_buf;
 737
 738         if (!ASSERT_EQ(skel->bss->map2_accessed, 2, "map2_accessed"))
 739                 goto free_buf;
 740
 741         ASSERT_EQ(skel->bss->map2_seqnum1, skel->bss->map2_seqnum2, "map2_seqnum");
 742
 743 free_buf:
 744         free(buf);
 745 close_iter:
 746         close(iter_fd);
 747 free_link:
 748         bpf_link__destroy(link);
 749 free_map2:
 750         close(map2_fd);
 751 free_map1:
 752         close(map1_fd);
 753 out:
 754         bpf_iter_test_kern4__destroy(skel);
 755 }
 756
 757 static void test_bpf_hash_map(void)
 758 {
 759         __u32 expected_key_a = 0, expected_key_b = 0;
 760         DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 761         struct bpf_iter_bpf_hash_map *skel;
 762         int err, i, len, map_fd, iter_fd;
 763         union bpf_iter_link_info linfo;
 764         __u64 val, expected_val = 0;
 765         struct bpf_link *link;
 766         struct key_t {
 767                 int a;
 768                 int b;
 769                 int c;
 770         } key;
 771         char buf[64];
 772
 773         skel = bpf_iter_bpf_hash_map__open();
 774         if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_hash_map__open"))
 775                 return;
 776
 777         skel->bss->in_test_mode = true;
 778
 779         err = bpf_iter_bpf_hash_map__load(skel);
 780         if (!ASSERT_OK(err, "bpf_iter_bpf_hash_map__load"))
 781                 goto out;
 782
 783         /* iterator with hashmap2 and hashmap3 should fail */
 784         memset(&linfo, 0, sizeof(linfo));
 785         linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap2);
 786         opts.link_info = &linfo;
 787         opts.link_info_len = sizeof(linfo);
 788         link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
 789         if (!ASSERT_ERR_PTR(link, "attach_iter"))
 790                 goto out;
 791
 792         linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap3);
 793         link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
 794         if (!ASSERT_ERR_PTR(link, "attach_iter"))
 795                 goto out;
 796
 797         /* hashmap1 should be good, update map values here */
 798         map_fd = bpf_map__fd(skel->maps.hashmap1);
 799         for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) {
 800                 key.a = i + 1;
 801                 key.b = i + 2;
 802                 key.c = i + 3;
 803                 val = i + 4;
 804                 expected_key_a += key.a;
 805                 expected_key_b += key.b;
 806                 expected_val += val;
 807
 808                 err = bpf_map_update_elem(map_fd, &key, &val, BPF_ANY);
 809                 if (!ASSERT_OK(err, "map_update"))
 810                         goto out;
 811         }
 812
 813         /* Sleepable program is prohibited for hash map iterator */
 814         linfo.map.map_fd = map_fd;
 815         link = bpf_program__attach_iter(skel->progs.sleepable_dummy_dump, &opts);
 816         if (!ASSERT_ERR_PTR(link, "attach_sleepable_prog_to_iter"))
 817                 goto out;
 818
 819         linfo.map.map_fd = map_fd;
 820         link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
 821         if (!ASSERT_OK_PTR(link, "attach_iter"))
 822                 goto out;
 823
 824         iter_fd = bpf_iter_create(bpf_link__fd(link));
 825         if (!ASSERT_GE(iter_fd, 0, "create_iter"))
 826                 goto free_link;
 827
 828         /* do some tests */
 829         while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
 830                 ;
 831         if (!ASSERT_GE(len, 0, "read"))
 832                 goto close_iter;
 833
 834         /* test results */
 835         if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a"))
 836                 goto close_iter;
 837         if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b"))
 838                 goto close_iter;
 839         if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
 840                 goto close_iter;
 841
 842 close_iter:
 843         close(iter_fd);
 844 free_link:
 845         bpf_link__destroy(link);
 846 out:
 847         bpf_iter_bpf_hash_map__destroy(skel);
 848 }
 849
 850 static void test_bpf_percpu_hash_map(void)
 851 {
 852         __u32 expected_key_a = 0, expected_key_b = 0;
 853         DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 854         struct bpf_iter_bpf_percpu_hash_map *skel;
 855         int err, i, j, len, map_fd, iter_fd;
 856         union bpf_iter_link_info linfo;
 857         __u32 expected_val = 0;
 858         struct bpf_link *link;
 859         struct key_t {
 860                 int a;
 861                 int b;
 862                 int c;
 863         } key;
 864         char buf[64];
 865         void *val;
 866
 867         skel = bpf_iter_bpf_percpu_hash_map__open();
 868         if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__open"))
 869                 return;
 870
 871         skel->rodata->num_cpus = bpf_num_possible_cpus();
 872         val = malloc(8 * bpf_num_possible_cpus());
 873         if (!ASSERT_OK_PTR(val, "malloc"))
 874                 goto out;
 875
 876         err = bpf_iter_bpf_percpu_hash_map__load(skel);
 877         if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__load"))
 878                 goto out;
 879
 880         /* update map values here */
 881         map_fd = bpf_map__fd(skel->maps.hashmap1);
 882         for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) {
 883                 key.a = i + 1;
 884                 key.b = i + 2;
 885                 key.c = i + 3;
 886                 expected_key_a += key.a;
 887                 expected_key_b += key.b;
 888
 889                 for (j = 0; j < bpf_num_possible_cpus(); j++) {
 890                         *(__u32 *)(val + j * 8) = i + j;
 891                         expected_val += i + j;
 892                 }
 893
 894                 err = bpf_map_update_elem(map_fd, &key, val, BPF_ANY);
 895                 if (!ASSERT_OK(err, "map_update"))
 896                         goto out;
 897         }
 898
 899         memset(&linfo, 0, sizeof(linfo));
 900         linfo.map.map_fd = map_fd;
 901         opts.link_info = &linfo;
 902         opts.link_info_len = sizeof(linfo);
 903         link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_hash_map, &opts);
 904         if (!ASSERT_OK_PTR(link, "attach_iter"))
 905                 goto out;
 906
 907         iter_fd = bpf_iter_create(bpf_link__fd(link));
 908         if (!ASSERT_GE(iter_fd, 0, "create_iter"))
 909                 goto free_link;
 910
 911         /* do some tests */
 912         while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
 913                 ;
 914         if (!ASSERT_GE(len, 0, "read"))
 915                 goto close_iter;
 916
 917         /* test results */
 918         if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a"))
 919                 goto close_iter;
 920         if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b"))
 921                 goto close_iter;
 922         if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
 923                 goto close_iter;
 924
 925 close_iter:
 926         close(iter_fd);
 927 free_link:
 928         bpf_link__destroy(link);
 929 out:
 930         bpf_iter_bpf_percpu_hash_map__destroy(skel);
 931         free(val);
 932 }
 933
 934 static void test_bpf_array_map(void)
 935 {
 936         __u64 val, expected_val = 0, res_first_val, first_val = 0;
 937         DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 938         __u32 key, expected_key = 0, res_first_key;
 939         int err, i, map_fd, hash_fd, iter_fd;
 940         struct bpf_iter_bpf_array_map *skel;
 941         union bpf_iter_link_info linfo;
 942         struct bpf_link *link;
 943         char buf[64] = {};
 944         int len, start;
 945
 946         skel = bpf_iter_bpf_array_map__open_and_load();
 947         if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load"))
 948                 return;
 949
 950         map_fd = bpf_map__fd(skel->maps.arraymap1);
 951         for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
 952                 val = i + 4;
 953                 expected_key += i;
 954                 expected_val += val;
 955
 956                 if (i == 0)
 957                         first_val = val;
 958
 959                 err = bpf_map_update_elem(map_fd, &i, &val, BPF_ANY);
 960                 if (!ASSERT_OK(err, "map_update"))
 961                         goto out;
 962         }
 963
 964         memset(&linfo, 0, sizeof(linfo));
 965         linfo.map.map_fd = map_fd;
 966         opts.link_info = &linfo;
 967         opts.link_info_len = sizeof(linfo);
 968         link = bpf_program__attach_iter(skel->progs.dump_bpf_array_map, &opts);
 969         if (!ASSERT_OK_PTR(link, "attach_iter"))
 970                 goto out;
 971
 972         iter_fd = bpf_iter_create(bpf_link__fd(link));
 973         if (!ASSERT_GE(iter_fd, 0, "create_iter"))
 974                 goto free_link;
 975
 976         /* do some tests */
 977         start = 0;
 978         while ((len = read(iter_fd, buf + start, sizeof(buf) - start)) > 0)
 979                 start += len;
 980         if (!ASSERT_GE(len, 0, "read"))
 981                 goto close_iter;
 982
 983         /* test results */
 984         res_first_key = *(__u32 *)buf;
 985         res_first_val = *(__u64 *)(buf + sizeof(__u32));
 986         if (!ASSERT_EQ(res_first_key, 0, "bpf_seq_write") ||
 987                         !ASSERT_EQ(res_first_val, first_val, "bpf_seq_write"))
 988                 goto close_iter;
 989
 990         if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum"))
 991                 goto close_iter;
 992         if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
 993                 goto close_iter;
 994
 995         hash_fd = bpf_map__fd(skel->maps.hashmap1);
 996         for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
 997                 err = bpf_map_lookup_elem(map_fd, &i, &val);
 998                 if (!ASSERT_OK(err, "map_lookup arraymap1"))
 999                         goto close_iter;
1000                 if (!ASSERT_EQ(i, val, "invalid_val arraymap1"))
1001                         goto close_iter;
1002
1003                 val = i + 4;
1004                 err = bpf_map_lookup_elem(hash_fd, &val, &key);
1005                 if (!ASSERT_OK(err, "map_lookup hashmap1"))
1006                         goto close_iter;
1007                 if (!ASSERT_EQ(key, val - 4, "invalid_val hashmap1"))
1008                         goto close_iter;
1009         }
1010
1011 close_iter:
1012         close(iter_fd);
1013 free_link:
1014         bpf_link__destroy(link);
1015 out:
1016         bpf_iter_bpf_array_map__destroy(skel);
1017 }
1018
1019 static void test_bpf_array_map_iter_fd(void)
1020 {
1021         struct bpf_iter_bpf_array_map *skel;
1022
1023         skel = bpf_iter_bpf_array_map__open_and_load();
1024         if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load"))
1025                 return;
1026
1027         do_read_map_iter_fd(&skel->skeleton, skel->progs.dump_bpf_array_map,
1028                             skel->maps.arraymap1);
1029
1030         bpf_iter_bpf_array_map__destroy(skel);
1031 }
1032
1033 static void test_bpf_percpu_array_map(void)
1034 {
1035         DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1036         struct bpf_iter_bpf_percpu_array_map *skel;
1037         __u32 expected_key = 0, expected_val = 0;
1038         union bpf_iter_link_info linfo;
1039         int err, i, j, map_fd, iter_fd;
1040         struct bpf_link *link;
1041         char buf[64];
1042         void *val;
1043         int len;
1044
1045         skel = bpf_iter_bpf_percpu_array_map__open();
1046         if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__open"))
1047                 return;
1048
1049         skel->rodata->num_cpus = bpf_num_possible_cpus();
1050         val = malloc(8 * bpf_num_possible_cpus());
1051         if (!ASSERT_OK_PTR(val, "malloc"))
1052                 goto out;
1053
1054         err = bpf_iter_bpf_percpu_array_map__load(skel);
1055         if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__load"))
1056                 goto out;
1057
1058         /* update map values here */
1059         map_fd = bpf_map__fd(skel->maps.arraymap1);
1060         for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
1061                 expected_key += i;
1062
1063                 for (j = 0; j < bpf_num_possible_cpus(); j++) {
1064                         *(__u32 *)(val + j * 8) = i + j;
1065                         expected_val += i + j;
1066                 }
1067
1068                 err = bpf_map_update_elem(map_fd, &i, val, BPF_ANY);
1069                 if (!ASSERT_OK(err, "map_update"))
1070                         goto out;
1071         }
1072
1073         memset(&linfo, 0, sizeof(linfo));
1074         linfo.map.map_fd = map_fd;
1075         opts.link_info = &linfo;
1076         opts.link_info_len = sizeof(linfo);
1077         link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_array_map, &opts);
1078         if (!ASSERT_OK_PTR(link, "attach_iter"))
1079                 goto out;
1080
1081         iter_fd = bpf_iter_create(bpf_link__fd(link));
1082         if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1083                 goto free_link;
1084
1085         /* do some tests */
1086         while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1087                 ;
1088         if (!ASSERT_GE(len, 0, "read"))
1089                 goto close_iter;
1090
1091         /* test results */
1092         if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum"))
1093                 goto close_iter;
1094         if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1095                 goto close_iter;
1096
1097 close_iter:
1098         close(iter_fd);
1099 free_link:
1100         bpf_link__destroy(link);
1101 out:
1102         bpf_iter_bpf_percpu_array_map__destroy(skel);
1103         free(val);
1104 }
1105
1106 /* An iterator program deletes all local storage in a map. */
1107 static void test_bpf_sk_storage_delete(void)
1108 {
1109         DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1110         struct bpf_iter_bpf_sk_storage_helpers *skel;
1111         union bpf_iter_link_info linfo;
1112         int err, len, map_fd, iter_fd;
1113         struct bpf_link *link;
1114         int sock_fd = -1;
1115         __u32 val = 42;
1116         char buf[64];
1117
1118         skel = bpf_iter_bpf_sk_storage_helpers__open_and_load();
1119         if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load"))
1120                 return;
1121
1122         map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1123
1124         sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
1125         if (!ASSERT_GE(sock_fd, 0, "socket"))
1126                 goto out;
1127
1128         err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST);
1129         if (!ASSERT_OK(err, "map_update"))
1130                 goto out;
1131
1132         memset(&linfo, 0, sizeof(linfo));
1133         linfo.map.map_fd = map_fd;
1134         opts.link_info = &linfo;
1135         opts.link_info_len = sizeof(linfo);
1136         link = bpf_program__attach_iter(skel->progs.delete_bpf_sk_storage_map,
1137                                         &opts);
1138         if (!ASSERT_OK_PTR(link, "attach_iter"))
1139                 goto out;
1140
1141         iter_fd = bpf_iter_create(bpf_link__fd(link));
1142         if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1143                 goto free_link;
1144
1145         /* do some tests */
1146         while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1147                 ;
1148         if (!ASSERT_GE(len, 0, "read"))
1149                 goto close_iter;
1150
1151         /* test results */
1152         err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1153
1154          /* Note: The following assertions serve to ensure
1155           * the value was deleted. It does so by asserting
1156           * that bpf_map_lookup_elem has failed. This might
1157           * seem counterintuitive at first.
1158           */
1159         ASSERT_ERR(err, "bpf_map_lookup_elem");
1160         ASSERT_EQ(errno, ENOENT, "bpf_map_lookup_elem");
1161
1162 close_iter:
1163         close(iter_fd);
1164 free_link:
1165         bpf_link__destroy(link);
1166 out:
1167         if (sock_fd >= 0)
1168                 close(sock_fd);
1169         bpf_iter_bpf_sk_storage_helpers__destroy(skel);
1170 }
1171
1172 /* This creates a socket and its local storage. It then runs a task_iter BPF
1173  * program that replaces the existing socket local storage with the tgid of the
1174  * only task owning a file descriptor to this socket, this process, prog_tests.
1175  * It then runs a tcp socket iterator that negates the value in the existing
1176  * socket local storage, the test verifies that the resulting value is -pid.
1177  */
1178 static void test_bpf_sk_storage_get(void)
1179 {
1180         struct bpf_iter_bpf_sk_storage_helpers *skel;
1181         int err, map_fd, val = -1;
1182         int sock_fd = -1;
1183
1184         skel = bpf_iter_bpf_sk_storage_helpers__open_and_load();
1185         if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load"))
1186                 return;
1187
1188         sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
1189         if (!ASSERT_GE(sock_fd, 0, "socket"))
1190                 goto out;
1191
1192         err = listen(sock_fd, 1);
1193         if (!ASSERT_OK(err, "listen"))
1194                 goto close_socket;
1195
1196         map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1197
1198         err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST);
1199         if (!ASSERT_OK(err, "bpf_map_update_elem"))
1200                 goto close_socket;
1201
1202         do_dummy_read(skel->progs.fill_socket_owner);
1203
1204         err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1205         if (!ASSERT_OK(err, "bpf_map_lookup_elem") ||
1206                         !ASSERT_EQ(val, getpid(), "bpf_map_lookup_elem"))
1207                 goto close_socket;
1208
1209         do_dummy_read(skel->progs.negate_socket_local_storage);
1210
1211         err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1212         ASSERT_OK(err, "bpf_map_lookup_elem");
1213         ASSERT_EQ(val, -getpid(), "bpf_map_lookup_elem");
1214
1215 close_socket:
1216         close(sock_fd);
1217 out:
1218         bpf_iter_bpf_sk_storage_helpers__destroy(skel);
1219 }
1220
1221 static void test_bpf_sk_stoarge_map_iter_fd(void)
1222 {
1223         struct bpf_iter_bpf_sk_storage_map *skel;
1224
1225         skel = bpf_iter_bpf_sk_storage_map__open_and_load();
1226         if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load"))
1227                 return;
1228
1229         do_read_map_iter_fd(&skel->skeleton, skel->progs.rw_bpf_sk_storage_map,
1230                             skel->maps.sk_stg_map);
1231
1232         bpf_iter_bpf_sk_storage_map__destroy(skel);
1233 }
1234
1235 static void test_bpf_sk_storage_map(void)
1236 {
1237         DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1238         int err, i, len, map_fd, iter_fd, num_sockets;
1239         struct bpf_iter_bpf_sk_storage_map *skel;
1240         union bpf_iter_link_info linfo;
1241         int sock_fd[3] = {-1, -1, -1};
1242         __u32 val, expected_val = 0;
1243         struct bpf_link *link;
1244         char buf[64];
1245
1246         skel = bpf_iter_bpf_sk_storage_map__open_and_load();
1247         if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load"))
1248                 return;
1249
1250         map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1251         num_sockets = ARRAY_SIZE(sock_fd);
1252         for (i = 0; i < num_sockets; i++) {
1253                 sock_fd[i] = socket(AF_INET6, SOCK_STREAM, 0);
1254                 if (!ASSERT_GE(sock_fd[i], 0, "socket"))
1255                         goto out;
1256
1257                 val = i + 1;
1258                 expected_val += val;
1259
1260                 err = bpf_map_update_elem(map_fd, &sock_fd[i], &val,
1261                                           BPF_NOEXIST);
1262                 if (!ASSERT_OK(err, "map_update"))
1263                         goto out;
1264         }
1265
1266         memset(&linfo, 0, sizeof(linfo));
1267         linfo.map.map_fd = map_fd;
1268         opts.link_info = &linfo;
1269         opts.link_info_len = sizeof(linfo);
1270         link = bpf_program__attach_iter(skel->progs.oob_write_bpf_sk_storage_map, &opts);
1271         err = libbpf_get_error(link);
1272         if (!ASSERT_EQ(err, -EACCES, "attach_oob_write_iter")) {
1273                 if (!err)
1274                         bpf_link__destroy(link);
1275                 goto out;
1276         }
1277
1278         link = bpf_program__attach_iter(skel->progs.rw_bpf_sk_storage_map, &opts);
1279         if (!ASSERT_OK_PTR(link, "attach_iter"))
1280                 goto out;
1281
1282         iter_fd = bpf_iter_create(bpf_link__fd(link));
1283         if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1284                 goto free_link;
1285
1286         skel->bss->to_add_val = time(NULL);
1287         /* do some tests */
1288         while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1289                 ;
1290         if (!ASSERT_GE(len, 0, "read"))
1291                 goto close_iter;
1292
1293         /* test results */
1294         if (!ASSERT_EQ(skel->bss->ipv6_sk_count, num_sockets, "ipv6_sk_count"))
1295                 goto close_iter;
1296
1297         if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1298                 goto close_iter;
1299
1300         for (i = 0; i < num_sockets; i++) {
1301                 err = bpf_map_lookup_elem(map_fd, &sock_fd[i], &val);
1302                 if (!ASSERT_OK(err, "map_lookup") ||
1303                     !ASSERT_EQ(val, i + 1 + skel->bss->to_add_val, "check_map_value"))
1304                         break;
1305         }
1306
1307 close_iter:
1308         close(iter_fd);
1309 free_link:
1310         bpf_link__destroy(link);
1311 out:
1312         for (i = 0; i < num_sockets; i++) {
1313                 if (sock_fd[i] >= 0)
1314                         close(sock_fd[i]);
1315         }
1316         bpf_iter_bpf_sk_storage_map__destroy(skel);
1317 }
1318
1319 static void test_rdonly_buf_out_of_bound(void)
1320 {
1321         DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1322         struct bpf_iter_test_kern5 *skel;
1323         union bpf_iter_link_info linfo;
1324         struct bpf_link *link;
1325
1326         skel = bpf_iter_test_kern5__open_and_load();
1327         if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern5__open_and_load"))
1328                 return;
1329
1330         memset(&linfo, 0, sizeof(linfo));
1331         linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap1);
1332         opts.link_info = &linfo;
1333         opts.link_info_len = sizeof(linfo);
1334         link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
1335         if (!ASSERT_ERR_PTR(link, "attach_iter"))
1336                 bpf_link__destroy(link);
1337
1338         bpf_iter_test_kern5__destroy(skel);
1339 }
1340
1341 static void test_buf_neg_offset(void)
1342 {
1343         struct bpf_iter_test_kern6 *skel;
1344
1345         skel = bpf_iter_test_kern6__open_and_load();
1346         if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern6__open_and_load"))
1347                 bpf_iter_test_kern6__destroy(skel);
1348 }
1349
1350 static void test_link_iter(void)
1351 {
1352         struct bpf_iter_bpf_link *skel;
1353
1354         skel = bpf_iter_bpf_link__open_and_load();
1355         if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_link__open_and_load"))
1356                 return;
1357
1358         do_dummy_read(skel->progs.dump_bpf_link);
1359
1360         bpf_iter_bpf_link__destroy(skel);
1361 }
1362
1363 static void test_ksym_iter(void)
1364 {
1365         struct bpf_iter_ksym *skel;
1366
1367         skel = bpf_iter_ksym__open_and_load();
1368         if (!ASSERT_OK_PTR(skel, "bpf_iter_ksym__open_and_load"))
1369                 return;
1370
1371         do_dummy_read(skel->progs.dump_ksym);
1372
1373         bpf_iter_ksym__destroy(skel);
1374 }
1375
1376 #define CMP_BUFFER_SIZE 1024
1377 static char task_vma_output[CMP_BUFFER_SIZE];
1378 static char proc_maps_output[CMP_BUFFER_SIZE];
1379
1380 /* remove \0 and \t from str, and only keep the first line */
1381 static void str_strip_first_line(char *str)
1382 {
1383         char *dst = str, *src = str;
1384
1385         do {
1386                 if (*src == ' ' || *src == '\t')
1387                         src++;
1388                 else
1389                         *(dst++) = *(src++);
1390
1391         } while (*src != '\0' && *src != '\n');
1392
1393         *dst = '\0';
1394 }
1395
1396 static void test_task_vma_common(struct bpf_iter_attach_opts *opts)
1397 {
1398         int err, iter_fd = -1, proc_maps_fd = -1;
1399         struct bpf_iter_task_vmas *skel;
1400         int len, read_size = 4;
1401         char maps_path[64];
1402
1403         skel = bpf_iter_task_vmas__open();
1404         if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vmas__open"))
1405                 return;
1406
1407         skel->bss->pid = getpid();
1408         skel->bss->one_task = opts ? 1 : 0;
1409
1410         err = bpf_iter_task_vmas__load(skel);
1411         if (!ASSERT_OK(err, "bpf_iter_task_vmas__load"))
1412                 goto out;
1413
1414         skel->links.proc_maps = bpf_program__attach_iter(
1415                 skel->progs.proc_maps, opts);
1416
1417         if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) {
1418                 skel->links.proc_maps = NULL;
1419                 goto out;
1420         }
1421
1422         iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps));
1423         if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1424                 goto out;
1425
1426         /* Read CMP_BUFFER_SIZE (1kB) from bpf_iter. Read in small chunks
1427          * to trigger seq_file corner cases.
1428          */
1429         len = 0;
1430         while (len < CMP_BUFFER_SIZE) {
1431                 err = read_fd_into_buffer(iter_fd, task_vma_output + len,
1432                                           MIN(read_size, CMP_BUFFER_SIZE - len));
1433                 if (!err)
1434                         break;
1435                 if (!ASSERT_GE(err, 0, "read_iter_fd"))
1436                         goto out;
1437                 len += err;
1438         }
1439         if (opts)
1440                 ASSERT_EQ(skel->bss->one_task_error, 0, "unexpected task");
1441
1442         /* read CMP_BUFFER_SIZE (1kB) from /proc/pid/maps */
1443         snprintf(maps_path, 64, "/proc/%u/maps", skel->bss->pid);
1444         proc_maps_fd = open(maps_path, O_RDONLY);
1445         if (!ASSERT_GE(proc_maps_fd, 0, "open_proc_maps"))
1446                 goto out;
1447         err = read_fd_into_buffer(proc_maps_fd, proc_maps_output, CMP_BUFFER_SIZE);
1448         if (!ASSERT_GE(err, 0, "read_prog_maps_fd"))
1449                 goto out;
1450
1451         /* strip and compare the first line of the two files */
1452         str_strip_first_line(task_vma_output);
1453         str_strip_first_line(proc_maps_output);
1454
1455         ASSERT_STREQ(task_vma_output, proc_maps_output, "compare_output");
1456
1457         check_bpf_link_info(skel->progs.proc_maps);
1458
1459 out:
1460         close(proc_maps_fd);
1461         close(iter_fd);
1462         bpf_iter_task_vmas__destroy(skel);
1463 }
1464
1465 static void test_task_vma_dead_task(void)
1466 {
1467         struct bpf_iter_task_vmas *skel;
1468         int wstatus, child_pid = -1;
1469         time_t start_tm, cur_tm;
1470         int err, iter_fd = -1;
1471         int wait_sec = 3;
1472
1473         skel = bpf_iter_task_vmas__open();
1474         if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vmas__open"))
1475                 return;
1476
1477         skel->bss->pid = getpid();
1478
1479         err = bpf_iter_task_vmas__load(skel);
1480         if (!ASSERT_OK(err, "bpf_iter_task_vmas__load"))
1481                 goto out;
1482
1483         skel->links.proc_maps = bpf_program__attach_iter(
1484                 skel->progs.proc_maps, NULL);
1485
1486         if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) {
1487                 skel->links.proc_maps = NULL;
1488                 goto out;
1489         }
1490
1491         start_tm = time(NULL);
1492         cur_tm = start_tm;
1493
1494         child_pid = fork();
1495         if (child_pid == 0) {
1496                 /* Fork short-lived processes in the background. */
1497                 while (cur_tm < start_tm + wait_sec) {
1498                         system("echo > /dev/null");
1499                         cur_tm = time(NULL);
1500                 }
1501                 exit(0);
1502         }
1503
1504         if (!ASSERT_GE(child_pid, 0, "fork_child"))
1505                 goto out;
1506
1507         while (cur_tm < start_tm + wait_sec) {
1508                 iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps));
1509                 if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1510                         goto out;
1511
1512                 /* Drain all data from iter_fd. */
1513                 while (cur_tm < start_tm + wait_sec) {
1514                         err = read_fd_into_buffer(iter_fd, task_vma_output, CMP_BUFFER_SIZE);
1515                         if (!ASSERT_GE(err, 0, "read_iter_fd"))
1516                                 goto out;
1517
1518                         cur_tm = time(NULL);
1519
1520                         if (err == 0)
1521                                 break;
1522                 }
1523
1524                 close(iter_fd);
1525                 iter_fd = -1;
1526         }
1527
1528         check_bpf_link_info(skel->progs.proc_maps);
1529
1530 out:
1531         waitpid(child_pid, &wstatus, 0);
1532         close(iter_fd);
1533         bpf_iter_task_vmas__destroy(skel);
1534 }
1535
1536 void test_bpf_sockmap_map_iter_fd(void)
1537 {
1538         struct bpf_iter_sockmap *skel;
1539
1540         skel = bpf_iter_sockmap__open_and_load();
1541         if (!ASSERT_OK_PTR(skel, "bpf_iter_sockmap__open_and_load"))
1542                 return;
1543
1544         do_read_map_iter_fd(&skel->skeleton, skel->progs.copy, skel->maps.sockmap);
1545
1546         bpf_iter_sockmap__destroy(skel);
1547 }
1548
1549 static void test_task_vma(void)
1550 {
1551         LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1552         union bpf_iter_link_info linfo;
1553
1554         memset(&linfo, 0, sizeof(linfo));
1555         linfo.task.tid = getpid();
1556         opts.link_info = &linfo;
1557         opts.link_info_len = sizeof(linfo);
1558
1559         test_task_vma_common(&opts);
1560         test_task_vma_common(NULL);
1561 }
1562
1563 /* uprobe attach point */
1564 static noinline int trigger_func(int arg)
1565 {
1566         asm volatile ("");
1567         return arg + 1;
1568 }
1569
1570 static void test_task_vma_offset_common(struct bpf_iter_attach_opts *opts, bool one_proc)
1571 {
1572         struct bpf_iter_vma_offset *skel;
1573         char buf[16] = {};
1574         int iter_fd, len;
1575         int pgsz, shift;
1576
1577         skel = bpf_iter_vma_offset__open_and_load();
1578         if (!ASSERT_OK_PTR(skel, "bpf_iter_vma_offset__open_and_load"))
1579                 return;
1580
1581         skel->bss->pid = getpid();
1582         skel->bss->address = (uintptr_t)trigger_func;
1583         for (pgsz = getpagesize(), shift = 0; pgsz > 1; pgsz >>= 1, shift++)
1584                 ;
1585         skel->bss->page_shift = shift;
1586
1587         skel->links.get_vma_offset = bpf_program__attach_iter(skel->progs.get_vma_offset, opts);
1588         if (!ASSERT_OK_PTR(skel->links.get_vma_offset, "attach_iter"))
1589                 goto exit;
1590
1591         iter_fd = bpf_iter_create(bpf_link__fd(skel->links.get_vma_offset));
1592         if (!ASSERT_GT(iter_fd, 0, "create_iter"))
1593                 goto exit;
1594
1595         while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1596                 ;
1597         buf[15] = 0;
1598         ASSERT_EQ(strcmp(buf, "OK\n"), 0, "strcmp");
1599
1600         ASSERT_EQ(skel->bss->offset, get_uprobe_offset(trigger_func), "offset");
1601         if (one_proc)
1602                 ASSERT_EQ(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count");
1603         else
1604                 ASSERT_GT(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count");
1605
1606         close(iter_fd);
1607
1608 exit:
1609         bpf_iter_vma_offset__destroy(skel);
1610 }
1611
1612 static void test_task_vma_offset(void)
1613 {
1614         LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1615         union bpf_iter_link_info linfo;
1616
1617         memset(&linfo, 0, sizeof(linfo));
1618         linfo.task.pid = getpid();
1619         opts.link_info = &linfo;
1620         opts.link_info_len = sizeof(linfo);
1621
1622         test_task_vma_offset_common(&opts, true);
1623
1624         linfo.task.pid = 0;
1625         linfo.task.tid = getpid();
1626         test_task_vma_offset_common(&opts, true);
1627
1628         test_task_vma_offset_common(NULL, false);
1629 }
1630
1631 void test_bpf_iter(void)
1632 {
1633         ASSERT_OK(pthread_mutex_init(&do_nothing_mutex, NULL), "pthread_mutex_init");
1634
1635         if (test__start_subtest("btf_id_or_null"))
1636                 test_btf_id_or_null();
1637         if (test__start_subtest("ipv6_route"))
1638                 test_ipv6_route();
1639         if (test__start_subtest("netlink"))
1640                 test_netlink();
1641         if (test__start_subtest("bpf_map"))
1642                 test_bpf_map();
1643         if (test__start_subtest("task_tid"))
1644                 test_task_tid();
1645         if (test__start_subtest("task_pid"))
1646                 test_task_pid();
1647         if (test__start_subtest("task_pidfd"))
1648                 test_task_pidfd();
1649         if (test__start_subtest("task_sleepable"))
1650                 test_task_sleepable();
1651         if (test__start_subtest("task_stack"))
1652                 test_task_stack();
1653         if (test__start_subtest("task_file"))
1654                 test_task_file();
1655         if (test__start_subtest("task_vma"))
1656                 test_task_vma();
1657         if (test__start_subtest("task_vma_dead_task"))
1658                 test_task_vma_dead_task();
1659         if (test__start_subtest("task_btf"))
1660                 test_task_btf();
1661         if (test__start_subtest("tcp4"))
1662                 test_tcp4();
1663         if (test__start_subtest("tcp6"))
1664                 test_tcp6();
1665         if (test__start_subtest("udp4"))
1666                 test_udp4();
1667         if (test__start_subtest("udp6"))
1668                 test_udp6();
1669         if (test__start_subtest("unix"))
1670                 test_unix();
1671         if (test__start_subtest("anon"))
1672                 test_anon_iter(false);
1673         if (test__start_subtest("anon-read-one-char"))
1674                 test_anon_iter(true);
1675         if (test__start_subtest("file"))
1676                 test_file_iter();
1677         if (test__start_subtest("overflow"))
1678                 test_overflow(false, false);
1679         if (test__start_subtest("overflow-e2big"))
1680                 test_overflow(true, false);
1681         if (test__start_subtest("prog-ret-1"))
1682                 test_overflow(false, true);
1683         if (test__start_subtest("bpf_hash_map"))
1684                 test_bpf_hash_map();
1685         if (test__start_subtest("bpf_percpu_hash_map"))
1686                 test_bpf_percpu_hash_map();
1687         if (test__start_subtest("bpf_array_map"))
1688                 test_bpf_array_map();
1689         if (test__start_subtest("bpf_array_map_iter_fd"))
1690                 test_bpf_array_map_iter_fd();
1691         if (test__start_subtest("bpf_percpu_array_map"))
1692                 test_bpf_percpu_array_map();
1693         if (test__start_subtest("bpf_sk_storage_map"))
1694                 test_bpf_sk_storage_map();
1695         if (test__start_subtest("bpf_sk_storage_map_iter_fd"))
1696                 test_bpf_sk_stoarge_map_iter_fd();
1697         if (test__start_subtest("bpf_sk_storage_delete"))
1698                 test_bpf_sk_storage_delete();
1699         if (test__start_subtest("bpf_sk_storage_get"))
1700                 test_bpf_sk_storage_get();
1701         if (test__start_subtest("rdonly-buf-out-of-bound"))
1702                 test_rdonly_buf_out_of_bound();
1703         if (test__start_subtest("buf-neg-offset"))
1704                 test_buf_neg_offset();
1705         if (test__start_subtest("link-iter"))
1706                 test_link_iter();
1707         if (test__start_subtest("ksym"))
1708                 test_ksym_iter();
1709         if (test__start_subtest("bpf_sockmap_map_iter_fd"))
1710                 test_bpf_sockmap_map_iter_fd();
1711         if (test__start_subtest("vma_offset"))
1712                 test_task_vma_offset();
1713 }