tools/testing/selftests/bpf/prog_tests/select_reuseport.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2018 Facebook */
   3
   4 #include <stdlib.h>
   5 #include <unistd.h>
   6 #include <stdbool.h>
   7 #include <string.h>
   8 #include <errno.h>
   9 #include <assert.h>
  10 #include <fcntl.h>
  11 #include <linux/bpf.h>
  12 #include <linux/err.h>
  13 #include <linux/types.h>
  14 #include <linux/if_ether.h>
  15 #include <sys/types.h>
  16 #include <sys/epoll.h>
  17 #include <sys/socket.h>
  18 #include <netinet/in.h>
  19 #include <bpf/bpf.h>
  20 #include <bpf/libbpf.h>
  21 #include "bpf_rlimit.h"
  22 #include "bpf_util.h"
  23
  24 #include "test_progs.h"
  25 #include "test_select_reuseport_common.h"
  26
  27 #define MAX_TEST_NAME 80
  28 #define MIN_TCPHDR_LEN 20
  29 #define UDPHDR_LEN 8
  30
  31 #define TCP_SYNCOOKIE_SYSCTL "/proc/sys/net/ipv4/tcp_syncookies"
  32 #define TCP_FO_SYSCTL "/proc/sys/net/ipv4/tcp_fastopen"
  33 #define REUSEPORT_ARRAY_SIZE 32
  34
  35 static int result_map, tmp_index_ovr_map, linum_map, data_check_map;
  36 static __u32 expected_results[NR_RESULTS];
  37 static int sk_fds[REUSEPORT_ARRAY_SIZE];
  38 static int reuseport_array = -1, outer_map = -1;
  39 static int select_by_skb_data_prog;
  40 static int saved_tcp_syncookie = -1;
  41 static struct bpf_object *obj;
  42 static int saved_tcp_fo = -1;
  43 static __u32 index_zero;
  44 static int epfd;
  45
  46 static union sa46 {
  47         struct sockaddr_in6 v6;
  48         struct sockaddr_in v4;
  49         sa_family_t family;
  50 } srv_sa;
  51
  52 #define RET_IF(condition, tag, format...) ({                            \
  53         if (CHECK_FAIL(condition)) {                                    \
  54                 printf(tag " " format);                                 \
  55                 return;                                                 \
  56         }                                                               \
  57 })
  58
  59 #define RET_ERR(condition, tag, format...) ({                           \
  60         if (CHECK_FAIL(condition)) {                                    \
  61                 printf(tag " " format);                                 \
  62                 return -1;                                              \
  63         }                                                               \
  64 })
  65
  66 static int create_maps(void)
  67 {
  68         struct bpf_create_map_attr attr = {};
  69
  70         /* Creating reuseport_array */
  71         attr.name = "reuseport_array";
  72         attr.map_type = BPF_MAP_TYPE_REUSEPORT_SOCKARRAY;
  73         attr.key_size = sizeof(__u32);
  74         attr.value_size = sizeof(__u32);
  75         attr.max_entries = REUSEPORT_ARRAY_SIZE;
  76
  77         reuseport_array = bpf_create_map_xattr(&attr);
  78         RET_ERR(reuseport_array == -1, "creating reuseport_array",
  79                 "reuseport_array:%d errno:%d\n", reuseport_array, errno);
  80
  81         /* Creating outer_map */
  82         attr.name = "outer_map";
  83         attr.map_type = BPF_MAP_TYPE_ARRAY_OF_MAPS;
  84         attr.key_size = sizeof(__u32);
  85         attr.value_size = sizeof(__u32);
  86         attr.max_entries = 1;
  87         attr.inner_map_fd = reuseport_array;
  88         outer_map = bpf_create_map_xattr(&attr);
  89         RET_ERR(outer_map == -1, "creating outer_map",
  90                 "outer_map:%d errno:%d\n", outer_map, errno);
  91
  92         return 0;
  93 }
  94
  95 static int prepare_bpf_obj(void)
  96 {
  97         struct bpf_program *prog;
  98         struct bpf_map *map;
  99         int err;
 100
 101         obj = bpf_object__open("test_select_reuseport_kern.o");
 102         RET_ERR(IS_ERR_OR_NULL(obj), "open test_select_reuseport_kern.o",
 103                 "obj:%p PTR_ERR(obj):%ld\n", obj, PTR_ERR(obj));
 104
 105         map = bpf_object__find_map_by_name(obj, "outer_map");
 106         RET_ERR(!map, "find outer_map", "!map\n");
 107         err = bpf_map__reuse_fd(map, outer_map);
 108         RET_ERR(err, "reuse outer_map", "err:%d\n", err);
 109
 110         err = bpf_object__load(obj);
 111         RET_ERR(err, "load bpf_object", "err:%d\n", err);
 112
 113         prog = bpf_program__next(NULL, obj);
 114         RET_ERR(!prog, "get first bpf_program", "!prog\n");
 115         select_by_skb_data_prog = bpf_program__fd(prog);
 116         RET_ERR(select_by_skb_data_prog == -1, "get prog fd",
 117                 "select_by_skb_data_prog:%d\n", select_by_skb_data_prog);
 118
 119         map = bpf_object__find_map_by_name(obj, "result_map");
 120         RET_ERR(!map, "find result_map", "!map\n");
 121         result_map = bpf_map__fd(map);
 122         RET_ERR(result_map == -1, "get result_map fd",
 123                 "result_map:%d\n", result_map);
 124
 125         map = bpf_object__find_map_by_name(obj, "tmp_index_ovr_map");
 126         RET_ERR(!map, "find tmp_index_ovr_map\n", "!map");
 127         tmp_index_ovr_map = bpf_map__fd(map);
 128         RET_ERR(tmp_index_ovr_map == -1, "get tmp_index_ovr_map fd",
 129                 "tmp_index_ovr_map:%d\n", tmp_index_ovr_map);
 130
 131         map = bpf_object__find_map_by_name(obj, "linum_map");
 132         RET_ERR(!map, "find linum_map", "!map\n");
 133         linum_map = bpf_map__fd(map);
 134         RET_ERR(linum_map == -1, "get linum_map fd",
 135                 "linum_map:%d\n", linum_map);
 136
 137         map = bpf_object__find_map_by_name(obj, "data_check_map");
 138         RET_ERR(!map, "find data_check_map", "!map\n");
 139         data_check_map = bpf_map__fd(map);
 140         RET_ERR(data_check_map == -1, "get data_check_map fd",
 141                 "data_check_map:%d\n", data_check_map);
 142
 143         return 0;
 144 }
 145
 146 static void sa46_init_loopback(union sa46 *sa, sa_family_t family)
 147 {
 148         memset(sa, 0, sizeof(*sa));
 149         sa->family = family;
 150         if (sa->family == AF_INET6)
 151                 sa->v6.sin6_addr = in6addr_loopback;
 152         else
 153                 sa->v4.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
 154 }
 155
 156 static void sa46_init_inany(union sa46 *sa, sa_family_t family)
 157 {
 158         memset(sa, 0, sizeof(*sa));
 159         sa->family = family;
 160         if (sa->family == AF_INET6)
 161                 sa->v6.sin6_addr = in6addr_any;
 162         else
 163                 sa->v4.sin_addr.s_addr = INADDR_ANY;
 164 }
 165
 166 static int read_int_sysctl(const char *sysctl)
 167 {
 168         char buf[16];
 169         int fd, ret;
 170
 171         fd = open(sysctl, 0);
 172         RET_ERR(fd == -1, "open(sysctl)",
 173                 "sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno);
 174
 175         ret = read(fd, buf, sizeof(buf));
 176         RET_ERR(ret <= 0, "read(sysctl)",
 177                 "sysctl:%s ret:%d errno:%d\n", sysctl, ret, errno);
 178
 179         close(fd);
 180         return atoi(buf);
 181 }
 182
 183 static int write_int_sysctl(const char *sysctl, int v)
 184 {
 185         int fd, ret, size;
 186         char buf[16];
 187
 188         fd = open(sysctl, O_RDWR);
 189         RET_ERR(fd == -1, "open(sysctl)",
 190                 "sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno);
 191
 192         size = snprintf(buf, sizeof(buf), "%d", v);
 193         ret = write(fd, buf, size);
 194         RET_ERR(ret != size, "write(sysctl)",
 195                 "sysctl:%s ret:%d size:%d errno:%d\n",
 196                 sysctl, ret, size, errno);
 197
 198         close(fd);
 199         return 0;
 200 }
 201
 202 static void restore_sysctls(void)
 203 {
 204         if (saved_tcp_fo != -1)
 205                 write_int_sysctl(TCP_FO_SYSCTL, saved_tcp_fo);
 206         if (saved_tcp_syncookie != -1)
 207                 write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, saved_tcp_syncookie);
 208 }
 209
 210 static int enable_fastopen(void)
 211 {
 212         int fo;
 213
 214         fo = read_int_sysctl(TCP_FO_SYSCTL);
 215         if (fo < 0)
 216                 return -1;
 217
 218         return write_int_sysctl(TCP_FO_SYSCTL, fo | 7);
 219 }
 220
 221 static int enable_syncookie(void)
 222 {
 223         return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 2);
 224 }
 225
 226 static int disable_syncookie(void)
 227 {
 228         return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 0);
 229 }
 230
 231 static long get_linum(void)
 232 {
 233         __u32 linum;
 234         int err;
 235
 236         err = bpf_map_lookup_elem(linum_map, &index_zero, &linum);
 237         RET_ERR(err == -1, "lookup_elem(linum_map)", "err:%d errno:%d\n",
 238                 err, errno);
 239
 240         return linum;
 241 }
 242
 243 static void check_data(int type, sa_family_t family, const struct cmd *cmd,
 244                        int cli_fd)
 245 {
 246         struct data_check expected = {}, result;
 247         union sa46 cli_sa;
 248         socklen_t addrlen;
 249         int err;
 250
 251         addrlen = sizeof(cli_sa);
 252         err = getsockname(cli_fd, (struct sockaddr *)&cli_sa,
 253                           &addrlen);
 254         RET_IF(err == -1, "getsockname(cli_fd)", "err:%d errno:%d\n",
 255                err, errno);
 256
 257         err = bpf_map_lookup_elem(data_check_map, &index_zero, &result);
 258         RET_IF(err == -1, "lookup_elem(data_check_map)", "err:%d errno:%d\n",
 259                err, errno);
 260
 261         if (type == SOCK_STREAM) {
 262                 expected.len = MIN_TCPHDR_LEN;
 263                 expected.ip_protocol = IPPROTO_TCP;
 264         } else {
 265                 expected.len = UDPHDR_LEN;
 266                 expected.ip_protocol = IPPROTO_UDP;
 267         }
 268
 269         if (family == AF_INET6) {
 270                 expected.eth_protocol = htons(ETH_P_IPV6);
 271                 expected.bind_inany = !srv_sa.v6.sin6_addr.s6_addr32[3] &&
 272                         !srv_sa.v6.sin6_addr.s6_addr32[2] &&
 273                         !srv_sa.v6.sin6_addr.s6_addr32[1] &&
 274                         !srv_sa.v6.sin6_addr.s6_addr32[0];
 275
 276                 memcpy(&expected.skb_addrs[0], cli_sa.v6.sin6_addr.s6_addr32,
 277                        sizeof(cli_sa.v6.sin6_addr));
 278                 memcpy(&expected.skb_addrs[4], &in6addr_loopback,
 279                        sizeof(in6addr_loopback));
 280                 expected.skb_ports[0] = cli_sa.v6.sin6_port;
 281                 expected.skb_ports[1] = srv_sa.v6.sin6_port;
 282         } else {
 283                 expected.eth_protocol = htons(ETH_P_IP);
 284                 expected.bind_inany = !srv_sa.v4.sin_addr.s_addr;
 285
 286                 expected.skb_addrs[0] = cli_sa.v4.sin_addr.s_addr;
 287                 expected.skb_addrs[1] = htonl(INADDR_LOOPBACK);
 288                 expected.skb_ports[0] = cli_sa.v4.sin_port;
 289                 expected.skb_ports[1] = srv_sa.v4.sin_port;
 290         }
 291
 292         if (memcmp(&result, &expected, offsetof(struct data_check,
 293                                                 equal_check_end))) {
 294                 printf("unexpected data_check\n");
 295                 printf("  result: (0x%x, %u, %u)\n",
 296                        result.eth_protocol, result.ip_protocol,
 297                        result.bind_inany);
 298                 printf("expected: (0x%x, %u, %u)\n",
 299                        expected.eth_protocol, expected.ip_protocol,
 300                        expected.bind_inany);
 301                 RET_IF(1, "data_check result != expected",
 302                        "bpf_prog_linum:%ld\n", get_linum());
 303         }
 304
 305         RET_IF(!result.hash, "data_check result.hash empty",
 306                "result.hash:%u", result.hash);
 307
 308         expected.len += cmd ? sizeof(*cmd) : 0;
 309         if (type == SOCK_STREAM)
 310                 RET_IF(expected.len > result.len, "expected.len > result.len",
 311                        "expected.len:%u result.len:%u bpf_prog_linum:%ld\n",
 312                        expected.len, result.len, get_linum());
 313         else
 314                 RET_IF(expected.len != result.len, "expected.len != result.len",
 315                        "expected.len:%u result.len:%u bpf_prog_linum:%ld\n",
 316                        expected.len, result.len, get_linum());
 317 }
 318
 319 static const char *result_to_str(enum result res)
 320 {
 321         switch (res) {
 322         case DROP_ERR_INNER_MAP:
 323                 return "DROP_ERR_INNER_MAP";
 324         case DROP_ERR_SKB_DATA:
 325                 return "DROP_ERR_SKB_DATA";
 326         case DROP_ERR_SK_SELECT_REUSEPORT:
 327                 return "DROP_ERR_SK_SELECT_REUSEPORT";
 328         case DROP_MISC:
 329                 return "DROP_MISC";
 330         case PASS:
 331                 return "PASS";
 332         case PASS_ERR_SK_SELECT_REUSEPORT:
 333                 return "PASS_ERR_SK_SELECT_REUSEPORT";
 334         default:
 335                 return "UNKNOWN";
 336         }
 337 }
 338
 339 static void check_results(void)
 340 {
 341         __u32 results[NR_RESULTS];
 342         __u32 i, broken = 0;
 343         int err;
 344
 345         for (i = 0; i < NR_RESULTS; i++) {
 346                 err = bpf_map_lookup_elem(result_map, &i, &results[i]);
 347                 RET_IF(err == -1, "lookup_elem(result_map)",
 348                        "i:%u err:%d errno:%d\n", i, err, errno);
 349         }
 350
 351         for (i = 0; i < NR_RESULTS; i++) {
 352                 if (results[i] != expected_results[i]) {
 353                         broken = i;
 354                         break;
 355                 }
 356         }
 357
 358         if (i == NR_RESULTS)
 359                 return;
 360
 361         printf("unexpected result\n");
 362         printf(" result: [");
 363         printf("%u", results[0]);
 364         for (i = 1; i < NR_RESULTS; i++)
 365                 printf(", %u", results[i]);
 366         printf("]\n");
 367
 368         printf("expected: [");
 369         printf("%u", expected_results[0]);
 370         for (i = 1; i < NR_RESULTS; i++)
 371                 printf(", %u", expected_results[i]);
 372         printf("]\n");
 373
 374         printf("mismatch on %s (bpf_prog_linum:%ld)\n", result_to_str(broken),
 375                get_linum());
 376
 377         CHECK_FAIL(true);
 378 }
 379
 380 static int send_data(int type, sa_family_t family, void *data, size_t len,
 381                      enum result expected)
 382 {
 383         union sa46 cli_sa;
 384         int fd, err;
 385
 386         fd = socket(family, type, 0);
 387         RET_ERR(fd == -1, "socket()", "fd:%d errno:%d\n", fd, errno);
 388
 389         sa46_init_loopback(&cli_sa, family);
 390         err = bind(fd, (struct sockaddr *)&cli_sa, sizeof(cli_sa));
 391         RET_ERR(fd == -1, "bind(cli_sa)", "err:%d errno:%d\n", err, errno);
 392
 393         err = sendto(fd, data, len, MSG_FASTOPEN, (struct sockaddr *)&srv_sa,
 394                      sizeof(srv_sa));
 395         RET_ERR(err != len && expected >= PASS,
 396                 "sendto()", "family:%u err:%d errno:%d expected:%d\n",
 397                 family, err, errno, expected);
 398
 399         return fd;
 400 }
 401
 402 static void do_test(int type, sa_family_t family, struct cmd *cmd,
 403                     enum result expected)
 404 {
 405         int nev, srv_fd, cli_fd;
 406         struct epoll_event ev;
 407         struct cmd rcv_cmd;
 408         ssize_t nread;
 409
 410         cli_fd = send_data(type, family, cmd, cmd ? sizeof(*cmd) : 0,
 411                            expected);
 412         if (cli_fd < 0)
 413                 return;
 414         nev = epoll_wait(epfd, &ev, 1, expected >= PASS ? 5 : 0);
 415         RET_IF((nev <= 0 && expected >= PASS) ||
 416                (nev > 0 && expected < PASS),
 417                "nev <> expected",
 418                "nev:%d expected:%d type:%d family:%d data:(%d, %d)\n",
 419                nev, expected, type, family,
 420                cmd ? cmd->reuseport_index : -1,
 421                cmd ? cmd->pass_on_failure : -1);
 422         check_results();
 423         check_data(type, family, cmd, cli_fd);
 424
 425         if (expected < PASS)
 426                 return;
 427
 428         RET_IF(expected != PASS_ERR_SK_SELECT_REUSEPORT &&
 429                cmd->reuseport_index != ev.data.u32,
 430                "check cmd->reuseport_index",
 431                "cmd:(%u, %u) ev.data.u32:%u\n",
 432                cmd->pass_on_failure, cmd->reuseport_index, ev.data.u32);
 433
 434         srv_fd = sk_fds[ev.data.u32];
 435         if (type == SOCK_STREAM) {
 436                 int new_fd = accept(srv_fd, NULL, 0);
 437
 438                 RET_IF(new_fd == -1, "accept(srv_fd)",
 439                        "ev.data.u32:%u new_fd:%d errno:%d\n",
 440                        ev.data.u32, new_fd, errno);
 441
 442                 nread = recv(new_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT);
 443                 RET_IF(nread != sizeof(rcv_cmd),
 444                        "recv(new_fd)",
 445                        "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n",
 446                        ev.data.u32, nread, sizeof(rcv_cmd), errno);
 447
 448                 close(new_fd);
 449         } else {
 450                 nread = recv(srv_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT);
 451                 RET_IF(nread != sizeof(rcv_cmd),
 452                        "recv(sk_fds)",
 453                        "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n",
 454                        ev.data.u32, nread, sizeof(rcv_cmd), errno);
 455         }
 456
 457         close(cli_fd);
 458 }
 459
 460 static void test_err_inner_map(int type, sa_family_t family)
 461 {
 462         struct cmd cmd = {
 463                 .reuseport_index = 0,
 464                 .pass_on_failure = 0,
 465         };
 466
 467         expected_results[DROP_ERR_INNER_MAP]++;
 468         do_test(type, family, &cmd, DROP_ERR_INNER_MAP);
 469 }
 470
 471 static void test_err_skb_data(int type, sa_family_t family)
 472 {
 473         expected_results[DROP_ERR_SKB_DATA]++;
 474         do_test(type, family, NULL, DROP_ERR_SKB_DATA);
 475 }
 476
 477 static void test_err_sk_select_port(int type, sa_family_t family)
 478 {
 479         struct cmd cmd = {
 480                 .reuseport_index = REUSEPORT_ARRAY_SIZE,
 481                 .pass_on_failure = 0,
 482         };
 483
 484         expected_results[DROP_ERR_SK_SELECT_REUSEPORT]++;
 485         do_test(type, family, &cmd, DROP_ERR_SK_SELECT_REUSEPORT);
 486 }
 487
 488 static void test_pass(int type, sa_family_t family)
 489 {
 490         struct cmd cmd;
 491         int i;
 492
 493         cmd.pass_on_failure = 0;
 494         for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) {
 495                 expected_results[PASS]++;
 496                 cmd.reuseport_index = i;
 497                 do_test(type, family, &cmd, PASS);
 498         }
 499 }
 500
 501 static void test_syncookie(int type, sa_family_t family)
 502 {
 503         int err, tmp_index = 1;
 504         struct cmd cmd = {
 505                 .reuseport_index = 0,
 506                 .pass_on_failure = 0,
 507         };
 508
 509         if (type != SOCK_STREAM) {
 510                 test__skip();
 511                 return;
 512         }
 513
 514         /*
 515          * +1 for TCP-SYN and
 516          * +1 for the TCP-ACK (ack the syncookie)
 517          */
 518         expected_results[PASS] += 2;
 519         enable_syncookie();
 520         /*
 521          * Simulate TCP-SYN and TCP-ACK are handled by two different sk:
 522          * TCP-SYN: select sk_fds[tmp_index = 1] tmp_index is from the
 523          *          tmp_index_ovr_map
 524          * TCP-ACK: select sk_fds[reuseport_index = 0] reuseport_index
 525          *          is from the cmd.reuseport_index
 526          */
 527         err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero,
 528                                   &tmp_index, BPF_ANY);
 529         RET_IF(err == -1, "update_elem(tmp_index_ovr_map, 0, 1)",
 530                "err:%d errno:%d\n", err, errno);
 531         do_test(type, family, &cmd, PASS);
 532         err = bpf_map_lookup_elem(tmp_index_ovr_map, &index_zero,
 533                                   &tmp_index);
 534         RET_IF(err == -1 || tmp_index != -1,
 535                "lookup_elem(tmp_index_ovr_map)",
 536                "err:%d errno:%d tmp_index:%d\n",
 537                err, errno, tmp_index);
 538         disable_syncookie();
 539 }
 540
 541 static void test_pass_on_err(int type, sa_family_t family)
 542 {
 543         struct cmd cmd = {
 544                 .reuseport_index = REUSEPORT_ARRAY_SIZE,
 545                 .pass_on_failure = 1,
 546         };
 547
 548         expected_results[PASS_ERR_SK_SELECT_REUSEPORT] += 1;
 549         do_test(type, family, &cmd, PASS_ERR_SK_SELECT_REUSEPORT);
 550 }
 551
 552 static void test_detach_bpf(int type, sa_family_t family)
 553 {
 554 #ifdef SO_DETACH_REUSEPORT_BPF
 555         __u32 nr_run_before = 0, nr_run_after = 0, tmp, i;
 556         struct epoll_event ev;
 557         int cli_fd, err, nev;
 558         struct cmd cmd = {};
 559         int optvalue = 0;
 560
 561         err = setsockopt(sk_fds[0], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF,
 562                          &optvalue, sizeof(optvalue));
 563         RET_IF(err == -1, "setsockopt(SO_DETACH_REUSEPORT_BPF)",
 564                "err:%d errno:%d\n", err, errno);
 565
 566         err = setsockopt(sk_fds[1], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF,
 567                          &optvalue, sizeof(optvalue));
 568         RET_IF(err == 0 || errno != ENOENT,
 569                "setsockopt(SO_DETACH_REUSEPORT_BPF)",
 570                "err:%d errno:%d\n", err, errno);
 571
 572         for (i = 0; i < NR_RESULTS; i++) {
 573                 err = bpf_map_lookup_elem(result_map, &i, &tmp);
 574                 RET_IF(err == -1, "lookup_elem(result_map)",
 575                        "i:%u err:%d errno:%d\n", i, err, errno);
 576                 nr_run_before += tmp;
 577         }
 578
 579         cli_fd = send_data(type, family, &cmd, sizeof(cmd), PASS);
 580         if (cli_fd < 0)
 581                 return;
 582         nev = epoll_wait(epfd, &ev, 1, 5);
 583         RET_IF(nev <= 0, "nev <= 0",
 584                "nev:%d expected:1 type:%d family:%d data:(0, 0)\n",
 585                nev,  type, family);
 586
 587         for (i = 0; i < NR_RESULTS; i++) {
 588                 err = bpf_map_lookup_elem(result_map, &i, &tmp);
 589                 RET_IF(err == -1, "lookup_elem(result_map)",
 590                        "i:%u err:%d errno:%d\n", i, err, errno);
 591                 nr_run_after += tmp;
 592         }
 593
 594         RET_IF(nr_run_before != nr_run_after,
 595                "nr_run_before != nr_run_after",
 596                "nr_run_before:%u nr_run_after:%u\n",
 597                nr_run_before, nr_run_after);
 598
 599         close(cli_fd);
 600 #else
 601         test__skip();
 602 #endif
 603 }
 604
 605 static void prepare_sk_fds(int type, sa_family_t family, bool inany)
 606 {
 607         const int first = REUSEPORT_ARRAY_SIZE - 1;
 608         int i, err, optval = 1;
 609         struct epoll_event ev;
 610         socklen_t addrlen;
 611
 612         if (inany)
 613                 sa46_init_inany(&srv_sa, family);
 614         else
 615                 sa46_init_loopback(&srv_sa, family);
 616         addrlen = sizeof(srv_sa);
 617
 618         /*
 619          * The sk_fds[] is filled from the back such that the order
 620          * is exactly opposite to the (struct sock_reuseport *)reuse->socks[].
 621          */
 622         for (i = first; i >= 0; i--) {
 623                 sk_fds[i] = socket(family, type, 0);
 624                 RET_IF(sk_fds[i] == -1, "socket()", "sk_fds[%d]:%d errno:%d\n",
 625                        i, sk_fds[i], errno);
 626                 err = setsockopt(sk_fds[i], SOL_SOCKET, SO_REUSEPORT,
 627                                  &optval, sizeof(optval));
 628                 RET_IF(err == -1, "setsockopt(SO_REUSEPORT)",
 629                        "sk_fds[%d] err:%d errno:%d\n",
 630                        i, err, errno);
 631
 632                 if (i == first) {
 633                         err = setsockopt(sk_fds[i], SOL_SOCKET,
 634                                          SO_ATTACH_REUSEPORT_EBPF,
 635                                          &select_by_skb_data_prog,
 636                                          sizeof(select_by_skb_data_prog));
 637                         RET_IF(err == -1, "setsockopt(SO_ATTACH_REUEPORT_EBPF)",
 638                                "err:%d errno:%d\n", err, errno);
 639                 }
 640
 641                 err = bind(sk_fds[i], (struct sockaddr *)&srv_sa, addrlen);
 642                 RET_IF(err == -1, "bind()", "sk_fds[%d] err:%d errno:%d\n",
 643                        i, err, errno);
 644
 645                 if (type == SOCK_STREAM) {
 646                         err = listen(sk_fds[i], 10);
 647                         RET_IF(err == -1, "listen()",
 648                                "sk_fds[%d] err:%d errno:%d\n",
 649                                i, err, errno);
 650                 }
 651
 652                 err = bpf_map_update_elem(reuseport_array, &i, &sk_fds[i],
 653                                           BPF_NOEXIST);
 654                 RET_IF(err == -1, "update_elem(reuseport_array)",
 655                        "sk_fds[%d] err:%d errno:%d\n", i, err, errno);
 656
 657                 if (i == first) {
 658                         socklen_t addrlen = sizeof(srv_sa);
 659
 660                         err = getsockname(sk_fds[i], (struct sockaddr *)&srv_sa,
 661                                           &addrlen);
 662                         RET_IF(err == -1, "getsockname()",
 663                                "sk_fds[%d] err:%d errno:%d\n", i, err, errno);
 664                 }
 665         }
 666
 667         epfd = epoll_create(1);
 668         RET_IF(epfd == -1, "epoll_create(1)",
 669                "epfd:%d errno:%d\n", epfd, errno);
 670
 671         ev.events = EPOLLIN;
 672         for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) {
 673                 ev.data.u32 = i;
 674                 err = epoll_ctl(epfd, EPOLL_CTL_ADD, sk_fds[i], &ev);
 675                 RET_IF(err, "epoll_ctl(EPOLL_CTL_ADD)", "sk_fds[%d]\n", i);
 676         }
 677 }
 678
 679 static void setup_per_test(int type, sa_family_t family, bool inany,
 680                            bool no_inner_map)
 681 {
 682         int ovr = -1, err;
 683
 684         prepare_sk_fds(type, family, inany);
 685         err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, &ovr,
 686                                   BPF_ANY);
 687         RET_IF(err == -1, "update_elem(tmp_index_ovr_map, 0, -1)",
 688                "err:%d errno:%d\n", err, errno);
 689
 690         /* Install reuseport_array to outer_map? */
 691         if (no_inner_map)
 692                 return;
 693
 694         err = bpf_map_update_elem(outer_map, &index_zero, &reuseport_array,
 695                                   BPF_ANY);
 696         RET_IF(err == -1, "update_elem(outer_map, 0, reuseport_array)",
 697                "err:%d errno:%d\n", err, errno);
 698 }
 699
 700 static void cleanup_per_test(bool no_inner_map)
 701 {
 702         int i, err, zero = 0;
 703
 704         memset(expected_results, 0, sizeof(expected_results));
 705
 706         for (i = 0; i < NR_RESULTS; i++) {
 707                 err = bpf_map_update_elem(result_map, &i, &zero, BPF_ANY);
 708                 RET_IF(err, "reset elem in result_map",
 709                        "i:%u err:%d errno:%d\n", i, err, errno);
 710         }
 711
 712         err = bpf_map_update_elem(linum_map, &zero, &zero, BPF_ANY);
 713         RET_IF(err, "reset line number in linum_map", "err:%d errno:%d\n",
 714                err, errno);
 715
 716         for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++)
 717                 close(sk_fds[i]);
 718         close(epfd);
 719
 720         /* Delete reuseport_array from outer_map? */
 721         if (no_inner_map)
 722                 return;
 723
 724         err = bpf_map_delete_elem(outer_map, &index_zero);
 725         RET_IF(err == -1, "delete_elem(outer_map)",
 726                "err:%d errno:%d\n", err, errno);
 727 }
 728
 729 static void cleanup(void)
 730 {
 731         if (outer_map != -1)
 732                 close(outer_map);
 733         if (reuseport_array != -1)
 734                 close(reuseport_array);
 735         if (obj)
 736                 bpf_object__close(obj);
 737 }
 738
 739 static const char *family_str(sa_family_t family)
 740 {
 741         switch (family) {
 742         case AF_INET:
 743                 return "IPv4";
 744         case AF_INET6:
 745                 return "IPv6";
 746         default:
 747                 return "unknown";
 748         }
 749 }
 750
 751 static const char *sotype_str(int sotype)
 752 {
 753         switch (sotype) {
 754         case SOCK_STREAM:
 755                 return "TCP";
 756         case SOCK_DGRAM:
 757                 return "UDP";
 758         default:
 759                 return "unknown";
 760         }
 761 }
 762
 763 #define TEST_INIT(fn, ...) { fn, #fn, __VA_ARGS__ }
 764
 765 static void test_config(int sotype, sa_family_t family, bool inany)
 766 {
 767         const struct test {
 768                 void (*fn)(int sotype, sa_family_t family);
 769                 const char *name;
 770                 bool no_inner_map;
 771         } tests[] = {
 772                 TEST_INIT(test_err_inner_map, true /* no_inner_map */),
 773                 TEST_INIT(test_err_skb_data),
 774                 TEST_INIT(test_err_sk_select_port),
 775                 TEST_INIT(test_pass),
 776                 TEST_INIT(test_syncookie),
 777                 TEST_INIT(test_pass_on_err),
 778                 TEST_INIT(test_detach_bpf),
 779         };
 780         char s[MAX_TEST_NAME];
 781         const struct test *t;
 782
 783         for (t = tests; t < tests + ARRAY_SIZE(tests); t++) {
 784                 snprintf(s, sizeof(s), "%s/%s %s %s",
 785                          family_str(family), sotype_str(sotype),
 786                          inany ? "INANY" : "LOOPBACK", t->name);
 787
 788                 if (!test__start_subtest(s))
 789                         continue;
 790
 791                 setup_per_test(sotype, family, inany, t->no_inner_map);
 792                 t->fn(sotype, family);
 793                 cleanup_per_test(t->no_inner_map);
 794         }
 795 }
 796
 797 #define BIND_INANY true
 798
 799 static void test_all(void)
 800 {
 801         const struct config {
 802                 int sotype;
 803                 sa_family_t family;
 804                 bool inany;
 805         } configs[] = {
 806                 { SOCK_STREAM, AF_INET },
 807                 { SOCK_STREAM, AF_INET, BIND_INANY },
 808                 { SOCK_STREAM, AF_INET6 },
 809                 { SOCK_STREAM, AF_INET6, BIND_INANY },
 810                 { SOCK_DGRAM, AF_INET },
 811                 { SOCK_DGRAM, AF_INET6 },
 812         };
 813         const struct config *c;
 814
 815         for (c = configs; c < configs + ARRAY_SIZE(configs); c++)
 816                 test_config(c->sotype, c->family, c->inany);
 817 }
 818
 819 void test_select_reuseport(void)
 820 {
 821         if (create_maps())
 822                 goto out;
 823         if (prepare_bpf_obj())
 824                 goto out;
 825
 826         saved_tcp_fo = read_int_sysctl(TCP_FO_SYSCTL);
 827         if (saved_tcp_fo < 0)
 828                 goto out;
 829         saved_tcp_syncookie = read_int_sysctl(TCP_SYNCOOKIE_SYSCTL);
 830         if (saved_tcp_syncookie < 0)
 831                 goto out;
 832
 833         if (enable_fastopen())
 834                 goto out;
 835         if (disable_syncookie())
 836                 goto out;
 837
 838         test_all();
 839 out:
 840         cleanup();
 841         restore_sysctls();
 842 }