1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
5 #include <sys/socket.h>
7 #include <sys/select.h>
8 #include <netinet/in.h>
21 #include <sys/resource.h>
22 #include <sys/types.h>
23 #include <sys/sendfile.h>
25 #include <linux/netlink.h>
26 #include <linux/socket.h>
27 #include <linux/sock_diag.h>
28 #include <linux/bpf.h>
29 #include <linux/if_link.h>
30 #include <linux/tls.h>
37 #include <bpf/libbpf.h>
40 #include "bpf_rlimit.h"
41 #include "cgroup_helpers.h"
44 static void running_handler(int a);
53 /* randomly selected ports for testing on lo */
57 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o"
58 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o"
59 #define CG_PATH "/sockmap"
62 int s1, s2, c1, c2, p1, p2;
67 struct bpf_map *maps[9];
84 int txmsg_ktls_skb_drop;
85 int txmsg_ktls_skb_redir;
89 static const struct option long_options[] = {
90 {"help", no_argument, NULL, 'h' },
91 {"cgroup", required_argument, NULL, 'c' },
92 {"rate", required_argument, NULL, 'r' },
93 {"verbose", optional_argument, NULL, 'v' },
94 {"iov_count", required_argument, NULL, 'i' },
95 {"length", required_argument, NULL, 'l' },
96 {"test", required_argument, NULL, 't' },
97 {"data_test", no_argument, NULL, 'd' },
98 {"txmsg", no_argument, &txmsg_pass, 1 },
99 {"txmsg_redir", no_argument, &txmsg_redir, 1 },
100 {"txmsg_drop", no_argument, &txmsg_drop, 1 },
101 {"txmsg_apply", required_argument, NULL, 'a'},
102 {"txmsg_cork", required_argument, NULL, 'k'},
103 {"txmsg_start", required_argument, NULL, 's'},
104 {"txmsg_end", required_argument, NULL, 'e'},
105 {"txmsg_start_push", required_argument, NULL, 'p'},
106 {"txmsg_end_push", required_argument, NULL, 'q'},
107 {"txmsg_start_pop", required_argument, NULL, 'w'},
108 {"txmsg_pop", required_argument, NULL, 'x'},
109 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
110 {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 },
111 {"ktls", no_argument, &ktls, 1 },
112 {"peek", no_argument, &peek_flag, 1 },
113 {"whitelist", required_argument, NULL, 'n' },
114 {"blacklist", required_argument, NULL, 'b' },
133 struct sockmap_options {
150 void (*tester)(int cg_fd, struct sockmap_options *opt);
153 static void test_start(void)
158 static void test_fail(void)
163 static void test_pass(void)
168 static void test_reset(void)
170 txmsg_start = txmsg_end = 0;
171 txmsg_start_pop = txmsg_pop = 0;
172 txmsg_start_push = txmsg_end_push = 0;
173 txmsg_pass = txmsg_drop = txmsg_redir = 0;
174 txmsg_apply = txmsg_cork = 0;
175 txmsg_ingress = txmsg_redir_skb = 0;
176 txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0;
179 static int test_start_subtest(const struct _test *t, struct sockmap_options *o)
182 env.subtest = t->title;
183 env.prepend = o->prepend;
186 env.fail_last = env.fail_cnt;
191 static void test_end_subtest(void)
193 int error = env.fail_cnt - env.fail_last;
194 int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);
199 fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n",
200 env.test_num, env.subtest_num,
201 !type ? "sockmap" : "sockhash",
203 env.subtest, error ? "FAIL" : "OK");
206 static void test_print_results(void)
208 fprintf(stdout, "Pass: %d Fail: %d\n",
209 env.succ_cnt, env.fail_cnt);
212 static void usage(char *argv[])
216 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
217 printf(" options:\n");
218 for (i = 0; long_options[i].name != 0; i++) {
219 printf(" --%-12s", long_options[i].name);
220 if (long_options[i].flag != NULL)
221 printf(" flag (internal value:%d)\n",
222 *long_options[i].flag);
224 printf(" -%c\n", long_options[i].val);
229 char *sock_to_string(int s)
247 static int sockmap_init_ktls(int verbose, int s)
249 struct tls12_crypto_info_aes_gcm_128 tls_tx = {
251 .version = TLS_1_2_VERSION,
252 .cipher_type = TLS_CIPHER_AES_GCM_128,
255 struct tls12_crypto_info_aes_gcm_128 tls_rx = {
257 .version = TLS_1_2_VERSION,
258 .cipher_type = TLS_CIPHER_AES_GCM_128,
261 int so_buf = 6553500;
264 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));
266 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);
269 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));
271 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);
274 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));
276 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);
279 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));
281 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);
284 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));
286 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);
291 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));
294 static int sockmap_init_sockets(int verbose)
297 struct sockaddr_in addr;
298 int *fds[4] = {&s1, &s2, &c1, &c2};
300 s1 = s2 = p1 = p2 = c1 = c2 = 0;
303 for (i = 0; i < 4; i++) {
304 *fds[i] = socket(AF_INET, SOCK_STREAM, 0);
306 perror("socket s1 failed()");
312 for (i = 0; i < 2; i++) {
313 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
314 (char *)&one, sizeof(one));
316 perror("setsockopt failed()");
321 /* Non-blocking sockets */
322 for (i = 0; i < 2; i++) {
323 err = ioctl(*fds[i], FIONBIO, (char *)&one);
325 perror("ioctl s1 failed()");
330 /* Bind server sockets */
331 memset(&addr, 0, sizeof(struct sockaddr_in));
332 addr.sin_family = AF_INET;
333 addr.sin_addr.s_addr = inet_addr("127.0.0.1");
335 addr.sin_port = htons(S1_PORT);
336 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
338 perror("bind s1 failed()");
342 addr.sin_port = htons(S2_PORT);
343 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
345 perror("bind s2 failed()");
349 /* Listen server sockets */
350 addr.sin_port = htons(S1_PORT);
351 err = listen(s1, 32);
353 perror("listen s1 failed()");
357 addr.sin_port = htons(S2_PORT);
358 err = listen(s2, 32);
360 perror("listen s1 failed()");
364 /* Initiate Connect */
365 addr.sin_port = htons(S1_PORT);
366 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
367 if (err < 0 && errno != EINPROGRESS) {
368 perror("connect c1 failed()");
372 addr.sin_port = htons(S2_PORT);
373 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
374 if (err < 0 && errno != EINPROGRESS) {
375 perror("connect c2 failed()");
377 } else if (err < 0) {
381 /* Accept Connecrtions */
382 p1 = accept(s1, NULL, NULL);
384 perror("accept s1 failed()");
388 p2 = accept(s2, NULL, NULL);
390 perror("accept s1 failed()");
395 printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
396 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
405 struct timespec start;
409 static int msg_loop_sendpage(int fd, int iov_length, int cnt,
411 struct sockmap_options *opt)
413 bool drop = opt->drop_expected;
420 perror("create file for sendpage");
423 for (i = 0; i < iov_length * cnt; i++, k++)
424 fwrite(&k, sizeof(char), 1, file);
426 fseek(file, 0, SEEK_SET);
430 clock_gettime(CLOCK_MONOTONIC, &s->start);
431 for (i = 0; i < cnt; i++) {
435 sent = sendfile(fd, fp, NULL, iov_length);
437 if (!drop && sent < 0) {
438 perror("sendpage loop error");
441 } else if (drop && sent >= 0) {
442 printf("sendpage loop error expected: %i errno %i\n",
449 s->bytes_sent += sent;
451 clock_gettime(CLOCK_MONOTONIC, &s->end);
456 static void msg_free_iov(struct msghdr *msg)
460 for (i = 0; i < msg->msg_iovlen; i++)
461 free(msg->msg_iov[i].iov_base);
467 static int msg_alloc_iov(struct msghdr *msg,
468 int iov_count, int iov_length,
469 bool data, bool xmit)
475 iov = calloc(iov_count, sizeof(struct iovec));
479 for (i = 0; i < iov_count; i++) {
480 unsigned char *d = calloc(iov_length, sizeof(char));
483 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
487 iov[i].iov_len = iov_length;
492 for (j = 0; j < iov_length; j++)
498 msg->msg_iovlen = iov_count;
502 for (i--; i >= 0 ; i--)
503 free(msg->msg_iov[i].iov_base);
507 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz)
509 int i, j = 0, bytes_cnt = 0;
512 for (i = 0; i < msg->msg_iovlen; i++) {
513 unsigned char *d = msg->msg_iov[i].iov_base;
515 /* Special case test for skb ingress + ktls */
516 if (i == 0 && txmsg_ktls_skb) {
517 if (msg->msg_iov[i].iov_len < 4)
519 if (txmsg_ktls_skb_redir) {
520 if (memcmp(&d[13], "PASS", 4) != 0) {
522 "detected redirect ktls_skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n", i, 0, d[13], d[14], d[15], d[16]);
530 } else if (txmsg_ktls_skb) {
531 if (memcmp(d, "PASS", 4) != 0) {
533 "detected ktls_skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n", i, 0, d[0], d[1], d[2], d[3]);
543 for (; j < msg->msg_iov[i].iov_len && size; j++) {
546 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
547 i, j, d[j], k - 1, d[j+1], k);
551 if (bytes_cnt == chunk_sz) {
561 static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
562 struct msg_stats *s, bool tx,
563 struct sockmap_options *opt)
565 struct msghdr msg = {0}, msg_peek = {0};
566 int err, i, flags = MSG_NOSIGNAL;
567 bool drop = opt->drop_expected;
568 bool data = opt->data_test;
570 err = msg_alloc_iov(&msg, iov_count, iov_length, data, tx);
574 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
580 clock_gettime(CLOCK_MONOTONIC, &s->start);
581 for (i = 0; i < cnt; i++) {
585 sent = sendmsg(fd, &msg, flags);
587 if (!drop && sent < 0) {
588 perror("sendmsg loop error");
590 } else if (drop && sent >= 0) {
592 "sendmsg loop error expected: %i errno %i\n",
598 s->bytes_sent += sent;
600 clock_gettime(CLOCK_MONOTONIC, &s->end);
602 int slct, recvp = 0, recv, max_fd = fd;
603 float total_bytes, txmsg_pop_total;
604 int fd_flags = O_NONBLOCK;
605 struct timeval timeout;
609 /* Account for pop bytes noting each iteration of apply will
610 * call msg_pop_data helper so we need to account for this
611 * by calculating the number of apply iterations. Note user
612 * of the tool can create cases where no data is sent by
613 * manipulating pop/push/pull/etc. For example txmsg_apply 1
614 * with txmsg_pop 1 will try to apply 1B at a time but each
615 * iteration will then pop 1B so no data will ever be sent.
616 * This is really only useful for testing edge cases in code
619 total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
621 txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply);
623 txmsg_pop_total = txmsg_pop * cnt;
624 total_bytes -= txmsg_pop_total;
625 err = clock_gettime(CLOCK_MONOTONIC, &s->start);
627 perror("recv start time");
628 while (s->bytes_recvd < total_bytes) {
631 timeout.tv_usec = 300000;
641 slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
644 clock_gettime(CLOCK_MONOTONIC, &s->end);
648 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
650 clock_gettime(CLOCK_MONOTONIC, &s->end);
657 recvp = recvmsg(fd, &msg_peek, flags);
659 if (errno != EWOULDBLOCK) {
660 clock_gettime(CLOCK_MONOTONIC, &s->end);
667 recv = recvmsg(fd, &msg, flags);
669 if (errno != EWOULDBLOCK) {
670 clock_gettime(CLOCK_MONOTONIC, &s->end);
671 perror("recv failed()");
676 s->bytes_recvd += recv;
679 int chunk_sz = opt->sendpage ?
681 iov_length * iov_count;
683 errno = msg_verify_data(&msg, recv, chunk_sz);
685 perror("data verify msg failed");
689 errno = msg_verify_data(&msg_peek,
693 perror("data verify msg_peek failed");
699 clock_gettime(CLOCK_MONOTONIC, &s->end);
703 msg_free_iov(&msg_peek);
707 msg_free_iov(&msg_peek);
711 static float giga = 1000000000;
713 static inline float sentBps(struct msg_stats s)
715 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
718 static inline float recvdBps(struct msg_stats s)
720 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
723 static int sendmsg_test(struct sockmap_options *opt)
725 float sent_Bps = 0, recvd_Bps = 0;
726 int rx_fd, txpid, rxpid, err = 0;
727 struct msg_stats s = {0};
728 int iov_count = opt->iov_count;
729 int iov_buf = opt->iov_length;
730 int rx_status, tx_status;
741 /* Redirecting into non-TLS socket which sends into a TLS
742 * socket is not a valid test. So in this case lets not
743 * enable kTLS but still run the test.
745 if (!txmsg_redir || (txmsg_redir && txmsg_ingress)) {
746 err = sockmap_init_ktls(opt->verbose, rx_fd);
750 err = sockmap_init_ktls(opt->verbose, c1);
757 iov_buf -= (txmsg_pop - txmsg_start_pop + 1);
758 if (opt->drop_expected || txmsg_ktls_skb_drop)
761 if (!iov_buf) /* zero bytes sent case */
766 err = msg_loop(rx_fd, iov_count, iov_buf,
767 cnt, &s, false, opt);
768 if (opt->verbose > 1)
770 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
771 iov_count, iov_buf, cnt, err);
772 if (s.end.tv_sec - s.start.tv_sec) {
773 sent_Bps = sentBps(s);
774 recvd_Bps = recvdBps(s);
776 if (opt->verbose > 1)
778 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
779 s.bytes_sent, sent_Bps, sent_Bps/giga,
780 s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
781 peek_flag ? "(peek_msg)" : "");
782 if (err && txmsg_cork)
785 } else if (rxpid == -1) {
786 perror("msg_loop_rx");
793 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
795 err = msg_loop(c1, iov_count, iov_buf,
800 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
801 iov_count, iov_buf, cnt, err);
802 if (s.end.tv_sec - s.start.tv_sec) {
803 sent_Bps = sentBps(s);
804 recvd_Bps = recvdBps(s);
806 if (opt->verbose > 1)
808 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
809 s.bytes_sent, sent_Bps, sent_Bps/giga,
810 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
812 } else if (txpid == -1) {
813 perror("msg_loop_tx");
817 assert(waitpid(rxpid, &rx_status, 0) == rxpid);
818 assert(waitpid(txpid, &tx_status, 0) == txpid);
819 if (WIFEXITED(rx_status)) {
820 err = WEXITSTATUS(rx_status);
822 fprintf(stderr, "rx thread exited with err %d.\n", err);
826 if (WIFEXITED(tx_status)) {
827 err = WEXITSTATUS(tx_status);
829 fprintf(stderr, "tx thread exited with err %d.\n", err);
835 static int forever_ping_pong(int rate, struct sockmap_options *opt)
837 struct timeval timeout;
838 char buf[1024] = {0};
844 /* Ping/Pong data from client to server */
845 sc = send(c1, buf, sizeof(buf), 0);
847 perror("send failed()");
852 int s, rc, i, max_fd = p2;
862 s = select(max_fd + 1, &w, NULL, NULL, &timeout);
867 fprintf(stderr, "unexpected timeout\n");
871 for (i = 0; i <= max_fd && s > 0; ++i) {
872 if (!FD_ISSET(i, &w))
877 rc = recv(i, buf, sizeof(buf), 0);
879 if (errno != EWOULDBLOCK) {
880 perror("recv failed()");
890 sc = send(i, buf, rc, 0);
892 perror("send failed()");
919 static int run_options(struct sockmap_options *options, int cg_fd, int test)
921 int i, key, next_key, err, tx_prog_fd = -1, zero = 0;
923 /* If base test skip BPF setup */
924 if (test == BASE || test == BASE_SENDPAGE)
927 /* Attach programs to sockmap */
928 err = bpf_prog_attach(prog_fd[0], map_fd[0],
929 BPF_SK_SKB_STREAM_PARSER, 0);
932 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
933 prog_fd[0], map_fd[0], err, strerror(errno));
937 err = bpf_prog_attach(prog_fd[1], map_fd[0],
938 BPF_SK_SKB_STREAM_VERDICT, 0);
940 fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
941 err, strerror(errno));
945 /* Attach programs to TLS sockmap */
946 if (txmsg_ktls_skb) {
947 err = bpf_prog_attach(prog_fd[0], map_fd[8],
948 BPF_SK_SKB_STREAM_PARSER, 0);
951 "ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n",
952 prog_fd[0], map_fd[8], err, strerror(errno));
956 err = bpf_prog_attach(prog_fd[2], map_fd[8],
957 BPF_SK_SKB_STREAM_VERDICT, 0);
959 fprintf(stderr, "ERROR: bpf_prog_attach (TLS sockmap): %d (%s)\n",
960 err, strerror(errno));
965 /* Attach to cgroups */
966 err = bpf_prog_attach(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
968 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
969 err, strerror(errno));
974 err = sockmap_init_sockets(options->verbose);
976 fprintf(stderr, "ERROR: test socket failed: %d\n", err);
980 /* Attach txmsg program to sockmap */
982 tx_prog_fd = prog_fd[4];
983 else if (txmsg_redir)
984 tx_prog_fd = prog_fd[5];
985 else if (txmsg_apply)
986 tx_prog_fd = prog_fd[6];
988 tx_prog_fd = prog_fd[7];
990 tx_prog_fd = prog_fd[8];
997 err = bpf_prog_attach(tx_prog_fd,
998 map_fd[1], BPF_SK_MSG_VERDICT, 0);
1001 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
1002 err, strerror(errno));
1006 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
1009 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1010 err, strerror(errno));
1019 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
1022 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1023 err, strerror(errno));
1028 err = bpf_map_update_elem(map_fd[3],
1029 &i, &txmsg_apply, BPF_ANY);
1032 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
1033 err, strerror(errno));
1039 err = bpf_map_update_elem(map_fd[4],
1040 &i, &txmsg_cork, BPF_ANY);
1043 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
1044 err, strerror(errno));
1050 err = bpf_map_update_elem(map_fd[5],
1051 &i, &txmsg_start, BPF_ANY);
1054 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
1055 err, strerror(errno));
1062 err = bpf_map_update_elem(map_fd[5],
1063 &i, &txmsg_end, BPF_ANY);
1066 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
1067 err, strerror(errno));
1072 if (txmsg_start_push) {
1074 err = bpf_map_update_elem(map_fd[5],
1075 &i, &txmsg_start_push, BPF_ANY);
1078 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n",
1079 err, strerror(errno));
1084 if (txmsg_end_push) {
1086 err = bpf_map_update_elem(map_fd[5],
1087 &i, &txmsg_end_push, BPF_ANY);
1090 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n",
1091 txmsg_end_push, i, err, strerror(errno));
1096 if (txmsg_start_pop) {
1098 err = bpf_map_update_elem(map_fd[5],
1099 &i, &txmsg_start_pop, BPF_ANY);
1102 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",
1103 txmsg_start_pop, i, err, strerror(errno));
1108 bpf_map_update_elem(map_fd[5],
1109 &i, &txmsg_start_pop, BPF_ANY);
1114 err = bpf_map_update_elem(map_fd[5],
1115 &i, &txmsg_pop, BPF_ANY);
1118 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",
1119 txmsg_pop, i, err, strerror(errno));
1124 bpf_map_update_elem(map_fd[5],
1125 &i, &txmsg_pop, BPF_ANY);
1129 if (txmsg_ingress) {
1130 int in = BPF_F_INGRESS;
1133 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
1136 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1137 err, strerror(errno));
1140 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
1143 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
1144 err, strerror(errno));
1146 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
1149 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1150 err, strerror(errno));
1154 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
1157 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1158 err, strerror(errno));
1162 if (txmsg_ktls_skb) {
1163 int ingress = BPF_F_INGRESS;
1166 err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY);
1169 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1170 err, strerror(errno));
1173 if (txmsg_ktls_skb_redir) {
1175 err = bpf_map_update_elem(map_fd[7],
1176 &i, &ingress, BPF_ANY);
1179 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1180 err, strerror(errno));
1184 if (txmsg_ktls_skb_drop) {
1186 err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY);
1190 if (txmsg_redir_skb) {
1191 int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
1193 int ingress = BPF_F_INGRESS;
1196 err = bpf_map_update_elem(map_fd[7],
1197 &i, &ingress, BPF_ANY);
1200 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1201 err, strerror(errno));
1205 err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY);
1208 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1209 err, strerror(errno));
1215 options->drop_expected = true;
1217 if (test == PING_PONG)
1218 err = forever_ping_pong(options->rate, options);
1219 else if (test == SENDMSG) {
1220 options->base = false;
1221 options->sendpage = false;
1222 err = sendmsg_test(options);
1223 } else if (test == SENDPAGE) {
1224 options->base = false;
1225 options->sendpage = true;
1226 err = sendmsg_test(options);
1227 } else if (test == BASE) {
1228 options->base = true;
1229 options->sendpage = false;
1230 err = sendmsg_test(options);
1231 } else if (test == BASE_SENDPAGE) {
1232 options->base = true;
1233 options->sendpage = true;
1234 err = sendmsg_test(options);
1236 fprintf(stderr, "unknown test\n");
1238 /* Detatch and zero all the maps */
1239 bpf_prog_detach2(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS);
1240 bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER);
1241 bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT);
1242 bpf_prog_detach2(prog_fd[0], map_fd[8], BPF_SK_SKB_STREAM_PARSER);
1243 bpf_prog_detach2(prog_fd[2], map_fd[8], BPF_SK_SKB_STREAM_VERDICT);
1245 if (tx_prog_fd >= 0)
1246 bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT);
1248 for (i = 0; i < 8; i++) {
1250 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1251 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
1252 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1266 static char *test_to_str(int test)
1277 #define OPTSTRING 60
1278 static void test_options(char *options)
1280 char tstr[OPTSTRING];
1282 memset(options, 0, OPTSTRING);
1285 strncat(options, "pass,", OPTSTRING);
1287 strncat(options, "redir,", OPTSTRING);
1289 strncat(options, "drop,", OPTSTRING);
1291 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
1292 strncat(options, tstr, OPTSTRING);
1295 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
1296 strncat(options, tstr, OPTSTRING);
1299 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
1300 strncat(options, tstr, OPTSTRING);
1303 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1304 strncat(options, tstr, OPTSTRING);
1306 if (txmsg_start_pop) {
1307 snprintf(tstr, OPTSTRING, "pop (%d,%d),",
1308 txmsg_start_pop, txmsg_start_pop + txmsg_pop);
1309 strncat(options, tstr, OPTSTRING);
1312 strncat(options, "ingress,", OPTSTRING);
1313 if (txmsg_redir_skb)
1314 strncat(options, "redir_skb,", OPTSTRING);
1316 strncat(options, "ktls_skb,", OPTSTRING);
1318 strncat(options, "ktls,", OPTSTRING);
1320 strncat(options, "peek,", OPTSTRING);
1323 static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
1325 char *options = calloc(OPTSTRING, sizeof(char));
1328 if (test == SENDPAGE)
1329 opt->sendpage = true;
1331 opt->sendpage = false;
1334 opt->drop_expected = true;
1336 opt->drop_expected = false;
1338 test_options(options);
1342 " [TEST %i]: (%i, %i, %i, %s, %s): ",
1343 test_cnt, opt->rate, opt->iov_count, opt->iov_length,
1344 test_to_str(test), options);
1347 err = run_options(opt, cgrp, test);
1349 fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED");
1351 !err ? passed++ : failed++;
1356 static void test_exec(int cgrp, struct sockmap_options *opt)
1358 int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);
1363 err = __test_exec(cgrp, SENDMSG, opt);
1368 err = __test_exec(cgrp, SENDPAGE, opt);
1374 static void test_send_one(struct sockmap_options *opt, int cgrp)
1376 opt->iov_length = 1;
1379 test_exec(cgrp, opt);
1381 opt->iov_length = 1;
1382 opt->iov_count = 1024;
1384 test_exec(cgrp, opt);
1386 opt->iov_length = 1024;
1389 test_exec(cgrp, opt);
1393 static void test_send_many(struct sockmap_options *opt, int cgrp)
1395 opt->iov_length = 3;
1398 test_exec(cgrp, opt);
1402 opt->iov_length = 5;
1403 test_exec(cgrp, opt);
1406 static void test_send_large(struct sockmap_options *opt, int cgrp)
1408 opt->iov_length = 256;
1409 opt->iov_count = 1024;
1411 test_exec(cgrp, opt);
1414 static void test_send(struct sockmap_options *opt, int cgrp)
1416 test_send_one(opt, cgrp);
1417 test_send_many(opt, cgrp);
1418 test_send_large(opt, cgrp);
1422 static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)
1424 /* Test small and large iov_count values with pass/redir/apply/cork */
1426 test_send(opt, cgrp);
1429 static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)
1432 test_send(opt, cgrp);
1435 static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)
1438 test_send(opt, cgrp);
1441 static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)
1443 txmsg_pass = txmsg_drop = 0;
1444 txmsg_ingress = txmsg_redir = 1;
1445 test_send(opt, cgrp);
1448 static void test_txmsg_skb(int cgrp, struct sockmap_options *opt)
1450 bool data = opt->data_test;
1453 opt->data_test = true;
1456 txmsg_pass = txmsg_drop = 0;
1457 txmsg_ingress = txmsg_redir = 0;
1461 /* Using data verification so ensure iov layout is
1462 * expected from test receiver side. e.g. has enough
1463 * bytes to write test code.
1465 opt->iov_length = 100;
1468 test_exec(cgrp, opt);
1470 txmsg_ktls_skb_drop = 1;
1471 test_exec(cgrp, opt);
1473 txmsg_ktls_skb_drop = 0;
1474 txmsg_ktls_skb_redir = 1;
1475 test_exec(cgrp, opt);
1477 opt->data_test = data;
1482 /* Test cork with hung data. This tests poor usage patterns where
1483 * cork can leave data on the ring if user program is buggy and
1484 * doesn't flush them somehow. They do take some time however
1485 * because they wait for a timeout. Test pass, redir and cork with
1486 * apply logic. Use cork size of 4097 with send_large to avoid
1487 * aligning cork size with send size.
1489 static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)
1495 test_send_large(opt, cgrp);
1501 test_send_large(opt, cgrp);
1507 test_send_large(opt, cgrp);
1510 static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)
1512 /* Test basic start/end */
1515 test_send(opt, cgrp);
1520 test_send_large(opt, cgrp);
1522 /* Test pull + redirect */
1526 test_send(opt, cgrp);
1528 /* Test pull + cork */
1533 test_send_many(opt, cgrp);
1535 /* Test pull + cork + redirect */
1540 test_send_many(opt, cgrp);
1543 static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)
1545 /* Test basic pop */
1546 txmsg_start_pop = 1;
1548 test_send_many(opt, cgrp);
1550 /* Test pop with >4k */
1551 txmsg_start_pop = 4096;
1553 test_send_large(opt, cgrp);
1555 /* Test pop + redirect */
1557 txmsg_start_pop = 1;
1559 test_send_many(opt, cgrp);
1561 /* Test pop + cork */
1564 txmsg_start_pop = 1;
1566 test_send_many(opt, cgrp);
1568 /* Test pop + redirect + cork */
1571 txmsg_start_pop = 1;
1573 test_send_many(opt, cgrp);
1576 static void test_txmsg_push(int cgrp, struct sockmap_options *opt)
1578 /* Test basic push */
1579 txmsg_start_push = 1;
1581 test_send(opt, cgrp);
1583 /* Test push 4kB >4k */
1584 txmsg_start_push = 4096;
1585 txmsg_end_push = 4096;
1586 test_send_large(opt, cgrp);
1588 /* Test push + redirect */
1590 txmsg_start_push = 1;
1592 test_send_many(opt, cgrp);
1594 /* Test push + cork */
1597 txmsg_start_push = 1;
1599 test_send_many(opt, cgrp);
1602 static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)
1604 txmsg_start_push = 1;
1605 txmsg_end_push = 10;
1606 txmsg_start_pop = 5;
1608 test_send_large(opt, cgrp);
1611 static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)
1617 test_send_one(opt, cgrp);
1623 test_send_one(opt, cgrp);
1629 test_send_large(opt, cgrp);
1635 test_send_large(opt, cgrp);
1638 static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)
1644 test_send(opt, cgrp);
1650 test_send(opt, cgrp);
1653 char *map_names[] = {
1665 int prog_attach_type[] = {
1666 BPF_SK_SKB_STREAM_PARSER,
1667 BPF_SK_SKB_STREAM_VERDICT,
1668 BPF_SK_SKB_STREAM_VERDICT,
1669 BPF_CGROUP_SOCK_OPS,
1680 BPF_PROG_TYPE_SK_SKB,
1681 BPF_PROG_TYPE_SK_SKB,
1682 BPF_PROG_TYPE_SK_SKB,
1683 BPF_PROG_TYPE_SOCK_OPS,
1684 BPF_PROG_TYPE_SK_MSG,
1685 BPF_PROG_TYPE_SK_MSG,
1686 BPF_PROG_TYPE_SK_MSG,
1687 BPF_PROG_TYPE_SK_MSG,
1688 BPF_PROG_TYPE_SK_MSG,
1689 BPF_PROG_TYPE_SK_MSG,
1690 BPF_PROG_TYPE_SK_MSG,
1693 static int populate_progs(char *bpf_file)
1695 struct bpf_program *prog;
1696 struct bpf_object *obj;
1700 obj = bpf_object__open(bpf_file);
1701 err = libbpf_get_error(obj);
1705 libbpf_strerror(err, err_buf, sizeof(err_buf));
1706 printf("Unable to load eBPF objects in file '%s' : %s\n",
1711 bpf_object__for_each_program(prog, obj) {
1712 bpf_program__set_type(prog, prog_type[i]);
1713 bpf_program__set_expected_attach_type(prog,
1714 prog_attach_type[i]);
1718 i = bpf_object__load(obj);
1720 bpf_object__for_each_program(prog, obj) {
1721 prog_fd[i] = bpf_program__fd(prog);
1725 for (i = 0; i < sizeof(map_fd)/sizeof(int); i++) {
1726 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1727 map_fd[i] = bpf_map__fd(maps[i]);
1728 if (map_fd[i] < 0) {
1729 fprintf(stderr, "load_bpf_file: (%i) %s\n",
1730 map_fd[i], strerror(errno));
1738 struct _test test[] = {
1739 {"txmsg test passthrough", test_txmsg_pass},
1740 {"txmsg test redirect", test_txmsg_redir},
1741 {"txmsg test drop", test_txmsg_drop},
1742 {"txmsg test ingress redirect", test_txmsg_ingress_redir},
1743 {"txmsg test skb", test_txmsg_skb},
1744 {"txmsg test apply", test_txmsg_apply},
1745 {"txmsg test cork", test_txmsg_cork},
1746 {"txmsg test hanging corks", test_txmsg_cork_hangs},
1747 {"txmsg test push_data", test_txmsg_push},
1748 {"txmsg test pull-data", test_txmsg_pull},
1749 {"txmsg test pop-data", test_txmsg_pop},
1750 {"txmsg test push/pop data", test_txmsg_push_pop},
1753 static int check_whitelist(struct _test *t, struct sockmap_options *opt)
1757 if (!opt->whitelist)
1759 ptr = strdup(opt->whitelist);
1762 entry = strtok(ptr, ",");
1764 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1765 strstr(opt->map, entry) != 0 ||
1766 strstr(t->title, entry) != 0)
1768 entry = strtok(NULL, ",");
1773 static int check_blacklist(struct _test *t, struct sockmap_options *opt)
1777 if (!opt->blacklist)
1779 ptr = strdup(opt->blacklist);
1782 entry = strtok(ptr, ",");
1784 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1785 strstr(opt->map, entry) != 0 ||
1786 strstr(t->title, entry) != 0)
1788 entry = strtok(NULL, ",");
1793 static int __test_selftests(int cg_fd, struct sockmap_options *opt)
1797 err = populate_progs(opt->map);
1799 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
1803 /* Tests basic commands and APIs */
1804 for (i = 0; i < sizeof(test)/sizeof(struct _test); i++) {
1805 struct _test t = test[i];
1807 if (check_whitelist(&t, opt) != 0)
1809 if (check_blacklist(&t, opt) == 0)
1812 test_start_subtest(&t, opt);
1813 t.tester(cg_fd, opt);
1820 static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)
1822 opt->map = BPF_SOCKMAP_FILENAME;
1823 __test_selftests(cg_fd, opt);
1826 static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)
1828 opt->map = BPF_SOCKHASH_FILENAME;
1829 __test_selftests(cg_fd, opt);
1832 static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt)
1834 opt->map = BPF_SOCKHASH_FILENAME;
1835 opt->prepend = "ktls";
1837 __test_selftests(cg_fd, opt);
1841 static int test_selftest(int cg_fd, struct sockmap_options *opt)
1844 test_selftests_sockmap(cg_fd, opt);
1845 test_selftests_sockhash(cg_fd, opt);
1846 test_selftests_ktls(cg_fd, opt);
1847 test_print_results();
1851 int main(int argc, char **argv)
1853 int iov_count = 1, length = 1024, rate = 1;
1854 struct sockmap_options options = {0};
1855 int opt, longindex, err, cg_fd = 0;
1856 char *bpf_file = BPF_SOCKMAP_FILENAME;
1857 int test = SELFTESTS;
1858 bool cg_created = 0;
1860 while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",
1861 long_options, &longindex)) != -1) {
1864 txmsg_start = atoi(optarg);
1867 txmsg_end = atoi(optarg);
1870 txmsg_start_push = atoi(optarg);
1873 txmsg_end_push = atoi(optarg);
1876 txmsg_start_pop = atoi(optarg);
1879 txmsg_pop = atoi(optarg);
1882 txmsg_apply = atoi(optarg);
1885 txmsg_cork = atoi(optarg);
1888 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
1891 "ERROR: (%i) open cg path failed: %s\n",
1897 rate = atoi(optarg);
1900 options.verbose = 1;
1902 options.verbose = atoi(optarg);
1905 iov_count = atoi(optarg);
1908 length = atoi(optarg);
1911 options.data_test = true;
1914 if (strcmp(optarg, "ping") == 0) {
1916 } else if (strcmp(optarg, "sendmsg") == 0) {
1918 } else if (strcmp(optarg, "base") == 0) {
1920 } else if (strcmp(optarg, "base_sendpage") == 0) {
1921 test = BASE_SENDPAGE;
1922 } else if (strcmp(optarg, "sendpage") == 0) {
1930 options.whitelist = strdup(optarg);
1931 if (!options.whitelist)
1935 options.blacklist = strdup(optarg);
1936 if (!options.blacklist)
1948 if (setup_cgroup_environment()) {
1949 fprintf(stderr, "ERROR: cgroup env failed\n");
1953 cg_fd = create_and_get_cgroup(CG_PATH);
1956 "ERROR: (%i) open cg path failed: %s\n",
1957 cg_fd, strerror(errno));
1961 if (join_cgroup(CG_PATH)) {
1962 fprintf(stderr, "ERROR: failed to join cgroup\n");
1968 if (test == SELFTESTS) {
1969 err = test_selftest(cg_fd, &options);
1973 err = populate_progs(bpf_file);
1975 fprintf(stderr, "populate program: (%s) %s\n",
1976 bpf_file, strerror(errno));
1982 signal(SIGINT, running_handler);
1984 options.iov_count = iov_count;
1985 options.iov_length = length;
1986 options.rate = rate;
1988 err = run_options(&options, cg_fd, test);
1990 if (options.whitelist)
1991 free(options.whitelist);
1992 if (options.blacklist)
1993 free(options.blacklist);
1995 cleanup_cgroup_environment();
2000 void running_handler(int a)