samples/bpf/xdp_redirect_cpu_user.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2017 Jesper Dangaard Brouer, Red Hat, Inc.
   3  */
   4 static const char *__doc__ =
   5 "XDP CPU redirect tool, using BPF_MAP_TYPE_CPUMAP\n"
   6 "Usage: xdp_redirect_cpu -d <IFINDEX|IFNAME> -c 0 ... -c N\n"
   7 "Valid specification for CPUMAP BPF program:\n"
   8 "  --mprog-name/-e pass (use built-in XDP_PASS program)\n"
   9 "  --mprog-name/-e drop (use built-in XDP_DROP program)\n"
  10 "  --redirect-device/-r <ifindex|ifname> (use built-in DEVMAP redirect program)\n"
  11 "  Custom CPUMAP BPF program:\n"
  12 "    --mprog-filename/-f <filename> --mprog-name/-e <program>\n"
  13 "    Optionally, also pass --redirect-map/-m and --redirect-device/-r together\n"
  14 "    to configure DEVMAP in BPF object <filename>\n";
  15
  16 #include <errno.h>
  17 #include <signal.h>
  18 #include <stdio.h>
  19 #include <stdlib.h>
  20 #include <stdbool.h>
  21 #include <string.h>
  22 #include <unistd.h>
  23 #include <locale.h>
  24 #include <sys/resource.h>
  25 #include <sys/sysinfo.h>
  26 #include <getopt.h>
  27 #include <net/if.h>
  28 #include <time.h>
  29 #include <linux/limits.h>
  30 #include <arpa/inet.h>
  31 #include <linux/if_link.h>
  32 #include <bpf/bpf.h>
  33 #include <bpf/libbpf.h>
  34 #include "bpf_util.h"
  35 #include "xdp_sample_user.h"
  36 #include "xdp_redirect_cpu.skel.h"
  37
  38 static int map_fd;
  39 static int avail_fd;
  40 static int count_fd;
  41
  42 static int mask = SAMPLE_RX_CNT | SAMPLE_REDIRECT_ERR_MAP_CNT |
  43                   SAMPLE_CPUMAP_ENQUEUE_CNT | SAMPLE_CPUMAP_KTHREAD_CNT |
  44                   SAMPLE_EXCEPTION_CNT;
  45
  46 DEFINE_SAMPLE_INIT(xdp_redirect_cpu);
  47
  48 static const struct option long_options[] = {
  49         { "help", no_argument, NULL, 'h' },
  50         { "dev", required_argument, NULL, 'd' },
  51         { "skb-mode", no_argument, NULL, 'S' },
  52         { "progname", required_argument, NULL, 'p' },
  53         { "qsize", required_argument, NULL, 'q' },
  54         { "cpu", required_argument, NULL, 'c' },
  55         { "stress-mode", no_argument, NULL, 'x' },
  56         { "force", no_argument, NULL, 'F' },
  57         { "interval", required_argument, NULL, 'i' },
  58         { "verbose", no_argument, NULL, 'v' },
  59         { "stats", no_argument, NULL, 's' },
  60         { "mprog-name", required_argument, NULL, 'e' },
  61         { "mprog-filename", required_argument, NULL, 'f' },
  62         { "redirect-device", required_argument, NULL, 'r' },
  63         { "redirect-map", required_argument, NULL, 'm' },
  64         {}
  65 };
  66
  67 static void print_avail_progs(struct bpf_object *obj)
  68 {
  69         struct bpf_program *pos;
  70
  71         printf(" Programs to be used for -p/--progname:\n");
  72         bpf_object__for_each_program(pos, obj) {
  73                 if (bpf_program__is_xdp(pos)) {
  74                         if (!strncmp(bpf_program__name(pos), "xdp_prognum",
  75                                      sizeof("xdp_prognum") - 1))
  76                                 printf(" %s\n", bpf_program__name(pos));
  77                 }
  78         }
  79 }
  80
  81 static void usage(char *argv[], const struct option *long_options,
  82                   const char *doc, int mask, bool error, struct bpf_object *obj)
  83 {
  84         sample_usage(argv, long_options, doc, mask, error);
  85         print_avail_progs(obj);
  86 }
  87
  88 static int create_cpu_entry(__u32 cpu, struct bpf_cpumap_val *value,
  89                             __u32 avail_idx, bool new)
  90 {
  91         __u32 curr_cpus_count = 0;
  92         __u32 key = 0;
  93         int ret;
  94
  95         /* Add a CPU entry to cpumap, as this allocate a cpu entry in
  96          * the kernel for the cpu.
  97          */
  98         ret = bpf_map_update_elem(map_fd, &cpu, value, 0);
  99         if (ret < 0) {
 100                 fprintf(stderr, "Create CPU entry failed: %s\n", strerror(errno));
 101                 return ret;
 102         }
 103
 104         /* Inform bpf_prog's that a new CPU is available to select
 105          * from via some control maps.
 106          */
 107         ret = bpf_map_update_elem(avail_fd, &avail_idx, &cpu, 0);
 108         if (ret < 0) {
 109                 fprintf(stderr, "Add to avail CPUs failed: %s\n", strerror(errno));
 110                 return ret;
 111         }
 112
 113         /* When not replacing/updating existing entry, bump the count */
 114         ret = bpf_map_lookup_elem(count_fd, &key, &curr_cpus_count);
 115         if (ret < 0) {
 116                 fprintf(stderr, "Failed reading curr cpus_count: %s\n",
 117                         strerror(errno));
 118                 return ret;
 119         }
 120         if (new) {
 121                 curr_cpus_count++;
 122                 ret = bpf_map_update_elem(count_fd, &key,
 123                                           &curr_cpus_count, 0);
 124                 if (ret < 0) {
 125                         fprintf(stderr, "Failed write curr cpus_count: %s\n",
 126                                 strerror(errno));
 127                         return ret;
 128                 }
 129         }
 130
 131         printf("%s CPU: %u as idx: %u qsize: %d cpumap_prog_fd: %d (cpus_count: %u)\n",
 132                new ? "Add new" : "Replace", cpu, avail_idx,
 133                value->qsize, value->bpf_prog.fd, curr_cpus_count);
 134
 135         return 0;
 136 }
 137
 138 /* CPUs are zero-indexed. Thus, add a special sentinel default value
 139  * in map cpus_available to mark CPU index'es not configured
 140  */
 141 static int mark_cpus_unavailable(void)
 142 {
 143         int ret, i, n_cpus = libbpf_num_possible_cpus();
 144         __u32 invalid_cpu = n_cpus;
 145
 146         for (i = 0; i < n_cpus; i++) {
 147                 ret = bpf_map_update_elem(avail_fd, &i,
 148                                           &invalid_cpu, 0);
 149                 if (ret < 0) {
 150                         fprintf(stderr, "Failed marking CPU unavailable: %s\n",
 151                                 strerror(errno));
 152                         return ret;
 153                 }
 154         }
 155
 156         return 0;
 157 }
 158
 159 /* Stress cpumap management code by concurrently changing underlying cpumap */
 160 static void stress_cpumap(void *ctx)
 161 {
 162         struct bpf_cpumap_val *value = ctx;
 163
 164         /* Changing qsize will cause kernel to free and alloc a new
 165          * bpf_cpu_map_entry, with an associated/complicated tear-down
 166          * procedure.
 167          */
 168         value->qsize = 1024;
 169         create_cpu_entry(1, value, 0, false);
 170         value->qsize = 8;
 171         create_cpu_entry(1, value, 0, false);
 172         value->qsize = 16000;
 173         create_cpu_entry(1, value, 0, false);
 174 }
 175
 176 static int set_cpumap_prog(struct xdp_redirect_cpu *skel,
 177                            const char *redir_interface, const char *redir_map,
 178                            const char *mprog_filename, const char *mprog_name)
 179 {
 180         if (mprog_filename) {
 181                 struct bpf_program *prog;
 182                 struct bpf_object *obj;
 183                 int ret;
 184
 185                 if (!mprog_name) {
 186                         fprintf(stderr, "BPF program not specified for file %s\n",
 187                                 mprog_filename);
 188                         goto end;
 189                 }
 190                 if ((redir_interface && !redir_map) || (!redir_interface && redir_map)) {
 191                         fprintf(stderr, "--redirect-%s specified but --redirect-%s not specified\n",
 192                                 redir_interface ? "device" : "map", redir_interface ? "map" : "device");
 193                         goto end;
 194                 }
 195
 196                 /* Custom BPF program */
 197                 obj = bpf_object__open_file(mprog_filename, NULL);
 198                 if (!obj) {
 199                         ret = -errno;
 200                         fprintf(stderr, "Failed to bpf_prog_load_xattr: %s\n",
 201                                 strerror(errno));
 202                         return ret;
 203                 }
 204
 205                 ret = bpf_object__load(obj);
 206                 if (ret < 0) {
 207                         ret = -errno;
 208                         fprintf(stderr, "Failed to bpf_object__load: %s\n",
 209                                 strerror(errno));
 210                         return ret;
 211                 }
 212
 213                 if (redir_map) {
 214                         int err, redir_map_fd, ifindex_out, key = 0;
 215
 216                         redir_map_fd = bpf_object__find_map_fd_by_name(obj, redir_map);
 217                         if (redir_map_fd < 0) {
 218                                 fprintf(stderr, "Failed to bpf_object__find_map_fd_by_name: %s\n",
 219                                         strerror(errno));
 220                                 return redir_map_fd;
 221                         }
 222
 223                         ifindex_out = if_nametoindex(redir_interface);
 224                         if (!ifindex_out)
 225                                 ifindex_out = strtoul(redir_interface, NULL, 0);
 226                         if (!ifindex_out) {
 227                                 fprintf(stderr, "Bad interface name or index\n");
 228                                 return -EINVAL;
 229                         }
 230
 231                         err = bpf_map_update_elem(redir_map_fd, &key, &ifindex_out, 0);
 232                         if (err < 0)
 233                                 return err;
 234                 }
 235
 236                 prog = bpf_object__find_program_by_name(obj, mprog_name);
 237                 if (!prog) {
 238                         ret = -errno;
 239                         fprintf(stderr, "Failed to bpf_object__find_program_by_name: %s\n",
 240                                 strerror(errno));
 241                         return ret;
 242                 }
 243
 244                 return bpf_program__fd(prog);
 245         } else {
 246                 if (mprog_name) {
 247                         if (redir_interface || redir_map) {
 248                                 fprintf(stderr, "Need to specify --mprog-filename/-f\n");
 249                                 goto end;
 250                         }
 251                         if (!strcmp(mprog_name, "pass") || !strcmp(mprog_name, "drop")) {
 252                                 /* Use built-in pass/drop programs */
 253                                 return *mprog_name == 'p' ? bpf_program__fd(skel->progs.xdp_redirect_cpu_pass)
 254                                         : bpf_program__fd(skel->progs.xdp_redirect_cpu_drop);
 255                         } else {
 256                                 fprintf(stderr, "Unknown name \"%s\" for built-in BPF program\n",
 257                                         mprog_name);
 258                                 goto end;
 259                         }
 260                 } else {
 261                         if (redir_map) {
 262                                 fprintf(stderr, "Need to specify --mprog-filename, --mprog-name and"
 263                                         " --redirect-device with --redirect-map\n");
 264                                 goto end;
 265                         }
 266                         if (redir_interface) {
 267                                 /* Use built-in devmap redirect */
 268                                 struct bpf_devmap_val val = {};
 269                                 int ifindex_out, err;
 270                                 __u32 key = 0;
 271
 272                                 if (!redir_interface)
 273                                         return 0;
 274
 275                                 ifindex_out = if_nametoindex(redir_interface);
 276                                 if (!ifindex_out)
 277                                         ifindex_out = strtoul(redir_interface, NULL, 0);
 278                                 if (!ifindex_out) {
 279                                         fprintf(stderr, "Bad interface name or index\n");
 280                                         return -EINVAL;
 281                                 }
 282
 283                                 if (get_mac_addr(ifindex_out, skel->bss->tx_mac_addr) < 0) {
 284                                         printf("Get interface %d mac failed\n", ifindex_out);
 285                                         return -EINVAL;
 286                                 }
 287
 288                                 val.ifindex = ifindex_out;
 289                                 val.bpf_prog.fd = bpf_program__fd(skel->progs.xdp_redirect_egress_prog);
 290                                 err = bpf_map_update_elem(bpf_map__fd(skel->maps.tx_port), &key, &val, 0);
 291                                 if (err < 0)
 292                                         return -errno;
 293
 294                                 return bpf_program__fd(skel->progs.xdp_redirect_cpu_devmap);
 295                         }
 296                 }
 297         }
 298
 299         /* Disabled */
 300         return 0;
 301 end:
 302         fprintf(stderr, "Invalid options for CPUMAP BPF program\n");
 303         return -EINVAL;
 304 }
 305
 306 int main(int argc, char **argv)
 307 {
 308         const char *redir_interface = NULL, *redir_map = NULL;
 309         const char *mprog_filename = NULL, *mprog_name = NULL;
 310         struct xdp_redirect_cpu *skel;
 311         struct bpf_map_info info = {};
 312         char ifname_buf[IF_NAMESIZE];
 313         struct bpf_cpumap_val value;
 314         __u32 infosz = sizeof(info);
 315         int ret = EXIT_FAIL_OPTION;
 316         unsigned long interval = 2;
 317         bool stress_mode = false;
 318         struct bpf_program *prog;
 319         const char *prog_name;
 320         bool generic = false;
 321         bool force = false;
 322         int added_cpus = 0;
 323         bool error = true;
 324         int longindex = 0;
 325         int add_cpu = -1;
 326         int ifindex = -1;
 327         int *cpu, i, opt;
 328         char *ifname;
 329         __u32 qsize;
 330         int n_cpus;
 331
 332         n_cpus = libbpf_num_possible_cpus();
 333
 334         /* Notice: Choosing the queue size is very important when CPU is
 335          * configured with power-saving states.
 336          *
 337          * If deepest state take 133 usec to wakeup from (133/10^6). When link
 338          * speed is 10Gbit/s ((10*10^9/8) in bytes/sec). How many bytes can
 339          * arrive with in 133 usec at this speed: (10*10^9/8)*(133/10^6) =
 340          * 166250 bytes. With MTU size packets this is 110 packets, and with
 341          * minimum Ethernet (MAC-preamble + intergap) 84 bytes is 1979 packets.
 342          *
 343          * Setting default cpumap queue to 2048 as worst-case (small packet)
 344          * should be +64 packet due kthread wakeup call (due to xdp_do_flush)
 345          * worst-case is 2043 packets.
 346          *
 347          * Sysadm can configured system to avoid deep-sleep via:
 348          *   tuned-adm profile network-latency
 349          */
 350         qsize = 2048;
 351
 352         skel = xdp_redirect_cpu__open();
 353         if (!skel) {
 354                 fprintf(stderr, "Failed to xdp_redirect_cpu__open: %s\n",
 355                         strerror(errno));
 356                 ret = EXIT_FAIL_BPF;
 357                 goto end;
 358         }
 359
 360         ret = sample_init_pre_load(skel);
 361         if (ret < 0) {
 362                 fprintf(stderr, "Failed to sample_init_pre_load: %s\n", strerror(-ret));
 363                 ret = EXIT_FAIL_BPF;
 364                 goto end_destroy;
 365         }
 366
 367         if (bpf_map__set_max_entries(skel->maps.cpu_map, n_cpus) < 0) {
 368                 fprintf(stderr, "Failed to set max entries for cpu_map map: %s",
 369                         strerror(errno));
 370                 ret = EXIT_FAIL_BPF;
 371                 goto end_destroy;
 372         }
 373
 374         if (bpf_map__set_max_entries(skel->maps.cpus_available, n_cpus) < 0) {
 375                 fprintf(stderr, "Failed to set max entries for cpus_available map: %s",
 376                         strerror(errno));
 377                 ret = EXIT_FAIL_BPF;
 378                 goto end_destroy;
 379         }
 380
 381         cpu = calloc(n_cpus, sizeof(int));
 382         if (!cpu) {
 383                 fprintf(stderr, "Failed to allocate cpu array\n");
 384                 goto end_destroy;
 385         }
 386
 387         prog = skel->progs.xdp_prognum5_lb_hash_ip_pairs;
 388         while ((opt = getopt_long(argc, argv, "d:si:Sxp:f:e:r:m:c:q:Fvh",
 389                                   long_options, &longindex)) != -1) {
 390                 switch (opt) {
 391                 case 'd':
 392                         if (strlen(optarg) >= IF_NAMESIZE) {
 393                                 fprintf(stderr, "-d/--dev name too long\n");
 394                                 goto end_cpu;
 395                         }
 396                         ifname = (char *)&ifname_buf;
 397                         safe_strncpy(ifname, optarg, sizeof(ifname));
 398                         ifindex = if_nametoindex(ifname);
 399                         if (!ifindex)
 400                                 ifindex = strtoul(optarg, NULL, 0);
 401                         if (!ifindex) {
 402                                 fprintf(stderr, "Bad interface index or name (%d): %s\n",
 403                                         errno, strerror(errno));
 404                                 usage(argv, long_options, __doc__, mask, true, skel->obj);
 405                                 goto end_cpu;
 406                         }
 407                         break;
 408                 case 's':
 409                         mask |= SAMPLE_REDIRECT_MAP_CNT;
 410                         break;
 411                 case 'i':
 412                         interval = strtoul(optarg, NULL, 0);
 413                         break;
 414                 case 'S':
 415                         generic = true;
 416                         break;
 417                 case 'x':
 418                         stress_mode = true;
 419                         break;
 420                 case 'p':
 421                         /* Selecting eBPF prog to load */
 422                         prog_name = optarg;
 423                         prog = bpf_object__find_program_by_name(skel->obj,
 424                                                                 prog_name);
 425                         if (!prog) {
 426                                 fprintf(stderr,
 427                                         "Failed to find program %s specified by"
 428                                         " option -p/--progname\n",
 429                                         prog_name);
 430                                 print_avail_progs(skel->obj);
 431                                 goto end_cpu;
 432                         }
 433                         break;
 434                 case 'f':
 435                         mprog_filename = optarg;
 436                         break;
 437                 case 'e':
 438                         mprog_name = optarg;
 439                         break;
 440                 case 'r':
 441                         redir_interface = optarg;
 442                         mask |= SAMPLE_DEVMAP_XMIT_CNT_MULTI;
 443                         break;
 444                 case 'm':
 445                         redir_map = optarg;
 446                         break;
 447                 case 'c':
 448                         /* Add multiple CPUs */
 449                         add_cpu = strtoul(optarg, NULL, 0);
 450                         if (add_cpu >= n_cpus) {
 451                                 fprintf(stderr,
 452                                 "--cpu nr too large for cpumap err (%d):%s\n",
 453                                         errno, strerror(errno));
 454                                 usage(argv, long_options, __doc__, mask, true, skel->obj);
 455                                 goto end_cpu;
 456                         }
 457                         cpu[added_cpus++] = add_cpu;
 458                         break;
 459                 case 'q':
 460                         qsize = strtoul(optarg, NULL, 0);
 461                         break;
 462                 case 'F':
 463                         force = true;
 464                         break;
 465                 case 'v':
 466                         sample_switch_mode();
 467                         break;
 468                 case 'h':
 469                         error = false;
 470                 default:
 471                         usage(argv, long_options, __doc__, mask, error, skel->obj);
 472                         goto end_cpu;
 473                 }
 474         }
 475
 476         ret = EXIT_FAIL_OPTION;
 477         if (ifindex == -1) {
 478                 fprintf(stderr, "Required option --dev missing\n");
 479                 usage(argv, long_options, __doc__, mask, true, skel->obj);
 480                 goto end_cpu;
 481         }
 482
 483         if (add_cpu == -1) {
 484                 fprintf(stderr, "Required option --cpu missing\n"
 485                                 "Specify multiple --cpu option to add more\n");
 486                 usage(argv, long_options, __doc__, mask, true, skel->obj);
 487                 goto end_cpu;
 488         }
 489
 490         skel->rodata->from_match[0] = ifindex;
 491         if (redir_interface)
 492                 skel->rodata->to_match[0] = if_nametoindex(redir_interface);
 493
 494         ret = xdp_redirect_cpu__load(skel);
 495         if (ret < 0) {
 496                 fprintf(stderr, "Failed to xdp_redirect_cpu__load: %s\n",
 497                         strerror(errno));
 498                 goto end_cpu;
 499         }
 500
 501         ret = bpf_obj_get_info_by_fd(bpf_map__fd(skel->maps.cpu_map), &info, &infosz);
 502         if (ret < 0) {
 503                 fprintf(stderr, "Failed bpf_obj_get_info_by_fd for cpumap: %s\n",
 504                         strerror(errno));
 505                 goto end_cpu;
 506         }
 507
 508         skel->bss->cpumap_map_id = info.id;
 509
 510         map_fd = bpf_map__fd(skel->maps.cpu_map);
 511         avail_fd = bpf_map__fd(skel->maps.cpus_available);
 512         count_fd = bpf_map__fd(skel->maps.cpus_count);
 513
 514         ret = mark_cpus_unavailable();
 515         if (ret < 0) {
 516                 fprintf(stderr, "Unable to mark CPUs as unavailable\n");
 517                 goto end_cpu;
 518         }
 519
 520         ret = sample_init(skel, mask);
 521         if (ret < 0) {
 522                 fprintf(stderr, "Failed to initialize sample: %s\n", strerror(-ret));
 523                 ret = EXIT_FAIL;
 524                 goto end_cpu;
 525         }
 526
 527         value.bpf_prog.fd = set_cpumap_prog(skel, redir_interface, redir_map,
 528                                             mprog_filename, mprog_name);
 529         if (value.bpf_prog.fd < 0) {
 530                 fprintf(stderr, "Failed to set CPUMAP BPF program: %s\n",
 531                         strerror(-value.bpf_prog.fd));
 532                 usage(argv, long_options, __doc__, mask, true, skel->obj);
 533                 ret = EXIT_FAIL_BPF;
 534                 goto end_cpu;
 535         }
 536         value.qsize = qsize;
 537
 538         for (i = 0; i < added_cpus; i++) {
 539                 if (create_cpu_entry(cpu[i], &value, i, true) < 0) {
 540                         fprintf(stderr, "Cannot proceed, exiting\n");
 541                         usage(argv, long_options, __doc__, mask, true, skel->obj);
 542                         goto end_cpu;
 543                 }
 544         }
 545
 546         ret = EXIT_FAIL_XDP;
 547         if (sample_install_xdp(prog, ifindex, generic, force) < 0)
 548                 goto end_cpu;
 549
 550         ret = sample_run(interval, stress_mode ? stress_cpumap : NULL, &value);
 551         if (ret < 0) {
 552                 fprintf(stderr, "Failed during sample run: %s\n", strerror(-ret));
 553                 ret = EXIT_FAIL;
 554                 goto end_cpu;
 555         }
 556         ret = EXIT_OK;
 557 end_cpu:
 558         free(cpu);
 559 end_destroy:
 560         xdp_redirect_cpu__destroy(skel);
 561 end:
 562         sample_exit(ret);
 563 }