samples/bpf/xdp_sample_user.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 #define _GNU_SOURCE
   3
   4 #include <arpa/inet.h>
   5 #include <bpf/bpf.h>
   6 #include <bpf/libbpf.h>
   7 #include <errno.h>
   8 #include <fcntl.h>
   9 #include <getopt.h>
  10 #include <linux/ethtool.h>
  11 #include <linux/hashtable.h>
  12 #include <linux/if_link.h>
  13 #include <linux/jhash.h>
  14 #include <linux/limits.h>
  15 #include <linux/list.h>
  16 #include <linux/sockios.h>
  17 #include <locale.h>
  18 #include <math.h>
  19 #include <net/if.h>
  20 #include <poll.h>
  21 #include <signal.h>
  22 #include <stdbool.h>
  23 #include <stdio.h>
  24 #include <stdlib.h>
  25 #include <string.h>
  26 #include <sys/ioctl.h>
  27 #include <sys/mman.h>
  28 #include <sys/resource.h>
  29 #include <sys/signalfd.h>
  30 #include <sys/sysinfo.h>
  31 #include <sys/timerfd.h>
  32 #include <sys/utsname.h>
  33 #include <time.h>
  34 #include <unistd.h>
  35
  36 #include "bpf_util.h"
  37 #include "xdp_sample_user.h"
  38
  39 #define __sample_print(fmt, cond, ...)                                         \
  40         ({                                                                     \
  41                 if (cond)                                                      \
  42                         printf(fmt, ##__VA_ARGS__);                            \
  43         })
  44
  45 #define print_always(fmt, ...) __sample_print(fmt, 1, ##__VA_ARGS__)
  46 #define print_default(fmt, ...)                                                \
  47         __sample_print(fmt, sample_log_level & LL_DEFAULT, ##__VA_ARGS__)
  48 #define __print_err(err, fmt, ...)                                             \
  49         ({                                                                     \
  50                 __sample_print(fmt, err > 0 || sample_log_level & LL_DEFAULT,  \
  51                                ##__VA_ARGS__);                                 \
  52                 sample_err_exp = sample_err_exp ? true : err > 0;              \
  53         })
  54 #define print_err(err, fmt, ...) __print_err(err, fmt, ##__VA_ARGS__)
  55
  56 #define __COLUMN(x) "%'10" x " %-13s"
  57 #define FMT_COLUMNf __COLUMN(".0f")
  58 #define FMT_COLUMNd __COLUMN("d")
  59 #define FMT_COLUMNl __COLUMN("llu")
  60 #define RX(rx) rx, "rx/s"
  61 #define PPS(pps) pps, "pkt/s"
  62 #define DROP(drop) drop, "drop/s"
  63 #define ERR(err) err, "error/s"
  64 #define HITS(hits) hits, "hit/s"
  65 #define XMIT(xmit) xmit, "xmit/s"
  66 #define PASS(pass) pass, "pass/s"
  67 #define REDIR(redir) redir, "redir/s"
  68 #define NANOSEC_PER_SEC 1000000000 /* 10^9 */
  69
  70 #define XDP_UNKNOWN (XDP_REDIRECT + 1)
  71 #define XDP_ACTION_MAX (XDP_UNKNOWN + 1)
  72 #define XDP_REDIRECT_ERR_MAX 7
  73
  74 enum map_type {
  75         MAP_RX,
  76         MAP_REDIRECT_ERR,
  77         MAP_CPUMAP_ENQUEUE,
  78         MAP_CPUMAP_KTHREAD,
  79         MAP_EXCEPTION,
  80         MAP_DEVMAP_XMIT,
  81         MAP_DEVMAP_XMIT_MULTI,
  82         NUM_MAP,
  83 };
  84
  85 enum log_level {
  86         LL_DEFAULT = 1U << 0,
  87         LL_SIMPLE = 1U << 1,
  88         LL_DEBUG = 1U << 2,
  89 };
  90
  91 struct record {
  92         __u64 timestamp;
  93         struct datarec total;
  94         struct datarec *cpu;
  95 };
  96
  97 struct map_entry {
  98         struct hlist_node node;
  99         __u64 pair;
 100         struct record val;
 101 };
 102
 103 struct stats_record {
 104         struct record rx_cnt;
 105         struct record redir_err[XDP_REDIRECT_ERR_MAX];
 106         struct record kthread;
 107         struct record exception[XDP_ACTION_MAX];
 108         struct record devmap_xmit;
 109         DECLARE_HASHTABLE(xmit_map, 5);
 110         struct record enq[];
 111 };
 112
 113 struct sample_output {
 114         struct {
 115                 __u64 rx;
 116                 __u64 redir;
 117                 __u64 drop;
 118                 __u64 drop_xmit;
 119                 __u64 err;
 120                 __u64 xmit;
 121         } totals;
 122         struct {
 123                 __u64 pps;
 124                 __u64 drop;
 125                 __u64 err;
 126         } rx_cnt;
 127         struct {
 128                 __u64 suc;
 129                 __u64 err;
 130         } redir_cnt;
 131         struct {
 132                 __u64 hits;
 133         } except_cnt;
 134         struct {
 135                 __u64 pps;
 136                 __u64 drop;
 137                 __u64 err;
 138                 double bavg;
 139         } xmit_cnt;
 140 };
 141
 142 struct xdp_desc {
 143         int ifindex;
 144         __u32 prog_id;
 145         int flags;
 146 } sample_xdp_progs[32];
 147
 148 struct datarec *sample_mmap[NUM_MAP];
 149 struct bpf_map *sample_map[NUM_MAP];
 150 size_t sample_map_count[NUM_MAP];
 151 enum log_level sample_log_level;
 152 struct sample_output sample_out;
 153 unsigned long sample_interval;
 154 bool sample_err_exp;
 155 int sample_xdp_cnt;
 156 int sample_n_cpus;
 157 int sample_sig_fd;
 158 int sample_mask;
 159
 160 static const char *xdp_redirect_err_names[XDP_REDIRECT_ERR_MAX] = {
 161         /* Key=1 keeps unknown errors */
 162         "Success",
 163         "Unknown",
 164         "EINVAL",
 165         "ENETDOWN",
 166         "EMSGSIZE",
 167         "EOPNOTSUPP",
 168         "ENOSPC",
 169 };
 170
 171 /* Keyed from Unknown */
 172 static const char *xdp_redirect_err_help[XDP_REDIRECT_ERR_MAX - 1] = {
 173         "Unknown error",
 174         "Invalid redirection",
 175         "Device being redirected to is down",
 176         "Packet length too large for device",
 177         "Operation not supported",
 178         "No space in ptr_ring of cpumap kthread",
 179 };
 180
 181 static const char *xdp_action_names[XDP_ACTION_MAX] = {
 182         [XDP_ABORTED]  = "XDP_ABORTED",
 183         [XDP_DROP]     = "XDP_DROP",
 184         [XDP_PASS]     = "XDP_PASS",
 185         [XDP_TX]       = "XDP_TX",
 186         [XDP_REDIRECT] = "XDP_REDIRECT",
 187         [XDP_UNKNOWN]  = "XDP_UNKNOWN",
 188 };
 189
 190 static __u64 gettime(void)
 191 {
 192         struct timespec t;
 193         int res;
 194
 195         res = clock_gettime(CLOCK_MONOTONIC, &t);
 196         if (res < 0) {
 197                 fprintf(stderr, "Error with gettimeofday! (%i)\n", res);
 198                 return UINT64_MAX;
 199         }
 200         return (__u64)t.tv_sec * NANOSEC_PER_SEC + t.tv_nsec;
 201 }
 202
 203 static const char *action2str(int action)
 204 {
 205         if (action < XDP_ACTION_MAX)
 206                 return xdp_action_names[action];
 207         return NULL;
 208 }
 209
 210 static void sample_print_help(int mask)
 211 {
 212         printf("Output format description\n\n"
 213                "By default, redirect success statistics are disabled, use -s to enable.\n"
 214                "The terse output mode is default, verbose mode can be activated using -v\n"
 215                "Use SIGQUIT (Ctrl + \\) to switch the mode dynamically at runtime\n\n"
 216                "Terse mode displays at most the following fields:\n"
 217                "  rx/s        Number of packets received per second\n"
 218                "  redir/s     Number of packets successfully redirected per second\n"
 219                "  err,drop/s  Aggregated count of errors per second (including dropped packets)\n"
 220                "  xmit/s      Number of packets transmitted on the output device per second\n\n"
 221                "Output description for verbose mode:\n"
 222                "  FIELD                 DESCRIPTION\n");
 223
 224         if (mask & SAMPLE_RX_CNT) {
 225                 printf("  receive\t\tDisplays the number of packets received & errors encountered\n"
 226                        " \t\t\tWhenever an error or packet drop occurs, details of per CPU error\n"
 227                        " \t\t\tand drop statistics will be expanded inline in terse mode.\n"
 228                        " \t\t\t\tpkt/s     - Packets received per second\n"
 229                        " \t\t\t\tdrop/s    - Packets dropped per second\n"
 230                        " \t\t\t\terror/s   - Errors encountered per second\n\n");
 231         }
 232         if (mask & (SAMPLE_REDIRECT_CNT | SAMPLE_REDIRECT_ERR_CNT)) {
 233                 printf("  redirect\t\tDisplays the number of packets successfully redirected\n"
 234                        "  \t\t\tErrors encountered are expanded under redirect_err field\n"
 235                        "  \t\t\tNote that passing -s to enable it has a per packet overhead\n"
 236                        "  \t\t\t\tredir/s   - Packets redirected successfully per second\n\n"
 237                        "  redirect_err\t\tDisplays the number of packets that failed redirection\n"
 238                        "  \t\t\tThe errno is expanded under this field with per CPU count\n"
 239                        "  \t\t\tThe recognized errors are:\n");
 240
 241                 for (int i = 2; i < XDP_REDIRECT_ERR_MAX; i++)
 242                         printf("\t\t\t  %s: %s\n", xdp_redirect_err_names[i],
 243                                xdp_redirect_err_help[i - 1]);
 244
 245                 printf("  \n\t\t\t\terror/s   - Packets that failed redirection per second\n\n");
 246         }
 247
 248         if (mask & SAMPLE_CPUMAP_ENQUEUE_CNT) {
 249                 printf("  enqueue to cpu N\tDisplays the number of packets enqueued to bulk queue of CPU N\n"
 250                        "  \t\t\tExpands to cpu:FROM->N to display enqueue stats for each CPU enqueuing to CPU N\n"
 251                        "  \t\t\tReceived packets can be associated with the CPU redirect program is enqueuing \n"
 252                        "  \t\t\tpackets to.\n"
 253                        "  \t\t\t\tpkt/s    - Packets enqueued per second from other CPU to CPU N\n"
 254                        "  \t\t\t\tdrop/s   - Packets dropped when trying to enqueue to CPU N\n"
 255                        "  \t\t\t\tbulk-avg - Average number of packets processed for each event\n\n");
 256         }
 257
 258         if (mask & SAMPLE_CPUMAP_KTHREAD_CNT) {
 259                 printf("  kthread\t\tDisplays the number of packets processed in CPUMAP kthread for each CPU\n"
 260                        "  \t\t\tPackets consumed from ptr_ring in kthread, and its xdp_stats (after calling \n"
 261                        "  \t\t\tCPUMAP bpf prog) are expanded below this. xdp_stats are expanded as a total and\n"
 262                        "  \t\t\tthen per-CPU to associate it to each CPU's pinned CPUMAP kthread.\n"
 263                        "  \t\t\t\tpkt/s    - Packets consumed per second from ptr_ring\n"
 264                        "  \t\t\t\tdrop/s   - Packets dropped per second in kthread\n"
 265                        "  \t\t\t\tsched    - Number of times kthread called schedule()\n\n"
 266                        "  \t\t\txdp_stats (also expands to per-CPU counts)\n"
 267                        "  \t\t\t\tpass/s  - XDP_PASS count for CPUMAP program execution\n"
 268                        "  \t\t\t\tdrop/s  - XDP_DROP count for CPUMAP program execution\n"
 269                        "  \t\t\t\tredir/s - XDP_REDIRECT count for CPUMAP program execution\n\n");
 270         }
 271
 272         if (mask & SAMPLE_EXCEPTION_CNT) {
 273                 printf("  xdp_exception\t\tDisplays xdp_exception tracepoint events\n"
 274                        "  \t\t\tThis can occur due to internal driver errors, unrecognized\n"
 275                        "  \t\t\tXDP actions and due to explicit user trigger by use of XDP_ABORTED\n"
 276                        "  \t\t\tEach action is expanded below this field with its count\n"
 277                        "  \t\t\t\thit/s     - Number of times the tracepoint was hit per second\n\n");
 278         }
 279
 280         if (mask & SAMPLE_DEVMAP_XMIT_CNT) {
 281                 printf("  devmap_xmit\t\tDisplays devmap_xmit tracepoint events\n"
 282                        "  \t\t\tThis tracepoint is invoked for successful transmissions on output\n"
 283                        "  \t\t\tdevice but these statistics are not available for generic XDP mode,\n"
 284                        "  \t\t\thence they will be omitted from the output when using SKB mode\n"
 285                        "  \t\t\t\txmit/s    - Number of packets that were transmitted per second\n"
 286                        "  \t\t\t\tdrop/s    - Number of packets that failed transmissions per second\n"
 287                        "  \t\t\t\tdrv_err/s - Number of internal driver errors per second\n"
 288                        "  \t\t\t\tbulk-avg  - Average number of packets processed for each event\n\n");
 289         }
 290 }
 291
 292 void sample_usage(char *argv[], const struct option *long_options,
 293                   const char *doc, int mask, bool error)
 294 {
 295         int i;
 296
 297         if (!error)
 298                 sample_print_help(mask);
 299
 300         printf("\n%s\nOption for %s:\n", doc, argv[0]);
 301         for (i = 0; long_options[i].name != 0; i++) {
 302                 printf(" --%-15s", long_options[i].name);
 303                 if (long_options[i].flag != NULL)
 304                         printf(" flag (internal value: %d)",
 305                                *long_options[i].flag);
 306                 else
 307                         printf("\t short-option: -%c", long_options[i].val);
 308                 printf("\n");
 309         }
 310         printf("\n");
 311 }
 312
 313 static struct datarec *alloc_record_per_cpu(void)
 314 {
 315         unsigned int nr_cpus = libbpf_num_possible_cpus();
 316         struct datarec *array;
 317
 318         array = calloc(nr_cpus, sizeof(*array));
 319         if (!array) {
 320                 fprintf(stderr, "Failed to allocate memory (nr_cpus: %u)\n",
 321                         nr_cpus);
 322                 return NULL;
 323         }
 324         return array;
 325 }
 326
 327 static int map_entry_init(struct map_entry *e, __u64 pair)
 328 {
 329         e->pair = pair;
 330         INIT_HLIST_NODE(&e->node);
 331         e->val.timestamp = gettime();
 332         e->val.cpu = alloc_record_per_cpu();
 333         if (!e->val.cpu)
 334                 return -ENOMEM;
 335         return 0;
 336 }
 337
 338 static void map_collect_percpu(struct datarec *values, struct record *rec)
 339 {
 340         /* For percpu maps, userspace gets a value per possible CPU */
 341         unsigned int nr_cpus = libbpf_num_possible_cpus();
 342         __u64 sum_xdp_redirect = 0;
 343         __u64 sum_processed = 0;
 344         __u64 sum_xdp_pass = 0;
 345         __u64 sum_xdp_drop = 0;
 346         __u64 sum_dropped = 0;
 347         __u64 sum_issue = 0;
 348         int i;
 349
 350         /* Get time as close as possible to reading map contents */
 351         rec->timestamp = gettime();
 352
 353         /* Record and sum values from each CPU */
 354         for (i = 0; i < nr_cpus; i++) {
 355                 rec->cpu[i].processed = READ_ONCE(values[i].processed);
 356                 rec->cpu[i].dropped = READ_ONCE(values[i].dropped);
 357                 rec->cpu[i].issue = READ_ONCE(values[i].issue);
 358                 rec->cpu[i].xdp_pass = READ_ONCE(values[i].xdp_pass);
 359                 rec->cpu[i].xdp_drop = READ_ONCE(values[i].xdp_drop);
 360                 rec->cpu[i].xdp_redirect = READ_ONCE(values[i].xdp_redirect);
 361
 362                 sum_processed += rec->cpu[i].processed;
 363                 sum_dropped += rec->cpu[i].dropped;
 364                 sum_issue += rec->cpu[i].issue;
 365                 sum_xdp_pass += rec->cpu[i].xdp_pass;
 366                 sum_xdp_drop += rec->cpu[i].xdp_drop;
 367                 sum_xdp_redirect += rec->cpu[i].xdp_redirect;
 368         }
 369
 370         rec->total.processed = sum_processed;
 371         rec->total.dropped = sum_dropped;
 372         rec->total.issue = sum_issue;
 373         rec->total.xdp_pass = sum_xdp_pass;
 374         rec->total.xdp_drop = sum_xdp_drop;
 375         rec->total.xdp_redirect = sum_xdp_redirect;
 376 }
 377
 378 static int map_collect_percpu_devmap(int map_fd, struct stats_record *rec)
 379 {
 380         unsigned int nr_cpus = bpf_num_possible_cpus();
 381         __u32 batch, count = 32;
 382         struct datarec *values;
 383         bool init = false;
 384         __u64 *keys;
 385         int i, ret;
 386
 387         keys = calloc(count, sizeof(__u64));
 388         if (!keys)
 389                 return -ENOMEM;
 390         values = calloc(count * nr_cpus, sizeof(struct datarec));
 391         if (!values) {
 392                 free(keys);
 393                 return -ENOMEM;
 394         }
 395
 396         for (;;) {
 397                 bool exit = false;
 398
 399                 ret = bpf_map_lookup_batch(map_fd, init ? &batch : NULL, &batch,
 400                                            keys, values, &count, NULL);
 401                 if (ret < 0 && errno != ENOENT)
 402                         break;
 403                 if (errno == ENOENT)
 404                         exit = true;
 405
 406                 init = true;
 407                 for (i = 0; i < count; i++) {
 408                         struct map_entry *e, *x = NULL;
 409                         __u64 pair = keys[i];
 410                         struct datarec *arr;
 411
 412                         arr = &values[i * nr_cpus];
 413                         hash_for_each_possible(rec->xmit_map, e, node, pair) {
 414                                 if (e->pair == pair) {
 415                                         x = e;
 416                                         break;
 417                                 }
 418                         }
 419                         if (!x) {
 420                                 x = calloc(1, sizeof(*x));
 421                                 if (!x)
 422                                         goto cleanup;
 423                                 if (map_entry_init(x, pair) < 0) {
 424                                         free(x);
 425                                         goto cleanup;
 426                                 }
 427                                 hash_add(rec->xmit_map, &x->node, pair);
 428                         }
 429                         map_collect_percpu(arr, &x->val);
 430                 }
 431
 432                 if (exit)
 433                         break;
 434                 count = 32;
 435         }
 436
 437         free(values);
 438         free(keys);
 439         return 0;
 440 cleanup:
 441         free(values);
 442         free(keys);
 443         return -ENOMEM;
 444 }
 445
 446 static struct stats_record *alloc_stats_record(void)
 447 {
 448         struct stats_record *rec;
 449         int i;
 450
 451         rec = calloc(1, sizeof(*rec) + sample_n_cpus * sizeof(struct record));
 452         if (!rec) {
 453                 fprintf(stderr, "Failed to allocate memory\n");
 454                 return NULL;
 455         }
 456
 457         if (sample_mask & SAMPLE_RX_CNT) {
 458                 rec->rx_cnt.cpu = alloc_record_per_cpu();
 459                 if (!rec->rx_cnt.cpu) {
 460                         fprintf(stderr,
 461                                 "Failed to allocate rx_cnt per-CPU array\n");
 462                         goto end_rec;
 463                 }
 464         }
 465         if (sample_mask & (SAMPLE_REDIRECT_CNT | SAMPLE_REDIRECT_ERR_CNT)) {
 466                 for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++) {
 467                         rec->redir_err[i].cpu = alloc_record_per_cpu();
 468                         if (!rec->redir_err[i].cpu) {
 469                                 fprintf(stderr,
 470                                         "Failed to allocate redir_err per-CPU array for "
 471                                         "\"%s\" case\n",
 472                                         xdp_redirect_err_names[i]);
 473                                 while (i--)
 474                                         free(rec->redir_err[i].cpu);
 475                                 goto end_rx_cnt;
 476                         }
 477                 }
 478         }
 479         if (sample_mask & SAMPLE_CPUMAP_KTHREAD_CNT) {
 480                 rec->kthread.cpu = alloc_record_per_cpu();
 481                 if (!rec->kthread.cpu) {
 482                         fprintf(stderr,
 483                                 "Failed to allocate kthread per-CPU array\n");
 484                         goto end_redir;
 485                 }
 486         }
 487         if (sample_mask & SAMPLE_EXCEPTION_CNT) {
 488                 for (i = 0; i < XDP_ACTION_MAX; i++) {
 489                         rec->exception[i].cpu = alloc_record_per_cpu();
 490                         if (!rec->exception[i].cpu) {
 491                                 fprintf(stderr,
 492                                         "Failed to allocate exception per-CPU array for "
 493                                         "\"%s\" case\n",
 494                                         action2str(i));
 495                                 while (i--)
 496                                         free(rec->exception[i].cpu);
 497                                 goto end_kthread;
 498                         }
 499                 }
 500         }
 501         if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT) {
 502                 rec->devmap_xmit.cpu = alloc_record_per_cpu();
 503                 if (!rec->devmap_xmit.cpu) {
 504                         fprintf(stderr,
 505                                 "Failed to allocate devmap_xmit per-CPU array\n");
 506                         goto end_exception;
 507                 }
 508         }
 509         if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
 510                 hash_init(rec->xmit_map);
 511         if (sample_mask & SAMPLE_CPUMAP_ENQUEUE_CNT) {
 512                 for (i = 0; i < sample_n_cpus; i++) {
 513                         rec->enq[i].cpu = alloc_record_per_cpu();
 514                         if (!rec->enq[i].cpu) {
 515                                 fprintf(stderr,
 516                                         "Failed to allocate enqueue per-CPU array for "
 517                                         "CPU %d\n",
 518                                         i);
 519                                 while (i--)
 520                                         free(rec->enq[i].cpu);
 521                                 goto end_devmap_xmit;
 522                         }
 523                 }
 524         }
 525
 526         return rec;
 527
 528 end_devmap_xmit:
 529         free(rec->devmap_xmit.cpu);
 530 end_exception:
 531         for (i = 0; i < XDP_ACTION_MAX; i++)
 532                 free(rec->exception[i].cpu);
 533 end_kthread:
 534         free(rec->kthread.cpu);
 535 end_redir:
 536         for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++)
 537                 free(rec->redir_err[i].cpu);
 538 end_rx_cnt:
 539         free(rec->rx_cnt.cpu);
 540 end_rec:
 541         free(rec);
 542         return NULL;
 543 }
 544
 545 static void free_stats_record(struct stats_record *r)
 546 {
 547         struct hlist_node *tmp;
 548         struct map_entry *e;
 549         int i;
 550
 551         for (i = 0; i < sample_n_cpus; i++)
 552                 free(r->enq[i].cpu);
 553         hash_for_each_safe(r->xmit_map, i, tmp, e, node) {
 554                 hash_del(&e->node);
 555                 free(e->val.cpu);
 556                 free(e);
 557         }
 558         free(r->devmap_xmit.cpu);
 559         for (i = 0; i < XDP_ACTION_MAX; i++)
 560                 free(r->exception[i].cpu);
 561         free(r->kthread.cpu);
 562         for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++)
 563                 free(r->redir_err[i].cpu);
 564         free(r->rx_cnt.cpu);
 565         free(r);
 566 }
 567
 568 static double calc_period(struct record *r, struct record *p)
 569 {
 570         double period_ = 0;
 571         __u64 period = 0;
 572
 573         period = r->timestamp - p->timestamp;
 574         if (period > 0)
 575                 period_ = ((double)period / NANOSEC_PER_SEC);
 576
 577         return period_;
 578 }
 579
 580 static double sample_round(double val)
 581 {
 582         if (val - floor(val) < 0.5)
 583                 return floor(val);
 584         return ceil(val);
 585 }
 586
 587 static __u64 calc_pps(struct datarec *r, struct datarec *p, double period_)
 588 {
 589         __u64 packets = 0;
 590         __u64 pps = 0;
 591
 592         if (period_ > 0) {
 593                 packets = r->processed - p->processed;
 594                 pps = sample_round(packets / period_);
 595         }
 596         return pps;
 597 }
 598
 599 static __u64 calc_drop_pps(struct datarec *r, struct datarec *p, double period_)
 600 {
 601         __u64 packets = 0;
 602         __u64 pps = 0;
 603
 604         if (period_ > 0) {
 605                 packets = r->dropped - p->dropped;
 606                 pps = sample_round(packets / period_);
 607         }
 608         return pps;
 609 }
 610
 611 static __u64 calc_errs_pps(struct datarec *r, struct datarec *p, double period_)
 612 {
 613         __u64 packets = 0;
 614         __u64 pps = 0;
 615
 616         if (period_ > 0) {
 617                 packets = r->issue - p->issue;
 618                 pps = sample_round(packets / period_);
 619         }
 620         return pps;
 621 }
 622
 623 static __u64 calc_info_pps(struct datarec *r, struct datarec *p, double period_)
 624 {
 625         __u64 packets = 0;
 626         __u64 pps = 0;
 627
 628         if (period_ > 0) {
 629                 packets = r->info - p->info;
 630                 pps = sample_round(packets / period_);
 631         }
 632         return pps;
 633 }
 634
 635 static void calc_xdp_pps(struct datarec *r, struct datarec *p, double *xdp_pass,
 636                          double *xdp_drop, double *xdp_redirect, double period_)
 637 {
 638         *xdp_pass = 0, *xdp_drop = 0, *xdp_redirect = 0;
 639         if (period_ > 0) {
 640                 *xdp_redirect = (r->xdp_redirect - p->xdp_redirect) / period_;
 641                 *xdp_pass = (r->xdp_pass - p->xdp_pass) / period_;
 642                 *xdp_drop = (r->xdp_drop - p->xdp_drop) / period_;
 643         }
 644 }
 645
 646 static void stats_get_rx_cnt(struct stats_record *stats_rec,
 647                              struct stats_record *stats_prev,
 648                              unsigned int nr_cpus, struct sample_output *out)
 649 {
 650         struct record *rec, *prev;
 651         double t, pps, drop, err;
 652         int i;
 653
 654         rec = &stats_rec->rx_cnt;
 655         prev = &stats_prev->rx_cnt;
 656         t = calc_period(rec, prev);
 657
 658         for (i = 0; i < nr_cpus; i++) {
 659                 struct datarec *r = &rec->cpu[i];
 660                 struct datarec *p = &prev->cpu[i];
 661                 char str[64];
 662
 663                 pps = calc_pps(r, p, t);
 664                 drop = calc_drop_pps(r, p, t);
 665                 err = calc_errs_pps(r, p, t);
 666                 if (!pps && !drop && !err)
 667                         continue;
 668
 669                 snprintf(str, sizeof(str), "cpu:%d", i);
 670                 print_default("    %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
 671                               "\n",
 672                               str, PPS(pps), DROP(drop), ERR(err));
 673         }
 674
 675         if (out) {
 676                 pps = calc_pps(&rec->total, &prev->total, t);
 677                 drop = calc_drop_pps(&rec->total, &prev->total, t);
 678                 err = calc_errs_pps(&rec->total, &prev->total, t);
 679
 680                 out->rx_cnt.pps = pps;
 681                 out->rx_cnt.drop = drop;
 682                 out->rx_cnt.err = err;
 683                 out->totals.rx += pps;
 684                 out->totals.drop += drop;
 685                 out->totals.err += err;
 686         }
 687 }
 688
 689 static void stats_get_cpumap_enqueue(struct stats_record *stats_rec,
 690                                      struct stats_record *stats_prev,
 691                                      unsigned int nr_cpus)
 692 {
 693         struct record *rec, *prev;
 694         double t, pps, drop, err;
 695         int i, to_cpu;
 696
 697         /* cpumap enqueue stats */
 698         for (to_cpu = 0; to_cpu < sample_n_cpus; to_cpu++) {
 699                 rec = &stats_rec->enq[to_cpu];
 700                 prev = &stats_prev->enq[to_cpu];
 701                 t = calc_period(rec, prev);
 702
 703                 pps = calc_pps(&rec->total, &prev->total, t);
 704                 drop = calc_drop_pps(&rec->total, &prev->total, t);
 705                 err = calc_errs_pps(&rec->total, &prev->total, t);
 706
 707                 if (pps > 0 || drop > 0) {
 708                         char str[64];
 709
 710                         snprintf(str, sizeof(str), "enqueue to cpu %d", to_cpu);
 711
 712                         if (err > 0)
 713                                 err = pps / err; /* calc average bulk size */
 714
 715                         print_err(drop,
 716                                   "  %-20s " FMT_COLUMNf FMT_COLUMNf __COLUMN(
 717                                           ".2f") "\n",
 718                                   str, PPS(pps), DROP(drop), err, "bulk-avg");
 719                 }
 720
 721                 for (i = 0; i < nr_cpus; i++) {
 722                         struct datarec *r = &rec->cpu[i];
 723                         struct datarec *p = &prev->cpu[i];
 724                         char str[64];
 725
 726                         pps = calc_pps(r, p, t);
 727                         drop = calc_drop_pps(r, p, t);
 728                         err = calc_errs_pps(r, p, t);
 729                         if (!pps && !drop && !err)
 730                                 continue;
 731
 732                         snprintf(str, sizeof(str), "cpu:%d->%d", i, to_cpu);
 733                         if (err > 0)
 734                                 err = pps / err; /* calc average bulk size */
 735                         print_default(
 736                                 "    %-18s " FMT_COLUMNf FMT_COLUMNf __COLUMN(
 737                                         ".2f") "\n",
 738                                 str, PPS(pps), DROP(drop), err, "bulk-avg");
 739                 }
 740         }
 741 }
 742
 743 static void stats_get_cpumap_remote(struct stats_record *stats_rec,
 744                                     struct stats_record *stats_prev,
 745                                     unsigned int nr_cpus)
 746 {
 747         double xdp_pass, xdp_drop, xdp_redirect;
 748         struct record *rec, *prev;
 749         double t;
 750         int i;
 751
 752         rec = &stats_rec->kthread;
 753         prev = &stats_prev->kthread;
 754         t = calc_period(rec, prev);
 755
 756         calc_xdp_pps(&rec->total, &prev->total, &xdp_pass, &xdp_drop,
 757                      &xdp_redirect, t);
 758         if (xdp_pass || xdp_drop || xdp_redirect) {
 759                 print_err(xdp_drop,
 760                           "    %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf "\n",
 761                           "xdp_stats", PASS(xdp_pass), DROP(xdp_drop),
 762                           REDIR(xdp_redirect));
 763         }
 764
 765         for (i = 0; i < nr_cpus; i++) {
 766                 struct datarec *r = &rec->cpu[i];
 767                 struct datarec *p = &prev->cpu[i];
 768                 char str[64];
 769
 770                 calc_xdp_pps(r, p, &xdp_pass, &xdp_drop, &xdp_redirect, t);
 771                 if (!xdp_pass && !xdp_drop && !xdp_redirect)
 772                         continue;
 773
 774                 snprintf(str, sizeof(str), "cpu:%d", i);
 775                 print_default("      %-16s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
 776                               "\n",
 777                               str, PASS(xdp_pass), DROP(xdp_drop),
 778                               REDIR(xdp_redirect));
 779         }
 780 }
 781
 782 static void stats_get_cpumap_kthread(struct stats_record *stats_rec,
 783                                      struct stats_record *stats_prev,
 784                                      unsigned int nr_cpus)
 785 {
 786         struct record *rec, *prev;
 787         double t, pps, drop, err;
 788         int i;
 789
 790         rec = &stats_rec->kthread;
 791         prev = &stats_prev->kthread;
 792         t = calc_period(rec, prev);
 793
 794         pps = calc_pps(&rec->total, &prev->total, t);
 795         drop = calc_drop_pps(&rec->total, &prev->total, t);
 796         err = calc_errs_pps(&rec->total, &prev->total, t);
 797
 798         print_err(drop, "  %-20s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf "\n",
 799                   pps ? "kthread total" : "kthread", PPS(pps), DROP(drop), err,
 800                   "sched");
 801
 802         for (i = 0; i < nr_cpus; i++) {
 803                 struct datarec *r = &rec->cpu[i];
 804                 struct datarec *p = &prev->cpu[i];
 805                 char str[64];
 806
 807                 pps = calc_pps(r, p, t);
 808                 drop = calc_drop_pps(r, p, t);
 809                 err = calc_errs_pps(r, p, t);
 810                 if (!pps && !drop && !err)
 811                         continue;
 812
 813                 snprintf(str, sizeof(str), "cpu:%d", i);
 814                 print_default("    %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
 815                               "\n",
 816                               str, PPS(pps), DROP(drop), err, "sched");
 817         }
 818 }
 819
 820 static void stats_get_redirect_cnt(struct stats_record *stats_rec,
 821                                    struct stats_record *stats_prev,
 822                                    unsigned int nr_cpus,
 823                                    struct sample_output *out)
 824 {
 825         struct record *rec, *prev;
 826         double t, pps;
 827         int i;
 828
 829         rec = &stats_rec->redir_err[0];
 830         prev = &stats_prev->redir_err[0];
 831         t = calc_period(rec, prev);
 832         for (i = 0; i < nr_cpus; i++) {
 833                 struct datarec *r = &rec->cpu[i];
 834                 struct datarec *p = &prev->cpu[i];
 835                 char str[64];
 836
 837                 pps = calc_pps(r, p, t);
 838                 if (!pps)
 839                         continue;
 840
 841                 snprintf(str, sizeof(str), "cpu:%d", i);
 842                 print_default("    %-18s " FMT_COLUMNf "\n", str, REDIR(pps));
 843         }
 844
 845         if (out) {
 846                 pps = calc_pps(&rec->total, &prev->total, t);
 847                 out->redir_cnt.suc = pps;
 848                 out->totals.redir += pps;
 849         }
 850 }
 851
 852 static void stats_get_redirect_err_cnt(struct stats_record *stats_rec,
 853                                        struct stats_record *stats_prev,
 854                                        unsigned int nr_cpus,
 855                                        struct sample_output *out)
 856 {
 857         struct record *rec, *prev;
 858         double t, drop, sum = 0;
 859         int rec_i, i;
 860
 861         for (rec_i = 1; rec_i < XDP_REDIRECT_ERR_MAX; rec_i++) {
 862                 char str[64];
 863
 864                 rec = &stats_rec->redir_err[rec_i];
 865                 prev = &stats_prev->redir_err[rec_i];
 866                 t = calc_period(rec, prev);
 867
 868                 drop = calc_drop_pps(&rec->total, &prev->total, t);
 869                 if (drop > 0 && !out) {
 870                         snprintf(str, sizeof(str),
 871                                  sample_log_level & LL_DEFAULT ? "%s total" :
 872                                                                        "%s",
 873                                  xdp_redirect_err_names[rec_i]);
 874                         print_err(drop, "    %-18s " FMT_COLUMNf "\n", str,
 875                                   ERR(drop));
 876                 }
 877
 878                 for (i = 0; i < nr_cpus; i++) {
 879                         struct datarec *r = &rec->cpu[i];
 880                         struct datarec *p = &prev->cpu[i];
 881                         double drop;
 882
 883                         drop = calc_drop_pps(r, p, t);
 884                         if (!drop)
 885                                 continue;
 886
 887                         snprintf(str, sizeof(str), "cpu:%d", i);
 888                         print_default("       %-16s" FMT_COLUMNf "\n", str,
 889                                       ERR(drop));
 890                 }
 891
 892                 sum += drop;
 893         }
 894
 895         if (out) {
 896                 out->redir_cnt.err = sum;
 897                 out->totals.err += sum;
 898         }
 899 }
 900
 901 static void stats_get_exception_cnt(struct stats_record *stats_rec,
 902                                     struct stats_record *stats_prev,
 903                                     unsigned int nr_cpus,
 904                                     struct sample_output *out)
 905 {
 906         double t, drop, sum = 0;
 907         struct record *rec, *prev;
 908         int rec_i, i;
 909
 910         for (rec_i = 0; rec_i < XDP_ACTION_MAX; rec_i++) {
 911                 rec = &stats_rec->exception[rec_i];
 912                 prev = &stats_prev->exception[rec_i];
 913                 t = calc_period(rec, prev);
 914
 915                 drop = calc_drop_pps(&rec->total, &prev->total, t);
 916                 /* Fold out errors after heading */
 917                 sum += drop;
 918
 919                 if (drop > 0 && !out) {
 920                         print_always("    %-18s " FMT_COLUMNf "\n",
 921                                      action2str(rec_i), ERR(drop));
 922
 923                         for (i = 0; i < nr_cpus; i++) {
 924                                 struct datarec *r = &rec->cpu[i];
 925                                 struct datarec *p = &prev->cpu[i];
 926                                 char str[64];
 927                                 double drop;
 928
 929                                 drop = calc_drop_pps(r, p, t);
 930                                 if (!drop)
 931                                         continue;
 932
 933                                 snprintf(str, sizeof(str), "cpu:%d", i);
 934                                 print_default("       %-16s" FMT_COLUMNf "\n",
 935                                               str, ERR(drop));
 936                         }
 937                 }
 938         }
 939
 940         if (out) {
 941                 out->except_cnt.hits = sum;
 942                 out->totals.err += sum;
 943         }
 944 }
 945
 946 static void stats_get_devmap_xmit(struct stats_record *stats_rec,
 947                                   struct stats_record *stats_prev,
 948                                   unsigned int nr_cpus,
 949                                   struct sample_output *out)
 950 {
 951         double pps, drop, info, err;
 952         struct record *rec, *prev;
 953         double t;
 954         int i;
 955
 956         rec = &stats_rec->devmap_xmit;
 957         prev = &stats_prev->devmap_xmit;
 958         t = calc_period(rec, prev);
 959         for (i = 0; i < nr_cpus; i++) {
 960                 struct datarec *r = &rec->cpu[i];
 961                 struct datarec *p = &prev->cpu[i];
 962                 char str[64];
 963
 964                 pps = calc_pps(r, p, t);
 965                 drop = calc_drop_pps(r, p, t);
 966                 err = calc_errs_pps(r, p, t);
 967
 968                 if (!pps && !drop && !err)
 969                         continue;
 970
 971                 snprintf(str, sizeof(str), "cpu:%d", i);
 972                 info = calc_info_pps(r, p, t);
 973                 if (info > 0)
 974                         info = (pps + drop) / info; /* calc avg bulk */
 975                 print_default("     %-18s" FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
 976                                       __COLUMN(".2f") "\n",
 977                               str, XMIT(pps), DROP(drop), err, "drv_err/s",
 978                               info, "bulk-avg");
 979         }
 980         if (out) {
 981                 pps = calc_pps(&rec->total, &prev->total, t);
 982                 drop = calc_drop_pps(&rec->total, &prev->total, t);
 983                 info = calc_info_pps(&rec->total, &prev->total, t);
 984                 if (info > 0)
 985                         info = (pps + drop) / info; /* calc avg bulk */
 986                 err = calc_errs_pps(&rec->total, &prev->total, t);
 987
 988                 out->xmit_cnt.pps = pps;
 989                 out->xmit_cnt.drop = drop;
 990                 out->xmit_cnt.bavg = info;
 991                 out->xmit_cnt.err = err;
 992                 out->totals.xmit += pps;
 993                 out->totals.drop_xmit += drop;
 994                 out->totals.err += err;
 995         }
 996 }
 997
 998 static void stats_get_devmap_xmit_multi(struct stats_record *stats_rec,
 999                                         struct stats_record *stats_prev,
1000                                         unsigned int nr_cpus,
1001                                         struct sample_output *out,
1002                                         bool xmit_total)
1003 {
1004         double pps, drop, info, err;
1005         struct map_entry *entry;
1006         struct record *r, *p;
1007         double t;
1008         int bkt;
1009
1010         hash_for_each(stats_rec->xmit_map, bkt, entry, node) {
1011                 struct map_entry *e, *x = NULL;
1012                 char ifname_from[IFNAMSIZ];
1013                 char ifname_to[IFNAMSIZ];
1014                 const char *fstr, *tstr;
1015                 unsigned long prev_time;
1016                 struct record beg = {};
1017                 __u32 from_idx, to_idx;
1018                 char str[128];
1019                 __u64 pair;
1020                 int i;
1021
1022                 prev_time = sample_interval * NANOSEC_PER_SEC;
1023
1024                 pair = entry->pair;
1025                 from_idx = pair >> 32;
1026                 to_idx = pair & 0xFFFFFFFF;
1027
1028                 r = &entry->val;
1029                 beg.timestamp = r->timestamp - prev_time;
1030
1031                 /* Find matching entry from stats_prev map */
1032                 hash_for_each_possible(stats_prev->xmit_map, e, node, pair) {
1033                         if (e->pair == pair) {
1034                                 x = e;
1035                                 break;
1036                         }
1037                 }
1038                 if (x)
1039                         p = &x->val;
1040                 else
1041                         p = &beg;
1042                 t = calc_period(r, p);
1043                 pps = calc_pps(&r->total, &p->total, t);
1044                 drop = calc_drop_pps(&r->total, &p->total, t);
1045                 info = calc_info_pps(&r->total, &p->total, t);
1046                 if (info > 0)
1047                         info = (pps + drop) / info; /* calc avg bulk */
1048                 err = calc_errs_pps(&r->total, &p->total, t);
1049
1050                 if (out) {
1051                         /* We are responsible for filling out totals */
1052                         out->totals.xmit += pps;
1053                         out->totals.drop_xmit += drop;
1054                         out->totals.err += err;
1055                         continue;
1056                 }
1057
1058                 fstr = tstr = NULL;
1059                 if (if_indextoname(from_idx, ifname_from))
1060                         fstr = ifname_from;
1061                 if (if_indextoname(to_idx, ifname_to))
1062                         tstr = ifname_to;
1063
1064                 snprintf(str, sizeof(str), "xmit %s->%s", fstr ?: "?",
1065                          tstr ?: "?");
1066                 /* Skip idle streams of redirection */
1067                 if (pps || drop || err) {
1068                         print_err(drop,
1069                                   "  %-20s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
1070                                   __COLUMN(".2f") "\n", str, XMIT(pps), DROP(drop),
1071                                   err, "drv_err/s", info, "bulk-avg");
1072                 }
1073
1074                 for (i = 0; i < nr_cpus; i++) {
1075                         struct datarec *rc = &r->cpu[i];
1076                         struct datarec *pc, p_beg = {};
1077                         char str[64];
1078
1079                         pc = p == &beg ? &p_beg : &p->cpu[i];
1080
1081                         pps = calc_pps(rc, pc, t);
1082                         drop = calc_drop_pps(rc, pc, t);
1083                         err = calc_errs_pps(rc, pc, t);
1084
1085                         if (!pps && !drop && !err)
1086                                 continue;
1087
1088                         snprintf(str, sizeof(str), "cpu:%d", i);
1089                         info = calc_info_pps(rc, pc, t);
1090                         if (info > 0)
1091                                 info = (pps + drop) / info; /* calc avg bulk */
1092
1093                         print_default("     %-18s" FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
1094                                       __COLUMN(".2f") "\n", str, XMIT(pps),
1095                                       DROP(drop), err, "drv_err/s", info, "bulk-avg");
1096                 }
1097         }
1098 }
1099
1100 static void stats_print(const char *prefix, int mask, struct stats_record *r,
1101                         struct stats_record *p, struct sample_output *out)
1102 {
1103         int nr_cpus = libbpf_num_possible_cpus();
1104         const char *str;
1105
1106         print_always("%-23s", prefix ?: "Summary");
1107         if (mask & SAMPLE_RX_CNT)
1108                 print_always(FMT_COLUMNl, RX(out->totals.rx));
1109         if (mask & SAMPLE_REDIRECT_CNT)
1110                 print_always(FMT_COLUMNl, REDIR(out->totals.redir));
1111         printf(FMT_COLUMNl,
1112                out->totals.err + out->totals.drop + out->totals.drop_xmit,
1113                "err,drop/s");
1114         if (mask & SAMPLE_DEVMAP_XMIT_CNT ||
1115             mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
1116                 printf(FMT_COLUMNl, XMIT(out->totals.xmit));
1117         printf("\n");
1118
1119         if (mask & SAMPLE_RX_CNT) {
1120                 str = (sample_log_level & LL_DEFAULT) && out->rx_cnt.pps ?
1121                                     "receive total" :
1122                                     "receive";
1123                 print_err((out->rx_cnt.err || out->rx_cnt.drop),
1124                           "  %-20s " FMT_COLUMNl FMT_COLUMNl FMT_COLUMNl "\n",
1125                           str, PPS(out->rx_cnt.pps), DROP(out->rx_cnt.drop),
1126                           ERR(out->rx_cnt.err));
1127
1128                 stats_get_rx_cnt(r, p, nr_cpus, NULL);
1129         }
1130
1131         if (mask & SAMPLE_CPUMAP_ENQUEUE_CNT)
1132                 stats_get_cpumap_enqueue(r, p, nr_cpus);
1133
1134         if (mask & SAMPLE_CPUMAP_KTHREAD_CNT) {
1135                 stats_get_cpumap_kthread(r, p, nr_cpus);
1136                 stats_get_cpumap_remote(r, p, nr_cpus);
1137         }
1138
1139         if (mask & SAMPLE_REDIRECT_CNT) {
1140                 str = out->redir_cnt.suc ? "redirect total" : "redirect";
1141                 print_default("  %-20s " FMT_COLUMNl "\n", str,
1142                               REDIR(out->redir_cnt.suc));
1143
1144                 stats_get_redirect_cnt(r, p, nr_cpus, NULL);
1145         }
1146
1147         if (mask & SAMPLE_REDIRECT_ERR_CNT) {
1148                 str = (sample_log_level & LL_DEFAULT) && out->redir_cnt.err ?
1149                                     "redirect_err total" :
1150                                     "redirect_err";
1151                 print_err(out->redir_cnt.err, "  %-20s " FMT_COLUMNl "\n", str,
1152                           ERR(out->redir_cnt.err));
1153
1154                 stats_get_redirect_err_cnt(r, p, nr_cpus, NULL);
1155         }
1156
1157         if (mask & SAMPLE_EXCEPTION_CNT) {
1158                 str = out->except_cnt.hits ? "xdp_exception total" :
1159                                                    "xdp_exception";
1160
1161                 print_err(out->except_cnt.hits, "  %-20s " FMT_COLUMNl "\n", str,
1162                           HITS(out->except_cnt.hits));
1163
1164                 stats_get_exception_cnt(r, p, nr_cpus, NULL);
1165         }
1166
1167         if (mask & SAMPLE_DEVMAP_XMIT_CNT) {
1168                 str = (sample_log_level & LL_DEFAULT) && out->xmit_cnt.pps ?
1169                                     "devmap_xmit total" :
1170                                     "devmap_xmit";
1171
1172                 print_err(out->xmit_cnt.err || out->xmit_cnt.drop,
1173                           "  %-20s " FMT_COLUMNl FMT_COLUMNl FMT_COLUMNl
1174                                   __COLUMN(".2f") "\n",
1175                           str, XMIT(out->xmit_cnt.pps),
1176                           DROP(out->xmit_cnt.drop), out->xmit_cnt.err,
1177                           "drv_err/s", out->xmit_cnt.bavg, "bulk-avg");
1178
1179                 stats_get_devmap_xmit(r, p, nr_cpus, NULL);
1180         }
1181
1182         if (mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
1183                 stats_get_devmap_xmit_multi(r, p, nr_cpus, NULL,
1184                                             mask & SAMPLE_DEVMAP_XMIT_CNT);
1185
1186         if (sample_log_level & LL_DEFAULT ||
1187             ((sample_log_level & LL_SIMPLE) && sample_err_exp)) {
1188                 sample_err_exp = false;
1189                 printf("\n");
1190         }
1191 }
1192
1193 int sample_setup_maps(struct bpf_map **maps)
1194 {
1195         sample_n_cpus = libbpf_num_possible_cpus();
1196
1197         for (int i = 0; i < MAP_DEVMAP_XMIT_MULTI; i++) {
1198                 sample_map[i] = maps[i];
1199
1200                 switch (i) {
1201                 case MAP_RX:
1202                 case MAP_CPUMAP_KTHREAD:
1203                 case MAP_DEVMAP_XMIT:
1204                         sample_map_count[i] = sample_n_cpus;
1205                         break;
1206                 case MAP_REDIRECT_ERR:
1207                         sample_map_count[i] =
1208                                 XDP_REDIRECT_ERR_MAX * sample_n_cpus;
1209                         break;
1210                 case MAP_EXCEPTION:
1211                         sample_map_count[i] = XDP_ACTION_MAX * sample_n_cpus;
1212                 case MAP_CPUMAP_ENQUEUE:
1213                         sample_map_count[i] = sample_n_cpus * sample_n_cpus;
1214                         break;
1215                 default:
1216                         return -EINVAL;
1217                 }
1218                 if (bpf_map__resize(sample_map[i], sample_map_count[i]) < 0)
1219                         return -errno;
1220         }
1221         sample_map[MAP_DEVMAP_XMIT_MULTI] = maps[MAP_DEVMAP_XMIT_MULTI];
1222         return 0;
1223 }
1224
1225 static int sample_setup_maps_mappings(void)
1226 {
1227         for (int i = 0; i < MAP_DEVMAP_XMIT_MULTI; i++) {
1228                 size_t size = sample_map_count[i] * sizeof(struct datarec);
1229
1230                 sample_mmap[i] = mmap(NULL, size, PROT_READ | PROT_WRITE,
1231                                       MAP_SHARED, bpf_map__fd(sample_map[i]), 0);
1232                 if (sample_mmap[i] == MAP_FAILED)
1233                         return -errno;
1234         }
1235         return 0;
1236 }
1237
1238 int __sample_init(int mask)
1239 {
1240         sigset_t st;
1241
1242         sigemptyset(&st);
1243         sigaddset(&st, SIGQUIT);
1244         sigaddset(&st, SIGINT);
1245         sigaddset(&st, SIGTERM);
1246
1247         if (sigprocmask(SIG_BLOCK, &st, NULL) < 0)
1248                 return -errno;
1249
1250         sample_sig_fd = signalfd(-1, &st, SFD_CLOEXEC | SFD_NONBLOCK);
1251         if (sample_sig_fd < 0)
1252                 return -errno;
1253
1254         sample_mask = mask;
1255
1256         return sample_setup_maps_mappings();
1257 }
1258
1259 static int __sample_remove_xdp(int ifindex, __u32 prog_id, int xdp_flags)
1260 {
1261         __u32 cur_prog_id = 0;
1262         int ret;
1263
1264         if (prog_id) {
1265                 ret = bpf_get_link_xdp_id(ifindex, &cur_prog_id, xdp_flags);
1266                 if (ret < 0)
1267                         return -errno;
1268
1269                 if (prog_id != cur_prog_id) {
1270                         print_always(
1271                                 "Program on ifindex %d does not match installed "
1272                                 "program, skipping unload\n",
1273                                 ifindex);
1274                         return -ENOENT;
1275                 }
1276         }
1277
1278         return bpf_set_link_xdp_fd(ifindex, -1, xdp_flags);
1279 }
1280
1281 int sample_install_xdp(struct bpf_program *xdp_prog, int ifindex, bool generic,
1282                        bool force)
1283 {
1284         int ret, xdp_flags = 0;
1285         __u32 prog_id = 0;
1286
1287         if (sample_xdp_cnt == 32) {
1288                 fprintf(stderr,
1289                         "Total limit for installed XDP programs in a sample reached\n");
1290                 return -ENOTSUP;
1291         }
1292
1293         xdp_flags |= !force ? XDP_FLAGS_UPDATE_IF_NOEXIST : 0;
1294         xdp_flags |= generic ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE;
1295         ret = bpf_set_link_xdp_fd(ifindex, bpf_program__fd(xdp_prog),
1296                                   xdp_flags);
1297         if (ret < 0) {
1298                 ret = -errno;
1299                 fprintf(stderr,
1300                         "Failed to install program \"%s\" on ifindex %d, mode = %s, "
1301                         "force = %s: %s\n",
1302                         bpf_program__name(xdp_prog), ifindex,
1303                         generic ? "skb" : "native", force ? "true" : "false",
1304                         strerror(-ret));
1305                 return ret;
1306         }
1307
1308         ret = bpf_get_link_xdp_id(ifindex, &prog_id, xdp_flags);
1309         if (ret < 0) {
1310                 ret = -errno;
1311                 fprintf(stderr,
1312                         "Failed to get XDP program id for ifindex %d, removing program: %s\n",
1313                         ifindex, strerror(errno));
1314                 __sample_remove_xdp(ifindex, 0, xdp_flags);
1315                 return ret;
1316         }
1317         sample_xdp_progs[sample_xdp_cnt++] =
1318                 (struct xdp_desc){ ifindex, prog_id, xdp_flags };
1319
1320         return 0;
1321 }
1322
1323 static void sample_summary_print(void)
1324 {
1325         double period = sample_out.rx_cnt.pps;
1326
1327         if (sample_out.totals.rx) {
1328                 double pkts = sample_out.totals.rx;
1329
1330                 print_always("  Packets received    : %'-10llu\n",
1331                              sample_out.totals.rx);
1332                 print_always("  Average packets/s   : %'-10.0f\n",
1333                              sample_round(pkts / period));
1334         }
1335         if (sample_out.totals.redir) {
1336                 double pkts = sample_out.totals.redir;
1337
1338                 print_always("  Packets redirected  : %'-10llu\n",
1339                              sample_out.totals.redir);
1340                 print_always("  Average redir/s     : %'-10.0f\n",
1341                              sample_round(pkts / period));
1342         }
1343         if (sample_out.totals.drop)
1344                 print_always("  Rx dropped          : %'-10llu\n",
1345                              sample_out.totals.drop);
1346         if (sample_out.totals.drop_xmit)
1347                 print_always("  Tx dropped          : %'-10llu\n",
1348                              sample_out.totals.drop_xmit);
1349         if (sample_out.totals.err)
1350                 print_always("  Errors recorded     : %'-10llu\n",
1351                              sample_out.totals.err);
1352         if (sample_out.totals.xmit) {
1353                 double pkts = sample_out.totals.xmit;
1354
1355                 print_always("  Packets transmitted : %'-10llu\n",
1356                              sample_out.totals.xmit);
1357                 print_always("  Average transmit/s  : %'-10.0f\n",
1358                              sample_round(pkts / period));
1359         }
1360 }
1361
1362 void sample_exit(int status)
1363 {
1364         size_t size;
1365
1366         for (int i = 0; i < NUM_MAP; i++) {
1367                 size = sample_map_count[i] * sizeof(**sample_mmap);
1368                 munmap(sample_mmap[i], size);
1369         }
1370         while (sample_xdp_cnt--) {
1371                 int i = sample_xdp_cnt, ifindex, xdp_flags;
1372                 __u32 prog_id;
1373
1374                 prog_id = sample_xdp_progs[i].prog_id;
1375                 ifindex = sample_xdp_progs[i].ifindex;
1376                 xdp_flags = sample_xdp_progs[i].flags;
1377
1378                 __sample_remove_xdp(ifindex, prog_id, xdp_flags);
1379         }
1380         sample_summary_print();
1381         close(sample_sig_fd);
1382         exit(status);
1383 }
1384
1385 static int sample_stats_collect(struct stats_record *rec)
1386 {
1387         int i;
1388
1389         if (sample_mask & SAMPLE_RX_CNT)
1390                 map_collect_percpu(sample_mmap[MAP_RX], &rec->rx_cnt);
1391
1392         if (sample_mask & SAMPLE_REDIRECT_CNT)
1393                 map_collect_percpu(sample_mmap[MAP_REDIRECT_ERR], &rec->redir_err[0]);
1394
1395         if (sample_mask & SAMPLE_REDIRECT_ERR_CNT) {
1396                 for (i = 1; i < XDP_REDIRECT_ERR_MAX; i++)
1397                         map_collect_percpu(&sample_mmap[MAP_REDIRECT_ERR][i * sample_n_cpus],
1398                                            &rec->redir_err[i]);
1399         }
1400
1401         if (sample_mask & SAMPLE_CPUMAP_ENQUEUE_CNT)
1402                 for (i = 0; i < sample_n_cpus; i++)
1403                         map_collect_percpu(&sample_mmap[MAP_CPUMAP_ENQUEUE][i * sample_n_cpus],
1404                                            &rec->enq[i]);
1405
1406         if (sample_mask & SAMPLE_CPUMAP_KTHREAD_CNT)
1407                 map_collect_percpu(sample_mmap[MAP_CPUMAP_KTHREAD],
1408                                    &rec->kthread);
1409
1410         if (sample_mask & SAMPLE_EXCEPTION_CNT)
1411                 for (i = 0; i < XDP_ACTION_MAX; i++)
1412                         map_collect_percpu(&sample_mmap[MAP_EXCEPTION][i * sample_n_cpus],
1413                                            &rec->exception[i]);
1414
1415         if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT)
1416                 map_collect_percpu(sample_mmap[MAP_DEVMAP_XMIT], &rec->devmap_xmit);
1417
1418         if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI) {
1419                 if (map_collect_percpu_devmap(bpf_map__fd(sample_map[MAP_DEVMAP_XMIT_MULTI]), rec) < 0)
1420                         return -EINVAL;
1421         }
1422         return 0;
1423 }
1424
1425 static void sample_summary_update(struct sample_output *out, int interval)
1426 {
1427         sample_out.totals.rx += out->totals.rx;
1428         sample_out.totals.redir += out->totals.redir;
1429         sample_out.totals.drop += out->totals.drop;
1430         sample_out.totals.drop_xmit += out->totals.drop_xmit;
1431         sample_out.totals.err += out->totals.err;
1432         sample_out.totals.xmit += out->totals.xmit;
1433         sample_out.rx_cnt.pps += interval;
1434 }
1435
1436 static void sample_stats_print(int mask, struct stats_record *cur,
1437                                struct stats_record *prev, char *prog_name,
1438                                int interval)
1439 {
1440         struct sample_output out = {};
1441
1442         if (mask & SAMPLE_RX_CNT)
1443                 stats_get_rx_cnt(cur, prev, 0, &out);
1444         if (mask & SAMPLE_REDIRECT_CNT)
1445                 stats_get_redirect_cnt(cur, prev, 0, &out);
1446         if (mask & SAMPLE_REDIRECT_ERR_CNT)
1447                 stats_get_redirect_err_cnt(cur, prev, 0, &out);
1448         if (mask & SAMPLE_EXCEPTION_CNT)
1449                 stats_get_exception_cnt(cur, prev, 0, &out);
1450         if (mask & SAMPLE_DEVMAP_XMIT_CNT)
1451                 stats_get_devmap_xmit(cur, prev, 0, &out);
1452         else if (mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
1453                 stats_get_devmap_xmit_multi(cur, prev, 0, &out,
1454                                             mask & SAMPLE_DEVMAP_XMIT_CNT);
1455         sample_summary_update(&out, interval);
1456
1457         stats_print(prog_name, mask, cur, prev, &out);
1458 }
1459
1460 void sample_switch_mode(void)
1461 {
1462         sample_log_level ^= LL_DEBUG - 1;
1463 }
1464
1465 static int sample_signal_cb(void)
1466 {
1467         struct signalfd_siginfo si;
1468         int r;
1469
1470         r = read(sample_sig_fd, &si, sizeof(si));
1471         if (r < 0)
1472                 return -errno;
1473
1474         switch (si.ssi_signo) {
1475         case SIGQUIT:
1476                 sample_switch_mode();
1477                 printf("\n");
1478                 break;
1479         default:
1480                 printf("\n");
1481                 return 1;
1482         }
1483
1484         return 0;
1485 }
1486
1487 /* Pointer swap trick */
1488 static void swap(struct stats_record **a, struct stats_record **b)
1489 {
1490         struct stats_record *tmp;
1491
1492         tmp = *a;
1493         *a = *b;
1494         *b = tmp;
1495 }
1496
1497 static int sample_timer_cb(int timerfd, struct stats_record **rec,
1498                            struct stats_record **prev, int interval)
1499 {
1500         char line[64] = "Summary";
1501         int ret;
1502         __u64 t;
1503
1504         ret = read(timerfd, &t, sizeof(t));
1505         if (ret < 0)
1506                 return -errno;
1507
1508         swap(prev, rec);
1509         ret = sample_stats_collect(*rec);
1510         if (ret < 0)
1511                 return ret;
1512
1513         if (sample_xdp_cnt == 2 && !(sample_mask & SAMPLE_SKIP_HEADING)) {
1514                 char fi[IFNAMSIZ];
1515                 char to[IFNAMSIZ];
1516                 const char *f, *t;
1517
1518                 f = t = NULL;
1519                 if (if_indextoname(sample_xdp_progs[0].ifindex, fi))
1520                         f = fi;
1521                 if (if_indextoname(sample_xdp_progs[1].ifindex, to))
1522                         t = to;
1523
1524                 snprintf(line, sizeof(line), "%s->%s", f ?: "?", t ?: "?");
1525         }
1526
1527         sample_stats_print(sample_mask, *rec, *prev, line, interval);
1528         return 0;
1529 }
1530
1531 int sample_run(int interval, void (*post_cb)(void *), void *ctx)
1532 {
1533         struct timespec ts = { interval, 0 };
1534         struct itimerspec its = { ts, ts };
1535         struct stats_record *rec, *prev;
1536         struct pollfd pfd[2] = {};
1537         int timerfd, ret;
1538
1539         if (!interval) {
1540                 fprintf(stderr, "Incorrect interval 0\n");
1541                 return -EINVAL;
1542         }
1543         sample_interval = interval;
1544         /* Pretty print numbers */
1545         setlocale(LC_NUMERIC, "en_US.UTF-8");
1546
1547         timerfd = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC | TFD_NONBLOCK);
1548         if (timerfd < 0)
1549                 return -errno;
1550         timerfd_settime(timerfd, 0, &its, NULL);
1551
1552         pfd[0].fd = sample_sig_fd;
1553         pfd[0].events = POLLIN;
1554
1555         pfd[1].fd = timerfd;
1556         pfd[1].events = POLLIN;
1557
1558         ret = -ENOMEM;
1559         rec = alloc_stats_record();
1560         if (!rec)
1561                 goto end;
1562         prev = alloc_stats_record();
1563         if (!prev)
1564                 goto end_rec;
1565
1566         ret = sample_stats_collect(rec);
1567         if (ret < 0)
1568                 goto end_rec_prev;
1569
1570         for (;;) {
1571                 ret = poll(pfd, 2, -1);
1572                 if (ret < 0) {
1573                         if (errno == EINTR)
1574                                 continue;
1575                         else
1576                                 break;
1577                 }
1578
1579                 if (pfd[0].revents & POLLIN)
1580                         ret = sample_signal_cb();
1581                 else if (pfd[1].revents & POLLIN)
1582                         ret = sample_timer_cb(timerfd, &rec, &prev, interval);
1583
1584                 if (ret)
1585                         break;
1586
1587                 if (post_cb)
1588                         post_cb(ctx);
1589         }
1590
1591 end_rec_prev:
1592         free_stats_record(prev);
1593 end_rec:
1594         free_stats_record(rec);
1595 end:
1596         close(timerfd);
1597
1598         return ret;
1599 }
1600
1601 const char *get_driver_name(int ifindex)
1602 {
1603         struct ethtool_drvinfo drv = {};
1604         char ifname[IF_NAMESIZE];
1605         static char drvname[32];
1606         struct ifreq ifr = {};
1607         int fd, r = 0;
1608
1609         fd = socket(AF_INET, SOCK_DGRAM, 0);
1610         if (fd < 0)
1611                 return "[error]";
1612
1613         if (!if_indextoname(ifindex, ifname))
1614                 goto end;
1615
1616         drv.cmd = ETHTOOL_GDRVINFO;
1617         safe_strncpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name));
1618         ifr.ifr_data = (void *)&drv;
1619
1620         r = ioctl(fd, SIOCETHTOOL, &ifr);
1621         if (r)
1622                 goto end;
1623
1624         safe_strncpy(drvname, drv.driver, sizeof(drvname));
1625
1626         close(fd);
1627         return drvname;
1628
1629 end:
1630         r = errno;
1631         close(fd);
1632         return r == EOPNOTSUPP ? "loopback" : "[error]";
1633 }
1634
1635 int get_mac_addr(int ifindex, void *mac_addr)
1636 {
1637         char ifname[IF_NAMESIZE];
1638         struct ifreq ifr = {};
1639         int fd, r;
1640
1641         fd = socket(AF_INET, SOCK_DGRAM, 0);
1642         if (fd < 0)
1643                 return -errno;
1644
1645         if (!if_indextoname(ifindex, ifname)) {
1646                 r = -errno;
1647                 goto end;
1648         }
1649
1650         safe_strncpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name));
1651
1652         r = ioctl(fd, SIOCGIFHWADDR, &ifr);
1653         if (r) {
1654                 r = -errno;
1655                 goto end;
1656         }
1657
1658         memcpy(mac_addr, ifr.ifr_hwaddr.sa_data, 6 * sizeof(char));
1659
1660 end:
1661         close(fd);
1662         return r;
1663 }
1664
1665 __attribute__((constructor)) static void sample_ctor(void)
1666 {
1667         if (libbpf_set_strict_mode(LIBBPF_STRICT_ALL) < 0) {
1668                 fprintf(stderr, "Failed to set libbpf strict mode: %s\n",
1669                         strerror(errno));
1670                 /* Just exit, nothing to cleanup right now */
1671                 exit(EXIT_FAIL_BPF);
1672         }
1673 }