tools/perf/tests/builtin-test.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * builtin-test.c
   4  *
   5  * Builtin regression testing command: ever growing number of sanity tests
   6  */
   7 #include <fcntl.h>
   8 #include <errno.h>
   9 #include <poll.h>
  10 #include <unistd.h>
  11 #include <string.h>
  12 #include <stdlib.h>
  13 #include <sys/types.h>
  14 #include <dirent.h>
  15 #include <sys/wait.h>
  16 #include <sys/stat.h>
  17 #include "builtin.h"
  18 #include "config.h"
  19 #include "hist.h"
  20 #include "intlist.h"
  21 #include "tests.h"
  22 #include "debug.h"
  23 #include "color.h"
  24 #include <subcmd/parse-options.h>
  25 #include <subcmd/run-command.h>
  26 #include "string2.h"
  27 #include "symbol.h"
  28 #include "util/rlimit.h"
  29 #include "util/strbuf.h"
  30 #include <linux/kernel.h>
  31 #include <linux/string.h>
  32 #include <subcmd/exec-cmd.h>
  33 #include <linux/zalloc.h>
  34
  35 #include "tests-scripts.h"
  36
  37 /*
  38  * Command line option to not fork the test running in the same process and
  39  * making them easier to debug.
  40  */
  41 static bool dont_fork;
  42 /* Fork the tests in parallel and then wait for their completion. */
  43 static bool parallel;
  44 const char *dso_to_test;
  45 const char *test_objdump_path = "objdump";
  46
  47 /*
  48  * List of architecture specific tests. Not a weak symbol as the array length is
  49  * dependent on the initialization, as such GCC with LTO complains of
  50  * conflicting definitions with a weak symbol.
  51  */
  52 #if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) || defined(__powerpc64__)
  53 extern struct test_suite *arch_tests[];
  54 #else
  55 static struct test_suite *arch_tests[] = {
  56         NULL,
  57 };
  58 #endif
  59
  60 static struct test_suite *generic_tests[] = {
  61         &suite__vmlinux_matches_kallsyms,
  62 #ifdef HAVE_LIBTRACEEVENT
  63         &suite__openat_syscall_event,
  64         &suite__openat_syscall_event_on_all_cpus,
  65         &suite__basic_mmap,
  66 #endif
  67         &suite__mem,
  68         &suite__parse_events,
  69         &suite__expr,
  70         &suite__PERF_RECORD,
  71         &suite__pmu,
  72         &suite__pmu_events,
  73         &suite__dso_data,
  74         &suite__perf_evsel__roundtrip_name_test,
  75 #ifdef HAVE_LIBTRACEEVENT
  76         &suite__perf_evsel__tp_sched_test,
  77         &suite__syscall_openat_tp_fields,
  78 #endif
  79         &suite__attr,
  80         &suite__hists_link,
  81         &suite__python_use,
  82         &suite__bp_signal,
  83         &suite__bp_signal_overflow,
  84         &suite__bp_accounting,
  85         &suite__wp,
  86         &suite__task_exit,
  87         &suite__sw_clock_freq,
  88         &suite__code_reading,
  89         &suite__sample_parsing,
  90         &suite__keep_tracking,
  91         &suite__parse_no_sample_id_all,
  92         &suite__hists_filter,
  93         &suite__mmap_thread_lookup,
  94         &suite__thread_maps_share,
  95         &suite__hists_output,
  96         &suite__hists_cumulate,
  97 #ifdef HAVE_LIBTRACEEVENT
  98         &suite__switch_tracking,
  99 #endif
 100         &suite__fdarray__filter,
 101         &suite__fdarray__add,
 102         &suite__kmod_path__parse,
 103         &suite__thread_map,
 104         &suite__session_topology,
 105         &suite__thread_map_synthesize,
 106         &suite__thread_map_remove,
 107         &suite__cpu_map,
 108         &suite__synthesize_stat_config,
 109         &suite__synthesize_stat,
 110         &suite__synthesize_stat_round,
 111         &suite__event_update,
 112         &suite__event_times,
 113         &suite__backward_ring_buffer,
 114         &suite__sdt_event,
 115         &suite__is_printable_array,
 116         &suite__bitmap_print,
 117         &suite__perf_hooks,
 118         &suite__unit_number__scnprint,
 119         &suite__mem2node,
 120         &suite__time_utils,
 121         &suite__jit_write_elf,
 122         &suite__pfm,
 123         &suite__api_io,
 124         &suite__maps__merge_in,
 125         &suite__demangle_java,
 126         &suite__demangle_ocaml,
 127         &suite__parse_metric,
 128         &suite__pe_file_parsing,
 129         &suite__expand_cgroup_events,
 130         &suite__perf_time_to_tsc,
 131         &suite__dlfilter,
 132         &suite__sigtrap,
 133         &suite__event_groups,
 134         &suite__symbols,
 135         &suite__util,
 136         NULL,
 137 };
 138
 139 static struct test_suite **tests[] = {
 140         generic_tests,
 141         arch_tests,
 142         NULL, /* shell tests created at runtime. */
 143 };
 144
 145 static struct test_workload *workloads[] = {
 146         &workload__noploop,
 147         &workload__thloop,
 148         &workload__leafloop,
 149         &workload__sqrtloop,
 150         &workload__brstack,
 151         &workload__datasym,
 152 };
 153
 154 static int num_subtests(const struct test_suite *t)
 155 {
 156         int num;
 157
 158         if (!t->test_cases)
 159                 return 0;
 160
 161         num = 0;
 162         while (t->test_cases[num].name)
 163                 num++;
 164
 165         return num;
 166 }
 167
 168 static bool has_subtests(const struct test_suite *t)
 169 {
 170         return num_subtests(t) > 1;
 171 }
 172
 173 static const char *skip_reason(const struct test_suite *t, int subtest)
 174 {
 175         if (!t->test_cases)
 176                 return NULL;
 177
 178         return t->test_cases[subtest >= 0 ? subtest : 0].skip_reason;
 179 }
 180
 181 static const char *test_description(const struct test_suite *t, int subtest)
 182 {
 183         if (t->test_cases && subtest >= 0)
 184                 return t->test_cases[subtest].desc;
 185
 186         return t->desc;
 187 }
 188
 189 static test_fnptr test_function(const struct test_suite *t, int subtest)
 190 {
 191         if (subtest <= 0)
 192                 return t->test_cases[0].run_case;
 193
 194         return t->test_cases[subtest].run_case;
 195 }
 196
 197 static bool perf_test__matches(const char *desc, int curr, int argc, const char *argv[])
 198 {
 199         int i;
 200
 201         if (argc == 0)
 202                 return true;
 203
 204         for (i = 0; i < argc; ++i) {
 205                 char *end;
 206                 long nr = strtoul(argv[i], &end, 10);
 207
 208                 if (*end == '\0') {
 209                         if (nr == curr + 1)
 210                                 return true;
 211                         continue;
 212                 }
 213
 214                 if (strcasestr(desc, argv[i]))
 215                         return true;
 216         }
 217
 218         return false;
 219 }
 220
 221 struct child_test {
 222         struct child_process process;
 223         struct test_suite *test;
 224         int test_num;
 225         int subtest;
 226 };
 227
 228 static int run_test_child(struct child_process *process)
 229 {
 230         struct child_test *child = container_of(process, struct child_test, process);
 231         int err;
 232
 233         pr_debug("--- start ---\n");
 234         pr_debug("test child forked, pid %d\n", getpid());
 235         err = test_function(child->test, child->subtest)(child->test, child->subtest);
 236         pr_debug("---- end(%d) ----\n", err);
 237         fflush(NULL);
 238         return -err;
 239 }
 240
 241 static int print_test_result(struct test_suite *t, int i, int subtest, int result, int width)
 242 {
 243         if (has_subtests(t)) {
 244                 int subw = width > 2 ? width - 2 : width;
 245
 246                 pr_info("%3d.%1d: %-*s:", i + 1, subtest + 1, subw, test_description(t, subtest));
 247         } else
 248                 pr_info("%3d: %-*s:", i + 1, width, test_description(t, subtest));
 249
 250         switch (result) {
 251         case TEST_OK:
 252                 pr_info(" Ok\n");
 253                 break;
 254         case TEST_SKIP: {
 255                 const char *reason = skip_reason(t, subtest);
 256
 257                 if (reason)
 258                         color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (%s)\n", reason);
 259                 else
 260                         color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip\n");
 261         }
 262                 break;
 263         case TEST_FAIL:
 264         default:
 265                 color_fprintf(stderr, PERF_COLOR_RED, " FAILED!\n");
 266                 break;
 267         }
 268
 269         return 0;
 270 }
 271
 272 static int finish_test(struct child_test *child_test, int width)
 273 {
 274         struct test_suite *t = child_test->test;
 275         int i = child_test->test_num;
 276         int subi = child_test->subtest;
 277         int out = child_test->process.out;
 278         int err = child_test->process.err;
 279         bool out_done = out <= 0;
 280         bool err_done = err <= 0;
 281         struct strbuf out_output = STRBUF_INIT;
 282         struct strbuf err_output = STRBUF_INIT;
 283         int ret;
 284
 285         /*
 286          * For test suites with subtests, display the suite name ahead of the
 287          * sub test names.
 288          */
 289         if (has_subtests(t) && subi == 0)
 290                 pr_info("%3d: %-*s:\n", i + 1, width, test_description(t, -1));
 291
 292         /*
 293          * Busy loop reading from the child's stdout and stderr that are set to
 294          * be non-blocking until EOF.
 295          */
 296         if (!out_done)
 297                 fcntl(out, F_SETFL, O_NONBLOCK);
 298         if (!err_done)
 299                 fcntl(err, F_SETFL, O_NONBLOCK);
 300         if (verbose > 1) {
 301                 if (has_subtests(t))
 302                         pr_info("%3d.%1d: %s:\n", i + 1, subi + 1, test_description(t, subi));
 303                 else
 304                         pr_info("%3d: %s:\n", i + 1, test_description(t, -1));
 305         }
 306         while (!out_done || !err_done) {
 307                 struct pollfd pfds[2] = {
 308                         { .fd = out,
 309                           .events = POLLIN | POLLERR | POLLHUP | POLLNVAL,
 310                         },
 311                         { .fd = err,
 312                           .events = POLLIN | POLLERR | POLLHUP | POLLNVAL,
 313                         },
 314                 };
 315                 char buf[512];
 316                 ssize_t len;
 317
 318                 /* Poll to avoid excessive spinning, timeout set for 1000ms. */
 319                 poll(pfds, ARRAY_SIZE(pfds), /*timeout=*/1000);
 320                 if (!out_done && pfds[0].revents) {
 321                         errno = 0;
 322                         len = read(out, buf, sizeof(buf) - 1);
 323
 324                         if (len <= 0) {
 325                                 out_done = errno != EAGAIN;
 326                         } else {
 327                                 buf[len] = '\0';
 328                                 if (verbose > 1)
 329                                         fprintf(stdout, "%s", buf);
 330                                 else
 331                                         strbuf_addstr(&out_output, buf);
 332                         }
 333                 }
 334                 if (!err_done && pfds[1].revents) {
 335                         errno = 0;
 336                         len = read(err, buf, sizeof(buf) - 1);
 337
 338                         if (len <= 0) {
 339                                 err_done = errno != EAGAIN;
 340                         } else {
 341                                 buf[len] = '\0';
 342                                 if (verbose > 1)
 343                                         fprintf(stdout, "%s", buf);
 344                                 else
 345                                         strbuf_addstr(&err_output, buf);
 346                         }
 347                 }
 348         }
 349         /* Clean up child process. */
 350         ret = finish_command(&child_test->process);
 351         if (verbose == 1 && ret == TEST_FAIL) {
 352                 /* Add header for test that was skipped above. */
 353                 if (has_subtests(t))
 354                         pr_info("%3d.%1d: %s:\n", i + 1, subi + 1, test_description(t, subi));
 355                 else
 356                         pr_info("%3d: %s:\n", i + 1, test_description(t, -1));
 357                 fprintf(stdout, "%s", out_output.buf);
 358                 fprintf(stderr, "%s", err_output.buf);
 359         }
 360         strbuf_release(&out_output);
 361         strbuf_release(&err_output);
 362         print_test_result(t, i, subi, ret, width);
 363         if (out > 0)
 364                 close(out);
 365         if (err > 0)
 366                 close(err);
 367         return 0;
 368 }
 369
 370 static int start_test(struct test_suite *test, int i, int subi, struct child_test **child,
 371                       int width)
 372 {
 373         int err;
 374
 375         *child = NULL;
 376         if (dont_fork) {
 377                 pr_debug("--- start ---\n");
 378                 err = test_function(test, subi)(test, subi);
 379                 pr_debug("---- end ----\n");
 380                 print_test_result(test, i, subi, err, width);
 381                 return 0;
 382         }
 383
 384         *child = zalloc(sizeof(**child));
 385         if (!*child)
 386                 return -ENOMEM;
 387
 388         (*child)->test = test;
 389         (*child)->test_num = i;
 390         (*child)->subtest = subi;
 391         (*child)->process.pid = -1;
 392         (*child)->process.no_stdin = 1;
 393         if (verbose <= 0) {
 394                 (*child)->process.no_stdout = 1;
 395                 (*child)->process.no_stderr = 1;
 396         } else {
 397                 (*child)->process.out = -1;
 398                 (*child)->process.err = -1;
 399         }
 400         (*child)->process.no_exec_cmd = run_test_child;
 401         err = start_command(&(*child)->process);
 402         if (err || parallel)
 403                 return  err;
 404         return finish_test(*child, width);
 405 }
 406
 407 #define for_each_test(j, k, t)                                  \
 408         for (j = 0, k = 0; j < ARRAY_SIZE(tests); j++, k = 0)   \
 409                 while ((t = tests[j][k++]) != NULL)
 410
 411 static int __cmd_test(int argc, const char *argv[], struct intlist *skiplist)
 412 {
 413         struct test_suite *t;
 414         unsigned int j, k;
 415         int i = 0;
 416         int width = 0;
 417         size_t num_tests = 0;
 418         struct child_test **child_tests;
 419         int child_test_num = 0;
 420
 421         for_each_test(j, k, t) {
 422                 int len = strlen(test_description(t, -1));
 423
 424                 if (width < len)
 425                         width = len;
 426
 427                 if (has_subtests(t)) {
 428                         for (int subi = 0, subn = num_subtests(t); subi < subn; subi++) {
 429                                 len = strlen(test_description(t, subi));
 430                                 if (width < len)
 431                                         width = len;
 432                                 num_tests++;
 433                         }
 434                 } else {
 435                         num_tests++;
 436                 }
 437         }
 438         child_tests = calloc(num_tests, sizeof(*child_tests));
 439         if (!child_tests)
 440                 return -ENOMEM;
 441
 442         for_each_test(j, k, t) {
 443                 int curr = i++;
 444
 445                 if (!perf_test__matches(test_description(t, -1), curr, argc, argv)) {
 446                         bool skip = true;
 447
 448                         for (int subi = 0, subn = num_subtests(t); subi < subn; subi++) {
 449                                 if (perf_test__matches(test_description(t, subi),
 450                                                         curr, argc, argv))
 451                                         skip = false;
 452                         }
 453
 454                         if (skip)
 455                                 continue;
 456                 }
 457
 458                 if (intlist__find(skiplist, i)) {
 459                         pr_info("%3d: %-*s:", curr + 1, width, test_description(t, -1));
 460                         color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (user override)\n");
 461                         continue;
 462                 }
 463
 464                 if (!has_subtests(t)) {
 465                         int err = start_test(t, curr, -1, &child_tests[child_test_num++], width);
 466
 467                         if (err) {
 468                                 /* TODO: if parallel waitpid the already forked children. */
 469                                 free(child_tests);
 470                                 return err;
 471                         }
 472                 } else {
 473                         for (int subi = 0, subn = num_subtests(t); subi < subn; subi++) {
 474                                 int err;
 475
 476                                 if (!perf_test__matches(test_description(t, subi),
 477                                                         curr, argc, argv))
 478                                         continue;
 479
 480                                 err = start_test(t, curr, subi, &child_tests[child_test_num++],
 481                                                  width);
 482                                 if (err)
 483                                         return err;
 484                         }
 485                 }
 486         }
 487         for (i = 0; i < child_test_num; i++) {
 488                 if (parallel) {
 489                         int ret  = finish_test(child_tests[i], width);
 490
 491                         if (ret)
 492                                 return ret;
 493                 }
 494                 free(child_tests[i]);
 495         }
 496         free(child_tests);
 497         return 0;
 498 }
 499
 500 static int perf_test__list(int argc, const char **argv)
 501 {
 502         unsigned int j, k;
 503         struct test_suite *t;
 504         int i = 0;
 505
 506         for_each_test(j, k, t) {
 507                 int curr = i++;
 508
 509                 if (!perf_test__matches(test_description(t, -1), curr, argc, argv))
 510                         continue;
 511
 512                 pr_info("%3d: %s\n", i, test_description(t, -1));
 513
 514                 if (has_subtests(t)) {
 515                         int subn = num_subtests(t);
 516                         int subi;
 517
 518                         for (subi = 0; subi < subn; subi++)
 519                                 pr_info("%3d:%1d: %s\n", i, subi + 1,
 520                                         test_description(t, subi));
 521                 }
 522         }
 523         return 0;
 524 }
 525
 526 static int run_workload(const char *work, int argc, const char **argv)
 527 {
 528         unsigned int i = 0;
 529         struct test_workload *twl;
 530
 531         for (i = 0; i < ARRAY_SIZE(workloads); i++) {
 532                 twl = workloads[i];
 533                 if (!strcmp(twl->name, work))
 534                         return twl->func(argc, argv);
 535         }
 536
 537         pr_info("No workload found: %s\n", work);
 538         return -1;
 539 }
 540
 541 static int perf_test__config(const char *var, const char *value,
 542                              void *data __maybe_unused)
 543 {
 544         if (!strcmp(var, "annotate.objdump"))
 545                 test_objdump_path = value;
 546
 547         return 0;
 548 }
 549
 550 int cmd_test(int argc, const char **argv)
 551 {
 552         const char *test_usage[] = {
 553         "perf test [<options>] [{list <test-name-fragment>|[<test-name-fragments>|<test-numbers>]}]",
 554         NULL,
 555         };
 556         const char *skip = NULL;
 557         const char *workload = NULL;
 558         const struct option test_options[] = {
 559         OPT_STRING('s', "skip", &skip, "tests", "tests to skip"),
 560         OPT_INCR('v', "verbose", &verbose,
 561                     "be more verbose (show symbol address, etc)"),
 562         OPT_BOOLEAN('F', "dont-fork", &dont_fork,
 563                     "Do not fork for testcase"),
 564         OPT_BOOLEAN('p', "parallel", &parallel,
 565                     "Run the tests altogether in parallel"),
 566         OPT_STRING('w', "workload", &workload, "work", "workload to run for testing"),
 567         OPT_STRING(0, "dso", &dso_to_test, "dso", "dso to test"),
 568         OPT_STRING(0, "objdump", &test_objdump_path, "path",
 569                    "objdump binary to use for disassembly and annotations"),
 570         OPT_END()
 571         };
 572         const char * const test_subcommands[] = { "list", NULL };
 573         struct intlist *skiplist = NULL;
 574         int ret = hists__init();
 575
 576         if (ret < 0)
 577                 return ret;
 578
 579         perf_config(perf_test__config, NULL);
 580
 581         /* Unbuffered output */
 582         setvbuf(stdout, NULL, _IONBF, 0);
 583
 584         tests[2] = create_script_test_suites();
 585         argc = parse_options_subcommand(argc, argv, test_options, test_subcommands, test_usage, 0);
 586         if (argc >= 1 && !strcmp(argv[0], "list"))
 587                 return perf_test__list(argc - 1, argv + 1);
 588
 589         if (workload)
 590                 return run_workload(workload, argc, argv);
 591
 592         symbol_conf.priv_size = sizeof(int);
 593         symbol_conf.try_vmlinux_path = true;
 594
 595         if (symbol__init(NULL) < 0)
 596                 return -1;
 597
 598         if (skip != NULL)
 599                 skiplist = intlist__new(skip);
 600         /*
 601          * Tests that create BPF maps, for instance, need more than the 64K
 602          * default:
 603          */
 604         rlimit__bump_memlock();
 605
 606         return __cmd_test(argc, argv, skiplist);
 607 }