tools/perf/builtin-record.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * builtin-record.c
   4  *
   5  * Builtin record command: Record the profile of a workload
   6  * (or a CPU, or a PID) into the perf.data output file - for
   7  * later analysis via perf report.
   8  */
   9 #include "builtin.h"
  10
  11 #include "util/build-id.h"
  12 #include <subcmd/parse-options.h>
  13 #include "util/parse-events.h"
  14 #include "util/config.h"
  15
  16 #include "util/callchain.h"
  17 #include "util/cgroup.h"
  18 #include "util/header.h"
  19 #include "util/event.h"
  20 #include "util/evlist.h"
  21 #include "util/evsel.h"
  22 #include "util/debug.h"
  23 #include "util/mmap.h"
  24 #include "util/target.h"
  25 #include "util/session.h"
  26 #include "util/tool.h"
  27 #include "util/symbol.h"
  28 #include "util/record.h"
  29 #include "util/cpumap.h"
  30 #include "util/thread_map.h"
  31 #include "util/data.h"
  32 #include "util/perf_regs.h"
  33 #include "util/auxtrace.h"
  34 #include "util/tsc.h"
  35 #include "util/parse-branch-options.h"
  36 #include "util/parse-regs-options.h"
  37 #include "util/perf_api_probe.h"
  38 #include "util/llvm-utils.h"
  39 #include "util/bpf-loader.h"
  40 #include "util/trigger.h"
  41 #include "util/perf-hooks.h"
  42 #include "util/cpu-set-sched.h"
  43 #include "util/synthetic-events.h"
  44 #include "util/time-utils.h"
  45 #include "util/units.h"
  46 #include "util/bpf-event.h"
  47 #include "util/util.h"
  48 #include "util/pfm.h"
  49 #include "util/clockid.h"
  50 #include "util/pmu-hybrid.h"
  51 #include "util/evlist-hybrid.h"
  52 #include "asm/bug.h"
  53 #include "perf.h"
  54
  55 #include <errno.h>
  56 #include <inttypes.h>
  57 #include <locale.h>
  58 #include <poll.h>
  59 #include <pthread.h>
  60 #include <unistd.h>
  61 #include <sched.h>
  62 #include <signal.h>
  63 #ifdef HAVE_EVENTFD_SUPPORT
  64 #include <sys/eventfd.h>
  65 #endif
  66 #include <sys/mman.h>
  67 #include <sys/wait.h>
  68 #include <sys/types.h>
  69 #include <sys/stat.h>
  70 #include <fcntl.h>
  71 #include <linux/err.h>
  72 #include <linux/string.h>
  73 #include <linux/time64.h>
  74 #include <linux/zalloc.h>
  75 #include <linux/bitmap.h>
  76 #include <sys/time.h>
  77
  78 struct switch_output {
  79         bool             enabled;
  80         bool             signal;
  81         unsigned long    size;
  82         unsigned long    time;
  83         const char      *str;
  84         bool             set;
  85         char             **filenames;
  86         int              num_files;
  87         int              cur_file;
  88 };
  89
  90 struct record {
  91         struct perf_tool        tool;
  92         struct record_opts      opts;
  93         u64                     bytes_written;
  94         struct perf_data        data;
  95         struct auxtrace_record  *itr;
  96         struct evlist   *evlist;
  97         struct perf_session     *session;
  98         struct evlist           *sb_evlist;
  99         pthread_t               thread_id;
 100         int                     realtime_prio;
 101         bool                    switch_output_event_set;
 102         bool                    no_buildid;
 103         bool                    no_buildid_set;
 104         bool                    no_buildid_cache;
 105         bool                    no_buildid_cache_set;
 106         bool                    buildid_all;
 107         bool                    buildid_mmap;
 108         bool                    timestamp_filename;
 109         bool                    timestamp_boundary;
 110         struct switch_output    switch_output;
 111         unsigned long long      samples;
 112         struct mmap_cpu_mask    affinity_mask;
 113         unsigned long           output_max_size;        /* = 0: unlimited */
 114 };
 115
 116 static volatile int done;
 117
 118 static volatile int auxtrace_record__snapshot_started;
 119 static DEFINE_TRIGGER(auxtrace_snapshot_trigger);
 120 static DEFINE_TRIGGER(switch_output_trigger);
 121
 122 static const char *affinity_tags[PERF_AFFINITY_MAX] = {
 123         "SYS", "NODE", "CPU"
 124 };
 125
 126 static bool switch_output_signal(struct record *rec)
 127 {
 128         return rec->switch_output.signal &&
 129                trigger_is_ready(&switch_output_trigger);
 130 }
 131
 132 static bool switch_output_size(struct record *rec)
 133 {
 134         return rec->switch_output.size &&
 135                trigger_is_ready(&switch_output_trigger) &&
 136                (rec->bytes_written >= rec->switch_output.size);
 137 }
 138
 139 static bool switch_output_time(struct record *rec)
 140 {
 141         return rec->switch_output.time &&
 142                trigger_is_ready(&switch_output_trigger);
 143 }
 144
 145 static bool record__output_max_size_exceeded(struct record *rec)
 146 {
 147         return rec->output_max_size &&
 148                (rec->bytes_written >= rec->output_max_size);
 149 }
 150
 151 static int record__write(struct record *rec, struct mmap *map __maybe_unused,
 152                          void *bf, size_t size)
 153 {
 154         struct perf_data_file *file = &rec->session->data->file;
 155
 156         if (perf_data_file__write(file, bf, size) < 0) {
 157                 pr_err("failed to write perf data, error: %m\n");
 158                 return -1;
 159         }
 160
 161         rec->bytes_written += size;
 162
 163         if (record__output_max_size_exceeded(rec) && !done) {
 164                 fprintf(stderr, "[ perf record: perf size limit reached (%" PRIu64 " KB),"
 165                                 " stopping session ]\n",
 166                                 rec->bytes_written >> 10);
 167                 done = 1;
 168         }
 169
 170         if (switch_output_size(rec))
 171                 trigger_hit(&switch_output_trigger);
 172
 173         return 0;
 174 }
 175
 176 static int record__aio_enabled(struct record *rec);
 177 static int record__comp_enabled(struct record *rec);
 178 static size_t zstd_compress(struct perf_session *session, void *dst, size_t dst_size,
 179                             void *src, size_t src_size);
 180
 181 #ifdef HAVE_AIO_SUPPORT
 182 static int record__aio_write(struct aiocb *cblock, int trace_fd,
 183                 void *buf, size_t size, off_t off)
 184 {
 185         int rc;
 186
 187         cblock->aio_fildes = trace_fd;
 188         cblock->aio_buf    = buf;
 189         cblock->aio_nbytes = size;
 190         cblock->aio_offset = off;
 191         cblock->aio_sigevent.sigev_notify = SIGEV_NONE;
 192
 193         do {
 194                 rc = aio_write(cblock);
 195                 if (rc == 0) {
 196                         break;
 197                 } else if (errno != EAGAIN) {
 198                         cblock->aio_fildes = -1;
 199                         pr_err("failed to queue perf data, error: %m\n");
 200                         break;
 201                 }
 202         } while (1);
 203
 204         return rc;
 205 }
 206
 207 static int record__aio_complete(struct mmap *md, struct aiocb *cblock)
 208 {
 209         void *rem_buf;
 210         off_t rem_off;
 211         size_t rem_size;
 212         int rc, aio_errno;
 213         ssize_t aio_ret, written;
 214
 215         aio_errno = aio_error(cblock);
 216         if (aio_errno == EINPROGRESS)
 217                 return 0;
 218
 219         written = aio_ret = aio_return(cblock);
 220         if (aio_ret < 0) {
 221                 if (aio_errno != EINTR)
 222                         pr_err("failed to write perf data, error: %m\n");
 223                 written = 0;
 224         }
 225
 226         rem_size = cblock->aio_nbytes - written;
 227
 228         if (rem_size == 0) {
 229                 cblock->aio_fildes = -1;
 230                 /*
 231                  * md->refcount is incremented in record__aio_pushfn() for
 232                  * every aio write request started in record__aio_push() so
 233                  * decrement it because the request is now complete.
 234                  */
 235                 perf_mmap__put(&md->core);
 236                 rc = 1;
 237         } else {
 238                 /*
 239                  * aio write request may require restart with the
 240                  * reminder if the kernel didn't write whole
 241                  * chunk at once.
 242                  */
 243                 rem_off = cblock->aio_offset + written;
 244                 rem_buf = (void *)(cblock->aio_buf + written);
 245                 record__aio_write(cblock, cblock->aio_fildes,
 246                                 rem_buf, rem_size, rem_off);
 247                 rc = 0;
 248         }
 249
 250         return rc;
 251 }
 252
 253 static int record__aio_sync(struct mmap *md, bool sync_all)
 254 {
 255         struct aiocb **aiocb = md->aio.aiocb;
 256         struct aiocb *cblocks = md->aio.cblocks;
 257         struct timespec timeout = { 0, 1000 * 1000  * 1 }; /* 1ms */
 258         int i, do_suspend;
 259
 260         do {
 261                 do_suspend = 0;
 262                 for (i = 0; i < md->aio.nr_cblocks; ++i) {
 263                         if (cblocks[i].aio_fildes == -1 || record__aio_complete(md, &cblocks[i])) {
 264                                 if (sync_all)
 265                                         aiocb[i] = NULL;
 266                                 else
 267                                         return i;
 268                         } else {
 269                                 /*
 270                                  * Started aio write is not complete yet
 271                                  * so it has to be waited before the
 272                                  * next allocation.
 273                                  */
 274                                 aiocb[i] = &cblocks[i];
 275                                 do_suspend = 1;
 276                         }
 277                 }
 278                 if (!do_suspend)
 279                         return -1;
 280
 281                 while (aio_suspend((const struct aiocb **)aiocb, md->aio.nr_cblocks, &timeout)) {
 282                         if (!(errno == EAGAIN || errno == EINTR))
 283                                 pr_err("failed to sync perf data, error: %m\n");
 284                 }
 285         } while (1);
 286 }
 287
 288 struct record_aio {
 289         struct record   *rec;
 290         void            *data;
 291         size_t          size;
 292 };
 293
 294 static int record__aio_pushfn(struct mmap *map, void *to, void *buf, size_t size)
 295 {
 296         struct record_aio *aio = to;
 297
 298         /*
 299          * map->core.base data pointed by buf is copied into free map->aio.data[] buffer
 300          * to release space in the kernel buffer as fast as possible, calling
 301          * perf_mmap__consume() from perf_mmap__push() function.
 302          *
 303          * That lets the kernel to proceed with storing more profiling data into
 304          * the kernel buffer earlier than other per-cpu kernel buffers are handled.
 305          *
 306          * Coping can be done in two steps in case the chunk of profiling data
 307          * crosses the upper bound of the kernel buffer. In this case we first move
 308          * part of data from map->start till the upper bound and then the reminder
 309          * from the beginning of the kernel buffer till the end of the data chunk.
 310          */
 311
 312         if (record__comp_enabled(aio->rec)) {
 313                 size = zstd_compress(aio->rec->session, aio->data + aio->size,
 314                                      mmap__mmap_len(map) - aio->size,
 315                                      buf, size);
 316         } else {
 317                 memcpy(aio->data + aio->size, buf, size);
 318         }
 319
 320         if (!aio->size) {
 321                 /*
 322                  * Increment map->refcount to guard map->aio.data[] buffer
 323                  * from premature deallocation because map object can be
 324                  * released earlier than aio write request started on
 325                  * map->aio.data[] buffer is complete.
 326                  *
 327                  * perf_mmap__put() is done at record__aio_complete()
 328                  * after started aio request completion or at record__aio_push()
 329                  * if the request failed to start.
 330                  */
 331                 perf_mmap__get(&map->core);
 332         }
 333
 334         aio->size += size;
 335
 336         return size;
 337 }
 338
 339 static int record__aio_push(struct record *rec, struct mmap *map, off_t *off)
 340 {
 341         int ret, idx;
 342         int trace_fd = rec->session->data->file.fd;
 343         struct record_aio aio = { .rec = rec, .size = 0 };
 344
 345         /*
 346          * Call record__aio_sync() to wait till map->aio.data[] buffer
 347          * becomes available after previous aio write operation.
 348          */
 349
 350         idx = record__aio_sync(map, false);
 351         aio.data = map->aio.data[idx];
 352         ret = perf_mmap__push(map, &aio, record__aio_pushfn);
 353         if (ret != 0) /* ret > 0 - no data, ret < 0 - error */
 354                 return ret;
 355
 356         rec->samples++;
 357         ret = record__aio_write(&(map->aio.cblocks[idx]), trace_fd, aio.data, aio.size, *off);
 358         if (!ret) {
 359                 *off += aio.size;
 360                 rec->bytes_written += aio.size;
 361                 if (switch_output_size(rec))
 362                         trigger_hit(&switch_output_trigger);
 363         } else {
 364                 /*
 365                  * Decrement map->refcount incremented in record__aio_pushfn()
 366                  * back if record__aio_write() operation failed to start, otherwise
 367                  * map->refcount is decremented in record__aio_complete() after
 368                  * aio write operation finishes successfully.
 369                  */
 370                 perf_mmap__put(&map->core);
 371         }
 372
 373         return ret;
 374 }
 375
 376 static off_t record__aio_get_pos(int trace_fd)
 377 {
 378         return lseek(trace_fd, 0, SEEK_CUR);
 379 }
 380
 381 static void record__aio_set_pos(int trace_fd, off_t pos)
 382 {
 383         lseek(trace_fd, pos, SEEK_SET);
 384 }
 385
 386 static void record__aio_mmap_read_sync(struct record *rec)
 387 {
 388         int i;
 389         struct evlist *evlist = rec->evlist;
 390         struct mmap *maps = evlist->mmap;
 391
 392         if (!record__aio_enabled(rec))
 393                 return;
 394
 395         for (i = 0; i < evlist->core.nr_mmaps; i++) {
 396                 struct mmap *map = &maps[i];
 397
 398                 if (map->core.base)
 399                         record__aio_sync(map, true);
 400         }
 401 }
 402
 403 static int nr_cblocks_default = 1;
 404 static int nr_cblocks_max = 4;
 405
 406 static int record__aio_parse(const struct option *opt,
 407                              const char *str,
 408                              int unset)
 409 {
 410         struct record_opts *opts = (struct record_opts *)opt->value;
 411
 412         if (unset) {
 413                 opts->nr_cblocks = 0;
 414         } else {
 415                 if (str)
 416                         opts->nr_cblocks = strtol(str, NULL, 0);
 417                 if (!opts->nr_cblocks)
 418                         opts->nr_cblocks = nr_cblocks_default;
 419         }
 420
 421         return 0;
 422 }
 423 #else /* HAVE_AIO_SUPPORT */
 424 static int nr_cblocks_max = 0;
 425
 426 static int record__aio_push(struct record *rec __maybe_unused, struct mmap *map __maybe_unused,
 427                             off_t *off __maybe_unused)
 428 {
 429         return -1;
 430 }
 431
 432 static off_t record__aio_get_pos(int trace_fd __maybe_unused)
 433 {
 434         return -1;
 435 }
 436
 437 static void record__aio_set_pos(int trace_fd __maybe_unused, off_t pos __maybe_unused)
 438 {
 439 }
 440
 441 static void record__aio_mmap_read_sync(struct record *rec __maybe_unused)
 442 {
 443 }
 444 #endif
 445
 446 static int record__aio_enabled(struct record *rec)
 447 {
 448         return rec->opts.nr_cblocks > 0;
 449 }
 450
 451 #define MMAP_FLUSH_DEFAULT 1
 452 static int record__mmap_flush_parse(const struct option *opt,
 453                                     const char *str,
 454                                     int unset)
 455 {
 456         int flush_max;
 457         struct record_opts *opts = (struct record_opts *)opt->value;
 458         static struct parse_tag tags[] = {
 459                         { .tag  = 'B', .mult = 1       },
 460                         { .tag  = 'K', .mult = 1 << 10 },
 461                         { .tag  = 'M', .mult = 1 << 20 },
 462                         { .tag  = 'G', .mult = 1 << 30 },
 463                         { .tag  = 0 },
 464         };
 465
 466         if (unset)
 467                 return 0;
 468
 469         if (str) {
 470                 opts->mmap_flush = parse_tag_value(str, tags);
 471                 if (opts->mmap_flush == (int)-1)
 472                         opts->mmap_flush = strtol(str, NULL, 0);
 473         }
 474
 475         if (!opts->mmap_flush)
 476                 opts->mmap_flush = MMAP_FLUSH_DEFAULT;
 477
 478         flush_max = evlist__mmap_size(opts->mmap_pages);
 479         flush_max /= 4;
 480         if (opts->mmap_flush > flush_max)
 481                 opts->mmap_flush = flush_max;
 482
 483         return 0;
 484 }
 485
 486 #ifdef HAVE_ZSTD_SUPPORT
 487 static unsigned int comp_level_default = 1;
 488
 489 static int record__parse_comp_level(const struct option *opt, const char *str, int unset)
 490 {
 491         struct record_opts *opts = opt->value;
 492
 493         if (unset) {
 494                 opts->comp_level = 0;
 495         } else {
 496                 if (str)
 497                         opts->comp_level = strtol(str, NULL, 0);
 498                 if (!opts->comp_level)
 499                         opts->comp_level = comp_level_default;
 500         }
 501
 502         return 0;
 503 }
 504 #endif
 505 static unsigned int comp_level_max = 22;
 506
 507 static int record__comp_enabled(struct record *rec)
 508 {
 509         return rec->opts.comp_level > 0;
 510 }
 511
 512 static int process_synthesized_event(struct perf_tool *tool,
 513                                      union perf_event *event,
 514                                      struct perf_sample *sample __maybe_unused,
 515                                      struct machine *machine __maybe_unused)
 516 {
 517         struct record *rec = container_of(tool, struct record, tool);
 518         return record__write(rec, NULL, event, event->header.size);
 519 }
 520
 521 static int process_locked_synthesized_event(struct perf_tool *tool,
 522                                      union perf_event *event,
 523                                      struct perf_sample *sample __maybe_unused,
 524                                      struct machine *machine __maybe_unused)
 525 {
 526         static pthread_mutex_t synth_lock = PTHREAD_MUTEX_INITIALIZER;
 527         int ret;
 528
 529         pthread_mutex_lock(&synth_lock);
 530         ret = process_synthesized_event(tool, event, sample, machine);
 531         pthread_mutex_unlock(&synth_lock);
 532         return ret;
 533 }
 534
 535 static int record__pushfn(struct mmap *map, void *to, void *bf, size_t size)
 536 {
 537         struct record *rec = to;
 538
 539         if (record__comp_enabled(rec)) {
 540                 size = zstd_compress(rec->session, map->data, mmap__mmap_len(map), bf, size);
 541                 bf   = map->data;
 542         }
 543
 544         rec->samples++;
 545         return record__write(rec, map, bf, size);
 546 }
 547
 548 static volatile int signr = -1;
 549 static volatile int child_finished;
 550 #ifdef HAVE_EVENTFD_SUPPORT
 551 static int done_fd = -1;
 552 #endif
 553
 554 static void sig_handler(int sig)
 555 {
 556         if (sig == SIGCHLD)
 557                 child_finished = 1;
 558         else
 559                 signr = sig;
 560
 561         done = 1;
 562 #ifdef HAVE_EVENTFD_SUPPORT
 563 {
 564         u64 tmp = 1;
 565         /*
 566          * It is possible for this signal handler to run after done is checked
 567          * in the main loop, but before the perf counter fds are polled. If this
 568          * happens, the poll() will continue to wait even though done is set,
 569          * and will only break out if either another signal is received, or the
 570          * counters are ready for read. To ensure the poll() doesn't sleep when
 571          * done is set, use an eventfd (done_fd) to wake up the poll().
 572          */
 573         if (write(done_fd, &tmp, sizeof(tmp)) < 0)
 574                 pr_err("failed to signal wakeup fd, error: %m\n");
 575 }
 576 #endif // HAVE_EVENTFD_SUPPORT
 577 }
 578
 579 static void sigsegv_handler(int sig)
 580 {
 581         perf_hooks__recover();
 582         sighandler_dump_stack(sig);
 583 }
 584
 585 static void record__sig_exit(void)
 586 {
 587         if (signr == -1)
 588                 return;
 589
 590         signal(signr, SIG_DFL);
 591         raise(signr);
 592 }
 593
 594 #ifdef HAVE_AUXTRACE_SUPPORT
 595
 596 static int record__process_auxtrace(struct perf_tool *tool,
 597                                     struct mmap *map,
 598                                     union perf_event *event, void *data1,
 599                                     size_t len1, void *data2, size_t len2)
 600 {
 601         struct record *rec = container_of(tool, struct record, tool);
 602         struct perf_data *data = &rec->data;
 603         size_t padding;
 604         u8 pad[8] = {0};
 605
 606         if (!perf_data__is_pipe(data) && perf_data__is_single_file(data)) {
 607                 off_t file_offset;
 608                 int fd = perf_data__fd(data);
 609                 int err;
 610
 611                 file_offset = lseek(fd, 0, SEEK_CUR);
 612                 if (file_offset == -1)
 613                         return -1;
 614                 err = auxtrace_index__auxtrace_event(&rec->session->auxtrace_index,
 615                                                      event, file_offset);
 616                 if (err)
 617                         return err;
 618         }
 619
 620         /* event.auxtrace.size includes padding, see __auxtrace_mmap__read() */
 621         padding = (len1 + len2) & 7;
 622         if (padding)
 623                 padding = 8 - padding;
 624
 625         record__write(rec, map, event, event->header.size);
 626         record__write(rec, map, data1, len1);
 627         if (len2)
 628                 record__write(rec, map, data2, len2);
 629         record__write(rec, map, &pad, padding);
 630
 631         return 0;
 632 }
 633
 634 static int record__auxtrace_mmap_read(struct record *rec,
 635                                       struct mmap *map)
 636 {
 637         int ret;
 638
 639         ret = auxtrace_mmap__read(map, rec->itr, &rec->tool,
 640                                   record__process_auxtrace);
 641         if (ret < 0)
 642                 return ret;
 643
 644         if (ret)
 645                 rec->samples++;
 646
 647         return 0;
 648 }
 649
 650 static int record__auxtrace_mmap_read_snapshot(struct record *rec,
 651                                                struct mmap *map)
 652 {
 653         int ret;
 654
 655         ret = auxtrace_mmap__read_snapshot(map, rec->itr, &rec->tool,
 656                                            record__process_auxtrace,
 657                                            rec->opts.auxtrace_snapshot_size);
 658         if (ret < 0)
 659                 return ret;
 660
 661         if (ret)
 662                 rec->samples++;
 663
 664         return 0;
 665 }
 666
 667 static int record__auxtrace_read_snapshot_all(struct record *rec)
 668 {
 669         int i;
 670         int rc = 0;
 671
 672         for (i = 0; i < rec->evlist->core.nr_mmaps; i++) {
 673                 struct mmap *map = &rec->evlist->mmap[i];
 674
 675                 if (!map->auxtrace_mmap.base)
 676                         continue;
 677
 678                 if (record__auxtrace_mmap_read_snapshot(rec, map) != 0) {
 679                         rc = -1;
 680                         goto out;
 681                 }
 682         }
 683 out:
 684         return rc;
 685 }
 686
 687 static void record__read_auxtrace_snapshot(struct record *rec, bool on_exit)
 688 {
 689         pr_debug("Recording AUX area tracing snapshot\n");
 690         if (record__auxtrace_read_snapshot_all(rec) < 0) {
 691                 trigger_error(&auxtrace_snapshot_trigger);
 692         } else {
 693                 if (auxtrace_record__snapshot_finish(rec->itr, on_exit))
 694                         trigger_error(&auxtrace_snapshot_trigger);
 695                 else
 696                         trigger_ready(&auxtrace_snapshot_trigger);
 697         }
 698 }
 699
 700 static int record__auxtrace_snapshot_exit(struct record *rec)
 701 {
 702         if (trigger_is_error(&auxtrace_snapshot_trigger))
 703                 return 0;
 704
 705         if (!auxtrace_record__snapshot_started &&
 706             auxtrace_record__snapshot_start(rec->itr))
 707                 return -1;
 708
 709         record__read_auxtrace_snapshot(rec, true);
 710         if (trigger_is_error(&auxtrace_snapshot_trigger))
 711                 return -1;
 712
 713         return 0;
 714 }
 715
 716 static int record__auxtrace_init(struct record *rec)
 717 {
 718         int err;
 719
 720         if (!rec->itr) {
 721                 rec->itr = auxtrace_record__init(rec->evlist, &err);
 722                 if (err)
 723                         return err;
 724         }
 725
 726         err = auxtrace_parse_snapshot_options(rec->itr, &rec->opts,
 727                                               rec->opts.auxtrace_snapshot_opts);
 728         if (err)
 729                 return err;
 730
 731         err = auxtrace_parse_sample_options(rec->itr, rec->evlist, &rec->opts,
 732                                             rec->opts.auxtrace_sample_opts);
 733         if (err)
 734                 return err;
 735
 736         auxtrace_regroup_aux_output(rec->evlist);
 737
 738         return auxtrace_parse_filters(rec->evlist);
 739 }
 740
 741 #else
 742
 743 static inline
 744 int record__auxtrace_mmap_read(struct record *rec __maybe_unused,
 745                                struct mmap *map __maybe_unused)
 746 {
 747         return 0;
 748 }
 749
 750 static inline
 751 void record__read_auxtrace_snapshot(struct record *rec __maybe_unused,
 752                                     bool on_exit __maybe_unused)
 753 {
 754 }
 755
 756 static inline
 757 int auxtrace_record__snapshot_start(struct auxtrace_record *itr __maybe_unused)
 758 {
 759         return 0;
 760 }
 761
 762 static inline
 763 int record__auxtrace_snapshot_exit(struct record *rec __maybe_unused)
 764 {
 765         return 0;
 766 }
 767
 768 static int record__auxtrace_init(struct record *rec __maybe_unused)
 769 {
 770         return 0;
 771 }
 772
 773 #endif
 774
 775 static int record__config_text_poke(struct evlist *evlist)
 776 {
 777         struct evsel *evsel;
 778         int err;
 779
 780         /* Nothing to do if text poke is already configured */
 781         evlist__for_each_entry(evlist, evsel) {
 782                 if (evsel->core.attr.text_poke)
 783                         return 0;
 784         }
 785
 786         err = parse_events(evlist, "dummy:u", NULL);
 787         if (err)
 788                 return err;
 789
 790         evsel = evlist__last(evlist);
 791
 792         evsel->core.attr.freq = 0;
 793         evsel->core.attr.sample_period = 1;
 794         evsel->core.attr.text_poke = 1;
 795         evsel->core.attr.ksymbol = 1;
 796
 797         evsel->core.system_wide = true;
 798         evsel->no_aux_samples = true;
 799         evsel->immediate = true;
 800
 801         /* Text poke must be collected on all CPUs */
 802         perf_cpu_map__put(evsel->core.own_cpus);
 803         evsel->core.own_cpus = perf_cpu_map__new(NULL);
 804         perf_cpu_map__put(evsel->core.cpus);
 805         evsel->core.cpus = perf_cpu_map__get(evsel->core.own_cpus);
 806
 807         evsel__set_sample_bit(evsel, TIME);
 808
 809         return 0;
 810 }
 811
 812 static bool record__kcore_readable(struct machine *machine)
 813 {
 814         char kcore[PATH_MAX];
 815         int fd;
 816
 817         scnprintf(kcore, sizeof(kcore), "%s/proc/kcore", machine->root_dir);
 818
 819         fd = open(kcore, O_RDONLY);
 820         if (fd < 0)
 821                 return false;
 822
 823         close(fd);
 824
 825         return true;
 826 }
 827
 828 static int record__kcore_copy(struct machine *machine, struct perf_data *data)
 829 {
 830         char from_dir[PATH_MAX];
 831         char kcore_dir[PATH_MAX];
 832         int ret;
 833
 834         snprintf(from_dir, sizeof(from_dir), "%s/proc", machine->root_dir);
 835
 836         ret = perf_data__make_kcore_dir(data, kcore_dir, sizeof(kcore_dir));
 837         if (ret)
 838                 return ret;
 839
 840         return kcore_copy(from_dir, kcore_dir);
 841 }
 842
 843 static int record__mmap_evlist(struct record *rec,
 844                                struct evlist *evlist)
 845 {
 846         struct record_opts *opts = &rec->opts;
 847         bool auxtrace_overwrite = opts->auxtrace_snapshot_mode ||
 848                                   opts->auxtrace_sample_mode;
 849         char msg[512];
 850
 851         if (opts->affinity != PERF_AFFINITY_SYS)
 852                 cpu__setup_cpunode_map();
 853
 854         if (evlist__mmap_ex(evlist, opts->mmap_pages,
 855                                  opts->auxtrace_mmap_pages,
 856                                  auxtrace_overwrite,
 857                                  opts->nr_cblocks, opts->affinity,
 858                                  opts->mmap_flush, opts->comp_level) < 0) {
 859                 if (errno == EPERM) {
 860                         pr_err("Permission error mapping pages.\n"
 861                                "Consider increasing "
 862                                "/proc/sys/kernel/perf_event_mlock_kb,\n"
 863                                "or try again with a smaller value of -m/--mmap_pages.\n"
 864                                "(current value: %u,%u)\n",
 865                                opts->mmap_pages, opts->auxtrace_mmap_pages);
 866                         return -errno;
 867                 } else {
 868                         pr_err("failed to mmap with %d (%s)\n", errno,
 869                                 str_error_r(errno, msg, sizeof(msg)));
 870                         if (errno)
 871                                 return -errno;
 872                         else
 873                                 return -EINVAL;
 874                 }
 875         }
 876         return 0;
 877 }
 878
 879 static int record__mmap(struct record *rec)
 880 {
 881         return record__mmap_evlist(rec, rec->evlist);
 882 }
 883
 884 static int record__open(struct record *rec)
 885 {
 886         char msg[BUFSIZ];
 887         struct evsel *pos;
 888         struct evlist *evlist = rec->evlist;
 889         struct perf_session *session = rec->session;
 890         struct record_opts *opts = &rec->opts;
 891         int rc = 0;
 892
 893         /*
 894          * For initial_delay or system wide, we need to add a dummy event so
 895          * that we can track PERF_RECORD_MMAP to cover the delay of waiting or
 896          * event synthesis.
 897          */
 898         if (opts->initial_delay || target__has_cpu(&opts->target)) {
 899                 pos = evlist__get_tracking_event(evlist);
 900                 if (!evsel__is_dummy_event(pos)) {
 901                         /* Set up dummy event. */
 902                         if (evlist__add_dummy(evlist))
 903                                 return -ENOMEM;
 904                         pos = evlist__last(evlist);
 905                         evlist__set_tracking_event(evlist, pos);
 906                 }
 907
 908                 /*
 909                  * Enable the dummy event when the process is forked for
 910                  * initial_delay, immediately for system wide.
 911                  */
 912                 if (opts->initial_delay && !pos->immediate)
 913                         pos->core.attr.enable_on_exec = 1;
 914                 else
 915                         pos->immediate = 1;
 916         }
 917
 918         evlist__config(evlist, opts, &callchain_param);
 919
 920         evlist__for_each_entry(evlist, pos) {
 921 try_again:
 922                 if (evsel__open(pos, pos->core.cpus, pos->core.threads) < 0) {
 923                         if (evsel__fallback(pos, errno, msg, sizeof(msg))) {
 924                                 if (verbose > 0)
 925                                         ui__warning("%s\n", msg);
 926                                 goto try_again;
 927                         }
 928                         if ((errno == EINVAL || errno == EBADF) &&
 929                             pos->leader != pos &&
 930                             pos->weak_group) {
 931                                 pos = evlist__reset_weak_group(evlist, pos, true);
 932                                 goto try_again;
 933                         }
 934                         rc = -errno;
 935                         evsel__open_strerror(pos, &opts->target, errno, msg, sizeof(msg));
 936                         ui__error("%s\n", msg);
 937                         goto out;
 938                 }
 939
 940                 pos->supported = true;
 941         }
 942
 943         if (symbol_conf.kptr_restrict && !evlist__exclude_kernel(evlist)) {
 944                 pr_warning(
 945 "WARNING: Kernel address maps (/proc/{kallsyms,modules}) are restricted,\n"
 946 "check /proc/sys/kernel/kptr_restrict and /proc/sys/kernel/perf_event_paranoid.\n\n"
 947 "Samples in kernel functions may not be resolved if a suitable vmlinux\n"
 948 "file is not found in the buildid cache or in the vmlinux path.\n\n"
 949 "Samples in kernel modules won't be resolved at all.\n\n"
 950 "If some relocation was applied (e.g. kexec) symbols may be misresolved\n"
 951 "even with a suitable vmlinux or kallsyms file.\n\n");
 952         }
 953
 954         if (evlist__apply_filters(evlist, &pos)) {
 955                 pr_err("failed to set filter \"%s\" on event %s with %d (%s)\n",
 956                         pos->filter, evsel__name(pos), errno,
 957                         str_error_r(errno, msg, sizeof(msg)));
 958                 rc = -1;
 959                 goto out;
 960         }
 961
 962         rc = record__mmap(rec);
 963         if (rc)
 964                 goto out;
 965
 966         session->evlist = evlist;
 967         perf_session__set_id_hdr_size(session);
 968 out:
 969         return rc;
 970 }
 971
 972 static void set_timestamp_boundary(struct record *rec, u64 sample_time)
 973 {
 974         if (rec->evlist->first_sample_time == 0)
 975                 rec->evlist->first_sample_time = sample_time;
 976
 977         if (sample_time)
 978                 rec->evlist->last_sample_time = sample_time;
 979 }
 980
 981 static int process_sample_event(struct perf_tool *tool,
 982                                 union perf_event *event,
 983                                 struct perf_sample *sample,
 984                                 struct evsel *evsel,
 985                                 struct machine *machine)
 986 {
 987         struct record *rec = container_of(tool, struct record, tool);
 988
 989         set_timestamp_boundary(rec, sample->time);
 990
 991         if (rec->buildid_all)
 992                 return 0;
 993
 994         rec->samples++;
 995         return build_id__mark_dso_hit(tool, event, sample, evsel, machine);
 996 }
 997
 998 static int process_buildids(struct record *rec)
 999 {
1000         struct perf_session *session = rec->session;
1001
1002         if (perf_data__size(&rec->data) == 0)
1003                 return 0;
1004
1005         /*
1006          * During this process, it'll load kernel map and replace the
1007          * dso->long_name to a real pathname it found.  In this case
1008          * we prefer the vmlinux path like
1009          *   /lib/modules/3.16.4/build/vmlinux
1010          *
1011          * rather than build-id path (in debug directory).
1012          *   $HOME/.debug/.build-id/f0/6e17aa50adf4d00b88925e03775de107611551
1013          */
1014         symbol_conf.ignore_vmlinux_buildid = true;
1015
1016         /*
1017          * If --buildid-all is given, it marks all DSO regardless of hits,
1018          * so no need to process samples. But if timestamp_boundary is enabled,
1019          * it still needs to walk on all samples to get the timestamps of
1020          * first/last samples.
1021          */
1022         if (rec->buildid_all && !rec->timestamp_boundary)
1023                 rec->tool.sample = NULL;
1024
1025         return perf_session__process_events(session);
1026 }
1027
1028 static void perf_event__synthesize_guest_os(struct machine *machine, void *data)
1029 {
1030         int err;
1031         struct perf_tool *tool = data;
1032         /*
1033          *As for guest kernel when processing subcommand record&report,
1034          *we arrange module mmap prior to guest kernel mmap and trigger
1035          *a preload dso because default guest module symbols are loaded
1036          *from guest kallsyms instead of /lib/modules/XXX/XXX. This
1037          *method is used to avoid symbol missing when the first addr is
1038          *in module instead of in guest kernel.
1039          */
1040         err = perf_event__synthesize_modules(tool, process_synthesized_event,
1041                                              machine);
1042         if (err < 0)
1043                 pr_err("Couldn't record guest kernel [%d]'s reference"
1044                        " relocation symbol.\n", machine->pid);
1045
1046         /*
1047          * We use _stext for guest kernel because guest kernel's /proc/kallsyms
1048          * have no _text sometimes.
1049          */
1050         err = perf_event__synthesize_kernel_mmap(tool, process_synthesized_event,
1051                                                  machine);
1052         if (err < 0)
1053                 pr_err("Couldn't record guest kernel [%d]'s reference"
1054                        " relocation symbol.\n", machine->pid);
1055 }
1056
1057 static struct perf_event_header finished_round_event = {
1058         .size = sizeof(struct perf_event_header),
1059         .type = PERF_RECORD_FINISHED_ROUND,
1060 };
1061
1062 static void record__adjust_affinity(struct record *rec, struct mmap *map)
1063 {
1064         if (rec->opts.affinity != PERF_AFFINITY_SYS &&
1065             !bitmap_equal(rec->affinity_mask.bits, map->affinity_mask.bits,
1066                           rec->affinity_mask.nbits)) {
1067                 bitmap_zero(rec->affinity_mask.bits, rec->affinity_mask.nbits);
1068                 bitmap_or(rec->affinity_mask.bits, rec->affinity_mask.bits,
1069                           map->affinity_mask.bits, rec->affinity_mask.nbits);
1070                 sched_setaffinity(0, MMAP_CPU_MASK_BYTES(&rec->affinity_mask),
1071                                   (cpu_set_t *)rec->affinity_mask.bits);
1072                 if (verbose == 2)
1073                         mmap_cpu_mask__scnprintf(&rec->affinity_mask, "thread");
1074         }
1075 }
1076
1077 static size_t process_comp_header(void *record, size_t increment)
1078 {
1079         struct perf_record_compressed *event = record;
1080         size_t size = sizeof(*event);
1081
1082         if (increment) {
1083                 event->header.size += increment;
1084                 return increment;
1085         }
1086
1087         event->header.type = PERF_RECORD_COMPRESSED;
1088         event->header.size = size;
1089
1090         return size;
1091 }
1092
1093 static size_t zstd_compress(struct perf_session *session, void *dst, size_t dst_size,
1094                             void *src, size_t src_size)
1095 {
1096         size_t compressed;
1097         size_t max_record_size = PERF_SAMPLE_MAX_SIZE - sizeof(struct perf_record_compressed) - 1;
1098
1099         compressed = zstd_compress_stream_to_records(&session->zstd_data, dst, dst_size, src, src_size,
1100                                                      max_record_size, process_comp_header);
1101
1102         session->bytes_transferred += src_size;
1103         session->bytes_compressed  += compressed;
1104
1105         return compressed;
1106 }
1107
1108 static int record__mmap_read_evlist(struct record *rec, struct evlist *evlist,
1109                                     bool overwrite, bool synch)
1110 {
1111         u64 bytes_written = rec->bytes_written;
1112         int i;
1113         int rc = 0;
1114         struct mmap *maps;
1115         int trace_fd = rec->data.file.fd;
1116         off_t off = 0;
1117
1118         if (!evlist)
1119                 return 0;
1120
1121         maps = overwrite ? evlist->overwrite_mmap : evlist->mmap;
1122         if (!maps)
1123                 return 0;
1124
1125         if (overwrite && evlist->bkw_mmap_state != BKW_MMAP_DATA_PENDING)
1126                 return 0;
1127
1128         if (record__aio_enabled(rec))
1129                 off = record__aio_get_pos(trace_fd);
1130
1131         for (i = 0; i < evlist->core.nr_mmaps; i++) {
1132                 u64 flush = 0;
1133                 struct mmap *map = &maps[i];
1134
1135                 if (map->core.base) {
1136                         record__adjust_affinity(rec, map);
1137                         if (synch) {
1138                                 flush = map->core.flush;
1139                                 map->core.flush = 1;
1140                         }
1141                         if (!record__aio_enabled(rec)) {
1142                                 if (perf_mmap__push(map, rec, record__pushfn) < 0) {
1143                                         if (synch)
1144                                                 map->core.flush = flush;
1145                                         rc = -1;
1146                                         goto out;
1147                                 }
1148                         } else {
1149                                 if (record__aio_push(rec, map, &off) < 0) {
1150                                         record__aio_set_pos(trace_fd, off);
1151                                         if (synch)
1152                                                 map->core.flush = flush;
1153                                         rc = -1;
1154                                         goto out;
1155                                 }
1156                         }
1157                         if (synch)
1158                                 map->core.flush = flush;
1159                 }
1160
1161                 if (map->auxtrace_mmap.base && !rec->opts.auxtrace_snapshot_mode &&
1162                     !rec->opts.auxtrace_sample_mode &&
1163                     record__auxtrace_mmap_read(rec, map) != 0) {
1164                         rc = -1;
1165                         goto out;
1166                 }
1167         }
1168
1169         if (record__aio_enabled(rec))
1170                 record__aio_set_pos(trace_fd, off);
1171
1172         /*
1173          * Mark the round finished in case we wrote
1174          * at least one event.
1175          */
1176         if (bytes_written != rec->bytes_written)
1177                 rc = record__write(rec, NULL, &finished_round_event, sizeof(finished_round_event));
1178
1179         if (overwrite)
1180                 evlist__toggle_bkw_mmap(evlist, BKW_MMAP_EMPTY);
1181 out:
1182         return rc;
1183 }
1184
1185 static int record__mmap_read_all(struct record *rec, bool synch)
1186 {
1187         int err;
1188
1189         err = record__mmap_read_evlist(rec, rec->evlist, false, synch);
1190         if (err)
1191                 return err;
1192
1193         return record__mmap_read_evlist(rec, rec->evlist, true, synch);
1194 }
1195
1196 static void record__init_features(struct record *rec)
1197 {
1198         struct perf_session *session = rec->session;
1199         int feat;
1200
1201         for (feat = HEADER_FIRST_FEATURE; feat < HEADER_LAST_FEATURE; feat++)
1202                 perf_header__set_feat(&session->header, feat);
1203
1204         if (rec->no_buildid)
1205                 perf_header__clear_feat(&session->header, HEADER_BUILD_ID);
1206
1207         if (!have_tracepoints(&rec->evlist->core.entries))
1208                 perf_header__clear_feat(&session->header, HEADER_TRACING_DATA);
1209
1210         if (!rec->opts.branch_stack)
1211                 perf_header__clear_feat(&session->header, HEADER_BRANCH_STACK);
1212
1213         if (!rec->opts.full_auxtrace)
1214                 perf_header__clear_feat(&session->header, HEADER_AUXTRACE);
1215
1216         if (!(rec->opts.use_clockid && rec->opts.clockid_res_ns))
1217                 perf_header__clear_feat(&session->header, HEADER_CLOCKID);
1218
1219         if (!rec->opts.use_clockid)
1220                 perf_header__clear_feat(&session->header, HEADER_CLOCK_DATA);
1221
1222         perf_header__clear_feat(&session->header, HEADER_DIR_FORMAT);
1223         if (!record__comp_enabled(rec))
1224                 perf_header__clear_feat(&session->header, HEADER_COMPRESSED);
1225
1226         perf_header__clear_feat(&session->header, HEADER_STAT);
1227 }
1228
1229 static void
1230 record__finish_output(struct record *rec)
1231 {
1232         struct perf_data *data = &rec->data;
1233         int fd = perf_data__fd(data);
1234
1235         if (data->is_pipe)
1236                 return;
1237
1238         rec->session->header.data_size += rec->bytes_written;
1239         data->file.size = lseek(perf_data__fd(data), 0, SEEK_CUR);
1240
1241         if (!rec->no_buildid) {
1242                 process_buildids(rec);
1243
1244                 if (rec->buildid_all)
1245                         dsos__hit_all(rec->session);
1246         }
1247         perf_session__write_header(rec->session, rec->evlist, fd, true);
1248
1249         return;
1250 }
1251
1252 static int record__synthesize_workload(struct record *rec, bool tail)
1253 {
1254         int err;
1255         struct perf_thread_map *thread_map;
1256
1257         if (rec->opts.tail_synthesize != tail)
1258                 return 0;
1259
1260         thread_map = thread_map__new_by_tid(rec->evlist->workload.pid);
1261         if (thread_map == NULL)
1262                 return -1;
1263
1264         err = perf_event__synthesize_thread_map(&rec->tool, thread_map,
1265                                                  process_synthesized_event,
1266                                                  &rec->session->machines.host,
1267                                                  rec->opts.sample_address);
1268         perf_thread_map__put(thread_map);
1269         return err;
1270 }
1271
1272 static int record__synthesize(struct record *rec, bool tail);
1273
1274 static int
1275 record__switch_output(struct record *rec, bool at_exit)
1276 {
1277         struct perf_data *data = &rec->data;
1278         int fd, err;
1279         char *new_filename;
1280
1281         /* Same Size:      "2015122520103046"*/
1282         char timestamp[] = "InvalidTimestamp";
1283
1284         record__aio_mmap_read_sync(rec);
1285
1286         record__synthesize(rec, true);
1287         if (target__none(&rec->opts.target))
1288                 record__synthesize_workload(rec, true);
1289
1290         rec->samples = 0;
1291         record__finish_output(rec);
1292         err = fetch_current_timestamp(timestamp, sizeof(timestamp));
1293         if (err) {
1294                 pr_err("Failed to get current timestamp\n");
1295                 return -EINVAL;
1296         }
1297
1298         fd = perf_data__switch(data, timestamp,
1299                                     rec->session->header.data_offset,
1300                                     at_exit, &new_filename);
1301         if (fd >= 0 && !at_exit) {
1302                 rec->bytes_written = 0;
1303                 rec->session->header.data_size = 0;
1304         }
1305
1306         if (!quiet)
1307                 fprintf(stderr, "[ perf record: Dump %s.%s ]\n",
1308                         data->path, timestamp);
1309
1310         if (rec->switch_output.num_files) {
1311                 int n = rec->switch_output.cur_file + 1;
1312
1313                 if (n >= rec->switch_output.num_files)
1314                         n = 0;
1315                 rec->switch_output.cur_file = n;
1316                 if (rec->switch_output.filenames[n]) {
1317                         remove(rec->switch_output.filenames[n]);
1318                         zfree(&rec->switch_output.filenames[n]);
1319                 }
1320                 rec->switch_output.filenames[n] = new_filename;
1321         } else {
1322                 free(new_filename);
1323         }
1324
1325         /* Output tracking events */
1326         if (!at_exit) {
1327                 record__synthesize(rec, false);
1328
1329                 /*
1330                  * In 'perf record --switch-output' without -a,
1331                  * record__synthesize() in record__switch_output() won't
1332                  * generate tracking events because there's no thread_map
1333                  * in evlist. Which causes newly created perf.data doesn't
1334                  * contain map and comm information.
1335                  * Create a fake thread_map and directly call
1336                  * perf_event__synthesize_thread_map() for those events.
1337                  */
1338                 if (target__none(&rec->opts.target))
1339                         record__synthesize_workload(rec, false);
1340         }
1341         return fd;
1342 }
1343
1344 static volatile int workload_exec_errno;
1345
1346 /*
1347  * evlist__prepare_workload will send a SIGUSR1
1348  * if the fork fails, since we asked by setting its
1349  * want_signal to true.
1350  */
1351 static void workload_exec_failed_signal(int signo __maybe_unused,
1352                                         siginfo_t *info,
1353                                         void *ucontext __maybe_unused)
1354 {
1355         workload_exec_errno = info->si_value.sival_int;
1356         done = 1;
1357         child_finished = 1;
1358 }
1359
1360 static void snapshot_sig_handler(int sig);
1361 static void alarm_sig_handler(int sig);
1362
1363 static const struct perf_event_mmap_page *evlist__pick_pc(struct evlist *evlist)
1364 {
1365         if (evlist) {
1366                 if (evlist->mmap && evlist->mmap[0].core.base)
1367                         return evlist->mmap[0].core.base;
1368                 if (evlist->overwrite_mmap && evlist->overwrite_mmap[0].core.base)
1369                         return evlist->overwrite_mmap[0].core.base;
1370         }
1371         return NULL;
1372 }
1373
1374 static const struct perf_event_mmap_page *record__pick_pc(struct record *rec)
1375 {
1376         const struct perf_event_mmap_page *pc = evlist__pick_pc(rec->evlist);
1377         if (pc)
1378                 return pc;
1379         return NULL;
1380 }
1381
1382 static int record__synthesize(struct record *rec, bool tail)
1383 {
1384         struct perf_session *session = rec->session;
1385         struct machine *machine = &session->machines.host;
1386         struct perf_data *data = &rec->data;
1387         struct record_opts *opts = &rec->opts;
1388         struct perf_tool *tool = &rec->tool;
1389         int fd = perf_data__fd(data);
1390         int err = 0;
1391         event_op f = process_synthesized_event;
1392
1393         if (rec->opts.tail_synthesize != tail)
1394                 return 0;
1395
1396         if (data->is_pipe) {
1397                 /*
1398                  * We need to synthesize events first, because some
1399                  * features works on top of them (on report side).
1400                  */
1401                 err = perf_event__synthesize_attrs(tool, rec->evlist,
1402                                                    process_synthesized_event);
1403                 if (err < 0) {
1404                         pr_err("Couldn't synthesize attrs.\n");
1405                         goto out;
1406                 }
1407
1408                 err = perf_event__synthesize_features(tool, session, rec->evlist,
1409                                                       process_synthesized_event);
1410                 if (err < 0) {
1411                         pr_err("Couldn't synthesize features.\n");
1412                         return err;
1413                 }
1414
1415                 if (have_tracepoints(&rec->evlist->core.entries)) {
1416                         /*
1417                          * FIXME err <= 0 here actually means that
1418                          * there were no tracepoints so its not really
1419                          * an error, just that we don't need to
1420                          * synthesize anything.  We really have to
1421                          * return this more properly and also
1422                          * propagate errors that now are calling die()
1423                          */
1424                         err = perf_event__synthesize_tracing_data(tool, fd, rec->evlist,
1425                                                                   process_synthesized_event);
1426                         if (err <= 0) {
1427                                 pr_err("Couldn't record tracing data.\n");
1428                                 goto out;
1429                         }
1430                         rec->bytes_written += err;
1431                 }
1432         }
1433
1434         err = perf_event__synth_time_conv(record__pick_pc(rec), tool,
1435                                           process_synthesized_event, machine);
1436         if (err)
1437                 goto out;
1438
1439         /* Synthesize id_index before auxtrace_info */
1440         if (rec->opts.auxtrace_sample_mode) {
1441                 err = perf_event__synthesize_id_index(tool,
1442                                                       process_synthesized_event,
1443                                                       session->evlist, machine);
1444                 if (err)
1445                         goto out;
1446         }
1447
1448         if (rec->opts.full_auxtrace) {
1449                 err = perf_event__synthesize_auxtrace_info(rec->itr, tool,
1450                                         session, process_synthesized_event);
1451                 if (err)
1452                         goto out;
1453         }
1454
1455         if (!evlist__exclude_kernel(rec->evlist)) {
1456                 err = perf_event__synthesize_kernel_mmap(tool, process_synthesized_event,
1457                                                          machine);
1458                 WARN_ONCE(err < 0, "Couldn't record kernel reference relocation symbol\n"
1459                                    "Symbol resolution may be skewed if relocation was used (e.g. kexec).\n"
1460                                    "Check /proc/kallsyms permission or run as root.\n");
1461
1462                 err = perf_event__synthesize_modules(tool, process_synthesized_event,
1463                                                      machine);
1464                 WARN_ONCE(err < 0, "Couldn't record kernel module information.\n"
1465                                    "Symbol resolution may be skewed if relocation was used (e.g. kexec).\n"
1466                                    "Check /proc/modules permission or run as root.\n");
1467         }
1468
1469         if (perf_guest) {
1470                 machines__process_guests(&session->machines,
1471                                          perf_event__synthesize_guest_os, tool);
1472         }
1473
1474         err = perf_event__synthesize_extra_attr(&rec->tool,
1475                                                 rec->evlist,
1476                                                 process_synthesized_event,
1477                                                 data->is_pipe);
1478         if (err)
1479                 goto out;
1480
1481         err = perf_event__synthesize_thread_map2(&rec->tool, rec->evlist->core.threads,
1482                                                  process_synthesized_event,
1483                                                 NULL);
1484         if (err < 0) {
1485                 pr_err("Couldn't synthesize thread map.\n");
1486                 return err;
1487         }
1488
1489         err = perf_event__synthesize_cpu_map(&rec->tool, rec->evlist->core.cpus,
1490                                              process_synthesized_event, NULL);
1491         if (err < 0) {
1492                 pr_err("Couldn't synthesize cpu map.\n");
1493                 return err;
1494         }
1495
1496         err = perf_event__synthesize_bpf_events(session, process_synthesized_event,
1497                                                 machine, opts);
1498         if (err < 0)
1499                 pr_warning("Couldn't synthesize bpf events.\n");
1500
1501         err = perf_event__synthesize_cgroups(tool, process_synthesized_event,
1502                                              machine);
1503         if (err < 0)
1504                 pr_warning("Couldn't synthesize cgroup events.\n");
1505
1506         if (rec->opts.nr_threads_synthesize > 1) {
1507                 perf_set_multithreaded();
1508                 f = process_locked_synthesized_event;
1509         }
1510
1511         err = __machine__synthesize_threads(machine, tool, &opts->target, rec->evlist->core.threads,
1512                                             f, opts->sample_address,
1513                                             rec->opts.nr_threads_synthesize);
1514
1515         if (rec->opts.nr_threads_synthesize > 1)
1516                 perf_set_singlethreaded();
1517
1518 out:
1519         return err;
1520 }
1521
1522 static int record__process_signal_event(union perf_event *event __maybe_unused, void *data)
1523 {
1524         struct record *rec = data;
1525         pthread_kill(rec->thread_id, SIGUSR2);
1526         return 0;
1527 }
1528
1529 static int record__setup_sb_evlist(struct record *rec)
1530 {
1531         struct record_opts *opts = &rec->opts;
1532
1533         if (rec->sb_evlist != NULL) {
1534                 /*
1535                  * We get here if --switch-output-event populated the
1536                  * sb_evlist, so associate a callback that will send a SIGUSR2
1537                  * to the main thread.
1538                  */
1539                 evlist__set_cb(rec->sb_evlist, record__process_signal_event, rec);
1540                 rec->thread_id = pthread_self();
1541         }
1542 #ifdef HAVE_LIBBPF_SUPPORT
1543         if (!opts->no_bpf_event) {
1544                 if (rec->sb_evlist == NULL) {
1545                         rec->sb_evlist = evlist__new();
1546
1547                         if (rec->sb_evlist == NULL) {
1548                                 pr_err("Couldn't create side band evlist.\n.");
1549                                 return -1;
1550                         }
1551                 }
1552
1553                 if (evlist__add_bpf_sb_event(rec->sb_evlist, &rec->session->header.env)) {
1554                         pr_err("Couldn't ask for PERF_RECORD_BPF_EVENT side band events.\n.");
1555                         return -1;
1556                 }
1557         }
1558 #endif
1559         if (evlist__start_sb_thread(rec->sb_evlist, &rec->opts.target)) {
1560                 pr_debug("Couldn't start the BPF side band thread:\nBPF programs starting from now on won't be annotatable\n");
1561                 opts->no_bpf_event = true;
1562         }
1563
1564         return 0;
1565 }
1566
1567 static int record__init_clock(struct record *rec)
1568 {
1569         struct perf_session *session = rec->session;
1570         struct timespec ref_clockid;
1571         struct timeval ref_tod;
1572         u64 ref;
1573
1574         if (!rec->opts.use_clockid)
1575                 return 0;
1576
1577         if (rec->opts.use_clockid && rec->opts.clockid_res_ns)
1578                 session->header.env.clock.clockid_res_ns = rec->opts.clockid_res_ns;
1579
1580         session->header.env.clock.clockid = rec->opts.clockid;
1581
1582         if (gettimeofday(&ref_tod, NULL) != 0) {
1583                 pr_err("gettimeofday failed, cannot set reference time.\n");
1584                 return -1;
1585         }
1586
1587         if (clock_gettime(rec->opts.clockid, &ref_clockid)) {
1588                 pr_err("clock_gettime failed, cannot set reference time.\n");
1589                 return -1;
1590         }
1591
1592         ref = (u64) ref_tod.tv_sec * NSEC_PER_SEC +
1593               (u64) ref_tod.tv_usec * NSEC_PER_USEC;
1594
1595         session->header.env.clock.tod_ns = ref;
1596
1597         ref = (u64) ref_clockid.tv_sec * NSEC_PER_SEC +
1598               (u64) ref_clockid.tv_nsec;
1599
1600         session->header.env.clock.clockid_ns = ref;
1601         return 0;
1602 }
1603
1604 static void hit_auxtrace_snapshot_trigger(struct record *rec)
1605 {
1606         if (trigger_is_ready(&auxtrace_snapshot_trigger)) {
1607                 trigger_hit(&auxtrace_snapshot_trigger);
1608                 auxtrace_record__snapshot_started = 1;
1609                 if (auxtrace_record__snapshot_start(rec->itr))
1610                         trigger_error(&auxtrace_snapshot_trigger);
1611         }
1612 }
1613
1614 static void record__uniquify_name(struct record *rec)
1615 {
1616         struct evsel *pos;
1617         struct evlist *evlist = rec->evlist;
1618         char *new_name;
1619         int ret;
1620
1621         if (!perf_pmu__has_hybrid())
1622                 return;
1623
1624         evlist__for_each_entry(evlist, pos) {
1625                 if (!evsel__is_hybrid(pos))
1626                         continue;
1627
1628                 if (strchr(pos->name, '/'))
1629                         continue;
1630
1631                 ret = asprintf(&new_name, "%s/%s/",
1632                                pos->pmu_name, pos->name);
1633                 if (ret) {
1634                         free(pos->name);
1635                         pos->name = new_name;
1636                 }
1637         }
1638 }
1639
1640 static int __cmd_record(struct record *rec, int argc, const char **argv)
1641 {
1642         int err;
1643         int status = 0;
1644         unsigned long waking = 0;
1645         const bool forks = argc > 0;
1646         struct perf_tool *tool = &rec->tool;
1647         struct record_opts *opts = &rec->opts;
1648         struct perf_data *data = &rec->data;
1649         struct perf_session *session;
1650         bool disabled = false, draining = false;
1651         int fd;
1652         float ratio = 0;
1653         enum evlist_ctl_cmd cmd = EVLIST_CTL_CMD_UNSUPPORTED;
1654
1655         atexit(record__sig_exit);
1656         signal(SIGCHLD, sig_handler);
1657         signal(SIGINT, sig_handler);
1658         signal(SIGTERM, sig_handler);
1659         signal(SIGSEGV, sigsegv_handler);
1660
1661         if (rec->opts.record_namespaces)
1662                 tool->namespace_events = true;
1663
1664         if (rec->opts.record_cgroup) {
1665 #ifdef HAVE_FILE_HANDLE
1666                 tool->cgroup_events = true;
1667 #else
1668                 pr_err("cgroup tracking is not supported\n");
1669                 return -1;
1670 #endif
1671         }
1672
1673         if (rec->opts.auxtrace_snapshot_mode || rec->switch_output.enabled) {
1674                 signal(SIGUSR2, snapshot_sig_handler);
1675                 if (rec->opts.auxtrace_snapshot_mode)
1676                         trigger_on(&auxtrace_snapshot_trigger);
1677                 if (rec->switch_output.enabled)
1678                         trigger_on(&switch_output_trigger);
1679         } else {
1680                 signal(SIGUSR2, SIG_IGN);
1681         }
1682
1683         session = perf_session__new(data, false, tool);
1684         if (IS_ERR(session)) {
1685                 pr_err("Perf session creation failed.\n");
1686                 return PTR_ERR(session);
1687         }
1688
1689         fd = perf_data__fd(data);
1690         rec->session = session;
1691
1692         if (zstd_init(&session->zstd_data, rec->opts.comp_level) < 0) {
1693                 pr_err("Compression initialization failed.\n");
1694                 return -1;
1695         }
1696 #ifdef HAVE_EVENTFD_SUPPORT
1697         done_fd = eventfd(0, EFD_NONBLOCK);
1698         if (done_fd < 0) {
1699                 pr_err("Failed to create wakeup eventfd, error: %m\n");
1700                 status = -1;
1701                 goto out_delete_session;
1702         }
1703         err = evlist__add_wakeup_eventfd(rec->evlist, done_fd);
1704         if (err < 0) {
1705                 pr_err("Failed to add wakeup eventfd to poll list\n");
1706                 status = err;
1707                 goto out_delete_session;
1708         }
1709 #endif // HAVE_EVENTFD_SUPPORT
1710
1711         session->header.env.comp_type  = PERF_COMP_ZSTD;
1712         session->header.env.comp_level = rec->opts.comp_level;
1713
1714         if (rec->opts.kcore &&
1715             !record__kcore_readable(&session->machines.host)) {
1716                 pr_err("ERROR: kcore is not readable.\n");
1717                 return -1;
1718         }
1719
1720         if (record__init_clock(rec))
1721                 return -1;
1722
1723         record__init_features(rec);
1724
1725         if (forks) {
1726                 err = evlist__prepare_workload(rec->evlist, &opts->target, argv, data->is_pipe,
1727                                                workload_exec_failed_signal);
1728                 if (err < 0) {
1729                         pr_err("Couldn't run the workload!\n");
1730                         status = err;
1731                         goto out_delete_session;
1732                 }
1733         }
1734
1735         /*
1736          * If we have just single event and are sending data
1737          * through pipe, we need to force the ids allocation,
1738          * because we synthesize event name through the pipe
1739          * and need the id for that.
1740          */
1741         if (data->is_pipe && rec->evlist->core.nr_entries == 1)
1742                 rec->opts.sample_id = true;
1743
1744         record__uniquify_name(rec);
1745
1746         if (record__open(rec) != 0) {
1747                 err = -1;
1748                 goto out_child;
1749         }
1750         session->header.env.comp_mmap_len = session->evlist->core.mmap_len;
1751
1752         if (rec->opts.kcore) {
1753                 err = record__kcore_copy(&session->machines.host, data);
1754                 if (err) {
1755                         pr_err("ERROR: Failed to copy kcore\n");
1756                         goto out_child;
1757                 }
1758         }
1759
1760         err = bpf__apply_obj_config();
1761         if (err) {
1762                 char errbuf[BUFSIZ];
1763
1764                 bpf__strerror_apply_obj_config(err, errbuf, sizeof(errbuf));
1765                 pr_err("ERROR: Apply config to BPF failed: %s\n",
1766                          errbuf);
1767                 goto out_child;
1768         }
1769
1770         /*
1771          * Normally perf_session__new would do this, but it doesn't have the
1772          * evlist.
1773          */
1774         if (rec->tool.ordered_events && !evlist__sample_id_all(rec->evlist)) {
1775                 pr_warning("WARNING: No sample_id_all support, falling back to unordered processing\n");
1776                 rec->tool.ordered_events = false;
1777         }
1778
1779         if (!rec->evlist->nr_groups)
1780                 perf_header__clear_feat(&session->header, HEADER_GROUP_DESC);
1781
1782         if (data->is_pipe) {
1783                 err = perf_header__write_pipe(fd);
1784                 if (err < 0)
1785                         goto out_child;
1786         } else {
1787                 err = perf_session__write_header(session, rec->evlist, fd, false);
1788                 if (err < 0)
1789                         goto out_child;
1790         }
1791
1792         err = -1;
1793         if (!rec->no_buildid
1794             && !perf_header__has_feat(&session->header, HEADER_BUILD_ID)) {
1795                 pr_err("Couldn't generate buildids. "
1796                        "Use --no-buildid to profile anyway.\n");
1797                 goto out_child;
1798         }
1799
1800         err = record__setup_sb_evlist(rec);
1801         if (err)
1802                 goto out_child;
1803
1804         err = record__synthesize(rec, false);
1805         if (err < 0)
1806                 goto out_child;
1807
1808         if (rec->realtime_prio) {
1809                 struct sched_param param;
1810
1811                 param.sched_priority = rec->realtime_prio;
1812                 if (sched_setscheduler(0, SCHED_FIFO, &param)) {
1813                         pr_err("Could not set realtime priority.\n");
1814                         err = -1;
1815                         goto out_child;
1816                 }
1817         }
1818
1819         /*
1820          * When perf is starting the traced process, all the events
1821          * (apart from group members) have enable_on_exec=1 set,
1822          * so don't spoil it by prematurely enabling them.
1823          */
1824         if (!target__none(&opts->target) && !opts->initial_delay)
1825                 evlist__enable(rec->evlist);
1826
1827         /*
1828          * Let the child rip
1829          */
1830         if (forks) {
1831                 struct machine *machine = &session->machines.host;
1832                 union perf_event *event;
1833                 pid_t tgid;
1834
1835                 event = malloc(sizeof(event->comm) + machine->id_hdr_size);
1836                 if (event == NULL) {
1837                         err = -ENOMEM;
1838                         goto out_child;
1839                 }
1840
1841                 /*
1842                  * Some H/W events are generated before COMM event
1843                  * which is emitted during exec(), so perf script
1844                  * cannot see a correct process name for those events.
1845                  * Synthesize COMM event to prevent it.
1846                  */
1847                 tgid = perf_event__synthesize_comm(tool, event,
1848                                                    rec->evlist->workload.pid,
1849                                                    process_synthesized_event,
1850                                                    machine);
1851                 free(event);
1852
1853                 if (tgid == -1)
1854                         goto out_child;
1855
1856                 event = malloc(sizeof(event->namespaces) +
1857                                (NR_NAMESPACES * sizeof(struct perf_ns_link_info)) +
1858                                machine->id_hdr_size);
1859                 if (event == NULL) {
1860                         err = -ENOMEM;
1861                         goto out_child;
1862                 }
1863
1864                 /*
1865                  * Synthesize NAMESPACES event for the command specified.
1866                  */
1867                 perf_event__synthesize_namespaces(tool, event,
1868                                                   rec->evlist->workload.pid,
1869                                                   tgid, process_synthesized_event,
1870                                                   machine);
1871                 free(event);
1872
1873                 evlist__start_workload(rec->evlist);
1874         }
1875
1876         if (evlist__initialize_ctlfd(rec->evlist, opts->ctl_fd, opts->ctl_fd_ack))
1877                 goto out_child;
1878
1879         if (opts->initial_delay) {
1880                 pr_info(EVLIST_DISABLED_MSG);
1881                 if (opts->initial_delay > 0) {
1882                         usleep(opts->initial_delay * USEC_PER_MSEC);
1883                         evlist__enable(rec->evlist);
1884                         pr_info(EVLIST_ENABLED_MSG);
1885                 }
1886         }
1887
1888         trigger_ready(&auxtrace_snapshot_trigger);
1889         trigger_ready(&switch_output_trigger);
1890         perf_hooks__invoke_record_start();
1891         for (;;) {
1892                 unsigned long long hits = rec->samples;
1893
1894                 /*
1895                  * rec->evlist->bkw_mmap_state is possible to be
1896                  * BKW_MMAP_EMPTY here: when done == true and
1897                  * hits != rec->samples in previous round.
1898                  *
1899                  * evlist__toggle_bkw_mmap ensure we never
1900                  * convert BKW_MMAP_EMPTY to BKW_MMAP_DATA_PENDING.
1901                  */
1902                 if (trigger_is_hit(&switch_output_trigger) || done || draining)
1903                         evlist__toggle_bkw_mmap(rec->evlist, BKW_MMAP_DATA_PENDING);
1904
1905                 if (record__mmap_read_all(rec, false) < 0) {
1906                         trigger_error(&auxtrace_snapshot_trigger);
1907                         trigger_error(&switch_output_trigger);
1908                         err = -1;
1909                         goto out_child;
1910                 }
1911
1912                 if (auxtrace_record__snapshot_started) {
1913                         auxtrace_record__snapshot_started = 0;
1914                         if (!trigger_is_error(&auxtrace_snapshot_trigger))
1915                                 record__read_auxtrace_snapshot(rec, false);
1916                         if (trigger_is_error(&auxtrace_snapshot_trigger)) {
1917                                 pr_err("AUX area tracing snapshot failed\n");
1918                                 err = -1;
1919                                 goto out_child;
1920                         }
1921                 }
1922
1923                 if (trigger_is_hit(&switch_output_trigger)) {
1924                         /*
1925                          * If switch_output_trigger is hit, the data in
1926                          * overwritable ring buffer should have been collected,
1927                          * so bkw_mmap_state should be set to BKW_MMAP_EMPTY.
1928                          *
1929                          * If SIGUSR2 raise after or during record__mmap_read_all(),
1930                          * record__mmap_read_all() didn't collect data from
1931                          * overwritable ring buffer. Read again.
1932                          */
1933                         if (rec->evlist->bkw_mmap_state == BKW_MMAP_RUNNING)
1934                                 continue;
1935                         trigger_ready(&switch_output_trigger);
1936
1937                         /*
1938                          * Reenable events in overwrite ring buffer after
1939                          * record__mmap_read_all(): we should have collected
1940                          * data from it.
1941                          */
1942                         evlist__toggle_bkw_mmap(rec->evlist, BKW_MMAP_RUNNING);
1943
1944                         if (!quiet)
1945                                 fprintf(stderr, "[ perf record: dump data: Woken up %ld times ]\n",
1946                                         waking);
1947                         waking = 0;
1948                         fd = record__switch_output(rec, false);
1949                         if (fd < 0) {
1950                                 pr_err("Failed to switch to new file\n");
1951                                 trigger_error(&switch_output_trigger);
1952                                 err = fd;
1953                                 goto out_child;
1954                         }
1955
1956                         /* re-arm the alarm */
1957                         if (rec->switch_output.time)
1958                                 alarm(rec->switch_output.time);
1959                 }
1960
1961                 if (hits == rec->samples) {
1962                         if (done || draining)
1963                                 break;
1964                         err = evlist__poll(rec->evlist, -1);
1965                         /*
1966                          * Propagate error, only if there's any. Ignore positive
1967                          * number of returned events and interrupt error.
1968                          */
1969                         if (err > 0 || (err < 0 && errno == EINTR))
1970                                 err = 0;
1971                         waking++;
1972
1973                         if (evlist__filter_pollfd(rec->evlist, POLLERR | POLLHUP) == 0)
1974                                 draining = true;
1975                 }
1976
1977                 if (evlist__ctlfd_process(rec->evlist, &cmd) > 0) {
1978                         switch (cmd) {
1979                         case EVLIST_CTL_CMD_SNAPSHOT:
1980                                 hit_auxtrace_snapshot_trigger(rec);
1981                                 evlist__ctlfd_ack(rec->evlist);
1982                                 break;
1983                         case EVLIST_CTL_CMD_STOP:
1984                                 done = 1;
1985                                 break;
1986                         case EVLIST_CTL_CMD_ACK:
1987                         case EVLIST_CTL_CMD_UNSUPPORTED:
1988                         case EVLIST_CTL_CMD_ENABLE:
1989                         case EVLIST_CTL_CMD_DISABLE:
1990                         case EVLIST_CTL_CMD_EVLIST:
1991                         case EVLIST_CTL_CMD_PING:
1992                         default:
1993                                 break;
1994                         }
1995                 }
1996
1997                 /*
1998                  * When perf is starting the traced process, at the end events
1999                  * die with the process and we wait for that. Thus no need to
2000                  * disable events in this case.
2001                  */
2002                 if (done && !disabled && !target__none(&opts->target)) {
2003                         trigger_off(&auxtrace_snapshot_trigger);
2004                         evlist__disable(rec->evlist);
2005                         disabled = true;
2006                 }
2007         }
2008
2009         trigger_off(&auxtrace_snapshot_trigger);
2010         trigger_off(&switch_output_trigger);
2011
2012         if (opts->auxtrace_snapshot_on_exit)
2013                 record__auxtrace_snapshot_exit(rec);
2014
2015         if (forks && workload_exec_errno) {
2016                 char msg[STRERR_BUFSIZE], strevsels[2048];
2017                 const char *emsg = str_error_r(workload_exec_errno, msg, sizeof(msg));
2018
2019                 evlist__scnprintf_evsels(rec->evlist, sizeof(strevsels), strevsels);
2020
2021                 pr_err("Failed to collect '%s' for the '%s' workload: %s\n",
2022                         strevsels, argv[0], emsg);
2023                 err = -1;
2024                 goto out_child;
2025         }
2026
2027         if (!quiet)
2028                 fprintf(stderr, "[ perf record: Woken up %ld times to write data ]\n", waking);
2029
2030         if (target__none(&rec->opts.target))
2031                 record__synthesize_workload(rec, true);
2032
2033 out_child:
2034         evlist__finalize_ctlfd(rec->evlist);
2035         record__mmap_read_all(rec, true);
2036         record__aio_mmap_read_sync(rec);
2037
2038         if (rec->session->bytes_transferred && rec->session->bytes_compressed) {
2039                 ratio = (float)rec->session->bytes_transferred/(float)rec->session->bytes_compressed;
2040                 session->header.env.comp_ratio = ratio + 0.5;
2041         }
2042
2043         if (forks) {
2044                 int exit_status;
2045
2046                 if (!child_finished)
2047                         kill(rec->evlist->workload.pid, SIGTERM);
2048
2049                 wait(&exit_status);
2050
2051                 if (err < 0)
2052                         status = err;
2053                 else if (WIFEXITED(exit_status))
2054                         status = WEXITSTATUS(exit_status);
2055                 else if (WIFSIGNALED(exit_status))
2056                         signr = WTERMSIG(exit_status);
2057         } else
2058                 status = err;
2059
2060         record__synthesize(rec, true);
2061         /* this will be recalculated during process_buildids() */
2062         rec->samples = 0;
2063
2064         if (!err) {
2065                 if (!rec->timestamp_filename) {
2066                         record__finish_output(rec);
2067                 } else {
2068                         fd = record__switch_output(rec, true);
2069                         if (fd < 0) {
2070                                 status = fd;
2071                                 goto out_delete_session;
2072                         }
2073                 }
2074         }
2075
2076         perf_hooks__invoke_record_end();
2077
2078         if (!err && !quiet) {
2079                 char samples[128];
2080                 const char *postfix = rec->timestamp_filename ?
2081                                         ".<timestamp>" : "";
2082
2083                 if (rec->samples && !rec->opts.full_auxtrace)
2084                         scnprintf(samples, sizeof(samples),
2085                                   " (%" PRIu64 " samples)", rec->samples);
2086                 else
2087                         samples[0] = '\0';
2088
2089                 fprintf(stderr, "[ perf record: Captured and wrote %.3f MB %s%s%s",
2090                         perf_data__size(data) / 1024.0 / 1024.0,
2091                         data->path, postfix, samples);
2092                 if (ratio) {
2093                         fprintf(stderr, ", compressed (original %.3f MB, ratio is %.3f)",
2094                                         rec->session->bytes_transferred / 1024.0 / 1024.0,
2095                                         ratio);
2096                 }
2097                 fprintf(stderr, " ]\n");
2098         }
2099
2100 out_delete_session:
2101 #ifdef HAVE_EVENTFD_SUPPORT
2102         if (done_fd >= 0)
2103                 close(done_fd);
2104 #endif
2105         zstd_fini(&session->zstd_data);
2106         perf_session__delete(session);
2107
2108         if (!opts->no_bpf_event)
2109                 evlist__stop_sb_thread(rec->sb_evlist);
2110         return status;
2111 }
2112
2113 static void callchain_debug(struct callchain_param *callchain)
2114 {
2115         static const char *str[CALLCHAIN_MAX] = { "NONE", "FP", "DWARF", "LBR" };
2116
2117         pr_debug("callchain: type %s\n", str[callchain->record_mode]);
2118
2119         if (callchain->record_mode == CALLCHAIN_DWARF)
2120                 pr_debug("callchain: stack dump size %d\n",
2121                          callchain->dump_size);
2122 }
2123
2124 int record_opts__parse_callchain(struct record_opts *record,
2125                                  struct callchain_param *callchain,
2126                                  const char *arg, bool unset)
2127 {
2128         int ret;
2129         callchain->enabled = !unset;
2130
2131         /* --no-call-graph */
2132         if (unset) {
2133                 callchain->record_mode = CALLCHAIN_NONE;
2134                 pr_debug("callchain: disabled\n");
2135                 return 0;
2136         }
2137
2138         ret = parse_callchain_record_opt(arg, callchain);
2139         if (!ret) {
2140                 /* Enable data address sampling for DWARF unwind. */
2141                 if (callchain->record_mode == CALLCHAIN_DWARF)
2142                         record->sample_address = true;
2143                 callchain_debug(callchain);
2144         }
2145
2146         return ret;
2147 }
2148
2149 int record_parse_callchain_opt(const struct option *opt,
2150                                const char *arg,
2151                                int unset)
2152 {
2153         return record_opts__parse_callchain(opt->value, &callchain_param, arg, unset);
2154 }
2155
2156 int record_callchain_opt(const struct option *opt,
2157                          const char *arg __maybe_unused,
2158                          int unset __maybe_unused)
2159 {
2160         struct callchain_param *callchain = opt->value;
2161
2162         callchain->enabled = true;
2163
2164         if (callchain->record_mode == CALLCHAIN_NONE)
2165                 callchain->record_mode = CALLCHAIN_FP;
2166
2167         callchain_debug(callchain);
2168         return 0;
2169 }
2170
2171 static int perf_record_config(const char *var, const char *value, void *cb)
2172 {
2173         struct record *rec = cb;
2174
2175         if (!strcmp(var, "record.build-id")) {
2176                 if (!strcmp(value, "cache"))
2177                         rec->no_buildid_cache = false;
2178                 else if (!strcmp(value, "no-cache"))
2179                         rec->no_buildid_cache = true;
2180                 else if (!strcmp(value, "skip"))
2181                         rec->no_buildid = true;
2182                 else if (!strcmp(value, "mmap"))
2183                         rec->buildid_mmap = true;
2184                 else
2185                         return -1;
2186                 return 0;
2187         }
2188         if (!strcmp(var, "record.call-graph")) {
2189                 var = "call-graph.record-mode";
2190                 return perf_default_config(var, value, cb);
2191         }
2192 #ifdef HAVE_AIO_SUPPORT
2193         if (!strcmp(var, "record.aio")) {
2194                 rec->opts.nr_cblocks = strtol(value, NULL, 0);
2195                 if (!rec->opts.nr_cblocks)
2196                         rec->opts.nr_cblocks = nr_cblocks_default;
2197         }
2198 #endif
2199
2200         return 0;
2201 }
2202
2203
2204 static int record__parse_affinity(const struct option *opt, const char *str, int unset)
2205 {
2206         struct record_opts *opts = (struct record_opts *)opt->value;
2207
2208         if (unset || !str)
2209                 return 0;
2210
2211         if (!strcasecmp(str, "node"))
2212                 opts->affinity = PERF_AFFINITY_NODE;
2213         else if (!strcasecmp(str, "cpu"))
2214                 opts->affinity = PERF_AFFINITY_CPU;
2215
2216         return 0;
2217 }
2218
2219 static int parse_output_max_size(const struct option *opt,
2220                                  const char *str, int unset)
2221 {
2222         unsigned long *s = (unsigned long *)opt->value;
2223         static struct parse_tag tags_size[] = {
2224                 { .tag  = 'B', .mult = 1       },
2225                 { .tag  = 'K', .mult = 1 << 10 },
2226                 { .tag  = 'M', .mult = 1 << 20 },
2227                 { .tag  = 'G', .mult = 1 << 30 },
2228                 { .tag  = 0 },
2229         };
2230         unsigned long val;
2231
2232         if (unset) {
2233                 *s = 0;
2234                 return 0;
2235         }
2236
2237         val = parse_tag_value(str, tags_size);
2238         if (val != (unsigned long) -1) {
2239                 *s = val;
2240                 return 0;
2241         }
2242
2243         return -1;
2244 }
2245
2246 static int record__parse_mmap_pages(const struct option *opt,
2247                                     const char *str,
2248                                     int unset __maybe_unused)
2249 {
2250         struct record_opts *opts = opt->value;
2251         char *s, *p;
2252         unsigned int mmap_pages;
2253         int ret;
2254
2255         if (!str)
2256                 return -EINVAL;
2257
2258         s = strdup(str);
2259         if (!s)
2260                 return -ENOMEM;
2261
2262         p = strchr(s, ',');
2263         if (p)
2264                 *p = '\0';
2265
2266         if (*s) {
2267                 ret = __evlist__parse_mmap_pages(&mmap_pages, s);
2268                 if (ret)
2269                         goto out_free;
2270                 opts->mmap_pages = mmap_pages;
2271         }
2272
2273         if (!p) {
2274                 ret = 0;
2275                 goto out_free;
2276         }
2277
2278         ret = __evlist__parse_mmap_pages(&mmap_pages, p + 1);
2279         if (ret)
2280                 goto out_free;
2281
2282         opts->auxtrace_mmap_pages = mmap_pages;
2283
2284 out_free:
2285         free(s);
2286         return ret;
2287 }
2288
2289 static int parse_control_option(const struct option *opt,
2290                                 const char *str,
2291                                 int unset __maybe_unused)
2292 {
2293         struct record_opts *opts = opt->value;
2294
2295         return evlist__parse_control(str, &opts->ctl_fd, &opts->ctl_fd_ack, &opts->ctl_fd_close);
2296 }
2297
2298 static void switch_output_size_warn(struct record *rec)
2299 {
2300         u64 wakeup_size = evlist__mmap_size(rec->opts.mmap_pages);
2301         struct switch_output *s = &rec->switch_output;
2302
2303         wakeup_size /= 2;
2304
2305         if (s->size < wakeup_size) {
2306                 char buf[100];
2307
2308                 unit_number__scnprintf(buf, sizeof(buf), wakeup_size);
2309                 pr_warning("WARNING: switch-output data size lower than "
2310                            "wakeup kernel buffer size (%s) "
2311                            "expect bigger perf.data sizes\n", buf);
2312         }
2313 }
2314
2315 static int switch_output_setup(struct record *rec)
2316 {
2317         struct switch_output *s = &rec->switch_output;
2318         static struct parse_tag tags_size[] = {
2319                 { .tag  = 'B', .mult = 1       },
2320                 { .tag  = 'K', .mult = 1 << 10 },
2321                 { .tag  = 'M', .mult = 1 << 20 },
2322                 { .tag  = 'G', .mult = 1 << 30 },
2323                 { .tag  = 0 },
2324         };
2325         static struct parse_tag tags_time[] = {
2326                 { .tag  = 's', .mult = 1        },
2327                 { .tag  = 'm', .mult = 60       },
2328                 { .tag  = 'h', .mult = 60*60    },
2329                 { .tag  = 'd', .mult = 60*60*24 },
2330                 { .tag  = 0 },
2331         };
2332         unsigned long val;
2333
2334         /*
2335          * If we're using --switch-output-events, then we imply its
2336          * --switch-output=signal, as we'll send a SIGUSR2 from the side band
2337          *  thread to its parent.
2338          */
2339         if (rec->switch_output_event_set)
2340                 goto do_signal;
2341
2342         if (!s->set)
2343                 return 0;
2344
2345         if (!strcmp(s->str, "signal")) {
2346 do_signal:
2347                 s->signal = true;
2348                 pr_debug("switch-output with SIGUSR2 signal\n");
2349                 goto enabled;
2350         }
2351
2352         val = parse_tag_value(s->str, tags_size);
2353         if (val != (unsigned long) -1) {
2354                 s->size = val;
2355                 pr_debug("switch-output with %s size threshold\n", s->str);
2356                 goto enabled;
2357         }
2358
2359         val = parse_tag_value(s->str, tags_time);
2360         if (val != (unsigned long) -1) {
2361                 s->time = val;
2362                 pr_debug("switch-output with %s time threshold (%lu seconds)\n",
2363                          s->str, s->time);
2364                 goto enabled;
2365         }
2366
2367         return -1;
2368
2369 enabled:
2370         rec->timestamp_filename = true;
2371         s->enabled              = true;
2372
2373         if (s->size && !rec->opts.no_buffering)
2374                 switch_output_size_warn(rec);
2375
2376         return 0;
2377 }
2378
2379 static const char * const __record_usage[] = {
2380         "perf record [<options>] [<command>]",
2381         "perf record [<options>] -- <command> [<options>]",
2382         NULL
2383 };
2384 const char * const *record_usage = __record_usage;
2385
2386 static int build_id__process_mmap(struct perf_tool *tool, union perf_event *event,
2387                                   struct perf_sample *sample, struct machine *machine)
2388 {
2389         /*
2390          * We already have the kernel maps, put in place via perf_session__create_kernel_maps()
2391          * no need to add them twice.
2392          */
2393         if (!(event->header.misc & PERF_RECORD_MISC_USER))
2394                 return 0;
2395         return perf_event__process_mmap(tool, event, sample, machine);
2396 }
2397
2398 static int build_id__process_mmap2(struct perf_tool *tool, union perf_event *event,
2399                                    struct perf_sample *sample, struct machine *machine)
2400 {
2401         /*
2402          * We already have the kernel maps, put in place via perf_session__create_kernel_maps()
2403          * no need to add them twice.
2404          */
2405         if (!(event->header.misc & PERF_RECORD_MISC_USER))
2406                 return 0;
2407
2408         return perf_event__process_mmap2(tool, event, sample, machine);
2409 }
2410
2411 static int process_timestamp_boundary(struct perf_tool *tool,
2412                                       union perf_event *event __maybe_unused,
2413                                       struct perf_sample *sample,
2414                                       struct machine *machine __maybe_unused)
2415 {
2416         struct record *rec = container_of(tool, struct record, tool);
2417
2418         set_timestamp_boundary(rec, sample->time);
2419         return 0;
2420 }
2421
2422 /*
2423  * XXX Ideally would be local to cmd_record() and passed to a record__new
2424  * because we need to have access to it in record__exit, that is called
2425  * after cmd_record() exits, but since record_options need to be accessible to
2426  * builtin-script, leave it here.
2427  *
2428  * At least we don't ouch it in all the other functions here directly.
2429  *
2430  * Just say no to tons of global variables, sigh.
2431  */
2432 static struct record record = {
2433         .opts = {
2434                 .sample_time         = true,
2435                 .mmap_pages          = UINT_MAX,
2436                 .user_freq           = UINT_MAX,
2437                 .user_interval       = ULLONG_MAX,
2438                 .freq                = 4000,
2439                 .target              = {
2440                         .uses_mmap   = true,
2441                         .default_per_cpu = true,
2442                 },
2443                 .mmap_flush          = MMAP_FLUSH_DEFAULT,
2444                 .nr_threads_synthesize = 1,
2445                 .ctl_fd              = -1,
2446                 .ctl_fd_ack          = -1,
2447         },
2448         .tool = {
2449                 .sample         = process_sample_event,
2450                 .fork           = perf_event__process_fork,
2451                 .exit           = perf_event__process_exit,
2452                 .comm           = perf_event__process_comm,
2453                 .namespaces     = perf_event__process_namespaces,
2454                 .mmap           = build_id__process_mmap,
2455                 .mmap2          = build_id__process_mmap2,
2456                 .itrace_start   = process_timestamp_boundary,
2457                 .aux            = process_timestamp_boundary,
2458                 .ordered_events = true,
2459         },
2460 };
2461
2462 const char record_callchain_help[] = CALLCHAIN_RECORD_HELP
2463         "\n\t\t\t\tDefault: fp";
2464
2465 static bool dry_run;
2466
2467 /*
2468  * XXX Will stay a global variable till we fix builtin-script.c to stop messing
2469  * with it and switch to use the library functions in perf_evlist that came
2470  * from builtin-record.c, i.e. use record_opts,
2471  * evlist__prepare_workload, etc instead of fork+exec'in 'perf record',
2472  * using pipes, etc.
2473  */
2474 static struct option __record_options[] = {
2475         OPT_CALLBACK('e', "event", &record.evlist, "event",
2476                      "event selector. use 'perf list' to list available events",
2477                      parse_events_option),
2478         OPT_CALLBACK(0, "filter", &record.evlist, "filter",
2479                      "event filter", parse_filter),
2480         OPT_CALLBACK_NOOPT(0, "exclude-perf", &record.evlist,
2481                            NULL, "don't record events from perf itself",
2482                            exclude_perf),
2483         OPT_STRING('p', "pid", &record.opts.target.pid, "pid",
2484                     "record events on existing process id"),
2485         OPT_STRING('t', "tid", &record.opts.target.tid, "tid",
2486                     "record events on existing thread id"),
2487         OPT_INTEGER('r', "realtime", &record.realtime_prio,
2488                     "collect data with this RT SCHED_FIFO priority"),
2489         OPT_BOOLEAN(0, "no-buffering", &record.opts.no_buffering,
2490                     "collect data without buffering"),
2491         OPT_BOOLEAN('R', "raw-samples", &record.opts.raw_samples,
2492                     "collect raw sample records from all opened counters"),
2493         OPT_BOOLEAN('a', "all-cpus", &record.opts.target.system_wide,
2494                             "system-wide collection from all CPUs"),
2495         OPT_STRING('C', "cpu", &record.opts.target.cpu_list, "cpu",
2496                     "list of cpus to monitor"),
2497         OPT_U64('c', "count", &record.opts.user_interval, "event period to sample"),
2498         OPT_STRING('o', "output", &record.data.path, "file",
2499                     "output file name"),
2500         OPT_BOOLEAN_SET('i', "no-inherit", &record.opts.no_inherit,
2501                         &record.opts.no_inherit_set,
2502                         "child tasks do not inherit counters"),
2503         OPT_BOOLEAN(0, "tail-synthesize", &record.opts.tail_synthesize,
2504                     "synthesize non-sample events at the end of output"),
2505         OPT_BOOLEAN(0, "overwrite", &record.opts.overwrite, "use overwrite mode"),
2506         OPT_BOOLEAN(0, "no-bpf-event", &record.opts.no_bpf_event, "do not record bpf events"),
2507         OPT_BOOLEAN(0, "strict-freq", &record.opts.strict_freq,
2508                     "Fail if the specified frequency can't be used"),
2509         OPT_CALLBACK('F', "freq", &record.opts, "freq or 'max'",
2510                      "profile at this frequency",
2511                       record__parse_freq),
2512         OPT_CALLBACK('m', "mmap-pages", &record.opts, "pages[,pages]",
2513                      "number of mmap data pages and AUX area tracing mmap pages",
2514                      record__parse_mmap_pages),
2515         OPT_CALLBACK(0, "mmap-flush", &record.opts, "number",
2516                      "Minimal number of bytes that is extracted from mmap data pages (default: 1)",
2517                      record__mmap_flush_parse),
2518         OPT_BOOLEAN(0, "group", &record.opts.group,
2519                     "put the counters into a counter group"),
2520         OPT_CALLBACK_NOOPT('g', NULL, &callchain_param,
2521                            NULL, "enables call-graph recording" ,
2522                            &record_callchain_opt),
2523         OPT_CALLBACK(0, "call-graph", &record.opts,
2524                      "record_mode[,record_size]", record_callchain_help,
2525                      &record_parse_callchain_opt),
2526         OPT_INCR('v', "verbose", &verbose,
2527                     "be more verbose (show counter open errors, etc)"),
2528         OPT_BOOLEAN('q', "quiet", &quiet, "don't print any message"),
2529         OPT_BOOLEAN('s', "stat", &record.opts.inherit_stat,
2530                     "per thread counts"),
2531         OPT_BOOLEAN('d', "data", &record.opts.sample_address, "Record the sample addresses"),
2532         OPT_BOOLEAN(0, "phys-data", &record.opts.sample_phys_addr,
2533                     "Record the sample physical addresses"),
2534         OPT_BOOLEAN(0, "data-page-size", &record.opts.sample_data_page_size,
2535                     "Record the sampled data address data page size"),
2536         OPT_BOOLEAN(0, "code-page-size", &record.opts.sample_code_page_size,
2537                     "Record the sampled code address (ip) page size"),
2538         OPT_BOOLEAN(0, "sample-cpu", &record.opts.sample_cpu, "Record the sample cpu"),
2539         OPT_BOOLEAN_SET('T', "timestamp", &record.opts.sample_time,
2540                         &record.opts.sample_time_set,
2541                         "Record the sample timestamps"),
2542         OPT_BOOLEAN_SET('P', "period", &record.opts.period, &record.opts.period_set,
2543                         "Record the sample period"),
2544         OPT_BOOLEAN('n', "no-samples", &record.opts.no_samples,
2545                     "don't sample"),
2546         OPT_BOOLEAN_SET('N', "no-buildid-cache", &record.no_buildid_cache,
2547                         &record.no_buildid_cache_set,
2548                         "do not update the buildid cache"),
2549         OPT_BOOLEAN_SET('B', "no-buildid", &record.no_buildid,
2550                         &record.no_buildid_set,
2551                         "do not collect buildids in perf.data"),
2552         OPT_CALLBACK('G', "cgroup", &record.evlist, "name",
2553                      "monitor event in cgroup name only",
2554                      parse_cgroups),
2555         OPT_INTEGER('D', "delay", &record.opts.initial_delay,
2556                   "ms to wait before starting measurement after program start (-1: start with events disabled)"),
2557         OPT_BOOLEAN(0, "kcore", &record.opts.kcore, "copy /proc/kcore"),
2558         OPT_STRING('u', "uid", &record.opts.target.uid_str, "user",
2559                    "user to profile"),
2560
2561         OPT_CALLBACK_NOOPT('b', "branch-any", &record.opts.branch_stack,
2562                      "branch any", "sample any taken branches",
2563                      parse_branch_stack),
2564
2565         OPT_CALLBACK('j', "branch-filter", &record.opts.branch_stack,
2566                      "branch filter mask", "branch stack filter modes",
2567                      parse_branch_stack),
2568         OPT_BOOLEAN('W', "weight", &record.opts.sample_weight,
2569                     "sample by weight (on special events only)"),
2570         OPT_BOOLEAN(0, "transaction", &record.opts.sample_transaction,
2571                     "sample transaction flags (special events only)"),
2572         OPT_BOOLEAN(0, "per-thread", &record.opts.target.per_thread,
2573                     "use per-thread mmaps"),
2574         OPT_CALLBACK_OPTARG('I', "intr-regs", &record.opts.sample_intr_regs, NULL, "any register",
2575                     "sample selected machine registers on interrupt,"
2576                     " use '-I?' to list register names", parse_intr_regs),
2577         OPT_CALLBACK_OPTARG(0, "user-regs", &record.opts.sample_user_regs, NULL, "any register",
2578                     "sample selected machine registers on interrupt,"
2579                     " use '--user-regs=?' to list register names", parse_user_regs),
2580         OPT_BOOLEAN(0, "running-time", &record.opts.running_time,
2581                     "Record running/enabled time of read (:S) events"),
2582         OPT_CALLBACK('k', "clockid", &record.opts,
2583         "clockid", "clockid to use for events, see clock_gettime()",
2584         parse_clockid),
2585         OPT_STRING_OPTARG('S', "snapshot", &record.opts.auxtrace_snapshot_opts,
2586                           "opts", "AUX area tracing Snapshot Mode", ""),
2587         OPT_STRING_OPTARG(0, "aux-sample", &record.opts.auxtrace_sample_opts,
2588                           "opts", "sample AUX area", ""),
2589         OPT_UINTEGER(0, "proc-map-timeout", &proc_map_timeout,
2590                         "per thread proc mmap processing timeout in ms"),
2591         OPT_BOOLEAN(0, "namespaces", &record.opts.record_namespaces,
2592                     "Record namespaces events"),
2593         OPT_BOOLEAN(0, "all-cgroups", &record.opts.record_cgroup,
2594                     "Record cgroup events"),
2595         OPT_BOOLEAN_SET(0, "switch-events", &record.opts.record_switch_events,
2596                         &record.opts.record_switch_events_set,
2597                         "Record context switch events"),
2598         OPT_BOOLEAN_FLAG(0, "all-kernel", &record.opts.all_kernel,
2599                          "Configure all used events to run in kernel space.",
2600                          PARSE_OPT_EXCLUSIVE),
2601         OPT_BOOLEAN_FLAG(0, "all-user", &record.opts.all_user,
2602                          "Configure all used events to run in user space.",
2603                          PARSE_OPT_EXCLUSIVE),
2604         OPT_BOOLEAN(0, "kernel-callchains", &record.opts.kernel_callchains,
2605                     "collect kernel callchains"),
2606         OPT_BOOLEAN(0, "user-callchains", &record.opts.user_callchains,
2607                     "collect user callchains"),
2608         OPT_STRING(0, "clang-path", &llvm_param.clang_path, "clang path",
2609                    "clang binary to use for compiling BPF scriptlets"),
2610         OPT_STRING(0, "clang-opt", &llvm_param.clang_opt, "clang options",
2611                    "options passed to clang when compiling BPF scriptlets"),
2612         OPT_STRING(0, "vmlinux", &symbol_conf.vmlinux_name,
2613                    "file", "vmlinux pathname"),
2614         OPT_BOOLEAN(0, "buildid-all", &record.buildid_all,
2615                     "Record build-id of all DSOs regardless of hits"),
2616         OPT_BOOLEAN(0, "buildid-mmap", &record.buildid_mmap,
2617                     "Record build-id in map events"),
2618         OPT_BOOLEAN(0, "timestamp-filename", &record.timestamp_filename,
2619                     "append timestamp to output filename"),
2620         OPT_BOOLEAN(0, "timestamp-boundary", &record.timestamp_boundary,
2621                     "Record timestamp boundary (time of first/last samples)"),
2622         OPT_STRING_OPTARG_SET(0, "switch-output", &record.switch_output.str,
2623                           &record.switch_output.set, "signal or size[BKMG] or time[smhd]",
2624                           "Switch output when receiving SIGUSR2 (signal) or cross a size or time threshold",
2625                           "signal"),
2626         OPT_CALLBACK_SET(0, "switch-output-event", &record.sb_evlist, &record.switch_output_event_set, "switch output event",
2627                          "switch output event selector. use 'perf list' to list available events",
2628                          parse_events_option_new_evlist),
2629         OPT_INTEGER(0, "switch-max-files", &record.switch_output.num_files,
2630                    "Limit number of switch output generated files"),
2631         OPT_BOOLEAN(0, "dry-run", &dry_run,
2632                     "Parse options then exit"),
2633 #ifdef HAVE_AIO_SUPPORT
2634         OPT_CALLBACK_OPTARG(0, "aio", &record.opts,
2635                      &nr_cblocks_default, "n", "Use <n> control blocks in asynchronous trace writing mode (default: 1, max: 4)",
2636                      record__aio_parse),
2637 #endif
2638         OPT_CALLBACK(0, "affinity", &record.opts, "node|cpu",
2639                      "Set affinity mask of trace reading thread to NUMA node cpu mask or cpu of processed mmap buffer",
2640                      record__parse_affinity),
2641 #ifdef HAVE_ZSTD_SUPPORT
2642         OPT_CALLBACK_OPTARG('z', "compression-level", &record.opts, &comp_level_default,
2643                             "n", "Compressed records using specified level (default: 1 - fastest compression, 22 - greatest compression)",
2644                             record__parse_comp_level),
2645 #endif
2646         OPT_CALLBACK(0, "max-size", &record.output_max_size,
2647                      "size", "Limit the maximum size of the output file", parse_output_max_size),
2648         OPT_UINTEGER(0, "num-thread-synthesize",
2649                      &record.opts.nr_threads_synthesize,
2650                      "number of threads to run for event synthesis"),
2651 #ifdef HAVE_LIBPFM
2652         OPT_CALLBACK(0, "pfm-events", &record.evlist, "event",
2653                 "libpfm4 event selector. use 'perf list' to list available events",
2654                 parse_libpfm_events_option),
2655 #endif
2656         OPT_CALLBACK(0, "control", &record.opts, "fd:ctl-fd[,ack-fd] or fifo:ctl-fifo[,ack-fifo]",
2657                      "Listen on ctl-fd descriptor for command to control measurement ('enable': enable events, 'disable': disable events,\n"
2658                      "\t\t\t  'snapshot': AUX area tracing snapshot).\n"
2659                      "\t\t\t  Optionally send control command completion ('ack\\n') to ack-fd descriptor.\n"
2660                      "\t\t\t  Alternatively, ctl-fifo / ack-fifo will be opened and used as ctl-fd / ack-fd.",
2661                       parse_control_option),
2662         OPT_END()
2663 };
2664
2665 struct option *record_options = __record_options;
2666
2667 int cmd_record(int argc, const char **argv)
2668 {
2669         int err;
2670         struct record *rec = &record;
2671         char errbuf[BUFSIZ];
2672
2673         setlocale(LC_ALL, "");
2674
2675 #ifndef HAVE_LIBBPF_SUPPORT
2676 # define set_nobuild(s, l, c) set_option_nobuild(record_options, s, l, "NO_LIBBPF=1", c)
2677         set_nobuild('\0', "clang-path", true);
2678         set_nobuild('\0', "clang-opt", true);
2679 # undef set_nobuild
2680 #endif
2681
2682 #ifndef HAVE_BPF_PROLOGUE
2683 # if !defined (HAVE_DWARF_SUPPORT)
2684 #  define REASON  "NO_DWARF=1"
2685 # elif !defined (HAVE_LIBBPF_SUPPORT)
2686 #  define REASON  "NO_LIBBPF=1"
2687 # else
2688 #  define REASON  "this architecture doesn't support BPF prologue"
2689 # endif
2690 # define set_nobuild(s, l, c) set_option_nobuild(record_options, s, l, REASON, c)
2691         set_nobuild('\0', "vmlinux", true);
2692 # undef set_nobuild
2693 # undef REASON
2694 #endif
2695
2696         rec->opts.affinity = PERF_AFFINITY_SYS;
2697
2698         rec->evlist = evlist__new();
2699         if (rec->evlist == NULL)
2700                 return -ENOMEM;
2701
2702         err = perf_config(perf_record_config, rec);
2703         if (err)
2704                 return err;
2705
2706         argc = parse_options(argc, argv, record_options, record_usage,
2707                             PARSE_OPT_STOP_AT_NON_OPTION);
2708         if (quiet)
2709                 perf_quiet_option();
2710
2711         /* Make system wide (-a) the default target. */
2712         if (!argc && target__none(&rec->opts.target))
2713                 rec->opts.target.system_wide = true;
2714
2715         if (nr_cgroups && !rec->opts.target.system_wide) {
2716                 usage_with_options_msg(record_usage, record_options,
2717                         "cgroup monitoring only available in system-wide mode");
2718
2719         }
2720
2721         if (rec->buildid_mmap) {
2722                 if (!perf_can_record_build_id()) {
2723                         pr_err("Failed: no support to record build id in mmap events, update your kernel.\n");
2724                         err = -EINVAL;
2725                         goto out_opts;
2726                 }
2727                 pr_debug("Enabling build id in mmap2 events.\n");
2728                 /* Enable mmap build id synthesizing. */
2729                 symbol_conf.buildid_mmap2 = true;
2730                 /* Enable perf_event_attr::build_id bit. */
2731                 rec->opts.build_id = true;
2732                 /* Disable build id cache. */
2733                 rec->no_buildid = true;
2734         }
2735
2736         if (rec->opts.record_cgroup && !perf_can_record_cgroup()) {
2737                 pr_err("Kernel has no cgroup sampling support.\n");
2738                 err = -EINVAL;
2739                 goto out_opts;
2740         }
2741
2742         if (rec->opts.kcore)
2743                 rec->data.is_dir = true;
2744
2745         if (rec->opts.comp_level != 0) {
2746                 pr_debug("Compression enabled, disabling build id collection at the end of the session.\n");
2747                 rec->no_buildid = true;
2748         }
2749
2750         if (rec->opts.record_switch_events &&
2751             !perf_can_record_switch_events()) {
2752                 ui__error("kernel does not support recording context switch events\n");
2753                 parse_options_usage(record_usage, record_options, "switch-events", 0);
2754                 err = -EINVAL;
2755                 goto out_opts;
2756         }
2757
2758         if (switch_output_setup(rec)) {
2759                 parse_options_usage(record_usage, record_options, "switch-output", 0);
2760                 err = -EINVAL;
2761                 goto out_opts;
2762         }
2763
2764         if (rec->switch_output.time) {
2765                 signal(SIGALRM, alarm_sig_handler);
2766                 alarm(rec->switch_output.time);
2767         }
2768
2769         if (rec->switch_output.num_files) {
2770                 rec->switch_output.filenames = calloc(sizeof(char *),
2771                                                       rec->switch_output.num_files);
2772                 if (!rec->switch_output.filenames) {
2773                         err = -EINVAL;
2774                         goto out_opts;
2775                 }
2776         }
2777
2778         /*
2779          * Allow aliases to facilitate the lookup of symbols for address
2780          * filters. Refer to auxtrace_parse_filters().
2781          */
2782         symbol_conf.allow_aliases = true;
2783
2784         symbol__init(NULL);
2785
2786         if (rec->opts.affinity != PERF_AFFINITY_SYS) {
2787                 rec->affinity_mask.nbits = cpu__max_cpu();
2788                 rec->affinity_mask.bits = bitmap_alloc(rec->affinity_mask.nbits);
2789                 if (!rec->affinity_mask.bits) {
2790                         pr_err("Failed to allocate thread mask for %zd cpus\n", rec->affinity_mask.nbits);
2791                         err = -ENOMEM;
2792                         goto out_opts;
2793                 }
2794                 pr_debug2("thread mask[%zd]: empty\n", rec->affinity_mask.nbits);
2795         }
2796
2797         err = record__auxtrace_init(rec);
2798         if (err)
2799                 goto out;
2800
2801         if (dry_run)
2802                 goto out;
2803
2804         err = bpf__setup_stdout(rec->evlist);
2805         if (err) {
2806                 bpf__strerror_setup_stdout(rec->evlist, err, errbuf, sizeof(errbuf));
2807                 pr_err("ERROR: Setup BPF stdout failed: %s\n",
2808                          errbuf);
2809                 goto out;
2810         }
2811
2812         err = -ENOMEM;
2813
2814         if (rec->no_buildid_cache || rec->no_buildid) {
2815                 disable_buildid_cache();
2816         } else if (rec->switch_output.enabled) {
2817                 /*
2818                  * In 'perf record --switch-output', disable buildid
2819                  * generation by default to reduce data file switching
2820                  * overhead. Still generate buildid if they are required
2821                  * explicitly using
2822                  *
2823                  *  perf record --switch-output --no-no-buildid \
2824                  *              --no-no-buildid-cache
2825                  *
2826                  * Following code equals to:
2827                  *
2828                  * if ((rec->no_buildid || !rec->no_buildid_set) &&
2829                  *     (rec->no_buildid_cache || !rec->no_buildid_cache_set))
2830                  *         disable_buildid_cache();
2831                  */
2832                 bool disable = true;
2833
2834                 if (rec->no_buildid_set && !rec->no_buildid)
2835                         disable = false;
2836                 if (rec->no_buildid_cache_set && !rec->no_buildid_cache)
2837                         disable = false;
2838                 if (disable) {
2839                         rec->no_buildid = true;
2840                         rec->no_buildid_cache = true;
2841                         disable_buildid_cache();
2842                 }
2843         }
2844
2845         if (record.opts.overwrite)
2846                 record.opts.tail_synthesize = true;
2847
2848         if (rec->evlist->core.nr_entries == 0) {
2849                 if (perf_pmu__has_hybrid()) {
2850                         err = evlist__add_default_hybrid(rec->evlist,
2851                                                          !record.opts.no_samples);
2852                 } else {
2853                         err = __evlist__add_default(rec->evlist,
2854                                                     !record.opts.no_samples);
2855                 }
2856
2857                 if (err < 0) {
2858                         pr_err("Not enough memory for event selector list\n");
2859                         goto out;
2860                 }
2861         }
2862
2863         if (rec->opts.target.tid && !rec->opts.no_inherit_set)
2864                 rec->opts.no_inherit = true;
2865
2866         err = target__validate(&rec->opts.target);
2867         if (err) {
2868                 target__strerror(&rec->opts.target, err, errbuf, BUFSIZ);
2869                 ui__warning("%s\n", errbuf);
2870         }
2871
2872         err = target__parse_uid(&rec->opts.target);
2873         if (err) {
2874                 int saved_errno = errno;
2875
2876                 target__strerror(&rec->opts.target, err, errbuf, BUFSIZ);
2877                 ui__error("%s", errbuf);
2878
2879                 err = -saved_errno;
2880                 goto out;
2881         }
2882
2883         /* Enable ignoring missing threads when -u/-p option is defined. */
2884         rec->opts.ignore_missing_thread = rec->opts.target.uid != UINT_MAX || rec->opts.target.pid;
2885
2886         err = -ENOMEM;
2887         if (evlist__create_maps(rec->evlist, &rec->opts.target) < 0)
2888                 usage_with_options(record_usage, record_options);
2889
2890         err = auxtrace_record__options(rec->itr, rec->evlist, &rec->opts);
2891         if (err)
2892                 goto out;
2893
2894         /*
2895          * We take all buildids when the file contains
2896          * AUX area tracing data because we do not decode the
2897          * trace because it would take too long.
2898          */
2899         if (rec->opts.full_auxtrace)
2900                 rec->buildid_all = true;
2901
2902         if (rec->opts.text_poke) {
2903                 err = record__config_text_poke(rec->evlist);
2904                 if (err) {
2905                         pr_err("record__config_text_poke failed, error %d\n", err);
2906                         goto out;
2907                 }
2908         }
2909
2910         if (record_opts__config(&rec->opts)) {
2911                 err = -EINVAL;
2912                 goto out;
2913         }
2914
2915         if (rec->opts.nr_cblocks > nr_cblocks_max)
2916                 rec->opts.nr_cblocks = nr_cblocks_max;
2917         pr_debug("nr_cblocks: %d\n", rec->opts.nr_cblocks);
2918
2919         pr_debug("affinity: %s\n", affinity_tags[rec->opts.affinity]);
2920         pr_debug("mmap flush: %d\n", rec->opts.mmap_flush);
2921
2922         if (rec->opts.comp_level > comp_level_max)
2923                 rec->opts.comp_level = comp_level_max;
2924         pr_debug("comp level: %d\n", rec->opts.comp_level);
2925
2926         err = __cmd_record(&record, argc, argv);
2927 out:
2928         bitmap_free(rec->affinity_mask.bits);
2929         evlist__delete(rec->evlist);
2930         symbol__exit();
2931         auxtrace_record__free(rec->itr);
2932 out_opts:
2933         evlist__close_control(rec->opts.ctl_fd, rec->opts.ctl_fd_ack, &rec->opts.ctl_fd_close);
2934         return err;
2935 }
2936
2937 static void snapshot_sig_handler(int sig __maybe_unused)
2938 {
2939         struct record *rec = &record;
2940
2941         hit_auxtrace_snapshot_trigger(rec);
2942
2943         if (switch_output_signal(rec))
2944                 trigger_hit(&switch_output_trigger);
2945 }
2946
2947 static void alarm_sig_handler(int sig __maybe_unused)
2948 {
2949         struct record *rec = &record;
2950
2951         if (switch_output_time(rec))
2952                 trigger_hit(&switch_output_trigger);
2953 }