tools/perf/util/arm-spe.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Arm Statistical Profiling Extensions (SPE) support
   4  * Copyright (c) 2017-2018, Arm Ltd.
   5  */
   6
   7 #include <byteswap.h>
   8 #include <endian.h>
   9 #include <errno.h>
  10 #include <inttypes.h>
  11 #include <linux/bitops.h>
  12 #include <linux/kernel.h>
  13 #include <linux/log2.h>
  14 #include <linux/types.h>
  15 #include <linux/zalloc.h>
  16 #include <stdlib.h>
  17 #include <unistd.h>
  18
  19 #include "auxtrace.h"
  20 #include "color.h"
  21 #include "debug.h"
  22 #include "evlist.h"
  23 #include "evsel.h"
  24 #include "machine.h"
  25 #include "session.h"
  26 #include "symbol.h"
  27 #include "thread.h"
  28 #include "thread-stack.h"
  29 #include "tsc.h"
  30 #include "tool.h"
  31 #include "util/synthetic-events.h"
  32
  33 #include "arm-spe.h"
  34 #include "arm-spe-decoder/arm-spe-decoder.h"
  35 #include "arm-spe-decoder/arm-spe-pkt-decoder.h"
  36
  37 #define MAX_TIMESTAMP (~0ULL)
  38
  39 struct arm_spe {
  40         struct auxtrace                 auxtrace;
  41         struct auxtrace_queues          queues;
  42         struct auxtrace_heap            heap;
  43         struct itrace_synth_opts        synth_opts;
  44         u32                             auxtrace_type;
  45         struct perf_session             *session;
  46         struct machine                  *machine;
  47         u32                             pmu_type;
  48
  49         struct perf_tsc_conversion      tc;
  50
  51         u8                              timeless_decoding;
  52         u8                              data_queued;
  53
  54         u64                             sample_type;
  55         u8                              sample_flc;
  56         u8                              sample_llc;
  57         u8                              sample_tlb;
  58         u8                              sample_branch;
  59         u8                              sample_remote_access;
  60         u8                              sample_memory;
  61         u8                              sample_instructions;
  62         u64                             instructions_sample_period;
  63
  64         u64                             l1d_miss_id;
  65         u64                             l1d_access_id;
  66         u64                             llc_miss_id;
  67         u64                             llc_access_id;
  68         u64                             tlb_miss_id;
  69         u64                             tlb_access_id;
  70         u64                             branch_miss_id;
  71         u64                             remote_access_id;
  72         u64                             memory_id;
  73         u64                             instructions_id;
  74
  75         u64                             kernel_start;
  76
  77         unsigned long                   num_events;
  78         u8                              use_ctx_pkt_for_pid;
  79 };
  80
  81 struct arm_spe_queue {
  82         struct arm_spe                  *spe;
  83         unsigned int                    queue_nr;
  84         struct auxtrace_buffer          *buffer;
  85         struct auxtrace_buffer          *old_buffer;
  86         union perf_event                *event_buf;
  87         bool                            on_heap;
  88         bool                            done;
  89         pid_t                           pid;
  90         pid_t                           tid;
  91         int                             cpu;
  92         struct arm_spe_decoder          *decoder;
  93         u64                             time;
  94         u64                             timestamp;
  95         struct thread                   *thread;
  96         u64                             period_instructions;
  97 };
  98
  99 static void arm_spe_dump(struct arm_spe *spe __maybe_unused,
 100                          unsigned char *buf, size_t len)
 101 {
 102         struct arm_spe_pkt packet;
 103         size_t pos = 0;
 104         int ret, pkt_len, i;
 105         char desc[ARM_SPE_PKT_DESC_MAX];
 106         const char *color = PERF_COLOR_BLUE;
 107
 108         color_fprintf(stdout, color,
 109                       ". ... ARM SPE data: size %#zx bytes\n",
 110                       len);
 111
 112         while (len) {
 113                 ret = arm_spe_get_packet(buf, len, &packet);
 114                 if (ret > 0)
 115                         pkt_len = ret;
 116                 else
 117                         pkt_len = 1;
 118                 printf(".");
 119                 color_fprintf(stdout, color, "  %08x: ", pos);
 120                 for (i = 0; i < pkt_len; i++)
 121                         color_fprintf(stdout, color, " %02x", buf[i]);
 122                 for (; i < 16; i++)
 123                         color_fprintf(stdout, color, "   ");
 124                 if (ret > 0) {
 125                         ret = arm_spe_pkt_desc(&packet, desc,
 126                                                ARM_SPE_PKT_DESC_MAX);
 127                         if (!ret)
 128                                 color_fprintf(stdout, color, " %s\n", desc);
 129                 } else {
 130                         color_fprintf(stdout, color, " Bad packet!\n");
 131                 }
 132                 pos += pkt_len;
 133                 buf += pkt_len;
 134                 len -= pkt_len;
 135         }
 136 }
 137
 138 static void arm_spe_dump_event(struct arm_spe *spe, unsigned char *buf,
 139                                size_t len)
 140 {
 141         printf(".\n");
 142         arm_spe_dump(spe, buf, len);
 143 }
 144
 145 static int arm_spe_get_trace(struct arm_spe_buffer *b, void *data)
 146 {
 147         struct arm_spe_queue *speq = data;
 148         struct auxtrace_buffer *buffer = speq->buffer;
 149         struct auxtrace_buffer *old_buffer = speq->old_buffer;
 150         struct auxtrace_queue *queue;
 151
 152         queue = &speq->spe->queues.queue_array[speq->queue_nr];
 153
 154         buffer = auxtrace_buffer__next(queue, buffer);
 155         /* If no more data, drop the previous auxtrace_buffer and return */
 156         if (!buffer) {
 157                 if (old_buffer)
 158                         auxtrace_buffer__drop_data(old_buffer);
 159                 b->len = 0;
 160                 return 0;
 161         }
 162
 163         speq->buffer = buffer;
 164
 165         /* If the aux_buffer doesn't have data associated, try to load it */
 166         if (!buffer->data) {
 167                 /* get the file desc associated with the perf data file */
 168                 int fd = perf_data__fd(speq->spe->session->data);
 169
 170                 buffer->data = auxtrace_buffer__get_data(buffer, fd);
 171                 if (!buffer->data)
 172                         return -ENOMEM;
 173         }
 174
 175         b->len = buffer->size;
 176         b->buf = buffer->data;
 177
 178         if (b->len) {
 179                 if (old_buffer)
 180                         auxtrace_buffer__drop_data(old_buffer);
 181                 speq->old_buffer = buffer;
 182         } else {
 183                 auxtrace_buffer__drop_data(buffer);
 184                 return arm_spe_get_trace(b, data);
 185         }
 186
 187         return 0;
 188 }
 189
 190 static struct arm_spe_queue *arm_spe__alloc_queue(struct arm_spe *spe,
 191                 unsigned int queue_nr)
 192 {
 193         struct arm_spe_params params = { .get_trace = 0, };
 194         struct arm_spe_queue *speq;
 195
 196         speq = zalloc(sizeof(*speq));
 197         if (!speq)
 198                 return NULL;
 199
 200         speq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
 201         if (!speq->event_buf)
 202                 goto out_free;
 203
 204         speq->spe = spe;
 205         speq->queue_nr = queue_nr;
 206         speq->pid = -1;
 207         speq->tid = -1;
 208         speq->cpu = -1;
 209         speq->period_instructions = 0;
 210
 211         /* params set */
 212         params.get_trace = arm_spe_get_trace;
 213         params.data = speq;
 214
 215         /* create new decoder */
 216         speq->decoder = arm_spe_decoder_new(&params);
 217         if (!speq->decoder)
 218                 goto out_free;
 219
 220         return speq;
 221
 222 out_free:
 223         zfree(&speq->event_buf);
 224         free(speq);
 225
 226         return NULL;
 227 }
 228
 229 static inline u8 arm_spe_cpumode(struct arm_spe *spe, u64 ip)
 230 {
 231         return ip >= spe->kernel_start ?
 232                 PERF_RECORD_MISC_KERNEL :
 233                 PERF_RECORD_MISC_USER;
 234 }
 235
 236 static void arm_spe_set_pid_tid_cpu(struct arm_spe *spe,
 237                                     struct auxtrace_queue *queue)
 238 {
 239         struct arm_spe_queue *speq = queue->priv;
 240         pid_t tid;
 241
 242         tid = machine__get_current_tid(spe->machine, speq->cpu);
 243         if (tid != -1) {
 244                 speq->tid = tid;
 245                 thread__zput(speq->thread);
 246         } else
 247                 speq->tid = queue->tid;
 248
 249         if ((!speq->thread) && (speq->tid != -1)) {
 250                 speq->thread = machine__find_thread(spe->machine, -1,
 251                                                     speq->tid);
 252         }
 253
 254         if (speq->thread) {
 255                 speq->pid = speq->thread->pid_;
 256                 if (queue->cpu == -1)
 257                         speq->cpu = speq->thread->cpu;
 258         }
 259 }
 260
 261 static int arm_spe_set_tid(struct arm_spe_queue *speq, pid_t tid)
 262 {
 263         struct arm_spe *spe = speq->spe;
 264         int err = machine__set_current_tid(spe->machine, speq->cpu, -1, tid);
 265
 266         if (err)
 267                 return err;
 268
 269         arm_spe_set_pid_tid_cpu(spe, &spe->queues.queue_array[speq->queue_nr]);
 270
 271         return 0;
 272 }
 273
 274 static void arm_spe_prep_sample(struct arm_spe *spe,
 275                                 struct arm_spe_queue *speq,
 276                                 union perf_event *event,
 277                                 struct perf_sample *sample)
 278 {
 279         struct arm_spe_record *record = &speq->decoder->record;
 280
 281         if (!spe->timeless_decoding)
 282                 sample->time = tsc_to_perf_time(record->timestamp, &spe->tc);
 283
 284         sample->ip = record->from_ip;
 285         sample->cpumode = arm_spe_cpumode(spe, sample->ip);
 286         sample->pid = speq->pid;
 287         sample->tid = speq->tid;
 288         sample->period = 1;
 289         sample->cpu = speq->cpu;
 290
 291         event->sample.header.type = PERF_RECORD_SAMPLE;
 292         event->sample.header.misc = sample->cpumode;
 293         event->sample.header.size = sizeof(struct perf_event_header);
 294 }
 295
 296 static int arm_spe__inject_event(union perf_event *event, struct perf_sample *sample, u64 type)
 297 {
 298         event->header.size = perf_event__sample_event_size(sample, type, 0);
 299         return perf_event__synthesize_sample(event, type, 0, sample);
 300 }
 301
 302 static inline int
 303 arm_spe_deliver_synth_event(struct arm_spe *spe,
 304                             struct arm_spe_queue *speq __maybe_unused,
 305                             union perf_event *event,
 306                             struct perf_sample *sample)
 307 {
 308         int ret;
 309
 310         if (spe->synth_opts.inject) {
 311                 ret = arm_spe__inject_event(event, sample, spe->sample_type);
 312                 if (ret)
 313                         return ret;
 314         }
 315
 316         ret = perf_session__deliver_synth_event(spe->session, event, sample);
 317         if (ret)
 318                 pr_err("ARM SPE: failed to deliver event, error %d\n", ret);
 319
 320         return ret;
 321 }
 322
 323 static int arm_spe__synth_mem_sample(struct arm_spe_queue *speq,
 324                                      u64 spe_events_id, u64 data_src)
 325 {
 326         struct arm_spe *spe = speq->spe;
 327         struct arm_spe_record *record = &speq->decoder->record;
 328         union perf_event *event = speq->event_buf;
 329         struct perf_sample sample = { .ip = 0, };
 330
 331         arm_spe_prep_sample(spe, speq, event, &sample);
 332
 333         sample.id = spe_events_id;
 334         sample.stream_id = spe_events_id;
 335         sample.addr = record->virt_addr;
 336         sample.phys_addr = record->phys_addr;
 337         sample.data_src = data_src;
 338         sample.weight = record->latency;
 339
 340         return arm_spe_deliver_synth_event(spe, speq, event, &sample);
 341 }
 342
 343 static int arm_spe__synth_branch_sample(struct arm_spe_queue *speq,
 344                                         u64 spe_events_id)
 345 {
 346         struct arm_spe *spe = speq->spe;
 347         struct arm_spe_record *record = &speq->decoder->record;
 348         union perf_event *event = speq->event_buf;
 349         struct perf_sample sample = { .ip = 0, };
 350
 351         arm_spe_prep_sample(spe, speq, event, &sample);
 352
 353         sample.id = spe_events_id;
 354         sample.stream_id = spe_events_id;
 355         sample.addr = record->to_ip;
 356         sample.weight = record->latency;
 357
 358         return arm_spe_deliver_synth_event(spe, speq, event, &sample);
 359 }
 360
 361 static int arm_spe__synth_instruction_sample(struct arm_spe_queue *speq,
 362                                              u64 spe_events_id, u64 data_src)
 363 {
 364         struct arm_spe *spe = speq->spe;
 365         struct arm_spe_record *record = &speq->decoder->record;
 366         union perf_event *event = speq->event_buf;
 367         struct perf_sample sample = { .ip = 0, };
 368
 369         /*
 370          * Handles perf instruction sampling period.
 371          */
 372         speq->period_instructions++;
 373         if (speq->period_instructions < spe->instructions_sample_period)
 374                 return 0;
 375         speq->period_instructions = 0;
 376
 377         arm_spe_prep_sample(spe, speq, event, &sample);
 378
 379         sample.id = spe_events_id;
 380         sample.stream_id = spe_events_id;
 381         sample.addr = record->virt_addr;
 382         sample.phys_addr = record->phys_addr;
 383         sample.data_src = data_src;
 384         sample.period = spe->instructions_sample_period;
 385         sample.weight = record->latency;
 386
 387         return arm_spe_deliver_synth_event(spe, speq, event, &sample);
 388 }
 389
 390 static u64 arm_spe__synth_data_source(const struct arm_spe_record *record)
 391 {
 392         union perf_mem_data_src data_src = { 0 };
 393
 394         if (record->op == ARM_SPE_LD)
 395                 data_src.mem_op = PERF_MEM_OP_LOAD;
 396         else if (record->op == ARM_SPE_ST)
 397                 data_src.mem_op = PERF_MEM_OP_STORE;
 398         else
 399                 return 0;
 400
 401         if (record->type & (ARM_SPE_LLC_ACCESS | ARM_SPE_LLC_MISS)) {
 402                 data_src.mem_lvl = PERF_MEM_LVL_L3;
 403
 404                 if (record->type & ARM_SPE_LLC_MISS)
 405                         data_src.mem_lvl |= PERF_MEM_LVL_MISS;
 406                 else
 407                         data_src.mem_lvl |= PERF_MEM_LVL_HIT;
 408         } else if (record->type & (ARM_SPE_L1D_ACCESS | ARM_SPE_L1D_MISS)) {
 409                 data_src.mem_lvl = PERF_MEM_LVL_L1;
 410
 411                 if (record->type & ARM_SPE_L1D_MISS)
 412                         data_src.mem_lvl |= PERF_MEM_LVL_MISS;
 413                 else
 414                         data_src.mem_lvl |= PERF_MEM_LVL_HIT;
 415         }
 416
 417         if (record->type & ARM_SPE_REMOTE_ACCESS)
 418                 data_src.mem_lvl |= PERF_MEM_LVL_REM_CCE1;
 419
 420         if (record->type & (ARM_SPE_TLB_ACCESS | ARM_SPE_TLB_MISS)) {
 421                 data_src.mem_dtlb = PERF_MEM_TLB_WK;
 422
 423                 if (record->type & ARM_SPE_TLB_MISS)
 424                         data_src.mem_dtlb |= PERF_MEM_TLB_MISS;
 425                 else
 426                         data_src.mem_dtlb |= PERF_MEM_TLB_HIT;
 427         }
 428
 429         return data_src.val;
 430 }
 431
 432 static int arm_spe_sample(struct arm_spe_queue *speq)
 433 {
 434         const struct arm_spe_record *record = &speq->decoder->record;
 435         struct arm_spe *spe = speq->spe;
 436         u64 data_src;
 437         int err;
 438
 439         data_src = arm_spe__synth_data_source(record);
 440
 441         if (spe->sample_flc) {
 442                 if (record->type & ARM_SPE_L1D_MISS) {
 443                         err = arm_spe__synth_mem_sample(speq, spe->l1d_miss_id,
 444                                                         data_src);
 445                         if (err)
 446                                 return err;
 447                 }
 448
 449                 if (record->type & ARM_SPE_L1D_ACCESS) {
 450                         err = arm_spe__synth_mem_sample(speq, spe->l1d_access_id,
 451                                                         data_src);
 452                         if (err)
 453                                 return err;
 454                 }
 455         }
 456
 457         if (spe->sample_llc) {
 458                 if (record->type & ARM_SPE_LLC_MISS) {
 459                         err = arm_spe__synth_mem_sample(speq, spe->llc_miss_id,
 460                                                         data_src);
 461                         if (err)
 462                                 return err;
 463                 }
 464
 465                 if (record->type & ARM_SPE_LLC_ACCESS) {
 466                         err = arm_spe__synth_mem_sample(speq, spe->llc_access_id,
 467                                                         data_src);
 468                         if (err)
 469                                 return err;
 470                 }
 471         }
 472
 473         if (spe->sample_tlb) {
 474                 if (record->type & ARM_SPE_TLB_MISS) {
 475                         err = arm_spe__synth_mem_sample(speq, spe->tlb_miss_id,
 476                                                         data_src);
 477                         if (err)
 478                                 return err;
 479                 }
 480
 481                 if (record->type & ARM_SPE_TLB_ACCESS) {
 482                         err = arm_spe__synth_mem_sample(speq, spe->tlb_access_id,
 483                                                         data_src);
 484                         if (err)
 485                                 return err;
 486                 }
 487         }
 488
 489         if (spe->sample_branch && (record->type & ARM_SPE_BRANCH_MISS)) {
 490                 err = arm_spe__synth_branch_sample(speq, spe->branch_miss_id);
 491                 if (err)
 492                         return err;
 493         }
 494
 495         if (spe->sample_remote_access &&
 496             (record->type & ARM_SPE_REMOTE_ACCESS)) {
 497                 err = arm_spe__synth_mem_sample(speq, spe->remote_access_id,
 498                                                 data_src);
 499                 if (err)
 500                         return err;
 501         }
 502
 503         /*
 504          * When data_src is zero it means the record is not a memory operation,
 505          * skip to synthesize memory sample for this case.
 506          */
 507         if (spe->sample_memory && data_src) {
 508                 err = arm_spe__synth_mem_sample(speq, spe->memory_id, data_src);
 509                 if (err)
 510                         return err;
 511         }
 512
 513         if (spe->sample_instructions) {
 514                 err = arm_spe__synth_instruction_sample(speq, spe->instructions_id, data_src);
 515                 if (err)
 516                         return err;
 517         }
 518
 519         return 0;
 520 }
 521
 522 static int arm_spe_run_decoder(struct arm_spe_queue *speq, u64 *timestamp)
 523 {
 524         struct arm_spe *spe = speq->spe;
 525         struct arm_spe_record *record;
 526         int ret;
 527
 528         if (!spe->kernel_start)
 529                 spe->kernel_start = machine__kernel_start(spe->machine);
 530
 531         while (1) {
 532                 /*
 533                  * The usual logic is firstly to decode the packets, and then
 534                  * based the record to synthesize sample; but here the flow is
 535                  * reversed: it calls arm_spe_sample() for synthesizing samples
 536                  * prior to arm_spe_decode().
 537                  *
 538                  * Two reasons for this code logic:
 539                  * 1. Firstly, when setup queue in arm_spe__setup_queue(), it
 540                  * has decoded trace data and generated a record, but the record
 541                  * is left to generate sample until run to here, so it's correct
 542                  * to synthesize sample for the left record.
 543                  * 2. After decoding trace data, it needs to compare the record
 544                  * timestamp with the coming perf event, if the record timestamp
 545                  * is later than the perf event, it needs bail out and pushs the
 546                  * record into auxtrace heap, thus the record can be deferred to
 547                  * synthesize sample until run to here at the next time; so this
 548                  * can correlate samples between Arm SPE trace data and other
 549                  * perf events with correct time ordering.
 550                  */
 551
 552                 /*
 553                  * Update pid/tid info.
 554                  */
 555                 record = &speq->decoder->record;
 556                 if (!spe->timeless_decoding && record->context_id != (u64)-1) {
 557                         ret = arm_spe_set_tid(speq, record->context_id);
 558                         if (ret)
 559                                 return ret;
 560
 561                         spe->use_ctx_pkt_for_pid = true;
 562                 }
 563
 564                 ret = arm_spe_sample(speq);
 565                 if (ret)
 566                         return ret;
 567
 568                 ret = arm_spe_decode(speq->decoder);
 569                 if (!ret) {
 570                         pr_debug("No data or all data has been processed.\n");
 571                         return 1;
 572                 }
 573
 574                 /*
 575                  * Error is detected when decode SPE trace data, continue to
 576                  * the next trace data and find out more records.
 577                  */
 578                 if (ret < 0)
 579                         continue;
 580
 581                 record = &speq->decoder->record;
 582
 583                 /* Update timestamp for the last record */
 584                 if (record->timestamp > speq->timestamp)
 585                         speq->timestamp = record->timestamp;
 586
 587                 /*
 588                  * If the timestamp of the queue is later than timestamp of the
 589                  * coming perf event, bail out so can allow the perf event to
 590                  * be processed ahead.
 591                  */
 592                 if (!spe->timeless_decoding && speq->timestamp >= *timestamp) {
 593                         *timestamp = speq->timestamp;
 594                         return 0;
 595                 }
 596         }
 597
 598         return 0;
 599 }
 600
 601 static int arm_spe__setup_queue(struct arm_spe *spe,
 602                                struct auxtrace_queue *queue,
 603                                unsigned int queue_nr)
 604 {
 605         struct arm_spe_queue *speq = queue->priv;
 606         struct arm_spe_record *record;
 607
 608         if (list_empty(&queue->head) || speq)
 609                 return 0;
 610
 611         speq = arm_spe__alloc_queue(spe, queue_nr);
 612
 613         if (!speq)
 614                 return -ENOMEM;
 615
 616         queue->priv = speq;
 617
 618         if (queue->cpu != -1)
 619                 speq->cpu = queue->cpu;
 620
 621         if (!speq->on_heap) {
 622                 int ret;
 623
 624                 if (spe->timeless_decoding)
 625                         return 0;
 626
 627 retry:
 628                 ret = arm_spe_decode(speq->decoder);
 629
 630                 if (!ret)
 631                         return 0;
 632
 633                 if (ret < 0)
 634                         goto retry;
 635
 636                 record = &speq->decoder->record;
 637
 638                 speq->timestamp = record->timestamp;
 639                 ret = auxtrace_heap__add(&spe->heap, queue_nr, speq->timestamp);
 640                 if (ret)
 641                         return ret;
 642                 speq->on_heap = true;
 643         }
 644
 645         return 0;
 646 }
 647
 648 static int arm_spe__setup_queues(struct arm_spe *spe)
 649 {
 650         unsigned int i;
 651         int ret;
 652
 653         for (i = 0; i < spe->queues.nr_queues; i++) {
 654                 ret = arm_spe__setup_queue(spe, &spe->queues.queue_array[i], i);
 655                 if (ret)
 656                         return ret;
 657         }
 658
 659         return 0;
 660 }
 661
 662 static int arm_spe__update_queues(struct arm_spe *spe)
 663 {
 664         if (spe->queues.new_data) {
 665                 spe->queues.new_data = false;
 666                 return arm_spe__setup_queues(spe);
 667         }
 668
 669         return 0;
 670 }
 671
 672 static bool arm_spe__is_timeless_decoding(struct arm_spe *spe)
 673 {
 674         struct evsel *evsel;
 675         struct evlist *evlist = spe->session->evlist;
 676         bool timeless_decoding = true;
 677
 678         /*
 679          * Circle through the list of event and complain if we find one
 680          * with the time bit set.
 681          */
 682         evlist__for_each_entry(evlist, evsel) {
 683                 if ((evsel->core.attr.sample_type & PERF_SAMPLE_TIME))
 684                         timeless_decoding = false;
 685         }
 686
 687         return timeless_decoding;
 688 }
 689
 690 static int arm_spe_process_queues(struct arm_spe *spe, u64 timestamp)
 691 {
 692         unsigned int queue_nr;
 693         u64 ts;
 694         int ret;
 695
 696         while (1) {
 697                 struct auxtrace_queue *queue;
 698                 struct arm_spe_queue *speq;
 699
 700                 if (!spe->heap.heap_cnt)
 701                         return 0;
 702
 703                 if (spe->heap.heap_array[0].ordinal >= timestamp)
 704                         return 0;
 705
 706                 queue_nr = spe->heap.heap_array[0].queue_nr;
 707                 queue = &spe->queues.queue_array[queue_nr];
 708                 speq = queue->priv;
 709
 710                 auxtrace_heap__pop(&spe->heap);
 711
 712                 if (spe->heap.heap_cnt) {
 713                         ts = spe->heap.heap_array[0].ordinal + 1;
 714                         if (ts > timestamp)
 715                                 ts = timestamp;
 716                 } else {
 717                         ts = timestamp;
 718                 }
 719
 720                 /*
 721                  * A previous context-switch event has set pid/tid in the machine's context, so
 722                  * here we need to update the pid/tid in the thread and SPE queue.
 723                  */
 724                 if (!spe->use_ctx_pkt_for_pid)
 725                         arm_spe_set_pid_tid_cpu(spe, queue);
 726
 727                 ret = arm_spe_run_decoder(speq, &ts);
 728                 if (ret < 0) {
 729                         auxtrace_heap__add(&spe->heap, queue_nr, ts);
 730                         return ret;
 731                 }
 732
 733                 if (!ret) {
 734                         ret = auxtrace_heap__add(&spe->heap, queue_nr, ts);
 735                         if (ret < 0)
 736                                 return ret;
 737                 } else {
 738                         speq->on_heap = false;
 739                 }
 740         }
 741
 742         return 0;
 743 }
 744
 745 static int arm_spe_process_timeless_queues(struct arm_spe *spe, pid_t tid,
 746                                             u64 time_)
 747 {
 748         struct auxtrace_queues *queues = &spe->queues;
 749         unsigned int i;
 750         u64 ts = 0;
 751
 752         for (i = 0; i < queues->nr_queues; i++) {
 753                 struct auxtrace_queue *queue = &spe->queues.queue_array[i];
 754                 struct arm_spe_queue *speq = queue->priv;
 755
 756                 if (speq && (tid == -1 || speq->tid == tid)) {
 757                         speq->time = time_;
 758                         arm_spe_set_pid_tid_cpu(spe, queue);
 759                         arm_spe_run_decoder(speq, &ts);
 760                 }
 761         }
 762         return 0;
 763 }
 764
 765 static int arm_spe_context_switch(struct arm_spe *spe, union perf_event *event,
 766                                   struct perf_sample *sample)
 767 {
 768         pid_t pid, tid;
 769         int cpu;
 770
 771         if (!(event->header.misc & PERF_RECORD_MISC_SWITCH_OUT))
 772                 return 0;
 773
 774         pid = event->context_switch.next_prev_pid;
 775         tid = event->context_switch.next_prev_tid;
 776         cpu = sample->cpu;
 777
 778         if (tid == -1)
 779                 pr_warning("context_switch event has no tid\n");
 780
 781         return machine__set_current_tid(spe->machine, cpu, pid, tid);
 782 }
 783
 784 static int arm_spe_process_event(struct perf_session *session,
 785                                  union perf_event *event,
 786                                  struct perf_sample *sample,
 787                                  struct perf_tool *tool)
 788 {
 789         int err = 0;
 790         u64 timestamp;
 791         struct arm_spe *spe = container_of(session->auxtrace,
 792                         struct arm_spe, auxtrace);
 793
 794         if (dump_trace)
 795                 return 0;
 796
 797         if (!tool->ordered_events) {
 798                 pr_err("SPE trace requires ordered events\n");
 799                 return -EINVAL;
 800         }
 801
 802         if (sample->time && (sample->time != (u64) -1))
 803                 timestamp = perf_time_to_tsc(sample->time, &spe->tc);
 804         else
 805                 timestamp = 0;
 806
 807         if (timestamp || spe->timeless_decoding) {
 808                 err = arm_spe__update_queues(spe);
 809                 if (err)
 810                         return err;
 811         }
 812
 813         if (spe->timeless_decoding) {
 814                 if (event->header.type == PERF_RECORD_EXIT) {
 815                         err = arm_spe_process_timeless_queues(spe,
 816                                         event->fork.tid,
 817                                         sample->time);
 818                 }
 819         } else if (timestamp) {
 820                 err = arm_spe_process_queues(spe, timestamp);
 821                 if (err)
 822                         return err;
 823
 824                 if (!spe->use_ctx_pkt_for_pid &&
 825                     (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE ||
 826                     event->header.type == PERF_RECORD_SWITCH))
 827                         err = arm_spe_context_switch(spe, event, sample);
 828         }
 829
 830         return err;
 831 }
 832
 833 static int arm_spe_process_auxtrace_event(struct perf_session *session,
 834                                           union perf_event *event,
 835                                           struct perf_tool *tool __maybe_unused)
 836 {
 837         struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
 838                                              auxtrace);
 839
 840         if (!spe->data_queued) {
 841                 struct auxtrace_buffer *buffer;
 842                 off_t data_offset;
 843                 int fd = perf_data__fd(session->data);
 844                 int err;
 845
 846                 if (perf_data__is_pipe(session->data)) {
 847                         data_offset = 0;
 848                 } else {
 849                         data_offset = lseek(fd, 0, SEEK_CUR);
 850                         if (data_offset == -1)
 851                                 return -errno;
 852                 }
 853
 854                 err = auxtrace_queues__add_event(&spe->queues, session, event,
 855                                 data_offset, &buffer);
 856                 if (err)
 857                         return err;
 858
 859                 /* Dump here now we have copied a piped trace out of the pipe */
 860                 if (dump_trace) {
 861                         if (auxtrace_buffer__get_data(buffer, fd)) {
 862                                 arm_spe_dump_event(spe, buffer->data,
 863                                                 buffer->size);
 864                                 auxtrace_buffer__put_data(buffer);
 865                         }
 866                 }
 867         }
 868
 869         return 0;
 870 }
 871
 872 static int arm_spe_flush(struct perf_session *session __maybe_unused,
 873                          struct perf_tool *tool __maybe_unused)
 874 {
 875         struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
 876                         auxtrace);
 877         int ret;
 878
 879         if (dump_trace)
 880                 return 0;
 881
 882         if (!tool->ordered_events)
 883                 return -EINVAL;
 884
 885         ret = arm_spe__update_queues(spe);
 886         if (ret < 0)
 887                 return ret;
 888
 889         if (spe->timeless_decoding)
 890                 return arm_spe_process_timeless_queues(spe, -1,
 891                                 MAX_TIMESTAMP - 1);
 892
 893         ret = arm_spe_process_queues(spe, MAX_TIMESTAMP);
 894         if (ret)
 895                 return ret;
 896
 897         if (!spe->use_ctx_pkt_for_pid)
 898                 ui__warning("Arm SPE CONTEXT packets not found in the traces.\n"
 899                             "Matching of TIDs to SPE events could be inaccurate.\n");
 900
 901         return 0;
 902 }
 903
 904 static void arm_spe_free_queue(void *priv)
 905 {
 906         struct arm_spe_queue *speq = priv;
 907
 908         if (!speq)
 909                 return;
 910         thread__zput(speq->thread);
 911         arm_spe_decoder_free(speq->decoder);
 912         zfree(&speq->event_buf);
 913         free(speq);
 914 }
 915
 916 static void arm_spe_free_events(struct perf_session *session)
 917 {
 918         struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
 919                                              auxtrace);
 920         struct auxtrace_queues *queues = &spe->queues;
 921         unsigned int i;
 922
 923         for (i = 0; i < queues->nr_queues; i++) {
 924                 arm_spe_free_queue(queues->queue_array[i].priv);
 925                 queues->queue_array[i].priv = NULL;
 926         }
 927         auxtrace_queues__free(queues);
 928 }
 929
 930 static void arm_spe_free(struct perf_session *session)
 931 {
 932         struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
 933                                              auxtrace);
 934
 935         auxtrace_heap__free(&spe->heap);
 936         arm_spe_free_events(session);
 937         session->auxtrace = NULL;
 938         free(spe);
 939 }
 940
 941 static bool arm_spe_evsel_is_auxtrace(struct perf_session *session,
 942                                       struct evsel *evsel)
 943 {
 944         struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, auxtrace);
 945
 946         return evsel->core.attr.type == spe->pmu_type;
 947 }
 948
 949 static const char * const arm_spe_info_fmts[] = {
 950         [ARM_SPE_PMU_TYPE]              = "  PMU Type           %"PRId64"\n",
 951 };
 952
 953 static void arm_spe_print_info(__u64 *arr)
 954 {
 955         if (!dump_trace)
 956                 return;
 957
 958         fprintf(stdout, arm_spe_info_fmts[ARM_SPE_PMU_TYPE], arr[ARM_SPE_PMU_TYPE]);
 959 }
 960
 961 struct arm_spe_synth {
 962         struct perf_tool dummy_tool;
 963         struct perf_session *session;
 964 };
 965
 966 static int arm_spe_event_synth(struct perf_tool *tool,
 967                                union perf_event *event,
 968                                struct perf_sample *sample __maybe_unused,
 969                                struct machine *machine __maybe_unused)
 970 {
 971         struct arm_spe_synth *arm_spe_synth =
 972                       container_of(tool, struct arm_spe_synth, dummy_tool);
 973
 974         return perf_session__deliver_synth_event(arm_spe_synth->session,
 975                                                  event, NULL);
 976 }
 977
 978 static int arm_spe_synth_event(struct perf_session *session,
 979                                struct perf_event_attr *attr, u64 id)
 980 {
 981         struct arm_spe_synth arm_spe_synth;
 982
 983         memset(&arm_spe_synth, 0, sizeof(struct arm_spe_synth));
 984         arm_spe_synth.session = session;
 985
 986         return perf_event__synthesize_attr(&arm_spe_synth.dummy_tool, attr, 1,
 987                                            &id, arm_spe_event_synth);
 988 }
 989
 990 static void arm_spe_set_event_name(struct evlist *evlist, u64 id,
 991                                     const char *name)
 992 {
 993         struct evsel *evsel;
 994
 995         evlist__for_each_entry(evlist, evsel) {
 996                 if (evsel->core.id && evsel->core.id[0] == id) {
 997                         if (evsel->name)
 998                                 zfree(&evsel->name);
 999                         evsel->name = strdup(name);
1000                         break;
1001                 }
1002         }
1003 }
1004
1005 static int
1006 arm_spe_synth_events(struct arm_spe *spe, struct perf_session *session)
1007 {
1008         struct evlist *evlist = session->evlist;
1009         struct evsel *evsel;
1010         struct perf_event_attr attr;
1011         bool found = false;
1012         u64 id;
1013         int err;
1014
1015         evlist__for_each_entry(evlist, evsel) {
1016                 if (evsel->core.attr.type == spe->pmu_type) {
1017                         found = true;
1018                         break;
1019                 }
1020         }
1021
1022         if (!found) {
1023                 pr_debug("No selected events with SPE trace data\n");
1024                 return 0;
1025         }
1026
1027         memset(&attr, 0, sizeof(struct perf_event_attr));
1028         attr.size = sizeof(struct perf_event_attr);
1029         attr.type = PERF_TYPE_HARDWARE;
1030         attr.sample_type = evsel->core.attr.sample_type &
1031                                 (PERF_SAMPLE_MASK | PERF_SAMPLE_PHYS_ADDR);
1032         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1033                             PERF_SAMPLE_PERIOD | PERF_SAMPLE_DATA_SRC |
1034                             PERF_SAMPLE_WEIGHT | PERF_SAMPLE_ADDR;
1035         if (spe->timeless_decoding)
1036                 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1037         else
1038                 attr.sample_type |= PERF_SAMPLE_TIME;
1039
1040         spe->sample_type = attr.sample_type;
1041
1042         attr.exclude_user = evsel->core.attr.exclude_user;
1043         attr.exclude_kernel = evsel->core.attr.exclude_kernel;
1044         attr.exclude_hv = evsel->core.attr.exclude_hv;
1045         attr.exclude_host = evsel->core.attr.exclude_host;
1046         attr.exclude_guest = evsel->core.attr.exclude_guest;
1047         attr.sample_id_all = evsel->core.attr.sample_id_all;
1048         attr.read_format = evsel->core.attr.read_format;
1049
1050         /* create new id val to be a fixed offset from evsel id */
1051         id = evsel->core.id[0] + 1000000000;
1052
1053         if (!id)
1054                 id = 1;
1055
1056         if (spe->synth_opts.flc) {
1057                 spe->sample_flc = true;
1058
1059                 /* Level 1 data cache miss */
1060                 err = arm_spe_synth_event(session, &attr, id);
1061                 if (err)
1062                         return err;
1063                 spe->l1d_miss_id = id;
1064                 arm_spe_set_event_name(evlist, id, "l1d-miss");
1065                 id += 1;
1066
1067                 /* Level 1 data cache access */
1068                 err = arm_spe_synth_event(session, &attr, id);
1069                 if (err)
1070                         return err;
1071                 spe->l1d_access_id = id;
1072                 arm_spe_set_event_name(evlist, id, "l1d-access");
1073                 id += 1;
1074         }
1075
1076         if (spe->synth_opts.llc) {
1077                 spe->sample_llc = true;
1078
1079                 /* Last level cache miss */
1080                 err = arm_spe_synth_event(session, &attr, id);
1081                 if (err)
1082                         return err;
1083                 spe->llc_miss_id = id;
1084                 arm_spe_set_event_name(evlist, id, "llc-miss");
1085                 id += 1;
1086
1087                 /* Last level cache access */
1088                 err = arm_spe_synth_event(session, &attr, id);
1089                 if (err)
1090                         return err;
1091                 spe->llc_access_id = id;
1092                 arm_spe_set_event_name(evlist, id, "llc-access");
1093                 id += 1;
1094         }
1095
1096         if (spe->synth_opts.tlb) {
1097                 spe->sample_tlb = true;
1098
1099                 /* TLB miss */
1100                 err = arm_spe_synth_event(session, &attr, id);
1101                 if (err)
1102                         return err;
1103                 spe->tlb_miss_id = id;
1104                 arm_spe_set_event_name(evlist, id, "tlb-miss");
1105                 id += 1;
1106
1107                 /* TLB access */
1108                 err = arm_spe_synth_event(session, &attr, id);
1109                 if (err)
1110                         return err;
1111                 spe->tlb_access_id = id;
1112                 arm_spe_set_event_name(evlist, id, "tlb-access");
1113                 id += 1;
1114         }
1115
1116         if (spe->synth_opts.branches) {
1117                 spe->sample_branch = true;
1118
1119                 /* Branch miss */
1120                 err = arm_spe_synth_event(session, &attr, id);
1121                 if (err)
1122                         return err;
1123                 spe->branch_miss_id = id;
1124                 arm_spe_set_event_name(evlist, id, "branch-miss");
1125                 id += 1;
1126         }
1127
1128         if (spe->synth_opts.remote_access) {
1129                 spe->sample_remote_access = true;
1130
1131                 /* Remote access */
1132                 err = arm_spe_synth_event(session, &attr, id);
1133                 if (err)
1134                         return err;
1135                 spe->remote_access_id = id;
1136                 arm_spe_set_event_name(evlist, id, "remote-access");
1137                 id += 1;
1138         }
1139
1140         if (spe->synth_opts.mem) {
1141                 spe->sample_memory = true;
1142
1143                 err = arm_spe_synth_event(session, &attr, id);
1144                 if (err)
1145                         return err;
1146                 spe->memory_id = id;
1147                 arm_spe_set_event_name(evlist, id, "memory");
1148                 id += 1;
1149         }
1150
1151         if (spe->synth_opts.instructions) {
1152                 if (spe->synth_opts.period_type != PERF_ITRACE_PERIOD_INSTRUCTIONS) {
1153                         pr_warning("Only instruction-based sampling period is currently supported by Arm SPE.\n");
1154                         goto synth_instructions_out;
1155                 }
1156                 if (spe->synth_opts.period > 1)
1157                         pr_warning("Arm SPE has a hardware-based sample period.\n"
1158                                    "Additional instruction events will be discarded by --itrace\n");
1159
1160                 spe->sample_instructions = true;
1161                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1162                 attr.sample_period = spe->synth_opts.period;
1163                 spe->instructions_sample_period = attr.sample_period;
1164                 err = arm_spe_synth_event(session, &attr, id);
1165                 if (err)
1166                         return err;
1167                 spe->instructions_id = id;
1168                 arm_spe_set_event_name(evlist, id, "instructions");
1169         }
1170 synth_instructions_out:
1171
1172         return 0;
1173 }
1174
1175 int arm_spe_process_auxtrace_info(union perf_event *event,
1176                                   struct perf_session *session)
1177 {
1178         struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
1179         size_t min_sz = sizeof(u64) * ARM_SPE_AUXTRACE_PRIV_MAX;
1180         struct perf_record_time_conv *tc = &session->time_conv;
1181         struct arm_spe *spe;
1182         int err;
1183
1184         if (auxtrace_info->header.size < sizeof(struct perf_record_auxtrace_info) +
1185                                         min_sz)
1186                 return -EINVAL;
1187
1188         spe = zalloc(sizeof(struct arm_spe));
1189         if (!spe)
1190                 return -ENOMEM;
1191
1192         err = auxtrace_queues__init(&spe->queues);
1193         if (err)
1194                 goto err_free;
1195
1196         spe->session = session;
1197         spe->machine = &session->machines.host; /* No kvm support */
1198         spe->auxtrace_type = auxtrace_info->type;
1199         spe->pmu_type = auxtrace_info->priv[ARM_SPE_PMU_TYPE];
1200
1201         spe->timeless_decoding = arm_spe__is_timeless_decoding(spe);
1202
1203         /*
1204          * The synthesized event PERF_RECORD_TIME_CONV has been handled ahead
1205          * and the parameters for hardware clock are stored in the session
1206          * context.  Passes these parameters to the struct perf_tsc_conversion
1207          * in "spe->tc", which is used for later conversion between clock
1208          * counter and timestamp.
1209          *
1210          * For backward compatibility, copies the fields starting from
1211          * "time_cycles" only if they are contained in the event.
1212          */
1213         spe->tc.time_shift = tc->time_shift;
1214         spe->tc.time_mult = tc->time_mult;
1215         spe->tc.time_zero = tc->time_zero;
1216
1217         if (event_contains(*tc, time_cycles)) {
1218                 spe->tc.time_cycles = tc->time_cycles;
1219                 spe->tc.time_mask = tc->time_mask;
1220                 spe->tc.cap_user_time_zero = tc->cap_user_time_zero;
1221                 spe->tc.cap_user_time_short = tc->cap_user_time_short;
1222         }
1223
1224         spe->auxtrace.process_event = arm_spe_process_event;
1225         spe->auxtrace.process_auxtrace_event = arm_spe_process_auxtrace_event;
1226         spe->auxtrace.flush_events = arm_spe_flush;
1227         spe->auxtrace.free_events = arm_spe_free_events;
1228         spe->auxtrace.free = arm_spe_free;
1229         spe->auxtrace.evsel_is_auxtrace = arm_spe_evsel_is_auxtrace;
1230         session->auxtrace = &spe->auxtrace;
1231
1232         arm_spe_print_info(&auxtrace_info->priv[0]);
1233
1234         if (dump_trace)
1235                 return 0;
1236
1237         if (session->itrace_synth_opts && session->itrace_synth_opts->set)
1238                 spe->synth_opts = *session->itrace_synth_opts;
1239         else
1240                 itrace_synth_opts__set_default(&spe->synth_opts, false);
1241
1242         err = arm_spe_synth_events(spe, session);
1243         if (err)
1244                 goto err_free_queues;
1245
1246         err = auxtrace_queues__process_index(&spe->queues, session);
1247         if (err)
1248                 goto err_free_queues;
1249
1250         if (spe->queues.populated)
1251                 spe->data_queued = true;
1252
1253         return 0;
1254
1255 err_free_queues:
1256         auxtrace_queues__free(&spe->queues);
1257         session->auxtrace = NULL;
1258 err_free:
1259         free(spe);
1260         return err;
1261 }