tools/perf/util/cs-etm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright(C) 2015-2018 Linaro Limited.
   4  *
   5  * Author: Tor Jeremiassen <tor@ti.com>
   6  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
   7  */
   8
   9 #include <linux/bitops.h>
  10 #include <linux/err.h>
  11 #include <linux/kernel.h>
  12 #include <linux/log2.h>
  13 #include <linux/types.h>
  14
  15 #include <opencsd/ocsd_if_types.h>
  16 #include <stdlib.h>
  17
  18 #include "auxtrace.h"
  19 #include "color.h"
  20 #include "cs-etm.h"
  21 #include "cs-etm-decoder/cs-etm-decoder.h"
  22 #include "debug.h"
  23 #include "evlist.h"
  24 #include "intlist.h"
  25 #include "machine.h"
  26 #include "map.h"
  27 #include "perf.h"
  28 #include "symbol.h"
  29 #include "thread.h"
  30 #include "thread_map.h"
  31 #include "thread-stack.h"
  32 #include "util.h"
  33
  34 #define MAX_TIMESTAMP (~0ULL)
  35
  36 struct cs_etm_auxtrace {
  37         struct auxtrace auxtrace;
  38         struct auxtrace_queues queues;
  39         struct auxtrace_heap heap;
  40         struct itrace_synth_opts synth_opts;
  41         struct perf_session *session;
  42         struct machine *machine;
  43         struct thread *unknown_thread;
  44
  45         u8 timeless_decoding;
  46         u8 snapshot_mode;
  47         u8 data_queued;
  48         u8 sample_branches;
  49         u8 sample_instructions;
  50
  51         int num_cpu;
  52         u32 auxtrace_type;
  53         u64 branches_sample_type;
  54         u64 branches_id;
  55         u64 instructions_sample_type;
  56         u64 instructions_sample_period;
  57         u64 instructions_id;
  58         u64 **metadata;
  59         u64 kernel_start;
  60         unsigned int pmu_type;
  61 };
  62
  63 struct cs_etm_traceid_queue {
  64         u8 trace_chan_id;
  65         pid_t pid, tid;
  66         u64 period_instructions;
  67         size_t last_branch_pos;
  68         union perf_event *event_buf;
  69         struct thread *thread;
  70         struct branch_stack *last_branch;
  71         struct branch_stack *last_branch_rb;
  72         struct cs_etm_packet *prev_packet;
  73         struct cs_etm_packet *packet;
  74         struct cs_etm_packet_queue packet_queue;
  75 };
  76
  77 struct cs_etm_queue {
  78         struct cs_etm_auxtrace *etm;
  79         struct cs_etm_decoder *decoder;
  80         struct auxtrace_buffer *buffer;
  81         unsigned int queue_nr;
  82         u64 offset;
  83         const unsigned char *buf;
  84         size_t buf_len, buf_used;
  85         struct cs_etm_traceid_queue *traceid_queues;
  86 };
  87
  88 static int cs_etm__update_queues(struct cs_etm_auxtrace *etm);
  89 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
  90                                            pid_t tid);
  91
  92 /* PTMs ETMIDR [11:8] set to b0011 */
  93 #define ETMIDR_PTM_VERSION 0x00000300
  94
  95 static u32 cs_etm__get_v7_protocol_version(u32 etmidr)
  96 {
  97         etmidr &= ETMIDR_PTM_VERSION;
  98
  99         if (etmidr == ETMIDR_PTM_VERSION)
 100                 return CS_ETM_PROTO_PTM;
 101
 102         return CS_ETM_PROTO_ETMV3;
 103 }
 104
 105 static int cs_etm__get_magic(u8 trace_chan_id, u64 *magic)
 106 {
 107         struct int_node *inode;
 108         u64 *metadata;
 109
 110         inode = intlist__find(traceid_list, trace_chan_id);
 111         if (!inode)
 112                 return -EINVAL;
 113
 114         metadata = inode->priv;
 115         *magic = metadata[CS_ETM_MAGIC];
 116         return 0;
 117 }
 118
 119 int cs_etm__get_cpu(u8 trace_chan_id, int *cpu)
 120 {
 121         struct int_node *inode;
 122         u64 *metadata;
 123
 124         inode = intlist__find(traceid_list, trace_chan_id);
 125         if (!inode)
 126                 return -EINVAL;
 127
 128         metadata = inode->priv;
 129         *cpu = (int)metadata[CS_ETM_CPU];
 130         return 0;
 131 }
 132
 133 static void cs_etm__clear_packet_queue(struct cs_etm_packet_queue *queue)
 134 {
 135         int i;
 136
 137         queue->head = 0;
 138         queue->tail = 0;
 139         queue->packet_count = 0;
 140         for (i = 0; i < CS_ETM_PACKET_MAX_BUFFER; i++) {
 141                 queue->packet_buffer[i].isa = CS_ETM_ISA_UNKNOWN;
 142                 queue->packet_buffer[i].start_addr = CS_ETM_INVAL_ADDR;
 143                 queue->packet_buffer[i].end_addr = CS_ETM_INVAL_ADDR;
 144                 queue->packet_buffer[i].instr_count = 0;
 145                 queue->packet_buffer[i].last_instr_taken_branch = false;
 146                 queue->packet_buffer[i].last_instr_size = 0;
 147                 queue->packet_buffer[i].last_instr_type = 0;
 148                 queue->packet_buffer[i].last_instr_subtype = 0;
 149                 queue->packet_buffer[i].last_instr_cond = 0;
 150                 queue->packet_buffer[i].flags = 0;
 151                 queue->packet_buffer[i].exception_number = UINT32_MAX;
 152                 queue->packet_buffer[i].trace_chan_id = UINT8_MAX;
 153                 queue->packet_buffer[i].cpu = INT_MIN;
 154         }
 155 }
 156
 157 static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq,
 158                                       struct cs_etm_traceid_queue *tidq,
 159                                       u8 trace_chan_id)
 160 {
 161         int rc = -ENOMEM;
 162         struct auxtrace_queue *queue;
 163         struct cs_etm_auxtrace *etm = etmq->etm;
 164
 165         cs_etm__clear_packet_queue(&tidq->packet_queue);
 166
 167         queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
 168         tidq->tid = queue->tid;
 169         tidq->pid = -1;
 170         tidq->trace_chan_id = trace_chan_id;
 171
 172         tidq->packet = zalloc(sizeof(struct cs_etm_packet));
 173         if (!tidq->packet)
 174                 goto out;
 175
 176         tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet));
 177         if (!tidq->prev_packet)
 178                 goto out_free;
 179
 180         if (etm->synth_opts.last_branch) {
 181                 size_t sz = sizeof(struct branch_stack);
 182
 183                 sz += etm->synth_opts.last_branch_sz *
 184                       sizeof(struct branch_entry);
 185                 tidq->last_branch = zalloc(sz);
 186                 if (!tidq->last_branch)
 187                         goto out_free;
 188                 tidq->last_branch_rb = zalloc(sz);
 189                 if (!tidq->last_branch_rb)
 190                         goto out_free;
 191         }
 192
 193         tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
 194         if (!tidq->event_buf)
 195                 goto out_free;
 196
 197         return 0;
 198
 199 out_free:
 200         zfree(&tidq->last_branch_rb);
 201         zfree(&tidq->last_branch);
 202         zfree(&tidq->prev_packet);
 203         zfree(&tidq->packet);
 204 out:
 205         return rc;
 206 }
 207
 208 static struct cs_etm_traceid_queue
 209 *cs_etm__etmq_get_traceid_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
 210 {
 211         struct cs_etm_traceid_queue *tidq;
 212         struct cs_etm_auxtrace *etm = etmq->etm;
 213
 214         if (!etm->timeless_decoding)
 215                 return NULL;
 216
 217         tidq = etmq->traceid_queues;
 218
 219         if (tidq)
 220                 return tidq;
 221
 222         tidq = malloc(sizeof(*tidq));
 223         if (!tidq)
 224                 return NULL;
 225
 226         memset(tidq, 0, sizeof(*tidq));
 227
 228         if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id))
 229                 goto out_free;
 230
 231         etmq->traceid_queues = tidq;
 232
 233         return etmq->traceid_queues;
 234
 235 out_free:
 236         free(tidq);
 237
 238         return NULL;
 239 }
 240
 241 struct cs_etm_packet_queue
 242 *cs_etm__etmq_get_packet_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
 243 {
 244         struct cs_etm_traceid_queue *tidq;
 245
 246         tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
 247         if (tidq)
 248                 return &tidq->packet_queue;
 249
 250         return NULL;
 251 }
 252
 253 static void cs_etm__packet_dump(const char *pkt_string)
 254 {
 255         const char *color = PERF_COLOR_BLUE;
 256         int len = strlen(pkt_string);
 257
 258         if (len && (pkt_string[len-1] == '\n'))
 259                 color_fprintf(stdout, color, "  %s", pkt_string);
 260         else
 261                 color_fprintf(stdout, color, "  %s\n", pkt_string);
 262
 263         fflush(stdout);
 264 }
 265
 266 static void cs_etm__set_trace_param_etmv3(struct cs_etm_trace_params *t_params,
 267                                           struct cs_etm_auxtrace *etm, int idx,
 268                                           u32 etmidr)
 269 {
 270         u64 **metadata = etm->metadata;
 271
 272         t_params[idx].protocol = cs_etm__get_v7_protocol_version(etmidr);
 273         t_params[idx].etmv3.reg_ctrl = metadata[idx][CS_ETM_ETMCR];
 274         t_params[idx].etmv3.reg_trc_id = metadata[idx][CS_ETM_ETMTRACEIDR];
 275 }
 276
 277 static void cs_etm__set_trace_param_etmv4(struct cs_etm_trace_params *t_params,
 278                                           struct cs_etm_auxtrace *etm, int idx)
 279 {
 280         u64 **metadata = etm->metadata;
 281
 282         t_params[idx].protocol = CS_ETM_PROTO_ETMV4i;
 283         t_params[idx].etmv4.reg_idr0 = metadata[idx][CS_ETMV4_TRCIDR0];
 284         t_params[idx].etmv4.reg_idr1 = metadata[idx][CS_ETMV4_TRCIDR1];
 285         t_params[idx].etmv4.reg_idr2 = metadata[idx][CS_ETMV4_TRCIDR2];
 286         t_params[idx].etmv4.reg_idr8 = metadata[idx][CS_ETMV4_TRCIDR8];
 287         t_params[idx].etmv4.reg_configr = metadata[idx][CS_ETMV4_TRCCONFIGR];
 288         t_params[idx].etmv4.reg_traceidr = metadata[idx][CS_ETMV4_TRCTRACEIDR];
 289 }
 290
 291 static int cs_etm__init_trace_params(struct cs_etm_trace_params *t_params,
 292                                      struct cs_etm_auxtrace *etm)
 293 {
 294         int i;
 295         u32 etmidr;
 296         u64 architecture;
 297
 298         for (i = 0; i < etm->num_cpu; i++) {
 299                 architecture = etm->metadata[i][CS_ETM_MAGIC];
 300
 301                 switch (architecture) {
 302                 case __perf_cs_etmv3_magic:
 303                         etmidr = etm->metadata[i][CS_ETM_ETMIDR];
 304                         cs_etm__set_trace_param_etmv3(t_params, etm, i, etmidr);
 305                         break;
 306                 case __perf_cs_etmv4_magic:
 307                         cs_etm__set_trace_param_etmv4(t_params, etm, i);
 308                         break;
 309                 default:
 310                         return -EINVAL;
 311                 }
 312         }
 313
 314         return 0;
 315 }
 316
 317 static int cs_etm__init_decoder_params(struct cs_etm_decoder_params *d_params,
 318                                        struct cs_etm_queue *etmq,
 319                                        enum cs_etm_decoder_operation mode)
 320 {
 321         int ret = -EINVAL;
 322
 323         if (!(mode < CS_ETM_OPERATION_MAX))
 324                 goto out;
 325
 326         d_params->packet_printer = cs_etm__packet_dump;
 327         d_params->operation = mode;
 328         d_params->data = etmq;
 329         d_params->formatted = true;
 330         d_params->fsyncs = false;
 331         d_params->hsyncs = false;
 332         d_params->frame_aligned = true;
 333
 334         ret = 0;
 335 out:
 336         return ret;
 337 }
 338
 339 static void cs_etm__dump_event(struct cs_etm_auxtrace *etm,
 340                                struct auxtrace_buffer *buffer)
 341 {
 342         int ret;
 343         const char *color = PERF_COLOR_BLUE;
 344         struct cs_etm_decoder_params d_params;
 345         struct cs_etm_trace_params *t_params;
 346         struct cs_etm_decoder *decoder;
 347         size_t buffer_used = 0;
 348
 349         fprintf(stdout, "\n");
 350         color_fprintf(stdout, color,
 351                      ". ... CoreSight ETM Trace data: size %zu bytes\n",
 352                      buffer->size);
 353
 354         /* Use metadata to fill in trace parameters for trace decoder */
 355         t_params = zalloc(sizeof(*t_params) * etm->num_cpu);
 356
 357         if (!t_params)
 358                 return;
 359
 360         if (cs_etm__init_trace_params(t_params, etm))
 361                 goto out_free;
 362
 363         /* Set decoder parameters to simply print the trace packets */
 364         if (cs_etm__init_decoder_params(&d_params, NULL,
 365                                         CS_ETM_OPERATION_PRINT))
 366                 goto out_free;
 367
 368         decoder = cs_etm_decoder__new(etm->num_cpu, &d_params, t_params);
 369
 370         if (!decoder)
 371                 goto out_free;
 372         do {
 373                 size_t consumed;
 374
 375                 ret = cs_etm_decoder__process_data_block(
 376                                 decoder, buffer->offset,
 377                                 &((u8 *)buffer->data)[buffer_used],
 378                                 buffer->size - buffer_used, &consumed);
 379                 if (ret)
 380                         break;
 381
 382                 buffer_used += consumed;
 383         } while (buffer_used < buffer->size);
 384
 385         cs_etm_decoder__free(decoder);
 386
 387 out_free:
 388         zfree(&t_params);
 389 }
 390
 391 static int cs_etm__flush_events(struct perf_session *session,
 392                                 struct perf_tool *tool)
 393 {
 394         int ret;
 395         struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
 396                                                    struct cs_etm_auxtrace,
 397                                                    auxtrace);
 398         if (dump_trace)
 399                 return 0;
 400
 401         if (!tool->ordered_events)
 402                 return -EINVAL;
 403
 404         if (!etm->timeless_decoding)
 405                 return -EINVAL;
 406
 407         ret = cs_etm__update_queues(etm);
 408
 409         if (ret < 0)
 410                 return ret;
 411
 412         return cs_etm__process_timeless_queues(etm, -1);
 413 }
 414
 415 static void cs_etm__free_queue(void *priv)
 416 {
 417         struct cs_etm_queue *etmq = priv;
 418
 419         if (!etmq)
 420                 return;
 421
 422         thread__zput(etmq->traceid_queues->thread);
 423         cs_etm_decoder__free(etmq->decoder);
 424         zfree(&etmq->traceid_queues->event_buf);
 425         zfree(&etmq->traceid_queues->last_branch);
 426         zfree(&etmq->traceid_queues->last_branch_rb);
 427         zfree(&etmq->traceid_queues->prev_packet);
 428         zfree(&etmq->traceid_queues->packet);
 429         zfree(&etmq->traceid_queues);
 430         free(etmq);
 431 }
 432
 433 static void cs_etm__free_events(struct perf_session *session)
 434 {
 435         unsigned int i;
 436         struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
 437                                                    struct cs_etm_auxtrace,
 438                                                    auxtrace);
 439         struct auxtrace_queues *queues = &aux->queues;
 440
 441         for (i = 0; i < queues->nr_queues; i++) {
 442                 cs_etm__free_queue(queues->queue_array[i].priv);
 443                 queues->queue_array[i].priv = NULL;
 444         }
 445
 446         auxtrace_queues__free(queues);
 447 }
 448
 449 static void cs_etm__free(struct perf_session *session)
 450 {
 451         int i;
 452         struct int_node *inode, *tmp;
 453         struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
 454                                                    struct cs_etm_auxtrace,
 455                                                    auxtrace);
 456         cs_etm__free_events(session);
 457         session->auxtrace = NULL;
 458
 459         /* First remove all traceID/metadata nodes for the RB tree */
 460         intlist__for_each_entry_safe(inode, tmp, traceid_list)
 461                 intlist__remove(traceid_list, inode);
 462         /* Then the RB tree itself */
 463         intlist__delete(traceid_list);
 464
 465         for (i = 0; i < aux->num_cpu; i++)
 466                 zfree(&aux->metadata[i]);
 467
 468         thread__zput(aux->unknown_thread);
 469         zfree(&aux->metadata);
 470         zfree(&aux);
 471 }
 472
 473 static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address)
 474 {
 475         struct machine *machine;
 476
 477         machine = etmq->etm->machine;
 478
 479         if (address >= etmq->etm->kernel_start) {
 480                 if (machine__is_host(machine))
 481                         return PERF_RECORD_MISC_KERNEL;
 482                 else
 483                         return PERF_RECORD_MISC_GUEST_KERNEL;
 484         } else {
 485                 if (machine__is_host(machine))
 486                         return PERF_RECORD_MISC_USER;
 487                 else if (perf_guest)
 488                         return PERF_RECORD_MISC_GUEST_USER;
 489                 else
 490                         return PERF_RECORD_MISC_HYPERVISOR;
 491         }
 492 }
 493
 494 static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id,
 495                               u64 address, size_t size, u8 *buffer)
 496 {
 497         u8  cpumode;
 498         u64 offset;
 499         int len;
 500         struct   thread *thread;
 501         struct   machine *machine;
 502         struct   addr_location al;
 503
 504         (void)trace_chan_id;
 505
 506         if (!etmq)
 507                 return 0;
 508
 509         machine = etmq->etm->machine;
 510         cpumode = cs_etm__cpu_mode(etmq, address);
 511
 512         thread = etmq->traceid_queues->thread;
 513         if (!thread) {
 514                 if (cpumode != PERF_RECORD_MISC_KERNEL)
 515                         return 0;
 516                 thread = etmq->etm->unknown_thread;
 517         }
 518
 519         if (!thread__find_map(thread, cpumode, address, &al) || !al.map->dso)
 520                 return 0;
 521
 522         if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
 523             dso__data_status_seen(al.map->dso, DSO_DATA_STATUS_SEEN_ITRACE))
 524                 return 0;
 525
 526         offset = al.map->map_ip(al.map, address);
 527
 528         map__load(al.map);
 529
 530         len = dso__data_read_offset(al.map->dso, machine, offset, buffer, size);
 531
 532         if (len <= 0)
 533                 return 0;
 534
 535         return len;
 536 }
 537
 538 static struct cs_etm_queue *cs_etm__alloc_queue(struct cs_etm_auxtrace *etm)
 539 {
 540         struct cs_etm_decoder_params d_params;
 541         struct cs_etm_trace_params  *t_params = NULL;
 542         struct cs_etm_queue *etmq;
 543
 544         etmq = zalloc(sizeof(*etmq));
 545         if (!etmq)
 546                 return NULL;
 547
 548         /* Use metadata to fill in trace parameters for trace decoder */
 549         t_params = zalloc(sizeof(*t_params) * etm->num_cpu);
 550
 551         if (!t_params)
 552                 goto out_free;
 553
 554         if (cs_etm__init_trace_params(t_params, etm))
 555                 goto out_free;
 556
 557         /* Set decoder parameters to decode trace packets */
 558         if (cs_etm__init_decoder_params(&d_params, etmq,
 559                                         CS_ETM_OPERATION_DECODE))
 560                 goto out_free;
 561
 562         etmq->decoder = cs_etm_decoder__new(etm->num_cpu, &d_params, t_params);
 563
 564         if (!etmq->decoder)
 565                 goto out_free;
 566
 567         /*
 568          * Register a function to handle all memory accesses required by
 569          * the trace decoder library.
 570          */
 571         if (cs_etm_decoder__add_mem_access_cb(etmq->decoder,
 572                                               0x0L, ((u64) -1L),
 573                                               cs_etm__mem_access))
 574                 goto out_free_decoder;
 575
 576         zfree(&t_params);
 577         return etmq;
 578
 579 out_free_decoder:
 580         cs_etm_decoder__free(etmq->decoder);
 581 out_free:
 582         free(etmq);
 583
 584         return NULL;
 585 }
 586
 587 static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm,
 588                                struct auxtrace_queue *queue,
 589                                unsigned int queue_nr)
 590 {
 591         int ret = 0;
 592         struct cs_etm_queue *etmq = queue->priv;
 593
 594         if (list_empty(&queue->head) || etmq)
 595                 goto out;
 596
 597         etmq = cs_etm__alloc_queue(etm);
 598
 599         if (!etmq) {
 600                 ret = -ENOMEM;
 601                 goto out;
 602         }
 603
 604         queue->priv = etmq;
 605         etmq->etm = etm;
 606         etmq->queue_nr = queue_nr;
 607         etmq->offset = 0;
 608
 609 out:
 610         return ret;
 611 }
 612
 613 static int cs_etm__setup_queues(struct cs_etm_auxtrace *etm)
 614 {
 615         unsigned int i;
 616         int ret;
 617
 618         if (!etm->kernel_start)
 619                 etm->kernel_start = machine__kernel_start(etm->machine);
 620
 621         for (i = 0; i < etm->queues.nr_queues; i++) {
 622                 ret = cs_etm__setup_queue(etm, &etm->queues.queue_array[i], i);
 623                 if (ret)
 624                         return ret;
 625         }
 626
 627         return 0;
 628 }
 629
 630 static int cs_etm__update_queues(struct cs_etm_auxtrace *etm)
 631 {
 632         if (etm->queues.new_data) {
 633                 etm->queues.new_data = false;
 634                 return cs_etm__setup_queues(etm);
 635         }
 636
 637         return 0;
 638 }
 639
 640 static inline
 641 void cs_etm__copy_last_branch_rb(struct cs_etm_queue *etmq,
 642                                  struct cs_etm_traceid_queue *tidq)
 643 {
 644         struct branch_stack *bs_src = tidq->last_branch_rb;
 645         struct branch_stack *bs_dst = tidq->last_branch;
 646         size_t nr = 0;
 647
 648         /*
 649          * Set the number of records before early exit: ->nr is used to
 650          * determine how many branches to copy from ->entries.
 651          */
 652         bs_dst->nr = bs_src->nr;
 653
 654         /*
 655          * Early exit when there is nothing to copy.
 656          */
 657         if (!bs_src->nr)
 658                 return;
 659
 660         /*
 661          * As bs_src->entries is a circular buffer, we need to copy from it in
 662          * two steps.  First, copy the branches from the most recently inserted
 663          * branch ->last_branch_pos until the end of bs_src->entries buffer.
 664          */
 665         nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos;
 666         memcpy(&bs_dst->entries[0],
 667                &bs_src->entries[tidq->last_branch_pos],
 668                sizeof(struct branch_entry) * nr);
 669
 670         /*
 671          * If we wrapped around at least once, the branches from the beginning
 672          * of the bs_src->entries buffer and until the ->last_branch_pos element
 673          * are older valid branches: copy them over.  The total number of
 674          * branches copied over will be equal to the number of branches asked by
 675          * the user in last_branch_sz.
 676          */
 677         if (bs_src->nr >= etmq->etm->synth_opts.last_branch_sz) {
 678                 memcpy(&bs_dst->entries[nr],
 679                        &bs_src->entries[0],
 680                        sizeof(struct branch_entry) * tidq->last_branch_pos);
 681         }
 682 }
 683
 684 static inline
 685 void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq)
 686 {
 687         tidq->last_branch_pos = 0;
 688         tidq->last_branch_rb->nr = 0;
 689 }
 690
 691 static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq,
 692                                          u8 trace_chan_id, u64 addr)
 693 {
 694         u8 instrBytes[2];
 695
 696         cs_etm__mem_access(etmq, trace_chan_id, addr,
 697                            ARRAY_SIZE(instrBytes), instrBytes);
 698         /*
 699          * T32 instruction size is indicated by bits[15:11] of the first
 700          * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111
 701          * denote a 32-bit instruction.
 702          */
 703         return ((instrBytes[1] & 0xF8) >= 0xE8) ? 4 : 2;
 704 }
 705
 706 static inline u64 cs_etm__first_executed_instr(struct cs_etm_packet *packet)
 707 {
 708         /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
 709         if (packet->sample_type == CS_ETM_DISCONTINUITY)
 710                 return 0;
 711
 712         return packet->start_addr;
 713 }
 714
 715 static inline
 716 u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet)
 717 {
 718         /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
 719         if (packet->sample_type == CS_ETM_DISCONTINUITY)
 720                 return 0;
 721
 722         return packet->end_addr - packet->last_instr_size;
 723 }
 724
 725 static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq,
 726                                      u64 trace_chan_id,
 727                                      const struct cs_etm_packet *packet,
 728                                      u64 offset)
 729 {
 730         if (packet->isa == CS_ETM_ISA_T32) {
 731                 u64 addr = packet->start_addr;
 732
 733                 while (offset > 0) {
 734                         addr += cs_etm__t32_instr_size(etmq,
 735                                                        trace_chan_id, addr);
 736                         offset--;
 737                 }
 738                 return addr;
 739         }
 740
 741         /* Assume a 4 byte instruction size (A32/A64) */
 742         return packet->start_addr + offset * 4;
 743 }
 744
 745 static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq,
 746                                           struct cs_etm_traceid_queue *tidq)
 747 {
 748         struct branch_stack *bs = tidq->last_branch_rb;
 749         struct branch_entry *be;
 750
 751         /*
 752          * The branches are recorded in a circular buffer in reverse
 753          * chronological order: we start recording from the last element of the
 754          * buffer down.  After writing the first element of the stack, move the
 755          * insert position back to the end of the buffer.
 756          */
 757         if (!tidq->last_branch_pos)
 758                 tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;
 759
 760         tidq->last_branch_pos -= 1;
 761
 762         be       = &bs->entries[tidq->last_branch_pos];
 763         be->from = cs_etm__last_executed_instr(tidq->prev_packet);
 764         be->to   = cs_etm__first_executed_instr(tidq->packet);
 765         /* No support for mispredict */
 766         be->flags.mispred = 0;
 767         be->flags.predicted = 1;
 768
 769         /*
 770          * Increment bs->nr until reaching the number of last branches asked by
 771          * the user on the command line.
 772          */
 773         if (bs->nr < etmq->etm->synth_opts.last_branch_sz)
 774                 bs->nr += 1;
 775 }
 776
 777 static int cs_etm__inject_event(union perf_event *event,
 778                                struct perf_sample *sample, u64 type)
 779 {
 780         event->header.size = perf_event__sample_event_size(sample, type, 0);
 781         return perf_event__synthesize_sample(event, type, 0, sample);
 782 }
 783
 784
 785 static int
 786 cs_etm__get_trace(struct cs_etm_queue *etmq)
 787 {
 788         struct auxtrace_buffer *aux_buffer = etmq->buffer;
 789         struct auxtrace_buffer *old_buffer = aux_buffer;
 790         struct auxtrace_queue *queue;
 791
 792         queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
 793
 794         aux_buffer = auxtrace_buffer__next(queue, aux_buffer);
 795
 796         /* If no more data, drop the previous auxtrace_buffer and return */
 797         if (!aux_buffer) {
 798                 if (old_buffer)
 799                         auxtrace_buffer__drop_data(old_buffer);
 800                 etmq->buf_len = 0;
 801                 return 0;
 802         }
 803
 804         etmq->buffer = aux_buffer;
 805
 806         /* If the aux_buffer doesn't have data associated, try to load it */
 807         if (!aux_buffer->data) {
 808                 /* get the file desc associated with the perf data file */
 809                 int fd = perf_data__fd(etmq->etm->session->data);
 810
 811                 aux_buffer->data = auxtrace_buffer__get_data(aux_buffer, fd);
 812                 if (!aux_buffer->data)
 813                         return -ENOMEM;
 814         }
 815
 816         /* If valid, drop the previous buffer */
 817         if (old_buffer)
 818                 auxtrace_buffer__drop_data(old_buffer);
 819
 820         etmq->buf_used = 0;
 821         etmq->buf_len = aux_buffer->size;
 822         etmq->buf = aux_buffer->data;
 823
 824         return etmq->buf_len;
 825 }
 826
 827 static void cs_etm__set_pid_tid_cpu(struct cs_etm_auxtrace *etm,
 828                                     struct auxtrace_queue *queue,
 829                                     struct cs_etm_traceid_queue *tidq)
 830 {
 831         /* CPU-wide tracing isn't supported yet */
 832         if (queue->tid == -1)
 833                 return;
 834
 835         if ((!tidq->thread) && (tidq->tid != -1))
 836                 tidq->thread = machine__find_thread(etm->machine, -1,
 837                                                     tidq->tid);
 838
 839         if (tidq->thread)
 840                 tidq->pid = tidq->thread->pid_;
 841 }
 842
 843 static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq,
 844                                             struct cs_etm_traceid_queue *tidq,
 845                                             u64 addr, u64 period)
 846 {
 847         int ret = 0;
 848         struct cs_etm_auxtrace *etm = etmq->etm;
 849         union perf_event *event = tidq->event_buf;
 850         struct perf_sample sample = {.ip = 0,};
 851
 852         event->sample.header.type = PERF_RECORD_SAMPLE;
 853         event->sample.header.misc = cs_etm__cpu_mode(etmq, addr);
 854         event->sample.header.size = sizeof(struct perf_event_header);
 855
 856         sample.ip = addr;
 857         sample.pid = tidq->pid;
 858         sample.tid = tidq->tid;
 859         sample.id = etmq->etm->instructions_id;
 860         sample.stream_id = etmq->etm->instructions_id;
 861         sample.period = period;
 862         sample.cpu = tidq->packet->cpu;
 863         sample.flags = tidq->prev_packet->flags;
 864         sample.insn_len = 1;
 865         sample.cpumode = event->sample.header.misc;
 866
 867         if (etm->synth_opts.last_branch) {
 868                 cs_etm__copy_last_branch_rb(etmq, tidq);
 869                 sample.branch_stack = tidq->last_branch;
 870         }
 871
 872         if (etm->synth_opts.inject) {
 873                 ret = cs_etm__inject_event(event, &sample,
 874                                            etm->instructions_sample_type);
 875                 if (ret)
 876                         return ret;
 877         }
 878
 879         ret = perf_session__deliver_synth_event(etm->session, event, &sample);
 880
 881         if (ret)
 882                 pr_err(
 883                         "CS ETM Trace: failed to deliver instruction event, error %d\n",
 884                         ret);
 885
 886         if (etm->synth_opts.last_branch)
 887                 cs_etm__reset_last_branch_rb(tidq);
 888
 889         return ret;
 890 }
 891
 892 /*
 893  * The cs etm packet encodes an instruction range between a branch target
 894  * and the next taken branch. Generate sample accordingly.
 895  */
 896 static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq,
 897                                        struct cs_etm_traceid_queue *tidq)
 898 {
 899         int ret = 0;
 900         struct cs_etm_auxtrace *etm = etmq->etm;
 901         struct perf_sample sample = {.ip = 0,};
 902         union perf_event *event = tidq->event_buf;
 903         struct dummy_branch_stack {
 904                 u64                     nr;
 905                 struct branch_entry     entries;
 906         } dummy_bs;
 907         u64 ip;
 908
 909         ip = cs_etm__last_executed_instr(tidq->prev_packet);
 910
 911         event->sample.header.type = PERF_RECORD_SAMPLE;
 912         event->sample.header.misc = cs_etm__cpu_mode(etmq, ip);
 913         event->sample.header.size = sizeof(struct perf_event_header);
 914
 915         sample.ip = ip;
 916         sample.pid = tidq->pid;
 917         sample.tid = tidq->tid;
 918         sample.addr = cs_etm__first_executed_instr(tidq->packet);
 919         sample.id = etmq->etm->branches_id;
 920         sample.stream_id = etmq->etm->branches_id;
 921         sample.period = 1;
 922         sample.cpu = tidq->packet->cpu;
 923         sample.flags = tidq->prev_packet->flags;
 924         sample.cpumode = event->sample.header.misc;
 925
 926         /*
 927          * perf report cannot handle events without a branch stack
 928          */
 929         if (etm->synth_opts.last_branch) {
 930                 dummy_bs = (struct dummy_branch_stack){
 931                         .nr = 1,
 932                         .entries = {
 933                                 .from = sample.ip,
 934                                 .to = sample.addr,
 935                         },
 936                 };
 937                 sample.branch_stack = (struct branch_stack *)&dummy_bs;
 938         }
 939
 940         if (etm->synth_opts.inject) {
 941                 ret = cs_etm__inject_event(event, &sample,
 942                                            etm->branches_sample_type);
 943                 if (ret)
 944                         return ret;
 945         }
 946
 947         ret = perf_session__deliver_synth_event(etm->session, event, &sample);
 948
 949         if (ret)
 950                 pr_err(
 951                 "CS ETM Trace: failed to deliver instruction event, error %d\n",
 952                 ret);
 953
 954         return ret;
 955 }
 956
 957 struct cs_etm_synth {
 958         struct perf_tool dummy_tool;
 959         struct perf_session *session;
 960 };
 961
 962 static int cs_etm__event_synth(struct perf_tool *tool,
 963                                union perf_event *event,
 964                                struct perf_sample *sample __maybe_unused,
 965                                struct machine *machine __maybe_unused)
 966 {
 967         struct cs_etm_synth *cs_etm_synth =
 968                       container_of(tool, struct cs_etm_synth, dummy_tool);
 969
 970         return perf_session__deliver_synth_event(cs_etm_synth->session,
 971                                                  event, NULL);
 972 }
 973
 974 static int cs_etm__synth_event(struct perf_session *session,
 975                                struct perf_event_attr *attr, u64 id)
 976 {
 977         struct cs_etm_synth cs_etm_synth;
 978
 979         memset(&cs_etm_synth, 0, sizeof(struct cs_etm_synth));
 980         cs_etm_synth.session = session;
 981
 982         return perf_event__synthesize_attr(&cs_etm_synth.dummy_tool, attr, 1,
 983                                            &id, cs_etm__event_synth);
 984 }
 985
 986 static int cs_etm__synth_events(struct cs_etm_auxtrace *etm,
 987                                 struct perf_session *session)
 988 {
 989         struct perf_evlist *evlist = session->evlist;
 990         struct perf_evsel *evsel;
 991         struct perf_event_attr attr;
 992         bool found = false;
 993         u64 id;
 994         int err;
 995
 996         evlist__for_each_entry(evlist, evsel) {
 997                 if (evsel->attr.type == etm->pmu_type) {
 998                         found = true;
 999                         break;
1000                 }
1001         }
1002
1003         if (!found) {
1004                 pr_debug("No selected events with CoreSight Trace data\n");
1005                 return 0;
1006         }
1007
1008         memset(&attr, 0, sizeof(struct perf_event_attr));
1009         attr.size = sizeof(struct perf_event_attr);
1010         attr.type = PERF_TYPE_HARDWARE;
1011         attr.sample_type = evsel->attr.sample_type & PERF_SAMPLE_MASK;
1012         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1013                             PERF_SAMPLE_PERIOD;
1014         if (etm->timeless_decoding)
1015                 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1016         else
1017                 attr.sample_type |= PERF_SAMPLE_TIME;
1018
1019         attr.exclude_user = evsel->attr.exclude_user;
1020         attr.exclude_kernel = evsel->attr.exclude_kernel;
1021         attr.exclude_hv = evsel->attr.exclude_hv;
1022         attr.exclude_host = evsel->attr.exclude_host;
1023         attr.exclude_guest = evsel->attr.exclude_guest;
1024         attr.sample_id_all = evsel->attr.sample_id_all;
1025         attr.read_format = evsel->attr.read_format;
1026
1027         /* create new id val to be a fixed offset from evsel id */
1028         id = evsel->id[0] + 1000000000;
1029
1030         if (!id)
1031                 id = 1;
1032
1033         if (etm->synth_opts.branches) {
1034                 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
1035                 attr.sample_period = 1;
1036                 attr.sample_type |= PERF_SAMPLE_ADDR;
1037                 err = cs_etm__synth_event(session, &attr, id);
1038                 if (err)
1039                         return err;
1040                 etm->sample_branches = true;
1041                 etm->branches_sample_type = attr.sample_type;
1042                 etm->branches_id = id;
1043                 id += 1;
1044                 attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR;
1045         }
1046
1047         if (etm->synth_opts.last_branch)
1048                 attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1049
1050         if (etm->synth_opts.instructions) {
1051                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1052                 attr.sample_period = etm->synth_opts.period;
1053                 etm->instructions_sample_period = attr.sample_period;
1054                 err = cs_etm__synth_event(session, &attr, id);
1055                 if (err)
1056                         return err;
1057                 etm->sample_instructions = true;
1058                 etm->instructions_sample_type = attr.sample_type;
1059                 etm->instructions_id = id;
1060                 id += 1;
1061         }
1062
1063         return 0;
1064 }
1065
1066 static int cs_etm__sample(struct cs_etm_queue *etmq,
1067                           struct cs_etm_traceid_queue *tidq)
1068 {
1069         struct cs_etm_auxtrace *etm = etmq->etm;
1070         struct cs_etm_packet *tmp;
1071         int ret;
1072         u8 trace_chan_id = tidq->trace_chan_id;
1073         u64 instrs_executed = tidq->packet->instr_count;
1074
1075         tidq->period_instructions += instrs_executed;
1076
1077         /*
1078          * Record a branch when the last instruction in
1079          * PREV_PACKET is a branch.
1080          */
1081         if (etm->synth_opts.last_branch &&
1082             tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1083             tidq->prev_packet->last_instr_taken_branch)
1084                 cs_etm__update_last_branch_rb(etmq, tidq);
1085
1086         if (etm->sample_instructions &&
1087             tidq->period_instructions >= etm->instructions_sample_period) {
1088                 /*
1089                  * Emit instruction sample periodically
1090                  * TODO: allow period to be defined in cycles and clock time
1091                  */
1092
1093                 /* Get number of instructions executed after the sample point */
1094                 u64 instrs_over = tidq->period_instructions -
1095                         etm->instructions_sample_period;
1096
1097                 /*
1098                  * Calculate the address of the sampled instruction (-1 as
1099                  * sample is reported as though instruction has just been
1100                  * executed, but PC has not advanced to next instruction)
1101                  */
1102                 u64 offset = (instrs_executed - instrs_over - 1);
1103                 u64 addr = cs_etm__instr_addr(etmq, trace_chan_id,
1104                                               tidq->packet, offset);
1105
1106                 ret = cs_etm__synth_instruction_sample(
1107                         etmq, tidq, addr, etm->instructions_sample_period);
1108                 if (ret)
1109                         return ret;
1110
1111                 /* Carry remaining instructions into next sample period */
1112                 tidq->period_instructions = instrs_over;
1113         }
1114
1115         if (etm->sample_branches) {
1116                 bool generate_sample = false;
1117
1118                 /* Generate sample for tracing on packet */
1119                 if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY)
1120                         generate_sample = true;
1121
1122                 /* Generate sample for branch taken packet */
1123                 if (tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1124                     tidq->prev_packet->last_instr_taken_branch)
1125                         generate_sample = true;
1126
1127                 if (generate_sample) {
1128                         ret = cs_etm__synth_branch_sample(etmq, tidq);
1129                         if (ret)
1130                                 return ret;
1131                 }
1132         }
1133
1134         if (etm->sample_branches || etm->synth_opts.last_branch) {
1135                 /*
1136                  * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
1137                  * the next incoming packet.
1138                  */
1139                 tmp = tidq->packet;
1140                 tidq->packet = tidq->prev_packet;
1141                 tidq->prev_packet = tmp;
1142         }
1143
1144         return 0;
1145 }
1146
1147 static int cs_etm__exception(struct cs_etm_traceid_queue *tidq)
1148 {
1149         /*
1150          * When the exception packet is inserted, whether the last instruction
1151          * in previous range packet is taken branch or not, we need to force
1152          * to set 'prev_packet->last_instr_taken_branch' to true.  This ensures
1153          * to generate branch sample for the instruction range before the
1154          * exception is trapped to kernel or before the exception returning.
1155          *
1156          * The exception packet includes the dummy address values, so don't
1157          * swap PACKET with PREV_PACKET.  This keeps PREV_PACKET to be useful
1158          * for generating instruction and branch samples.
1159          */
1160         if (tidq->prev_packet->sample_type == CS_ETM_RANGE)
1161                 tidq->prev_packet->last_instr_taken_branch = true;
1162
1163         return 0;
1164 }
1165
1166 static int cs_etm__flush(struct cs_etm_queue *etmq,
1167                          struct cs_etm_traceid_queue *tidq)
1168 {
1169         int err = 0;
1170         struct cs_etm_auxtrace *etm = etmq->etm;
1171         struct cs_etm_packet *tmp;
1172
1173         /* Handle start tracing packet */
1174         if (tidq->prev_packet->sample_type == CS_ETM_EMPTY)
1175                 goto swap_packet;
1176
1177         if (etmq->etm->synth_opts.last_branch &&
1178             tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1179                 /*
1180                  * Generate a last branch event for the branches left in the
1181                  * circular buffer at the end of the trace.
1182                  *
1183                  * Use the address of the end of the last reported execution
1184                  * range
1185                  */
1186                 u64 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1187
1188                 err = cs_etm__synth_instruction_sample(
1189                         etmq, tidq, addr,
1190                         tidq->period_instructions);
1191                 if (err)
1192                         return err;
1193
1194                 tidq->period_instructions = 0;
1195
1196         }
1197
1198         if (etm->sample_branches &&
1199             tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1200                 err = cs_etm__synth_branch_sample(etmq, tidq);
1201                 if (err)
1202                         return err;
1203         }
1204
1205 swap_packet:
1206         if (etm->sample_branches || etm->synth_opts.last_branch) {
1207                 /*
1208                  * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
1209                  * the next incoming packet.
1210                  */
1211                 tmp = tidq->packet;
1212                 tidq->packet = tidq->prev_packet;
1213                 tidq->prev_packet = tmp;
1214         }
1215
1216         return err;
1217 }
1218
1219 static int cs_etm__end_block(struct cs_etm_queue *etmq,
1220                              struct cs_etm_traceid_queue *tidq)
1221 {
1222         int err;
1223
1224         /*
1225          * It has no new packet coming and 'etmq->packet' contains the stale
1226          * packet which was set at the previous time with packets swapping;
1227          * so skip to generate branch sample to avoid stale packet.
1228          *
1229          * For this case only flush branch stack and generate a last branch
1230          * event for the branches left in the circular buffer at the end of
1231          * the trace.
1232          */
1233         if (etmq->etm->synth_opts.last_branch &&
1234             tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1235                 /*
1236                  * Use the address of the end of the last reported execution
1237                  * range.
1238                  */
1239                 u64 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1240
1241                 err = cs_etm__synth_instruction_sample(
1242                         etmq, tidq, addr,
1243                         tidq->period_instructions);
1244                 if (err)
1245                         return err;
1246
1247                 tidq->period_instructions = 0;
1248         }
1249
1250         return 0;
1251 }
1252 /*
1253  * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue
1254  *                         if need be.
1255  * Returns:     < 0     if error
1256  *              = 0     if no more auxtrace_buffer to read
1257  *              > 0     if the current buffer isn't empty yet
1258  */
1259 static int cs_etm__get_data_block(struct cs_etm_queue *etmq)
1260 {
1261         int ret;
1262
1263         if (!etmq->buf_len) {
1264                 ret = cs_etm__get_trace(etmq);
1265                 if (ret <= 0)
1266                         return ret;
1267                 /*
1268                  * We cannot assume consecutive blocks in the data file
1269                  * are contiguous, reset the decoder to force re-sync.
1270                  */
1271                 ret = cs_etm_decoder__reset(etmq->decoder);
1272                 if (ret)
1273                         return ret;
1274         }
1275
1276         return etmq->buf_len;
1277 }
1278
1279 static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id,
1280                                  struct cs_etm_packet *packet,
1281                                  u64 end_addr)
1282 {
1283         u16 instr16;
1284         u32 instr32;
1285         u64 addr;
1286
1287         switch (packet->isa) {
1288         case CS_ETM_ISA_T32:
1289                 /*
1290                  * The SVC of T32 is defined in ARM DDI 0487D.a, F5.1.247:
1291                  *
1292                  *  b'15         b'8
1293                  * +-----------------+--------+
1294                  * | 1 1 0 1 1 1 1 1 |  imm8  |
1295                  * +-----------------+--------+
1296                  *
1297                  * According to the specifiction, it only defines SVC for T32
1298                  * with 16 bits instruction and has no definition for 32bits;
1299                  * so below only read 2 bytes as instruction size for T32.
1300                  */
1301                 addr = end_addr - 2;
1302                 cs_etm__mem_access(etmq, trace_chan_id, addr,
1303                                    sizeof(instr16), (u8 *)&instr16);
1304                 if ((instr16 & 0xFF00) == 0xDF00)
1305                         return true;
1306
1307                 break;
1308         case CS_ETM_ISA_A32:
1309                 /*
1310                  * The SVC of A32 is defined in ARM DDI 0487D.a, F5.1.247:
1311                  *
1312                  *  b'31 b'28 b'27 b'24
1313                  * +---------+---------+-------------------------+
1314                  * |  !1111  | 1 1 1 1 |        imm24            |
1315                  * +---------+---------+-------------------------+
1316                  */
1317                 addr = end_addr - 4;
1318                 cs_etm__mem_access(etmq, trace_chan_id, addr,
1319                                    sizeof(instr32), (u8 *)&instr32);
1320                 if ((instr32 & 0x0F000000) == 0x0F000000 &&
1321                     (instr32 & 0xF0000000) != 0xF0000000)
1322                         return true;
1323
1324                 break;
1325         case CS_ETM_ISA_A64:
1326                 /*
1327                  * The SVC of A64 is defined in ARM DDI 0487D.a, C6.2.294:
1328                  *
1329                  *  b'31               b'21           b'4     b'0
1330                  * +-----------------------+---------+-----------+
1331                  * | 1 1 0 1 0 1 0 0 0 0 0 |  imm16  | 0 0 0 0 1 |
1332                  * +-----------------------+---------+-----------+
1333                  */
1334                 addr = end_addr - 4;
1335                 cs_etm__mem_access(etmq, trace_chan_id, addr,
1336                                    sizeof(instr32), (u8 *)&instr32);
1337                 if ((instr32 & 0xFFE0001F) == 0xd4000001)
1338                         return true;
1339
1340                 break;
1341         case CS_ETM_ISA_UNKNOWN:
1342         default:
1343                 break;
1344         }
1345
1346         return false;
1347 }
1348
1349 static bool cs_etm__is_syscall(struct cs_etm_queue *etmq,
1350                                struct cs_etm_traceid_queue *tidq, u64 magic)
1351 {
1352         u8 trace_chan_id = tidq->trace_chan_id;
1353         struct cs_etm_packet *packet = tidq->packet;
1354         struct cs_etm_packet *prev_packet = tidq->prev_packet;
1355
1356         if (magic == __perf_cs_etmv3_magic)
1357                 if (packet->exception_number == CS_ETMV3_EXC_SVC)
1358                         return true;
1359
1360         /*
1361          * ETMv4 exception type CS_ETMV4_EXC_CALL covers SVC, SMC and
1362          * HVC cases; need to check if it's SVC instruction based on
1363          * packet address.
1364          */
1365         if (magic == __perf_cs_etmv4_magic) {
1366                 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
1367                     cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
1368                                          prev_packet->end_addr))
1369                         return true;
1370         }
1371
1372         return false;
1373 }
1374
1375 static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq,
1376                                        u64 magic)
1377 {
1378         struct cs_etm_packet *packet = tidq->packet;
1379
1380         if (magic == __perf_cs_etmv3_magic)
1381                 if (packet->exception_number == CS_ETMV3_EXC_DEBUG_HALT ||
1382                     packet->exception_number == CS_ETMV3_EXC_ASYNC_DATA_ABORT ||
1383                     packet->exception_number == CS_ETMV3_EXC_PE_RESET ||
1384                     packet->exception_number == CS_ETMV3_EXC_IRQ ||
1385                     packet->exception_number == CS_ETMV3_EXC_FIQ)
1386                         return true;
1387
1388         if (magic == __perf_cs_etmv4_magic)
1389                 if (packet->exception_number == CS_ETMV4_EXC_RESET ||
1390                     packet->exception_number == CS_ETMV4_EXC_DEBUG_HALT ||
1391                     packet->exception_number == CS_ETMV4_EXC_SYSTEM_ERROR ||
1392                     packet->exception_number == CS_ETMV4_EXC_INST_DEBUG ||
1393                     packet->exception_number == CS_ETMV4_EXC_DATA_DEBUG ||
1394                     packet->exception_number == CS_ETMV4_EXC_IRQ ||
1395                     packet->exception_number == CS_ETMV4_EXC_FIQ)
1396                         return true;
1397
1398         return false;
1399 }
1400
1401 static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq,
1402                                       struct cs_etm_traceid_queue *tidq,
1403                                       u64 magic)
1404 {
1405         u8 trace_chan_id = tidq->trace_chan_id;
1406         struct cs_etm_packet *packet = tidq->packet;
1407         struct cs_etm_packet *prev_packet = tidq->prev_packet;
1408
1409         if (magic == __perf_cs_etmv3_magic)
1410                 if (packet->exception_number == CS_ETMV3_EXC_SMC ||
1411                     packet->exception_number == CS_ETMV3_EXC_HYP ||
1412                     packet->exception_number == CS_ETMV3_EXC_JAZELLE_THUMBEE ||
1413                     packet->exception_number == CS_ETMV3_EXC_UNDEFINED_INSTR ||
1414                     packet->exception_number == CS_ETMV3_EXC_PREFETCH_ABORT ||
1415                     packet->exception_number == CS_ETMV3_EXC_DATA_FAULT ||
1416                     packet->exception_number == CS_ETMV3_EXC_GENERIC)
1417                         return true;
1418
1419         if (magic == __perf_cs_etmv4_magic) {
1420                 if (packet->exception_number == CS_ETMV4_EXC_TRAP ||
1421                     packet->exception_number == CS_ETMV4_EXC_ALIGNMENT ||
1422                     packet->exception_number == CS_ETMV4_EXC_INST_FAULT ||
1423                     packet->exception_number == CS_ETMV4_EXC_DATA_FAULT)
1424                         return true;
1425
1426                 /*
1427                  * For CS_ETMV4_EXC_CALL, except SVC other instructions
1428                  * (SMC, HVC) are taken as sync exceptions.
1429                  */
1430                 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
1431                     !cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
1432                                           prev_packet->end_addr))
1433                         return true;
1434
1435                 /*
1436                  * ETMv4 has 5 bits for exception number; if the numbers
1437                  * are in the range ( CS_ETMV4_EXC_FIQ, CS_ETMV4_EXC_END ]
1438                  * they are implementation defined exceptions.
1439                  *
1440                  * For this case, simply take it as sync exception.
1441                  */
1442                 if (packet->exception_number > CS_ETMV4_EXC_FIQ &&
1443                     packet->exception_number <= CS_ETMV4_EXC_END)
1444                         return true;
1445         }
1446
1447         return false;
1448 }
1449
1450 static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq,
1451                                     struct cs_etm_traceid_queue *tidq)
1452 {
1453         struct cs_etm_packet *packet = tidq->packet;
1454         struct cs_etm_packet *prev_packet = tidq->prev_packet;
1455         u8 trace_chan_id = tidq->trace_chan_id;
1456         u64 magic;
1457         int ret;
1458
1459         switch (packet->sample_type) {
1460         case CS_ETM_RANGE:
1461                 /*
1462                  * Immediate branch instruction without neither link nor
1463                  * return flag, it's normal branch instruction within
1464                  * the function.
1465                  */
1466                 if (packet->last_instr_type == OCSD_INSTR_BR &&
1467                     packet->last_instr_subtype == OCSD_S_INSTR_NONE) {
1468                         packet->flags = PERF_IP_FLAG_BRANCH;
1469
1470                         if (packet->last_instr_cond)
1471                                 packet->flags |= PERF_IP_FLAG_CONDITIONAL;
1472                 }
1473
1474                 /*
1475                  * Immediate branch instruction with link (e.g. BL), this is
1476                  * branch instruction for function call.
1477                  */
1478                 if (packet->last_instr_type == OCSD_INSTR_BR &&
1479                     packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
1480                         packet->flags = PERF_IP_FLAG_BRANCH |
1481                                         PERF_IP_FLAG_CALL;
1482
1483                 /*
1484                  * Indirect branch instruction with link (e.g. BLR), this is
1485                  * branch instruction for function call.
1486                  */
1487                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1488                     packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
1489                         packet->flags = PERF_IP_FLAG_BRANCH |
1490                                         PERF_IP_FLAG_CALL;
1491
1492                 /*
1493                  * Indirect branch instruction with subtype of
1494                  * OCSD_S_INSTR_V7_IMPLIED_RET, this is explicit hint for
1495                  * function return for A32/T32.
1496                  */
1497                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1498                     packet->last_instr_subtype == OCSD_S_INSTR_V7_IMPLIED_RET)
1499                         packet->flags = PERF_IP_FLAG_BRANCH |
1500                                         PERF_IP_FLAG_RETURN;
1501
1502                 /*
1503                  * Indirect branch instruction without link (e.g. BR), usually
1504                  * this is used for function return, especially for functions
1505                  * within dynamic link lib.
1506                  */
1507                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1508                     packet->last_instr_subtype == OCSD_S_INSTR_NONE)
1509                         packet->flags = PERF_IP_FLAG_BRANCH |
1510                                         PERF_IP_FLAG_RETURN;
1511
1512                 /* Return instruction for function return. */
1513                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1514                     packet->last_instr_subtype == OCSD_S_INSTR_V8_RET)
1515                         packet->flags = PERF_IP_FLAG_BRANCH |
1516                                         PERF_IP_FLAG_RETURN;
1517
1518                 /*
1519                  * Decoder might insert a discontinuity in the middle of
1520                  * instruction packets, fixup prev_packet with flag
1521                  * PERF_IP_FLAG_TRACE_BEGIN to indicate restarting trace.
1522                  */
1523                 if (prev_packet->sample_type == CS_ETM_DISCONTINUITY)
1524                         prev_packet->flags |= PERF_IP_FLAG_BRANCH |
1525                                               PERF_IP_FLAG_TRACE_BEGIN;
1526
1527                 /*
1528                  * If the previous packet is an exception return packet
1529                  * and the return address just follows SVC instuction,
1530                  * it needs to calibrate the previous packet sample flags
1531                  * as PERF_IP_FLAG_SYSCALLRET.
1532                  */
1533                 if (prev_packet->flags == (PERF_IP_FLAG_BRANCH |
1534                                            PERF_IP_FLAG_RETURN |
1535                                            PERF_IP_FLAG_INTERRUPT) &&
1536                     cs_etm__is_svc_instr(etmq, trace_chan_id,
1537                                          packet, packet->start_addr))
1538                         prev_packet->flags = PERF_IP_FLAG_BRANCH |
1539                                              PERF_IP_FLAG_RETURN |
1540                                              PERF_IP_FLAG_SYSCALLRET;
1541                 break;
1542         case CS_ETM_DISCONTINUITY:
1543                 /*
1544                  * The trace is discontinuous, if the previous packet is
1545                  * instruction packet, set flag PERF_IP_FLAG_TRACE_END
1546                  * for previous packet.
1547                  */
1548                 if (prev_packet->sample_type == CS_ETM_RANGE)
1549                         prev_packet->flags |= PERF_IP_FLAG_BRANCH |
1550                                               PERF_IP_FLAG_TRACE_END;
1551                 break;
1552         case CS_ETM_EXCEPTION:
1553                 ret = cs_etm__get_magic(packet->trace_chan_id, &magic);
1554                 if (ret)
1555                         return ret;
1556
1557                 /* The exception is for system call. */
1558                 if (cs_etm__is_syscall(etmq, tidq, magic))
1559                         packet->flags = PERF_IP_FLAG_BRANCH |
1560                                         PERF_IP_FLAG_CALL |
1561                                         PERF_IP_FLAG_SYSCALLRET;
1562                 /*
1563                  * The exceptions are triggered by external signals from bus,
1564                  * interrupt controller, debug module, PE reset or halt.
1565                  */
1566                 else if (cs_etm__is_async_exception(tidq, magic))
1567                         packet->flags = PERF_IP_FLAG_BRANCH |
1568                                         PERF_IP_FLAG_CALL |
1569                                         PERF_IP_FLAG_ASYNC |
1570                                         PERF_IP_FLAG_INTERRUPT;
1571                 /*
1572                  * Otherwise, exception is caused by trap, instruction &
1573                  * data fault, or alignment errors.
1574                  */
1575                 else if (cs_etm__is_sync_exception(etmq, tidq, magic))
1576                         packet->flags = PERF_IP_FLAG_BRANCH |
1577                                         PERF_IP_FLAG_CALL |
1578                                         PERF_IP_FLAG_INTERRUPT;
1579
1580                 /*
1581                  * When the exception packet is inserted, since exception
1582                  * packet is not used standalone for generating samples
1583                  * and it's affiliation to the previous instruction range
1584                  * packet; so set previous range packet flags to tell perf
1585                  * it is an exception taken branch.
1586                  */
1587                 if (prev_packet->sample_type == CS_ETM_RANGE)
1588                         prev_packet->flags = packet->flags;
1589                 break;
1590         case CS_ETM_EXCEPTION_RET:
1591                 /*
1592                  * When the exception return packet is inserted, since
1593                  * exception return packet is not used standalone for
1594                  * generating samples and it's affiliation to the previous
1595                  * instruction range packet; so set previous range packet
1596                  * flags to tell perf it is an exception return branch.
1597                  *
1598                  * The exception return can be for either system call or
1599                  * other exception types; unfortunately the packet doesn't
1600                  * contain exception type related info so we cannot decide
1601                  * the exception type purely based on exception return packet.
1602                  * If we record the exception number from exception packet and
1603                  * reuse it for excpetion return packet, this is not reliable
1604                  * due the trace can be discontinuity or the interrupt can
1605                  * be nested, thus the recorded exception number cannot be
1606                  * used for exception return packet for these two cases.
1607                  *
1608                  * For exception return packet, we only need to distinguish the
1609                  * packet is for system call or for other types.  Thus the
1610                  * decision can be deferred when receive the next packet which
1611                  * contains the return address, based on the return address we
1612                  * can read out the previous instruction and check if it's a
1613                  * system call instruction and then calibrate the sample flag
1614                  * as needed.
1615                  */
1616                 if (prev_packet->sample_type == CS_ETM_RANGE)
1617                         prev_packet->flags = PERF_IP_FLAG_BRANCH |
1618                                              PERF_IP_FLAG_RETURN |
1619                                              PERF_IP_FLAG_INTERRUPT;
1620                 break;
1621         case CS_ETM_EMPTY:
1622         default:
1623                 break;
1624         }
1625
1626         return 0;
1627 }
1628
1629 static int cs_etm__decode_data_block(struct cs_etm_queue *etmq)
1630 {
1631         int ret = 0;
1632         size_t processed = 0;
1633
1634         /*
1635          * Packets are decoded and added to the decoder's packet queue
1636          * until the decoder packet processing callback has requested that
1637          * processing stops or there is nothing left in the buffer.  Normal
1638          * operations that stop processing are a timestamp packet or a full
1639          * decoder buffer queue.
1640          */
1641         ret = cs_etm_decoder__process_data_block(etmq->decoder,
1642                                                  etmq->offset,
1643                                                  &etmq->buf[etmq->buf_used],
1644                                                  etmq->buf_len,
1645                                                  &processed);
1646         if (ret)
1647                 goto out;
1648
1649         etmq->offset += processed;
1650         etmq->buf_used += processed;
1651         etmq->buf_len -= processed;
1652
1653 out:
1654         return ret;
1655 }
1656
1657 static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq,
1658                                          struct cs_etm_traceid_queue *tidq)
1659 {
1660         int ret;
1661         struct cs_etm_packet_queue *packet_queue;
1662
1663         packet_queue = &tidq->packet_queue;
1664
1665         /* Process each packet in this chunk */
1666         while (1) {
1667                 ret = cs_etm_decoder__get_packet(packet_queue,
1668                                                  tidq->packet);
1669                 if (ret <= 0)
1670                         /*
1671                          * Stop processing this chunk on
1672                          * end of data or error
1673                          */
1674                         break;
1675
1676                 /*
1677                  * Since packet addresses are swapped in packet
1678                  * handling within below switch() statements,
1679                  * thus setting sample flags must be called
1680                  * prior to switch() statement to use address
1681                  * information before packets swapping.
1682                  */
1683                 ret = cs_etm__set_sample_flags(etmq, tidq);
1684                 if (ret < 0)
1685                         break;
1686
1687                 switch (tidq->packet->sample_type) {
1688                 case CS_ETM_RANGE:
1689                         /*
1690                          * If the packet contains an instruction
1691                          * range, generate instruction sequence
1692                          * events.
1693                          */
1694                         cs_etm__sample(etmq, tidq);
1695                         break;
1696                 case CS_ETM_EXCEPTION:
1697                 case CS_ETM_EXCEPTION_RET:
1698                         /*
1699                          * If the exception packet is coming,
1700                          * make sure the previous instruction
1701                          * range packet to be handled properly.
1702                          */
1703                         cs_etm__exception(tidq);
1704                         break;
1705                 case CS_ETM_DISCONTINUITY:
1706                         /*
1707                          * Discontinuity in trace, flush
1708                          * previous branch stack
1709                          */
1710                         cs_etm__flush(etmq, tidq);
1711                         break;
1712                 case CS_ETM_EMPTY:
1713                         /*
1714                          * Should not receive empty packet,
1715                          * report error.
1716                          */
1717                         pr_err("CS ETM Trace: empty packet\n");
1718                         return -EINVAL;
1719                 default:
1720                         break;
1721                 }
1722         }
1723
1724         return ret;
1725 }
1726
1727 static int cs_etm__run_decoder(struct cs_etm_queue *etmq)
1728 {
1729         int err = 0;
1730         struct cs_etm_traceid_queue *tidq;
1731
1732         tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID);
1733         if (!tidq)
1734                 return -EINVAL;
1735
1736         /* Go through each buffer in the queue and decode them one by one */
1737         while (1) {
1738                 err = cs_etm__get_data_block(etmq);
1739                 if (err <= 0)
1740                         return err;
1741
1742                 /* Run trace decoder until buffer consumed or end of trace */
1743                 do {
1744                         err = cs_etm__decode_data_block(etmq);
1745                         if (err)
1746                                 return err;
1747
1748                         /*
1749                          * Process each packet in this chunk, nothing to do if
1750                          * an error occurs other than hoping the next one will
1751                          * be better.
1752                          */
1753                         err = cs_etm__process_traceid_queue(etmq, tidq);
1754
1755                 } while (etmq->buf_len);
1756
1757                 if (err == 0)
1758                         /* Flush any remaining branch stack entries */
1759                         err = cs_etm__end_block(etmq, tidq);
1760         }
1761
1762         return err;
1763 }
1764
1765 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
1766                                            pid_t tid)
1767 {
1768         unsigned int i;
1769         struct auxtrace_queues *queues = &etm->queues;
1770
1771         for (i = 0; i < queues->nr_queues; i++) {
1772                 struct auxtrace_queue *queue = &etm->queues.queue_array[i];
1773                 struct cs_etm_queue *etmq = queue->priv;
1774                 struct cs_etm_traceid_queue *tidq;
1775
1776                 if (!etmq)
1777                         continue;
1778
1779                 tidq = cs_etm__etmq_get_traceid_queue(etmq,
1780                                                 CS_ETM_PER_THREAD_TRACEID);
1781
1782                 if (!tidq)
1783                         continue;
1784
1785                 if ((tid == -1) || (tidq->tid == tid)) {
1786                         cs_etm__set_pid_tid_cpu(etm, queue, tidq);
1787                         cs_etm__run_decoder(etmq);
1788                 }
1789         }
1790
1791         return 0;
1792 }
1793
1794 static int cs_etm__process_itrace_start(struct cs_etm_auxtrace *etm,
1795                                         union perf_event *event)
1796 {
1797         struct thread *th;
1798
1799         if (etm->timeless_decoding)
1800                 return 0;
1801
1802         /*
1803          * Add the tid/pid to the log so that we can get a match when
1804          * we get a contextID from the decoder.
1805          */
1806         th = machine__findnew_thread(etm->machine,
1807                                      event->itrace_start.pid,
1808                                      event->itrace_start.tid);
1809         if (!th)
1810                 return -ENOMEM;
1811
1812         thread__put(th);
1813
1814         return 0;
1815 }
1816
1817 static int cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace *etm,
1818                                            union perf_event *event)
1819 {
1820         struct thread *th;
1821         bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
1822
1823         /*
1824          * Context switch in per-thread mode are irrelevant since perf
1825          * will start/stop tracing as the process is scheduled.
1826          */
1827         if (etm->timeless_decoding)
1828                 return 0;
1829
1830         /*
1831          * SWITCH_IN events carry the next process to be switched out while
1832          * SWITCH_OUT events carry the process to be switched in.  As such
1833          * we don't care about IN events.
1834          */
1835         if (!out)
1836                 return 0;
1837
1838         /*
1839          * Add the tid/pid to the log so that we can get a match when
1840          * we get a contextID from the decoder.
1841          */
1842         th = machine__findnew_thread(etm->machine,
1843                                      event->context_switch.next_prev_pid,
1844                                      event->context_switch.next_prev_tid);
1845         if (!th)
1846                 return -ENOMEM;
1847
1848         thread__put(th);
1849
1850         return 0;
1851 }
1852
1853 static int cs_etm__process_event(struct perf_session *session,
1854                                  union perf_event *event,
1855                                  struct perf_sample *sample,
1856                                  struct perf_tool *tool)
1857 {
1858         int err = 0;
1859         u64 timestamp;
1860         struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
1861                                                    struct cs_etm_auxtrace,
1862                                                    auxtrace);
1863
1864         if (dump_trace)
1865                 return 0;
1866
1867         if (!tool->ordered_events) {
1868                 pr_err("CoreSight ETM Trace requires ordered events\n");
1869                 return -EINVAL;
1870         }
1871
1872         if (!etm->timeless_decoding)
1873                 return -EINVAL;
1874
1875         if (sample->time && (sample->time != (u64) -1))
1876                 timestamp = sample->time;
1877         else
1878                 timestamp = 0;
1879
1880         if (timestamp || etm->timeless_decoding) {
1881                 err = cs_etm__update_queues(etm);
1882                 if (err)
1883                         return err;
1884         }
1885
1886         if (event->header.type == PERF_RECORD_EXIT)
1887                 return cs_etm__process_timeless_queues(etm,
1888                                                        event->fork.tid);
1889
1890         if (event->header.type == PERF_RECORD_ITRACE_START)
1891                 return cs_etm__process_itrace_start(etm, event);
1892         else if (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
1893                 return cs_etm__process_switch_cpu_wide(etm, event);
1894
1895         return 0;
1896 }
1897
1898 static int cs_etm__process_auxtrace_event(struct perf_session *session,
1899                                           union perf_event *event,
1900                                           struct perf_tool *tool __maybe_unused)
1901 {
1902         struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
1903                                                    struct cs_etm_auxtrace,
1904                                                    auxtrace);
1905         if (!etm->data_queued) {
1906                 struct auxtrace_buffer *buffer;
1907                 off_t  data_offset;
1908                 int fd = perf_data__fd(session->data);
1909                 bool is_pipe = perf_data__is_pipe(session->data);
1910                 int err;
1911
1912                 if (is_pipe)
1913                         data_offset = 0;
1914                 else {
1915                         data_offset = lseek(fd, 0, SEEK_CUR);
1916                         if (data_offset == -1)
1917                                 return -errno;
1918                 }
1919
1920                 err = auxtrace_queues__add_event(&etm->queues, session,
1921                                                  event, data_offset, &buffer);
1922                 if (err)
1923                         return err;
1924
1925                 if (dump_trace)
1926                         if (auxtrace_buffer__get_data(buffer, fd)) {
1927                                 cs_etm__dump_event(etm, buffer);
1928                                 auxtrace_buffer__put_data(buffer);
1929                         }
1930         }
1931
1932         return 0;
1933 }
1934
1935 static bool cs_etm__is_timeless_decoding(struct cs_etm_auxtrace *etm)
1936 {
1937         struct perf_evsel *evsel;
1938         struct perf_evlist *evlist = etm->session->evlist;
1939         bool timeless_decoding = true;
1940
1941         /*
1942          * Circle through the list of event and complain if we find one
1943          * with the time bit set.
1944          */
1945         evlist__for_each_entry(evlist, evsel) {
1946                 if ((evsel->attr.sample_type & PERF_SAMPLE_TIME))
1947                         timeless_decoding = false;
1948         }
1949
1950         return timeless_decoding;
1951 }
1952
1953 static const char * const cs_etm_global_header_fmts[] = {
1954         [CS_HEADER_VERSION_0]   = "     Header version                 %llx\n",
1955         [CS_PMU_TYPE_CPUS]      = "     PMU type/num cpus              %llx\n",
1956         [CS_ETM_SNAPSHOT]       = "     Snapshot                       %llx\n",
1957 };
1958
1959 static const char * const cs_etm_priv_fmts[] = {
1960         [CS_ETM_MAGIC]          = "     Magic number                   %llx\n",
1961         [CS_ETM_CPU]            = "     CPU                            %lld\n",
1962         [CS_ETM_ETMCR]          = "     ETMCR                          %llx\n",
1963         [CS_ETM_ETMTRACEIDR]    = "     ETMTRACEIDR                    %llx\n",
1964         [CS_ETM_ETMCCER]        = "     ETMCCER                        %llx\n",
1965         [CS_ETM_ETMIDR]         = "     ETMIDR                         %llx\n",
1966 };
1967
1968 static const char * const cs_etmv4_priv_fmts[] = {
1969         [CS_ETM_MAGIC]          = "     Magic number                   %llx\n",
1970         [CS_ETM_CPU]            = "     CPU                            %lld\n",
1971         [CS_ETMV4_TRCCONFIGR]   = "     TRCCONFIGR                     %llx\n",
1972         [CS_ETMV4_TRCTRACEIDR]  = "     TRCTRACEIDR                    %llx\n",
1973         [CS_ETMV4_TRCIDR0]      = "     TRCIDR0                        %llx\n",
1974         [CS_ETMV4_TRCIDR1]      = "     TRCIDR1                        %llx\n",
1975         [CS_ETMV4_TRCIDR2]      = "     TRCIDR2                        %llx\n",
1976         [CS_ETMV4_TRCIDR8]      = "     TRCIDR8                        %llx\n",
1977         [CS_ETMV4_TRCAUTHSTATUS] = "    TRCAUTHSTATUS                  %llx\n",
1978 };
1979
1980 static void cs_etm__print_auxtrace_info(u64 *val, int num)
1981 {
1982         int i, j, cpu = 0;
1983
1984         for (i = 0; i < CS_HEADER_VERSION_0_MAX; i++)
1985                 fprintf(stdout, cs_etm_global_header_fmts[i], val[i]);
1986
1987         for (i = CS_HEADER_VERSION_0_MAX; cpu < num; cpu++) {
1988                 if (val[i] == __perf_cs_etmv3_magic)
1989                         for (j = 0; j < CS_ETM_PRIV_MAX; j++, i++)
1990                                 fprintf(stdout, cs_etm_priv_fmts[j], val[i]);
1991                 else if (val[i] == __perf_cs_etmv4_magic)
1992                         for (j = 0; j < CS_ETMV4_PRIV_MAX; j++, i++)
1993                                 fprintf(stdout, cs_etmv4_priv_fmts[j], val[i]);
1994                 else
1995                         /* failure.. return */
1996                         return;
1997         }
1998 }
1999
2000 int cs_etm__process_auxtrace_info(union perf_event *event,
2001                                   struct perf_session *session)
2002 {
2003         struct auxtrace_info_event *auxtrace_info = &event->auxtrace_info;
2004         struct cs_etm_auxtrace *etm = NULL;
2005         struct int_node *inode;
2006         unsigned int pmu_type;
2007         int event_header_size = sizeof(struct perf_event_header);
2008         int info_header_size;
2009         int total_size = auxtrace_info->header.size;
2010         int priv_size = 0;
2011         int num_cpu;
2012         int err = 0, idx = -1;
2013         int i, j, k;
2014         u64 *ptr, *hdr = NULL;
2015         u64 **metadata = NULL;
2016
2017         /*
2018          * sizeof(auxtrace_info_event::type) +
2019          * sizeof(auxtrace_info_event::reserved) == 8
2020          */
2021         info_header_size = 8;
2022
2023         if (total_size < (event_header_size + info_header_size))
2024                 return -EINVAL;
2025
2026         priv_size = total_size - event_header_size - info_header_size;
2027
2028         /* First the global part */
2029         ptr = (u64 *) auxtrace_info->priv;
2030
2031         /* Look for version '0' of the header */
2032         if (ptr[0] != 0)
2033                 return -EINVAL;
2034
2035         hdr = zalloc(sizeof(*hdr) * CS_HEADER_VERSION_0_MAX);
2036         if (!hdr)
2037                 return -ENOMEM;
2038
2039         /* Extract header information - see cs-etm.h for format */
2040         for (i = 0; i < CS_HEADER_VERSION_0_MAX; i++)
2041                 hdr[i] = ptr[i];
2042         num_cpu = hdr[CS_PMU_TYPE_CPUS] & 0xffffffff;
2043         pmu_type = (unsigned int) ((hdr[CS_PMU_TYPE_CPUS] >> 32) &
2044                                     0xffffffff);
2045
2046         /*
2047          * Create an RB tree for traceID-metadata tuple.  Since the conversion
2048          * has to be made for each packet that gets decoded, optimizing access
2049          * in anything other than a sequential array is worth doing.
2050          */
2051         traceid_list = intlist__new(NULL);
2052         if (!traceid_list) {
2053                 err = -ENOMEM;
2054                 goto err_free_hdr;
2055         }
2056
2057         metadata = zalloc(sizeof(*metadata) * num_cpu);
2058         if (!metadata) {
2059                 err = -ENOMEM;
2060                 goto err_free_traceid_list;
2061         }
2062
2063         /*
2064          * The metadata is stored in the auxtrace_info section and encodes
2065          * the configuration of the ARM embedded trace macrocell which is
2066          * required by the trace decoder to properly decode the trace due
2067          * to its highly compressed nature.
2068          */
2069         for (j = 0; j < num_cpu; j++) {
2070                 if (ptr[i] == __perf_cs_etmv3_magic) {
2071                         metadata[j] = zalloc(sizeof(*metadata[j]) *
2072                                              CS_ETM_PRIV_MAX);
2073                         if (!metadata[j]) {
2074                                 err = -ENOMEM;
2075                                 goto err_free_metadata;
2076                         }
2077                         for (k = 0; k < CS_ETM_PRIV_MAX; k++)
2078                                 metadata[j][k] = ptr[i + k];
2079
2080                         /* The traceID is our handle */
2081                         idx = metadata[j][CS_ETM_ETMTRACEIDR];
2082                         i += CS_ETM_PRIV_MAX;
2083                 } else if (ptr[i] == __perf_cs_etmv4_magic) {
2084                         metadata[j] = zalloc(sizeof(*metadata[j]) *
2085                                              CS_ETMV4_PRIV_MAX);
2086                         if (!metadata[j]) {
2087                                 err = -ENOMEM;
2088                                 goto err_free_metadata;
2089                         }
2090                         for (k = 0; k < CS_ETMV4_PRIV_MAX; k++)
2091                                 metadata[j][k] = ptr[i + k];
2092
2093                         /* The traceID is our handle */
2094                         idx = metadata[j][CS_ETMV4_TRCTRACEIDR];
2095                         i += CS_ETMV4_PRIV_MAX;
2096                 }
2097
2098                 /* Get an RB node for this CPU */
2099                 inode = intlist__findnew(traceid_list, idx);
2100
2101                 /* Something went wrong, no need to continue */
2102                 if (!inode) {
2103                         err = PTR_ERR(inode);
2104                         goto err_free_metadata;
2105                 }
2106
2107                 /*
2108                  * The node for that CPU should not be taken.
2109                  * Back out if that's the case.
2110                  */
2111                 if (inode->priv) {
2112                         err = -EINVAL;
2113                         goto err_free_metadata;
2114                 }
2115                 /* All good, associate the traceID with the metadata pointer */
2116                 inode->priv = metadata[j];
2117         }
2118
2119         /*
2120          * Each of CS_HEADER_VERSION_0_MAX, CS_ETM_PRIV_MAX and
2121          * CS_ETMV4_PRIV_MAX mark how many double words are in the
2122          * global metadata, and each cpu's metadata respectively.
2123          * The following tests if the correct number of double words was
2124          * present in the auxtrace info section.
2125          */
2126         if (i * 8 != priv_size) {
2127                 err = -EINVAL;
2128                 goto err_free_metadata;
2129         }
2130
2131         etm = zalloc(sizeof(*etm));
2132
2133         if (!etm) {
2134                 err = -ENOMEM;
2135                 goto err_free_metadata;
2136         }
2137
2138         err = auxtrace_queues__init(&etm->queues);
2139         if (err)
2140                 goto err_free_etm;
2141
2142         etm->session = session;
2143         etm->machine = &session->machines.host;
2144
2145         etm->num_cpu = num_cpu;
2146         etm->pmu_type = pmu_type;
2147         etm->snapshot_mode = (hdr[CS_ETM_SNAPSHOT] != 0);
2148         etm->metadata = metadata;
2149         etm->auxtrace_type = auxtrace_info->type;
2150         etm->timeless_decoding = cs_etm__is_timeless_decoding(etm);
2151
2152         etm->auxtrace.process_event = cs_etm__process_event;
2153         etm->auxtrace.process_auxtrace_event = cs_etm__process_auxtrace_event;
2154         etm->auxtrace.flush_events = cs_etm__flush_events;
2155         etm->auxtrace.free_events = cs_etm__free_events;
2156         etm->auxtrace.free = cs_etm__free;
2157         session->auxtrace = &etm->auxtrace;
2158
2159         etm->unknown_thread = thread__new(999999999, 999999999);
2160         if (!etm->unknown_thread)
2161                 goto err_free_queues;
2162
2163         /*
2164          * Initialize list node so that at thread__zput() we can avoid
2165          * segmentation fault at list_del_init().
2166          */
2167         INIT_LIST_HEAD(&etm->unknown_thread->node);
2168
2169         err = thread__set_comm(etm->unknown_thread, "unknown", 0);
2170         if (err)
2171                 goto err_delete_thread;
2172
2173         if (thread__init_map_groups(etm->unknown_thread, etm->machine))
2174                 goto err_delete_thread;
2175
2176         if (dump_trace) {
2177                 cs_etm__print_auxtrace_info(auxtrace_info->priv, num_cpu);
2178                 return 0;
2179         }
2180
2181         if (session->itrace_synth_opts && session->itrace_synth_opts->set) {
2182                 etm->synth_opts = *session->itrace_synth_opts;
2183         } else {
2184                 itrace_synth_opts__set_default(&etm->synth_opts,
2185                                 session->itrace_synth_opts->default_no_sample);
2186                 etm->synth_opts.callchain = false;
2187         }
2188
2189         err = cs_etm__synth_events(etm, session);
2190         if (err)
2191                 goto err_delete_thread;
2192
2193         err = auxtrace_queues__process_index(&etm->queues, session);
2194         if (err)
2195                 goto err_delete_thread;
2196
2197         etm->data_queued = etm->queues.populated;
2198
2199         return 0;
2200
2201 err_delete_thread:
2202         thread__zput(etm->unknown_thread);
2203 err_free_queues:
2204         auxtrace_queues__free(&etm->queues);
2205         session->auxtrace = NULL;
2206 err_free_etm:
2207         zfree(&etm);
2208 err_free_metadata:
2209         /* No need to check @metadata[j], free(NULL) is supported */
2210         for (j = 0; j < num_cpu; j++)
2211                 free(metadata[j]);
2212         zfree(&metadata);
2213 err_free_traceid_list:
2214         intlist__delete(traceid_list);
2215 err_free_hdr:
2216         zfree(&hdr);
2217
2218         return -EINVAL;
2219 }