tools/perf/util/cs-etm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright(C) 2015-2018 Linaro Limited.
   4  *
   5  * Author: Tor Jeremiassen <tor@ti.com>
   6  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
   7  */
   8
   9 #include <linux/bitops.h>
  10 #include <linux/err.h>
  11 #include <linux/kernel.h>
  12 #include <linux/log2.h>
  13 #include <linux/types.h>
  14 #include <linux/zalloc.h>
  15
  16 #include <opencsd/ocsd_if_types.h>
  17 #include <stdlib.h>
  18
  19 #include "auxtrace.h"
  20 #include "color.h"
  21 #include "cs-etm.h"
  22 #include "cs-etm-decoder/cs-etm-decoder.h"
  23 #include "debug.h"
  24 #include "dso.h"
  25 #include "evlist.h"
  26 #include "intlist.h"
  27 #include "machine.h"
  28 #include "map.h"
  29 #include "perf.h"
  30 #include "symbol.h"
  31 #include "tool.h"
  32 #include "thread.h"
  33 #include "thread_map.h"
  34 #include "thread-stack.h"
  35 #include <tools/libc_compat.h>
  36 #include "util.h"
  37
  38 #define MAX_TIMESTAMP (~0ULL)
  39
  40 struct cs_etm_auxtrace {
  41         struct auxtrace auxtrace;
  42         struct auxtrace_queues queues;
  43         struct auxtrace_heap heap;
  44         struct itrace_synth_opts synth_opts;
  45         struct perf_session *session;
  46         struct machine *machine;
  47         struct thread *unknown_thread;
  48
  49         u8 timeless_decoding;
  50         u8 snapshot_mode;
  51         u8 data_queued;
  52         u8 sample_branches;
  53         u8 sample_instructions;
  54
  55         int num_cpu;
  56         u32 auxtrace_type;
  57         u64 branches_sample_type;
  58         u64 branches_id;
  59         u64 instructions_sample_type;
  60         u64 instructions_sample_period;
  61         u64 instructions_id;
  62         u64 **metadata;
  63         u64 kernel_start;
  64         unsigned int pmu_type;
  65 };
  66
  67 struct cs_etm_traceid_queue {
  68         u8 trace_chan_id;
  69         pid_t pid, tid;
  70         u64 period_instructions;
  71         size_t last_branch_pos;
  72         union perf_event *event_buf;
  73         struct thread *thread;
  74         struct branch_stack *last_branch;
  75         struct branch_stack *last_branch_rb;
  76         struct cs_etm_packet *prev_packet;
  77         struct cs_etm_packet *packet;
  78         struct cs_etm_packet_queue packet_queue;
  79 };
  80
  81 struct cs_etm_queue {
  82         struct cs_etm_auxtrace *etm;
  83         struct cs_etm_decoder *decoder;
  84         struct auxtrace_buffer *buffer;
  85         unsigned int queue_nr;
  86         u8 pending_timestamp;
  87         u64 offset;
  88         const unsigned char *buf;
  89         size_t buf_len, buf_used;
  90         /* Conversion between traceID and index in traceid_queues array */
  91         struct intlist *traceid_queues_list;
  92         struct cs_etm_traceid_queue **traceid_queues;
  93 };
  94
  95 static int cs_etm__update_queues(struct cs_etm_auxtrace *etm);
  96 static int cs_etm__process_queues(struct cs_etm_auxtrace *etm);
  97 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
  98                                            pid_t tid);
  99 static int cs_etm__get_data_block(struct cs_etm_queue *etmq);
 100 static int cs_etm__decode_data_block(struct cs_etm_queue *etmq);
 101
 102 /* PTMs ETMIDR [11:8] set to b0011 */
 103 #define ETMIDR_PTM_VERSION 0x00000300
 104
 105 /*
 106  * A struct auxtrace_heap_item only has a queue_nr and a timestamp to
 107  * work with.  One option is to modify to auxtrace_heap_XYZ() API or simply
 108  * encode the etm queue number as the upper 16 bit and the channel as
 109  * the lower 16 bit.
 110  */
 111 #define TO_CS_QUEUE_NR(queue_nr, trace_id_chan) \
 112                       (queue_nr << 16 | trace_chan_id)
 113 #define TO_QUEUE_NR(cs_queue_nr) (cs_queue_nr >> 16)
 114 #define TO_TRACE_CHAN_ID(cs_queue_nr) (cs_queue_nr & 0x0000ffff)
 115
 116 static u32 cs_etm__get_v7_protocol_version(u32 etmidr)
 117 {
 118         etmidr &= ETMIDR_PTM_VERSION;
 119
 120         if (etmidr == ETMIDR_PTM_VERSION)
 121                 return CS_ETM_PROTO_PTM;
 122
 123         return CS_ETM_PROTO_ETMV3;
 124 }
 125
 126 static int cs_etm__get_magic(u8 trace_chan_id, u64 *magic)
 127 {
 128         struct int_node *inode;
 129         u64 *metadata;
 130
 131         inode = intlist__find(traceid_list, trace_chan_id);
 132         if (!inode)
 133                 return -EINVAL;
 134
 135         metadata = inode->priv;
 136         *magic = metadata[CS_ETM_MAGIC];
 137         return 0;
 138 }
 139
 140 int cs_etm__get_cpu(u8 trace_chan_id, int *cpu)
 141 {
 142         struct int_node *inode;
 143         u64 *metadata;
 144
 145         inode = intlist__find(traceid_list, trace_chan_id);
 146         if (!inode)
 147                 return -EINVAL;
 148
 149         metadata = inode->priv;
 150         *cpu = (int)metadata[CS_ETM_CPU];
 151         return 0;
 152 }
 153
 154 void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq,
 155                                               u8 trace_chan_id)
 156 {
 157         /*
 158          * Wnen a timestamp packet is encountered the backend code
 159          * is stopped so that the front end has time to process packets
 160          * that were accumulated in the traceID queue.  Since there can
 161          * be more than one channel per cs_etm_queue, we need to specify
 162          * what traceID queue needs servicing.
 163          */
 164         etmq->pending_timestamp = trace_chan_id;
 165 }
 166
 167 static u64 cs_etm__etmq_get_timestamp(struct cs_etm_queue *etmq,
 168                                       u8 *trace_chan_id)
 169 {
 170         struct cs_etm_packet_queue *packet_queue;
 171
 172         if (!etmq->pending_timestamp)
 173                 return 0;
 174
 175         if (trace_chan_id)
 176                 *trace_chan_id = etmq->pending_timestamp;
 177
 178         packet_queue = cs_etm__etmq_get_packet_queue(etmq,
 179                                                      etmq->pending_timestamp);
 180         if (!packet_queue)
 181                 return 0;
 182
 183         /* Acknowledge pending status */
 184         etmq->pending_timestamp = 0;
 185
 186         /* See function cs_etm_decoder__do_{hard|soft}_timestamp() */
 187         return packet_queue->timestamp;
 188 }
 189
 190 static void cs_etm__clear_packet_queue(struct cs_etm_packet_queue *queue)
 191 {
 192         int i;
 193
 194         queue->head = 0;
 195         queue->tail = 0;
 196         queue->packet_count = 0;
 197         for (i = 0; i < CS_ETM_PACKET_MAX_BUFFER; i++) {
 198                 queue->packet_buffer[i].isa = CS_ETM_ISA_UNKNOWN;
 199                 queue->packet_buffer[i].start_addr = CS_ETM_INVAL_ADDR;
 200                 queue->packet_buffer[i].end_addr = CS_ETM_INVAL_ADDR;
 201                 queue->packet_buffer[i].instr_count = 0;
 202                 queue->packet_buffer[i].last_instr_taken_branch = false;
 203                 queue->packet_buffer[i].last_instr_size = 0;
 204                 queue->packet_buffer[i].last_instr_type = 0;
 205                 queue->packet_buffer[i].last_instr_subtype = 0;
 206                 queue->packet_buffer[i].last_instr_cond = 0;
 207                 queue->packet_buffer[i].flags = 0;
 208                 queue->packet_buffer[i].exception_number = UINT32_MAX;
 209                 queue->packet_buffer[i].trace_chan_id = UINT8_MAX;
 210                 queue->packet_buffer[i].cpu = INT_MIN;
 211         }
 212 }
 213
 214 static void cs_etm__clear_all_packet_queues(struct cs_etm_queue *etmq)
 215 {
 216         int idx;
 217         struct int_node *inode;
 218         struct cs_etm_traceid_queue *tidq;
 219         struct intlist *traceid_queues_list = etmq->traceid_queues_list;
 220
 221         intlist__for_each_entry(inode, traceid_queues_list) {
 222                 idx = (int)(intptr_t)inode->priv;
 223                 tidq = etmq->traceid_queues[idx];
 224                 cs_etm__clear_packet_queue(&tidq->packet_queue);
 225         }
 226 }
 227
 228 static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq,
 229                                       struct cs_etm_traceid_queue *tidq,
 230                                       u8 trace_chan_id)
 231 {
 232         int rc = -ENOMEM;
 233         struct auxtrace_queue *queue;
 234         struct cs_etm_auxtrace *etm = etmq->etm;
 235
 236         cs_etm__clear_packet_queue(&tidq->packet_queue);
 237
 238         queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
 239         tidq->tid = queue->tid;
 240         tidq->pid = -1;
 241         tidq->trace_chan_id = trace_chan_id;
 242
 243         tidq->packet = zalloc(sizeof(struct cs_etm_packet));
 244         if (!tidq->packet)
 245                 goto out;
 246
 247         tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet));
 248         if (!tidq->prev_packet)
 249                 goto out_free;
 250
 251         if (etm->synth_opts.last_branch) {
 252                 size_t sz = sizeof(struct branch_stack);
 253
 254                 sz += etm->synth_opts.last_branch_sz *
 255                       sizeof(struct branch_entry);
 256                 tidq->last_branch = zalloc(sz);
 257                 if (!tidq->last_branch)
 258                         goto out_free;
 259                 tidq->last_branch_rb = zalloc(sz);
 260                 if (!tidq->last_branch_rb)
 261                         goto out_free;
 262         }
 263
 264         tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
 265         if (!tidq->event_buf)
 266                 goto out_free;
 267
 268         return 0;
 269
 270 out_free:
 271         zfree(&tidq->last_branch_rb);
 272         zfree(&tidq->last_branch);
 273         zfree(&tidq->prev_packet);
 274         zfree(&tidq->packet);
 275 out:
 276         return rc;
 277 }
 278
 279 static struct cs_etm_traceid_queue
 280 *cs_etm__etmq_get_traceid_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
 281 {
 282         int idx;
 283         struct int_node *inode;
 284         struct intlist *traceid_queues_list;
 285         struct cs_etm_traceid_queue *tidq, **traceid_queues;
 286         struct cs_etm_auxtrace *etm = etmq->etm;
 287
 288         if (etm->timeless_decoding)
 289                 trace_chan_id = CS_ETM_PER_THREAD_TRACEID;
 290
 291         traceid_queues_list = etmq->traceid_queues_list;
 292
 293         /*
 294          * Check if the traceid_queue exist for this traceID by looking
 295          * in the queue list.
 296          */
 297         inode = intlist__find(traceid_queues_list, trace_chan_id);
 298         if (inode) {
 299                 idx = (int)(intptr_t)inode->priv;
 300                 return etmq->traceid_queues[idx];
 301         }
 302
 303         /* We couldn't find a traceid_queue for this traceID, allocate one */
 304         tidq = malloc(sizeof(*tidq));
 305         if (!tidq)
 306                 return NULL;
 307
 308         memset(tidq, 0, sizeof(*tidq));
 309
 310         /* Get a valid index for the new traceid_queue */
 311         idx = intlist__nr_entries(traceid_queues_list);
 312         /* Memory for the inode is free'ed in cs_etm_free_traceid_queues () */
 313         inode = intlist__findnew(traceid_queues_list, trace_chan_id);
 314         if (!inode)
 315                 goto out_free;
 316
 317         /* Associate this traceID with this index */
 318         inode->priv = (void *)(intptr_t)idx;
 319
 320         if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id))
 321                 goto out_free;
 322
 323         /* Grow the traceid_queues array by one unit */
 324         traceid_queues = etmq->traceid_queues;
 325         traceid_queues = reallocarray(traceid_queues,
 326                                       idx + 1,
 327                                       sizeof(*traceid_queues));
 328
 329         /*
 330          * On failure reallocarray() returns NULL and the original block of
 331          * memory is left untouched.
 332          */
 333         if (!traceid_queues)
 334                 goto out_free;
 335
 336         traceid_queues[idx] = tidq;
 337         etmq->traceid_queues = traceid_queues;
 338
 339         return etmq->traceid_queues[idx];
 340
 341 out_free:
 342         /*
 343          * Function intlist__remove() removes the inode from the list
 344          * and delete the memory associated to it.
 345          */
 346         intlist__remove(traceid_queues_list, inode);
 347         free(tidq);
 348
 349         return NULL;
 350 }
 351
 352 struct cs_etm_packet_queue
 353 *cs_etm__etmq_get_packet_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
 354 {
 355         struct cs_etm_traceid_queue *tidq;
 356
 357         tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
 358         if (tidq)
 359                 return &tidq->packet_queue;
 360
 361         return NULL;
 362 }
 363
 364 static void cs_etm__packet_dump(const char *pkt_string)
 365 {
 366         const char *color = PERF_COLOR_BLUE;
 367         int len = strlen(pkt_string);
 368
 369         if (len && (pkt_string[len-1] == '\n'))
 370                 color_fprintf(stdout, color, "  %s", pkt_string);
 371         else
 372                 color_fprintf(stdout, color, "  %s\n", pkt_string);
 373
 374         fflush(stdout);
 375 }
 376
 377 static void cs_etm__set_trace_param_etmv3(struct cs_etm_trace_params *t_params,
 378                                           struct cs_etm_auxtrace *etm, int idx,
 379                                           u32 etmidr)
 380 {
 381         u64 **metadata = etm->metadata;
 382
 383         t_params[idx].protocol = cs_etm__get_v7_protocol_version(etmidr);
 384         t_params[idx].etmv3.reg_ctrl = metadata[idx][CS_ETM_ETMCR];
 385         t_params[idx].etmv3.reg_trc_id = metadata[idx][CS_ETM_ETMTRACEIDR];
 386 }
 387
 388 static void cs_etm__set_trace_param_etmv4(struct cs_etm_trace_params *t_params,
 389                                           struct cs_etm_auxtrace *etm, int idx)
 390 {
 391         u64 **metadata = etm->metadata;
 392
 393         t_params[idx].protocol = CS_ETM_PROTO_ETMV4i;
 394         t_params[idx].etmv4.reg_idr0 = metadata[idx][CS_ETMV4_TRCIDR0];
 395         t_params[idx].etmv4.reg_idr1 = metadata[idx][CS_ETMV4_TRCIDR1];
 396         t_params[idx].etmv4.reg_idr2 = metadata[idx][CS_ETMV4_TRCIDR2];
 397         t_params[idx].etmv4.reg_idr8 = metadata[idx][CS_ETMV4_TRCIDR8];
 398         t_params[idx].etmv4.reg_configr = metadata[idx][CS_ETMV4_TRCCONFIGR];
 399         t_params[idx].etmv4.reg_traceidr = metadata[idx][CS_ETMV4_TRCTRACEIDR];
 400 }
 401
 402 static int cs_etm__init_trace_params(struct cs_etm_trace_params *t_params,
 403                                      struct cs_etm_auxtrace *etm)
 404 {
 405         int i;
 406         u32 etmidr;
 407         u64 architecture;
 408
 409         for (i = 0; i < etm->num_cpu; i++) {
 410                 architecture = etm->metadata[i][CS_ETM_MAGIC];
 411
 412                 switch (architecture) {
 413                 case __perf_cs_etmv3_magic:
 414                         etmidr = etm->metadata[i][CS_ETM_ETMIDR];
 415                         cs_etm__set_trace_param_etmv3(t_params, etm, i, etmidr);
 416                         break;
 417                 case __perf_cs_etmv4_magic:
 418                         cs_etm__set_trace_param_etmv4(t_params, etm, i);
 419                         break;
 420                 default:
 421                         return -EINVAL;
 422                 }
 423         }
 424
 425         return 0;
 426 }
 427
 428 static int cs_etm__init_decoder_params(struct cs_etm_decoder_params *d_params,
 429                                        struct cs_etm_queue *etmq,
 430                                        enum cs_etm_decoder_operation mode)
 431 {
 432         int ret = -EINVAL;
 433
 434         if (!(mode < CS_ETM_OPERATION_MAX))
 435                 goto out;
 436
 437         d_params->packet_printer = cs_etm__packet_dump;
 438         d_params->operation = mode;
 439         d_params->data = etmq;
 440         d_params->formatted = true;
 441         d_params->fsyncs = false;
 442         d_params->hsyncs = false;
 443         d_params->frame_aligned = true;
 444
 445         ret = 0;
 446 out:
 447         return ret;
 448 }
 449
 450 static void cs_etm__dump_event(struct cs_etm_auxtrace *etm,
 451                                struct auxtrace_buffer *buffer)
 452 {
 453         int ret;
 454         const char *color = PERF_COLOR_BLUE;
 455         struct cs_etm_decoder_params d_params;
 456         struct cs_etm_trace_params *t_params;
 457         struct cs_etm_decoder *decoder;
 458         size_t buffer_used = 0;
 459
 460         fprintf(stdout, "\n");
 461         color_fprintf(stdout, color,
 462                      ". ... CoreSight ETM Trace data: size %zu bytes\n",
 463                      buffer->size);
 464
 465         /* Use metadata to fill in trace parameters for trace decoder */
 466         t_params = zalloc(sizeof(*t_params) * etm->num_cpu);
 467
 468         if (!t_params)
 469                 return;
 470
 471         if (cs_etm__init_trace_params(t_params, etm))
 472                 goto out_free;
 473
 474         /* Set decoder parameters to simply print the trace packets */
 475         if (cs_etm__init_decoder_params(&d_params, NULL,
 476                                         CS_ETM_OPERATION_PRINT))
 477                 goto out_free;
 478
 479         decoder = cs_etm_decoder__new(etm->num_cpu, &d_params, t_params);
 480
 481         if (!decoder)
 482                 goto out_free;
 483         do {
 484                 size_t consumed;
 485
 486                 ret = cs_etm_decoder__process_data_block(
 487                                 decoder, buffer->offset,
 488                                 &((u8 *)buffer->data)[buffer_used],
 489                                 buffer->size - buffer_used, &consumed);
 490                 if (ret)
 491                         break;
 492
 493                 buffer_used += consumed;
 494         } while (buffer_used < buffer->size);
 495
 496         cs_etm_decoder__free(decoder);
 497
 498 out_free:
 499         zfree(&t_params);
 500 }
 501
 502 static int cs_etm__flush_events(struct perf_session *session,
 503                                 struct perf_tool *tool)
 504 {
 505         int ret;
 506         struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
 507                                                    struct cs_etm_auxtrace,
 508                                                    auxtrace);
 509         if (dump_trace)
 510                 return 0;
 511
 512         if (!tool->ordered_events)
 513                 return -EINVAL;
 514
 515         ret = cs_etm__update_queues(etm);
 516
 517         if (ret < 0)
 518                 return ret;
 519
 520         if (etm->timeless_decoding)
 521                 return cs_etm__process_timeless_queues(etm, -1);
 522
 523         return cs_etm__process_queues(etm);
 524 }
 525
 526 static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq)
 527 {
 528         int idx;
 529         uintptr_t priv;
 530         struct int_node *inode, *tmp;
 531         struct cs_etm_traceid_queue *tidq;
 532         struct intlist *traceid_queues_list = etmq->traceid_queues_list;
 533
 534         intlist__for_each_entry_safe(inode, tmp, traceid_queues_list) {
 535                 priv = (uintptr_t)inode->priv;
 536                 idx = priv;
 537
 538                 /* Free this traceid_queue from the array */
 539                 tidq = etmq->traceid_queues[idx];
 540                 thread__zput(tidq->thread);
 541                 zfree(&tidq->event_buf);
 542                 zfree(&tidq->last_branch);
 543                 zfree(&tidq->last_branch_rb);
 544                 zfree(&tidq->prev_packet);
 545                 zfree(&tidq->packet);
 546                 zfree(&tidq);
 547
 548                 /*
 549                  * Function intlist__remove() removes the inode from the list
 550                  * and delete the memory associated to it.
 551                  */
 552                 intlist__remove(traceid_queues_list, inode);
 553         }
 554
 555         /* Then the RB tree itself */
 556         intlist__delete(traceid_queues_list);
 557         etmq->traceid_queues_list = NULL;
 558
 559         /* finally free the traceid_queues array */
 560         zfree(&etmq->traceid_queues);
 561 }
 562
 563 static void cs_etm__free_queue(void *priv)
 564 {
 565         struct cs_etm_queue *etmq = priv;
 566
 567         if (!etmq)
 568                 return;
 569
 570         cs_etm_decoder__free(etmq->decoder);
 571         cs_etm__free_traceid_queues(etmq);
 572         free(etmq);
 573 }
 574
 575 static void cs_etm__free_events(struct perf_session *session)
 576 {
 577         unsigned int i;
 578         struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
 579                                                    struct cs_etm_auxtrace,
 580                                                    auxtrace);
 581         struct auxtrace_queues *queues = &aux->queues;
 582
 583         for (i = 0; i < queues->nr_queues; i++) {
 584                 cs_etm__free_queue(queues->queue_array[i].priv);
 585                 queues->queue_array[i].priv = NULL;
 586         }
 587
 588         auxtrace_queues__free(queues);
 589 }
 590
 591 static void cs_etm__free(struct perf_session *session)
 592 {
 593         int i;
 594         struct int_node *inode, *tmp;
 595         struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
 596                                                    struct cs_etm_auxtrace,
 597                                                    auxtrace);
 598         cs_etm__free_events(session);
 599         session->auxtrace = NULL;
 600
 601         /* First remove all traceID/metadata nodes for the RB tree */
 602         intlist__for_each_entry_safe(inode, tmp, traceid_list)
 603                 intlist__remove(traceid_list, inode);
 604         /* Then the RB tree itself */
 605         intlist__delete(traceid_list);
 606
 607         for (i = 0; i < aux->num_cpu; i++)
 608                 zfree(&aux->metadata[i]);
 609
 610         thread__zput(aux->unknown_thread);
 611         zfree(&aux->metadata);
 612         zfree(&aux);
 613 }
 614
 615 static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address)
 616 {
 617         struct machine *machine;
 618
 619         machine = etmq->etm->machine;
 620
 621         if (address >= etmq->etm->kernel_start) {
 622                 if (machine__is_host(machine))
 623                         return PERF_RECORD_MISC_KERNEL;
 624                 else
 625                         return PERF_RECORD_MISC_GUEST_KERNEL;
 626         } else {
 627                 if (machine__is_host(machine))
 628                         return PERF_RECORD_MISC_USER;
 629                 else if (perf_guest)
 630                         return PERF_RECORD_MISC_GUEST_USER;
 631                 else
 632                         return PERF_RECORD_MISC_HYPERVISOR;
 633         }
 634 }
 635
 636 static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id,
 637                               u64 address, size_t size, u8 *buffer)
 638 {
 639         u8  cpumode;
 640         u64 offset;
 641         int len;
 642         struct thread *thread;
 643         struct machine *machine;
 644         struct addr_location al;
 645         struct cs_etm_traceid_queue *tidq;
 646
 647         if (!etmq)
 648                 return 0;
 649
 650         machine = etmq->etm->machine;
 651         cpumode = cs_etm__cpu_mode(etmq, address);
 652         tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
 653         if (!tidq)
 654                 return 0;
 655
 656         thread = tidq->thread;
 657         if (!thread) {
 658                 if (cpumode != PERF_RECORD_MISC_KERNEL)
 659                         return 0;
 660                 thread = etmq->etm->unknown_thread;
 661         }
 662
 663         if (!thread__find_map(thread, cpumode, address, &al) || !al.map->dso)
 664                 return 0;
 665
 666         if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
 667             dso__data_status_seen(al.map->dso, DSO_DATA_STATUS_SEEN_ITRACE))
 668                 return 0;
 669
 670         offset = al.map->map_ip(al.map, address);
 671
 672         map__load(al.map);
 673
 674         len = dso__data_read_offset(al.map->dso, machine, offset, buffer, size);
 675
 676         if (len <= 0)
 677                 return 0;
 678
 679         return len;
 680 }
 681
 682 static struct cs_etm_queue *cs_etm__alloc_queue(struct cs_etm_auxtrace *etm)
 683 {
 684         struct cs_etm_decoder_params d_params;
 685         struct cs_etm_trace_params  *t_params = NULL;
 686         struct cs_etm_queue *etmq;
 687
 688         etmq = zalloc(sizeof(*etmq));
 689         if (!etmq)
 690                 return NULL;
 691
 692         etmq->traceid_queues_list = intlist__new(NULL);
 693         if (!etmq->traceid_queues_list)
 694                 goto out_free;
 695
 696         /* Use metadata to fill in trace parameters for trace decoder */
 697         t_params = zalloc(sizeof(*t_params) * etm->num_cpu);
 698
 699         if (!t_params)
 700                 goto out_free;
 701
 702         if (cs_etm__init_trace_params(t_params, etm))
 703                 goto out_free;
 704
 705         /* Set decoder parameters to decode trace packets */
 706         if (cs_etm__init_decoder_params(&d_params, etmq,
 707                                         CS_ETM_OPERATION_DECODE))
 708                 goto out_free;
 709
 710         etmq->decoder = cs_etm_decoder__new(etm->num_cpu, &d_params, t_params);
 711
 712         if (!etmq->decoder)
 713                 goto out_free;
 714
 715         /*
 716          * Register a function to handle all memory accesses required by
 717          * the trace decoder library.
 718          */
 719         if (cs_etm_decoder__add_mem_access_cb(etmq->decoder,
 720                                               0x0L, ((u64) -1L),
 721                                               cs_etm__mem_access))
 722                 goto out_free_decoder;
 723
 724         zfree(&t_params);
 725         return etmq;
 726
 727 out_free_decoder:
 728         cs_etm_decoder__free(etmq->decoder);
 729 out_free:
 730         intlist__delete(etmq->traceid_queues_list);
 731         free(etmq);
 732
 733         return NULL;
 734 }
 735
 736 static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm,
 737                                struct auxtrace_queue *queue,
 738                                unsigned int queue_nr)
 739 {
 740         int ret = 0;
 741         unsigned int cs_queue_nr;
 742         u8 trace_chan_id;
 743         u64 timestamp;
 744         struct cs_etm_queue *etmq = queue->priv;
 745
 746         if (list_empty(&queue->head) || etmq)
 747                 goto out;
 748
 749         etmq = cs_etm__alloc_queue(etm);
 750
 751         if (!etmq) {
 752                 ret = -ENOMEM;
 753                 goto out;
 754         }
 755
 756         queue->priv = etmq;
 757         etmq->etm = etm;
 758         etmq->queue_nr = queue_nr;
 759         etmq->offset = 0;
 760
 761         if (etm->timeless_decoding)
 762                 goto out;
 763
 764         /*
 765          * We are under a CPU-wide trace scenario.  As such we need to know
 766          * when the code that generated the traces started to execute so that
 767          * it can be correlated with execution on other CPUs.  So we get a
 768          * handle on the beginning of traces and decode until we find a
 769          * timestamp.  The timestamp is then added to the auxtrace min heap
 770          * in order to know what nibble (of all the etmqs) to decode first.
 771          */
 772         while (1) {
 773                 /*
 774                  * Fetch an aux_buffer from this etmq.  Bail if no more
 775                  * blocks or an error has been encountered.
 776                  */
 777                 ret = cs_etm__get_data_block(etmq);
 778                 if (ret <= 0)
 779                         goto out;
 780
 781                 /*
 782                  * Run decoder on the trace block.  The decoder will stop when
 783                  * encountering a timestamp, a full packet queue or the end of
 784                  * trace for that block.
 785                  */
 786                 ret = cs_etm__decode_data_block(etmq);
 787                 if (ret)
 788                         goto out;
 789
 790                 /*
 791                  * Function cs_etm_decoder__do_{hard|soft}_timestamp() does all
 792                  * the timestamp calculation for us.
 793                  */
 794                 timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
 795
 796                 /* We found a timestamp, no need to continue. */
 797                 if (timestamp)
 798                         break;
 799
 800                 /*
 801                  * We didn't find a timestamp so empty all the traceid packet
 802                  * queues before looking for another timestamp packet, either
 803                  * in the current data block or a new one.  Packets that were
 804                  * just decoded are useless since no timestamp has been
 805                  * associated with them.  As such simply discard them.
 806                  */
 807                 cs_etm__clear_all_packet_queues(etmq);
 808         }
 809
 810         /*
 811          * We have a timestamp.  Add it to the min heap to reflect when
 812          * instructions conveyed by the range packets of this traceID queue
 813          * started to execute.  Once the same has been done for all the traceID
 814          * queues of each etmq, redenring and decoding can start in
 815          * chronological order.
 816          *
 817          * Note that packets decoded above are still in the traceID's packet
 818          * queue and will be processed in cs_etm__process_queues().
 819          */
 820         cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_id_chan);
 821         ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, timestamp);
 822 out:
 823         return ret;
 824 }
 825
 826 static int cs_etm__setup_queues(struct cs_etm_auxtrace *etm)
 827 {
 828         unsigned int i;
 829         int ret;
 830
 831         if (!etm->kernel_start)
 832                 etm->kernel_start = machine__kernel_start(etm->machine);
 833
 834         for (i = 0; i < etm->queues.nr_queues; i++) {
 835                 ret = cs_etm__setup_queue(etm, &etm->queues.queue_array[i], i);
 836                 if (ret)
 837                         return ret;
 838         }
 839
 840         return 0;
 841 }
 842
 843 static int cs_etm__update_queues(struct cs_etm_auxtrace *etm)
 844 {
 845         if (etm->queues.new_data) {
 846                 etm->queues.new_data = false;
 847                 return cs_etm__setup_queues(etm);
 848         }
 849
 850         return 0;
 851 }
 852
 853 static inline
 854 void cs_etm__copy_last_branch_rb(struct cs_etm_queue *etmq,
 855                                  struct cs_etm_traceid_queue *tidq)
 856 {
 857         struct branch_stack *bs_src = tidq->last_branch_rb;
 858         struct branch_stack *bs_dst = tidq->last_branch;
 859         size_t nr = 0;
 860
 861         /*
 862          * Set the number of records before early exit: ->nr is used to
 863          * determine how many branches to copy from ->entries.
 864          */
 865         bs_dst->nr = bs_src->nr;
 866
 867         /*
 868          * Early exit when there is nothing to copy.
 869          */
 870         if (!bs_src->nr)
 871                 return;
 872
 873         /*
 874          * As bs_src->entries is a circular buffer, we need to copy from it in
 875          * two steps.  First, copy the branches from the most recently inserted
 876          * branch ->last_branch_pos until the end of bs_src->entries buffer.
 877          */
 878         nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos;
 879         memcpy(&bs_dst->entries[0],
 880                &bs_src->entries[tidq->last_branch_pos],
 881                sizeof(struct branch_entry) * nr);
 882
 883         /*
 884          * If we wrapped around at least once, the branches from the beginning
 885          * of the bs_src->entries buffer and until the ->last_branch_pos element
 886          * are older valid branches: copy them over.  The total number of
 887          * branches copied over will be equal to the number of branches asked by
 888          * the user in last_branch_sz.
 889          */
 890         if (bs_src->nr >= etmq->etm->synth_opts.last_branch_sz) {
 891                 memcpy(&bs_dst->entries[nr],
 892                        &bs_src->entries[0],
 893                        sizeof(struct branch_entry) * tidq->last_branch_pos);
 894         }
 895 }
 896
 897 static inline
 898 void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq)
 899 {
 900         tidq->last_branch_pos = 0;
 901         tidq->last_branch_rb->nr = 0;
 902 }
 903
 904 static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq,
 905                                          u8 trace_chan_id, u64 addr)
 906 {
 907         u8 instrBytes[2];
 908
 909         cs_etm__mem_access(etmq, trace_chan_id, addr,
 910                            ARRAY_SIZE(instrBytes), instrBytes);
 911         /*
 912          * T32 instruction size is indicated by bits[15:11] of the first
 913          * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111
 914          * denote a 32-bit instruction.
 915          */
 916         return ((instrBytes[1] & 0xF8) >= 0xE8) ? 4 : 2;
 917 }
 918
 919 static inline u64 cs_etm__first_executed_instr(struct cs_etm_packet *packet)
 920 {
 921         /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
 922         if (packet->sample_type == CS_ETM_DISCONTINUITY)
 923                 return 0;
 924
 925         return packet->start_addr;
 926 }
 927
 928 static inline
 929 u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet)
 930 {
 931         /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
 932         if (packet->sample_type == CS_ETM_DISCONTINUITY)
 933                 return 0;
 934
 935         return packet->end_addr - packet->last_instr_size;
 936 }
 937
 938 static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq,
 939                                      u64 trace_chan_id,
 940                                      const struct cs_etm_packet *packet,
 941                                      u64 offset)
 942 {
 943         if (packet->isa == CS_ETM_ISA_T32) {
 944                 u64 addr = packet->start_addr;
 945
 946                 while (offset > 0) {
 947                         addr += cs_etm__t32_instr_size(etmq,
 948                                                        trace_chan_id, addr);
 949                         offset--;
 950                 }
 951                 return addr;
 952         }
 953
 954         /* Assume a 4 byte instruction size (A32/A64) */
 955         return packet->start_addr + offset * 4;
 956 }
 957
 958 static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq,
 959                                           struct cs_etm_traceid_queue *tidq)
 960 {
 961         struct branch_stack *bs = tidq->last_branch_rb;
 962         struct branch_entry *be;
 963
 964         /*
 965          * The branches are recorded in a circular buffer in reverse
 966          * chronological order: we start recording from the last element of the
 967          * buffer down.  After writing the first element of the stack, move the
 968          * insert position back to the end of the buffer.
 969          */
 970         if (!tidq->last_branch_pos)
 971                 tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;
 972
 973         tidq->last_branch_pos -= 1;
 974
 975         be       = &bs->entries[tidq->last_branch_pos];
 976         be->from = cs_etm__last_executed_instr(tidq->prev_packet);
 977         be->to   = cs_etm__first_executed_instr(tidq->packet);
 978         /* No support for mispredict */
 979         be->flags.mispred = 0;
 980         be->flags.predicted = 1;
 981
 982         /*
 983          * Increment bs->nr until reaching the number of last branches asked by
 984          * the user on the command line.
 985          */
 986         if (bs->nr < etmq->etm->synth_opts.last_branch_sz)
 987                 bs->nr += 1;
 988 }
 989
 990 static int cs_etm__inject_event(union perf_event *event,
 991                                struct perf_sample *sample, u64 type)
 992 {
 993         event->header.size = perf_event__sample_event_size(sample, type, 0);
 994         return perf_event__synthesize_sample(event, type, 0, sample);
 995 }
 996
 997
 998 static int
 999 cs_etm__get_trace(struct cs_etm_queue *etmq)
1000 {
1001         struct auxtrace_buffer *aux_buffer = etmq->buffer;
1002         struct auxtrace_buffer *old_buffer = aux_buffer;
1003         struct auxtrace_queue *queue;
1004
1005         queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
1006
1007         aux_buffer = auxtrace_buffer__next(queue, aux_buffer);
1008
1009         /* If no more data, drop the previous auxtrace_buffer and return */
1010         if (!aux_buffer) {
1011                 if (old_buffer)
1012                         auxtrace_buffer__drop_data(old_buffer);
1013                 etmq->buf_len = 0;
1014                 return 0;
1015         }
1016
1017         etmq->buffer = aux_buffer;
1018
1019         /* If the aux_buffer doesn't have data associated, try to load it */
1020         if (!aux_buffer->data) {
1021                 /* get the file desc associated with the perf data file */
1022                 int fd = perf_data__fd(etmq->etm->session->data);
1023
1024                 aux_buffer->data = auxtrace_buffer__get_data(aux_buffer, fd);
1025                 if (!aux_buffer->data)
1026                         return -ENOMEM;
1027         }
1028
1029         /* If valid, drop the previous buffer */
1030         if (old_buffer)
1031                 auxtrace_buffer__drop_data(old_buffer);
1032
1033         etmq->buf_used = 0;
1034         etmq->buf_len = aux_buffer->size;
1035         etmq->buf = aux_buffer->data;
1036
1037         return etmq->buf_len;
1038 }
1039
1040 static void cs_etm__set_pid_tid_cpu(struct cs_etm_auxtrace *etm,
1041                                     struct cs_etm_traceid_queue *tidq)
1042 {
1043         if ((!tidq->thread) && (tidq->tid != -1))
1044                 tidq->thread = machine__find_thread(etm->machine, -1,
1045                                                     tidq->tid);
1046
1047         if (tidq->thread)
1048                 tidq->pid = tidq->thread->pid_;
1049 }
1050
1051 int cs_etm__etmq_set_tid(struct cs_etm_queue *etmq,
1052                          pid_t tid, u8 trace_chan_id)
1053 {
1054         int cpu, err = -EINVAL;
1055         struct cs_etm_auxtrace *etm = etmq->etm;
1056         struct cs_etm_traceid_queue *tidq;
1057
1058         tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
1059         if (!tidq)
1060                 return err;
1061
1062         if (cs_etm__get_cpu(trace_chan_id, &cpu) < 0)
1063                 return err;
1064
1065         err = machine__set_current_tid(etm->machine, cpu, tid, tid);
1066         if (err)
1067                 return err;
1068
1069         tidq->tid = tid;
1070         thread__zput(tidq->thread);
1071
1072         cs_etm__set_pid_tid_cpu(etm, tidq);
1073         return 0;
1074 }
1075
1076 bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq)
1077 {
1078         return !!etmq->etm->timeless_decoding;
1079 }
1080
1081 static void cs_etm__copy_insn(struct cs_etm_queue *etmq,
1082                               u64 trace_chan_id,
1083                               const struct cs_etm_packet *packet,
1084                               struct perf_sample *sample)
1085 {
1086         /*
1087          * It's pointless to read instructions for the CS_ETM_DISCONTINUITY
1088          * packet, so directly bail out with 'insn_len' = 0.
1089          */
1090         if (packet->sample_type == CS_ETM_DISCONTINUITY) {
1091                 sample->insn_len = 0;
1092                 return;
1093         }
1094
1095         /*
1096          * T32 instruction size might be 32-bit or 16-bit, decide by calling
1097          * cs_etm__t32_instr_size().
1098          */
1099         if (packet->isa == CS_ETM_ISA_T32)
1100                 sample->insn_len = cs_etm__t32_instr_size(etmq, trace_chan_id,
1101                                                           sample->ip);
1102         /* Otherwise, A64 and A32 instruction size are always 32-bit. */
1103         else
1104                 sample->insn_len = 4;
1105
1106         cs_etm__mem_access(etmq, trace_chan_id, sample->ip,
1107                            sample->insn_len, (void *)sample->insn);
1108 }
1109
1110 static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq,
1111                                             struct cs_etm_traceid_queue *tidq,
1112                                             u64 addr, u64 period)
1113 {
1114         int ret = 0;
1115         struct cs_etm_auxtrace *etm = etmq->etm;
1116         union perf_event *event = tidq->event_buf;
1117         struct perf_sample sample = {.ip = 0,};
1118
1119         event->sample.header.type = PERF_RECORD_SAMPLE;
1120         event->sample.header.misc = cs_etm__cpu_mode(etmq, addr);
1121         event->sample.header.size = sizeof(struct perf_event_header);
1122
1123         sample.ip = addr;
1124         sample.pid = tidq->pid;
1125         sample.tid = tidq->tid;
1126         sample.id = etmq->etm->instructions_id;
1127         sample.stream_id = etmq->etm->instructions_id;
1128         sample.period = period;
1129         sample.cpu = tidq->packet->cpu;
1130         sample.flags = tidq->prev_packet->flags;
1131         sample.cpumode = event->sample.header.misc;
1132
1133         cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample);
1134
1135         if (etm->synth_opts.last_branch) {
1136                 cs_etm__copy_last_branch_rb(etmq, tidq);
1137                 sample.branch_stack = tidq->last_branch;
1138         }
1139
1140         if (etm->synth_opts.inject) {
1141                 ret = cs_etm__inject_event(event, &sample,
1142                                            etm->instructions_sample_type);
1143                 if (ret)
1144                         return ret;
1145         }
1146
1147         ret = perf_session__deliver_synth_event(etm->session, event, &sample);
1148
1149         if (ret)
1150                 pr_err(
1151                         "CS ETM Trace: failed to deliver instruction event, error %d\n",
1152                         ret);
1153
1154         if (etm->synth_opts.last_branch)
1155                 cs_etm__reset_last_branch_rb(tidq);
1156
1157         return ret;
1158 }
1159
1160 /*
1161  * The cs etm packet encodes an instruction range between a branch target
1162  * and the next taken branch. Generate sample accordingly.
1163  */
1164 static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq,
1165                                        struct cs_etm_traceid_queue *tidq)
1166 {
1167         int ret = 0;
1168         struct cs_etm_auxtrace *etm = etmq->etm;
1169         struct perf_sample sample = {.ip = 0,};
1170         union perf_event *event = tidq->event_buf;
1171         struct dummy_branch_stack {
1172                 u64                     nr;
1173                 struct branch_entry     entries;
1174         } dummy_bs;
1175         u64 ip;
1176
1177         ip = cs_etm__last_executed_instr(tidq->prev_packet);
1178
1179         event->sample.header.type = PERF_RECORD_SAMPLE;
1180         event->sample.header.misc = cs_etm__cpu_mode(etmq, ip);
1181         event->sample.header.size = sizeof(struct perf_event_header);
1182
1183         sample.ip = ip;
1184         sample.pid = tidq->pid;
1185         sample.tid = tidq->tid;
1186         sample.addr = cs_etm__first_executed_instr(tidq->packet);
1187         sample.id = etmq->etm->branches_id;
1188         sample.stream_id = etmq->etm->branches_id;
1189         sample.period = 1;
1190         sample.cpu = tidq->packet->cpu;
1191         sample.flags = tidq->prev_packet->flags;
1192         sample.cpumode = event->sample.header.misc;
1193
1194         cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet,
1195                           &sample);
1196
1197         /*
1198          * perf report cannot handle events without a branch stack
1199          */
1200         if (etm->synth_opts.last_branch) {
1201                 dummy_bs = (struct dummy_branch_stack){
1202                         .nr = 1,
1203                         .entries = {
1204                                 .from = sample.ip,
1205                                 .to = sample.addr,
1206                         },
1207                 };
1208                 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1209         }
1210
1211         if (etm->synth_opts.inject) {
1212                 ret = cs_etm__inject_event(event, &sample,
1213                                            etm->branches_sample_type);
1214                 if (ret)
1215                         return ret;
1216         }
1217
1218         ret = perf_session__deliver_synth_event(etm->session, event, &sample);
1219
1220         if (ret)
1221                 pr_err(
1222                 "CS ETM Trace: failed to deliver instruction event, error %d\n",
1223                 ret);
1224
1225         return ret;
1226 }
1227
1228 struct cs_etm_synth {
1229         struct perf_tool dummy_tool;
1230         struct perf_session *session;
1231 };
1232
1233 static int cs_etm__event_synth(struct perf_tool *tool,
1234                                union perf_event *event,
1235                                struct perf_sample *sample __maybe_unused,
1236                                struct machine *machine __maybe_unused)
1237 {
1238         struct cs_etm_synth *cs_etm_synth =
1239                       container_of(tool, struct cs_etm_synth, dummy_tool);
1240
1241         return perf_session__deliver_synth_event(cs_etm_synth->session,
1242                                                  event, NULL);
1243 }
1244
1245 static int cs_etm__synth_event(struct perf_session *session,
1246                                struct perf_event_attr *attr, u64 id)
1247 {
1248         struct cs_etm_synth cs_etm_synth;
1249
1250         memset(&cs_etm_synth, 0, sizeof(struct cs_etm_synth));
1251         cs_etm_synth.session = session;
1252
1253         return perf_event__synthesize_attr(&cs_etm_synth.dummy_tool, attr, 1,
1254                                            &id, cs_etm__event_synth);
1255 }
1256
1257 static int cs_etm__synth_events(struct cs_etm_auxtrace *etm,
1258                                 struct perf_session *session)
1259 {
1260         struct evlist *evlist = session->evlist;
1261         struct evsel *evsel;
1262         struct perf_event_attr attr;
1263         bool found = false;
1264         u64 id;
1265         int err;
1266
1267         evlist__for_each_entry(evlist, evsel) {
1268                 if (evsel->core.attr.type == etm->pmu_type) {
1269                         found = true;
1270                         break;
1271                 }
1272         }
1273
1274         if (!found) {
1275                 pr_debug("No selected events with CoreSight Trace data\n");
1276                 return 0;
1277         }
1278
1279         memset(&attr, 0, sizeof(struct perf_event_attr));
1280         attr.size = sizeof(struct perf_event_attr);
1281         attr.type = PERF_TYPE_HARDWARE;
1282         attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;
1283         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1284                             PERF_SAMPLE_PERIOD;
1285         if (etm->timeless_decoding)
1286                 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1287         else
1288                 attr.sample_type |= PERF_SAMPLE_TIME;
1289
1290         attr.exclude_user = evsel->core.attr.exclude_user;
1291         attr.exclude_kernel = evsel->core.attr.exclude_kernel;
1292         attr.exclude_hv = evsel->core.attr.exclude_hv;
1293         attr.exclude_host = evsel->core.attr.exclude_host;
1294         attr.exclude_guest = evsel->core.attr.exclude_guest;
1295         attr.sample_id_all = evsel->core.attr.sample_id_all;
1296         attr.read_format = evsel->core.attr.read_format;
1297
1298         /* create new id val to be a fixed offset from evsel id */
1299         id = evsel->id[0] + 1000000000;
1300
1301         if (!id)
1302                 id = 1;
1303
1304         if (etm->synth_opts.branches) {
1305                 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
1306                 attr.sample_period = 1;
1307                 attr.sample_type |= PERF_SAMPLE_ADDR;
1308                 err = cs_etm__synth_event(session, &attr, id);
1309                 if (err)
1310                         return err;
1311                 etm->sample_branches = true;
1312                 etm->branches_sample_type = attr.sample_type;
1313                 etm->branches_id = id;
1314                 id += 1;
1315                 attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR;
1316         }
1317
1318         if (etm->synth_opts.last_branch)
1319                 attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1320
1321         if (etm->synth_opts.instructions) {
1322                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1323                 attr.sample_period = etm->synth_opts.period;
1324                 etm->instructions_sample_period = attr.sample_period;
1325                 err = cs_etm__synth_event(session, &attr, id);
1326                 if (err)
1327                         return err;
1328                 etm->sample_instructions = true;
1329                 etm->instructions_sample_type = attr.sample_type;
1330                 etm->instructions_id = id;
1331                 id += 1;
1332         }
1333
1334         return 0;
1335 }
1336
1337 static int cs_etm__sample(struct cs_etm_queue *etmq,
1338                           struct cs_etm_traceid_queue *tidq)
1339 {
1340         struct cs_etm_auxtrace *etm = etmq->etm;
1341         struct cs_etm_packet *tmp;
1342         int ret;
1343         u8 trace_chan_id = tidq->trace_chan_id;
1344         u64 instrs_executed = tidq->packet->instr_count;
1345
1346         tidq->period_instructions += instrs_executed;
1347
1348         /*
1349          * Record a branch when the last instruction in
1350          * PREV_PACKET is a branch.
1351          */
1352         if (etm->synth_opts.last_branch &&
1353             tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1354             tidq->prev_packet->last_instr_taken_branch)
1355                 cs_etm__update_last_branch_rb(etmq, tidq);
1356
1357         if (etm->sample_instructions &&
1358             tidq->period_instructions >= etm->instructions_sample_period) {
1359                 /*
1360                  * Emit instruction sample periodically
1361                  * TODO: allow period to be defined in cycles and clock time
1362                  */
1363
1364                 /* Get number of instructions executed after the sample point */
1365                 u64 instrs_over = tidq->period_instructions -
1366                         etm->instructions_sample_period;
1367
1368                 /*
1369                  * Calculate the address of the sampled instruction (-1 as
1370                  * sample is reported as though instruction has just been
1371                  * executed, but PC has not advanced to next instruction)
1372                  */
1373                 u64 offset = (instrs_executed - instrs_over - 1);
1374                 u64 addr = cs_etm__instr_addr(etmq, trace_chan_id,
1375                                               tidq->packet, offset);
1376
1377                 ret = cs_etm__synth_instruction_sample(
1378                         etmq, tidq, addr, etm->instructions_sample_period);
1379                 if (ret)
1380                         return ret;
1381
1382                 /* Carry remaining instructions into next sample period */
1383                 tidq->period_instructions = instrs_over;
1384         }
1385
1386         if (etm->sample_branches) {
1387                 bool generate_sample = false;
1388
1389                 /* Generate sample for tracing on packet */
1390                 if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY)
1391                         generate_sample = true;
1392
1393                 /* Generate sample for branch taken packet */
1394                 if (tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1395                     tidq->prev_packet->last_instr_taken_branch)
1396                         generate_sample = true;
1397
1398                 if (generate_sample) {
1399                         ret = cs_etm__synth_branch_sample(etmq, tidq);
1400                         if (ret)
1401                                 return ret;
1402                 }
1403         }
1404
1405         if (etm->sample_branches || etm->synth_opts.last_branch) {
1406                 /*
1407                  * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
1408                  * the next incoming packet.
1409                  */
1410                 tmp = tidq->packet;
1411                 tidq->packet = tidq->prev_packet;
1412                 tidq->prev_packet = tmp;
1413         }
1414
1415         return 0;
1416 }
1417
1418 static int cs_etm__exception(struct cs_etm_traceid_queue *tidq)
1419 {
1420         /*
1421          * When the exception packet is inserted, whether the last instruction
1422          * in previous range packet is taken branch or not, we need to force
1423          * to set 'prev_packet->last_instr_taken_branch' to true.  This ensures
1424          * to generate branch sample for the instruction range before the
1425          * exception is trapped to kernel or before the exception returning.
1426          *
1427          * The exception packet includes the dummy address values, so don't
1428          * swap PACKET with PREV_PACKET.  This keeps PREV_PACKET to be useful
1429          * for generating instruction and branch samples.
1430          */
1431         if (tidq->prev_packet->sample_type == CS_ETM_RANGE)
1432                 tidq->prev_packet->last_instr_taken_branch = true;
1433
1434         return 0;
1435 }
1436
1437 static int cs_etm__flush(struct cs_etm_queue *etmq,
1438                          struct cs_etm_traceid_queue *tidq)
1439 {
1440         int err = 0;
1441         struct cs_etm_auxtrace *etm = etmq->etm;
1442         struct cs_etm_packet *tmp;
1443
1444         /* Handle start tracing packet */
1445         if (tidq->prev_packet->sample_type == CS_ETM_EMPTY)
1446                 goto swap_packet;
1447
1448         if (etmq->etm->synth_opts.last_branch &&
1449             tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1450                 /*
1451                  * Generate a last branch event for the branches left in the
1452                  * circular buffer at the end of the trace.
1453                  *
1454                  * Use the address of the end of the last reported execution
1455                  * range
1456                  */
1457                 u64 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1458
1459                 err = cs_etm__synth_instruction_sample(
1460                         etmq, tidq, addr,
1461                         tidq->period_instructions);
1462                 if (err)
1463                         return err;
1464
1465                 tidq->period_instructions = 0;
1466
1467         }
1468
1469         if (etm->sample_branches &&
1470             tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1471                 err = cs_etm__synth_branch_sample(etmq, tidq);
1472                 if (err)
1473                         return err;
1474         }
1475
1476 swap_packet:
1477         if (etm->sample_branches || etm->synth_opts.last_branch) {
1478                 /*
1479                  * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
1480                  * the next incoming packet.
1481                  */
1482                 tmp = tidq->packet;
1483                 tidq->packet = tidq->prev_packet;
1484                 tidq->prev_packet = tmp;
1485         }
1486
1487         return err;
1488 }
1489
1490 static int cs_etm__end_block(struct cs_etm_queue *etmq,
1491                              struct cs_etm_traceid_queue *tidq)
1492 {
1493         int err;
1494
1495         /*
1496          * It has no new packet coming and 'etmq->packet' contains the stale
1497          * packet which was set at the previous time with packets swapping;
1498          * so skip to generate branch sample to avoid stale packet.
1499          *
1500          * For this case only flush branch stack and generate a last branch
1501          * event for the branches left in the circular buffer at the end of
1502          * the trace.
1503          */
1504         if (etmq->etm->synth_opts.last_branch &&
1505             tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1506                 /*
1507                  * Use the address of the end of the last reported execution
1508                  * range.
1509                  */
1510                 u64 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1511
1512                 err = cs_etm__synth_instruction_sample(
1513                         etmq, tidq, addr,
1514                         tidq->period_instructions);
1515                 if (err)
1516                         return err;
1517
1518                 tidq->period_instructions = 0;
1519         }
1520
1521         return 0;
1522 }
1523 /*
1524  * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue
1525  *                         if need be.
1526  * Returns:     < 0     if error
1527  *              = 0     if no more auxtrace_buffer to read
1528  *              > 0     if the current buffer isn't empty yet
1529  */
1530 static int cs_etm__get_data_block(struct cs_etm_queue *etmq)
1531 {
1532         int ret;
1533
1534         if (!etmq->buf_len) {
1535                 ret = cs_etm__get_trace(etmq);
1536                 if (ret <= 0)
1537                         return ret;
1538                 /*
1539                  * We cannot assume consecutive blocks in the data file
1540                  * are contiguous, reset the decoder to force re-sync.
1541                  */
1542                 ret = cs_etm_decoder__reset(etmq->decoder);
1543                 if (ret)
1544                         return ret;
1545         }
1546
1547         return etmq->buf_len;
1548 }
1549
1550 static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id,
1551                                  struct cs_etm_packet *packet,
1552                                  u64 end_addr)
1553 {
1554         /* Initialise to keep compiler happy */
1555         u16 instr16 = 0;
1556         u32 instr32 = 0;
1557         u64 addr;
1558
1559         switch (packet->isa) {
1560         case CS_ETM_ISA_T32:
1561                 /*
1562                  * The SVC of T32 is defined in ARM DDI 0487D.a, F5.1.247:
1563                  *
1564                  *  b'15         b'8
1565                  * +-----------------+--------+
1566                  * | 1 1 0 1 1 1 1 1 |  imm8  |
1567                  * +-----------------+--------+
1568                  *
1569                  * According to the specifiction, it only defines SVC for T32
1570                  * with 16 bits instruction and has no definition for 32bits;
1571                  * so below only read 2 bytes as instruction size for T32.
1572                  */
1573                 addr = end_addr - 2;
1574                 cs_etm__mem_access(etmq, trace_chan_id, addr,
1575                                    sizeof(instr16), (u8 *)&instr16);
1576                 if ((instr16 & 0xFF00) == 0xDF00)
1577                         return true;
1578
1579                 break;
1580         case CS_ETM_ISA_A32:
1581                 /*
1582                  * The SVC of A32 is defined in ARM DDI 0487D.a, F5.1.247:
1583                  *
1584                  *  b'31 b'28 b'27 b'24
1585                  * +---------+---------+-------------------------+
1586                  * |  !1111  | 1 1 1 1 |        imm24            |
1587                  * +---------+---------+-------------------------+
1588                  */
1589                 addr = end_addr - 4;
1590                 cs_etm__mem_access(etmq, trace_chan_id, addr,
1591                                    sizeof(instr32), (u8 *)&instr32);
1592                 if ((instr32 & 0x0F000000) == 0x0F000000 &&
1593                     (instr32 & 0xF0000000) != 0xF0000000)
1594                         return true;
1595
1596                 break;
1597         case CS_ETM_ISA_A64:
1598                 /*
1599                  * The SVC of A64 is defined in ARM DDI 0487D.a, C6.2.294:
1600                  *
1601                  *  b'31               b'21           b'4     b'0
1602                  * +-----------------------+---------+-----------+
1603                  * | 1 1 0 1 0 1 0 0 0 0 0 |  imm16  | 0 0 0 0 1 |
1604                  * +-----------------------+---------+-----------+
1605                  */
1606                 addr = end_addr - 4;
1607                 cs_etm__mem_access(etmq, trace_chan_id, addr,
1608                                    sizeof(instr32), (u8 *)&instr32);
1609                 if ((instr32 & 0xFFE0001F) == 0xd4000001)
1610                         return true;
1611
1612                 break;
1613         case CS_ETM_ISA_UNKNOWN:
1614         default:
1615                 break;
1616         }
1617
1618         return false;
1619 }
1620
1621 static bool cs_etm__is_syscall(struct cs_etm_queue *etmq,
1622                                struct cs_etm_traceid_queue *tidq, u64 magic)
1623 {
1624         u8 trace_chan_id = tidq->trace_chan_id;
1625         struct cs_etm_packet *packet = tidq->packet;
1626         struct cs_etm_packet *prev_packet = tidq->prev_packet;
1627
1628         if (magic == __perf_cs_etmv3_magic)
1629                 if (packet->exception_number == CS_ETMV3_EXC_SVC)
1630                         return true;
1631
1632         /*
1633          * ETMv4 exception type CS_ETMV4_EXC_CALL covers SVC, SMC and
1634          * HVC cases; need to check if it's SVC instruction based on
1635          * packet address.
1636          */
1637         if (magic == __perf_cs_etmv4_magic) {
1638                 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
1639                     cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
1640                                          prev_packet->end_addr))
1641                         return true;
1642         }
1643
1644         return false;
1645 }
1646
1647 static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq,
1648                                        u64 magic)
1649 {
1650         struct cs_etm_packet *packet = tidq->packet;
1651
1652         if (magic == __perf_cs_etmv3_magic)
1653                 if (packet->exception_number == CS_ETMV3_EXC_DEBUG_HALT ||
1654                     packet->exception_number == CS_ETMV3_EXC_ASYNC_DATA_ABORT ||
1655                     packet->exception_number == CS_ETMV3_EXC_PE_RESET ||
1656                     packet->exception_number == CS_ETMV3_EXC_IRQ ||
1657                     packet->exception_number == CS_ETMV3_EXC_FIQ)
1658                         return true;
1659
1660         if (magic == __perf_cs_etmv4_magic)
1661                 if (packet->exception_number == CS_ETMV4_EXC_RESET ||
1662                     packet->exception_number == CS_ETMV4_EXC_DEBUG_HALT ||
1663                     packet->exception_number == CS_ETMV4_EXC_SYSTEM_ERROR ||
1664                     packet->exception_number == CS_ETMV4_EXC_INST_DEBUG ||
1665                     packet->exception_number == CS_ETMV4_EXC_DATA_DEBUG ||
1666                     packet->exception_number == CS_ETMV4_EXC_IRQ ||
1667                     packet->exception_number == CS_ETMV4_EXC_FIQ)
1668                         return true;
1669
1670         return false;
1671 }
1672
1673 static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq,
1674                                       struct cs_etm_traceid_queue *tidq,
1675                                       u64 magic)
1676 {
1677         u8 trace_chan_id = tidq->trace_chan_id;
1678         struct cs_etm_packet *packet = tidq->packet;
1679         struct cs_etm_packet *prev_packet = tidq->prev_packet;
1680
1681         if (magic == __perf_cs_etmv3_magic)
1682                 if (packet->exception_number == CS_ETMV3_EXC_SMC ||
1683                     packet->exception_number == CS_ETMV3_EXC_HYP ||
1684                     packet->exception_number == CS_ETMV3_EXC_JAZELLE_THUMBEE ||
1685                     packet->exception_number == CS_ETMV3_EXC_UNDEFINED_INSTR ||
1686                     packet->exception_number == CS_ETMV3_EXC_PREFETCH_ABORT ||
1687                     packet->exception_number == CS_ETMV3_EXC_DATA_FAULT ||
1688                     packet->exception_number == CS_ETMV3_EXC_GENERIC)
1689                         return true;
1690
1691         if (magic == __perf_cs_etmv4_magic) {
1692                 if (packet->exception_number == CS_ETMV4_EXC_TRAP ||
1693                     packet->exception_number == CS_ETMV4_EXC_ALIGNMENT ||
1694                     packet->exception_number == CS_ETMV4_EXC_INST_FAULT ||
1695                     packet->exception_number == CS_ETMV4_EXC_DATA_FAULT)
1696                         return true;
1697
1698                 /*
1699                  * For CS_ETMV4_EXC_CALL, except SVC other instructions
1700                  * (SMC, HVC) are taken as sync exceptions.
1701                  */
1702                 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
1703                     !cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
1704                                           prev_packet->end_addr))
1705                         return true;
1706
1707                 /*
1708                  * ETMv4 has 5 bits for exception number; if the numbers
1709                  * are in the range ( CS_ETMV4_EXC_FIQ, CS_ETMV4_EXC_END ]
1710                  * they are implementation defined exceptions.
1711                  *
1712                  * For this case, simply take it as sync exception.
1713                  */
1714                 if (packet->exception_number > CS_ETMV4_EXC_FIQ &&
1715                     packet->exception_number <= CS_ETMV4_EXC_END)
1716                         return true;
1717         }
1718
1719         return false;
1720 }
1721
1722 static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq,
1723                                     struct cs_etm_traceid_queue *tidq)
1724 {
1725         struct cs_etm_packet *packet = tidq->packet;
1726         struct cs_etm_packet *prev_packet = tidq->prev_packet;
1727         u8 trace_chan_id = tidq->trace_chan_id;
1728         u64 magic;
1729         int ret;
1730
1731         switch (packet->sample_type) {
1732         case CS_ETM_RANGE:
1733                 /*
1734                  * Immediate branch instruction without neither link nor
1735                  * return flag, it's normal branch instruction within
1736                  * the function.
1737                  */
1738                 if (packet->last_instr_type == OCSD_INSTR_BR &&
1739                     packet->last_instr_subtype == OCSD_S_INSTR_NONE) {
1740                         packet->flags = PERF_IP_FLAG_BRANCH;
1741
1742                         if (packet->last_instr_cond)
1743                                 packet->flags |= PERF_IP_FLAG_CONDITIONAL;
1744                 }
1745
1746                 /*
1747                  * Immediate branch instruction with link (e.g. BL), this is
1748                  * branch instruction for function call.
1749                  */
1750                 if (packet->last_instr_type == OCSD_INSTR_BR &&
1751                     packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
1752                         packet->flags = PERF_IP_FLAG_BRANCH |
1753                                         PERF_IP_FLAG_CALL;
1754
1755                 /*
1756                  * Indirect branch instruction with link (e.g. BLR), this is
1757                  * branch instruction for function call.
1758                  */
1759                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1760                     packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
1761                         packet->flags = PERF_IP_FLAG_BRANCH |
1762                                         PERF_IP_FLAG_CALL;
1763
1764                 /*
1765                  * Indirect branch instruction with subtype of
1766                  * OCSD_S_INSTR_V7_IMPLIED_RET, this is explicit hint for
1767                  * function return for A32/T32.
1768                  */
1769                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1770                     packet->last_instr_subtype == OCSD_S_INSTR_V7_IMPLIED_RET)
1771                         packet->flags = PERF_IP_FLAG_BRANCH |
1772                                         PERF_IP_FLAG_RETURN;
1773
1774                 /*
1775                  * Indirect branch instruction without link (e.g. BR), usually
1776                  * this is used for function return, especially for functions
1777                  * within dynamic link lib.
1778                  */
1779                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1780                     packet->last_instr_subtype == OCSD_S_INSTR_NONE)
1781                         packet->flags = PERF_IP_FLAG_BRANCH |
1782                                         PERF_IP_FLAG_RETURN;
1783
1784                 /* Return instruction for function return. */
1785                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1786                     packet->last_instr_subtype == OCSD_S_INSTR_V8_RET)
1787                         packet->flags = PERF_IP_FLAG_BRANCH |
1788                                         PERF_IP_FLAG_RETURN;
1789
1790                 /*
1791                  * Decoder might insert a discontinuity in the middle of
1792                  * instruction packets, fixup prev_packet with flag
1793                  * PERF_IP_FLAG_TRACE_BEGIN to indicate restarting trace.
1794                  */
1795                 if (prev_packet->sample_type == CS_ETM_DISCONTINUITY)
1796                         prev_packet->flags |= PERF_IP_FLAG_BRANCH |
1797                                               PERF_IP_FLAG_TRACE_BEGIN;
1798
1799                 /*
1800                  * If the previous packet is an exception return packet
1801                  * and the return address just follows SVC instuction,
1802                  * it needs to calibrate the previous packet sample flags
1803                  * as PERF_IP_FLAG_SYSCALLRET.
1804                  */
1805                 if (prev_packet->flags == (PERF_IP_FLAG_BRANCH |
1806                                            PERF_IP_FLAG_RETURN |
1807                                            PERF_IP_FLAG_INTERRUPT) &&
1808                     cs_etm__is_svc_instr(etmq, trace_chan_id,
1809                                          packet, packet->start_addr))
1810                         prev_packet->flags = PERF_IP_FLAG_BRANCH |
1811                                              PERF_IP_FLAG_RETURN |
1812                                              PERF_IP_FLAG_SYSCALLRET;
1813                 break;
1814         case CS_ETM_DISCONTINUITY:
1815                 /*
1816                  * The trace is discontinuous, if the previous packet is
1817                  * instruction packet, set flag PERF_IP_FLAG_TRACE_END
1818                  * for previous packet.
1819                  */
1820                 if (prev_packet->sample_type == CS_ETM_RANGE)
1821                         prev_packet->flags |= PERF_IP_FLAG_BRANCH |
1822                                               PERF_IP_FLAG_TRACE_END;
1823                 break;
1824         case CS_ETM_EXCEPTION:
1825                 ret = cs_etm__get_magic(packet->trace_chan_id, &magic);
1826                 if (ret)
1827                         return ret;
1828
1829                 /* The exception is for system call. */
1830                 if (cs_etm__is_syscall(etmq, tidq, magic))
1831                         packet->flags = PERF_IP_FLAG_BRANCH |
1832                                         PERF_IP_FLAG_CALL |
1833                                         PERF_IP_FLAG_SYSCALLRET;
1834                 /*
1835                  * The exceptions are triggered by external signals from bus,
1836                  * interrupt controller, debug module, PE reset or halt.
1837                  */
1838                 else if (cs_etm__is_async_exception(tidq, magic))
1839                         packet->flags = PERF_IP_FLAG_BRANCH |
1840                                         PERF_IP_FLAG_CALL |
1841                                         PERF_IP_FLAG_ASYNC |
1842                                         PERF_IP_FLAG_INTERRUPT;
1843                 /*
1844                  * Otherwise, exception is caused by trap, instruction &
1845                  * data fault, or alignment errors.
1846                  */
1847                 else if (cs_etm__is_sync_exception(etmq, tidq, magic))
1848                         packet->flags = PERF_IP_FLAG_BRANCH |
1849                                         PERF_IP_FLAG_CALL |
1850                                         PERF_IP_FLAG_INTERRUPT;
1851
1852                 /*
1853                  * When the exception packet is inserted, since exception
1854                  * packet is not used standalone for generating samples
1855                  * and it's affiliation to the previous instruction range
1856                  * packet; so set previous range packet flags to tell perf
1857                  * it is an exception taken branch.
1858                  */
1859                 if (prev_packet->sample_type == CS_ETM_RANGE)
1860                         prev_packet->flags = packet->flags;
1861                 break;
1862         case CS_ETM_EXCEPTION_RET:
1863                 /*
1864                  * When the exception return packet is inserted, since
1865                  * exception return packet is not used standalone for
1866                  * generating samples and it's affiliation to the previous
1867                  * instruction range packet; so set previous range packet
1868                  * flags to tell perf it is an exception return branch.
1869                  *
1870                  * The exception return can be for either system call or
1871                  * other exception types; unfortunately the packet doesn't
1872                  * contain exception type related info so we cannot decide
1873                  * the exception type purely based on exception return packet.
1874                  * If we record the exception number from exception packet and
1875                  * reuse it for excpetion return packet, this is not reliable
1876                  * due the trace can be discontinuity or the interrupt can
1877                  * be nested, thus the recorded exception number cannot be
1878                  * used for exception return packet for these two cases.
1879                  *
1880                  * For exception return packet, we only need to distinguish the
1881                  * packet is for system call or for other types.  Thus the
1882                  * decision can be deferred when receive the next packet which
1883                  * contains the return address, based on the return address we
1884                  * can read out the previous instruction and check if it's a
1885                  * system call instruction and then calibrate the sample flag
1886                  * as needed.
1887                  */
1888                 if (prev_packet->sample_type == CS_ETM_RANGE)
1889                         prev_packet->flags = PERF_IP_FLAG_BRANCH |
1890                                              PERF_IP_FLAG_RETURN |
1891                                              PERF_IP_FLAG_INTERRUPT;
1892                 break;
1893         case CS_ETM_EMPTY:
1894         default:
1895                 break;
1896         }
1897
1898         return 0;
1899 }
1900
1901 static int cs_etm__decode_data_block(struct cs_etm_queue *etmq)
1902 {
1903         int ret = 0;
1904         size_t processed = 0;
1905
1906         /*
1907          * Packets are decoded and added to the decoder's packet queue
1908          * until the decoder packet processing callback has requested that
1909          * processing stops or there is nothing left in the buffer.  Normal
1910          * operations that stop processing are a timestamp packet or a full
1911          * decoder buffer queue.
1912          */
1913         ret = cs_etm_decoder__process_data_block(etmq->decoder,
1914                                                  etmq->offset,
1915                                                  &etmq->buf[etmq->buf_used],
1916                                                  etmq->buf_len,
1917                                                  &processed);
1918         if (ret)
1919                 goto out;
1920
1921         etmq->offset += processed;
1922         etmq->buf_used += processed;
1923         etmq->buf_len -= processed;
1924
1925 out:
1926         return ret;
1927 }
1928
1929 static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq,
1930                                          struct cs_etm_traceid_queue *tidq)
1931 {
1932         int ret;
1933         struct cs_etm_packet_queue *packet_queue;
1934
1935         packet_queue = &tidq->packet_queue;
1936
1937         /* Process each packet in this chunk */
1938         while (1) {
1939                 ret = cs_etm_decoder__get_packet(packet_queue,
1940                                                  tidq->packet);
1941                 if (ret <= 0)
1942                         /*
1943                          * Stop processing this chunk on
1944                          * end of data or error
1945                          */
1946                         break;
1947
1948                 /*
1949                  * Since packet addresses are swapped in packet
1950                  * handling within below switch() statements,
1951                  * thus setting sample flags must be called
1952                  * prior to switch() statement to use address
1953                  * information before packets swapping.
1954                  */
1955                 ret = cs_etm__set_sample_flags(etmq, tidq);
1956                 if (ret < 0)
1957                         break;
1958
1959                 switch (tidq->packet->sample_type) {
1960                 case CS_ETM_RANGE:
1961                         /*
1962                          * If the packet contains an instruction
1963                          * range, generate instruction sequence
1964                          * events.
1965                          */
1966                         cs_etm__sample(etmq, tidq);
1967                         break;
1968                 case CS_ETM_EXCEPTION:
1969                 case CS_ETM_EXCEPTION_RET:
1970                         /*
1971                          * If the exception packet is coming,
1972                          * make sure the previous instruction
1973                          * range packet to be handled properly.
1974                          */
1975                         cs_etm__exception(tidq);
1976                         break;
1977                 case CS_ETM_DISCONTINUITY:
1978                         /*
1979                          * Discontinuity in trace, flush
1980                          * previous branch stack
1981                          */
1982                         cs_etm__flush(etmq, tidq);
1983                         break;
1984                 case CS_ETM_EMPTY:
1985                         /*
1986                          * Should not receive empty packet,
1987                          * report error.
1988                          */
1989                         pr_err("CS ETM Trace: empty packet\n");
1990                         return -EINVAL;
1991                 default:
1992                         break;
1993                 }
1994         }
1995
1996         return ret;
1997 }
1998
1999 static void cs_etm__clear_all_traceid_queues(struct cs_etm_queue *etmq)
2000 {
2001         int idx;
2002         struct int_node *inode;
2003         struct cs_etm_traceid_queue *tidq;
2004         struct intlist *traceid_queues_list = etmq->traceid_queues_list;
2005
2006         intlist__for_each_entry(inode, traceid_queues_list) {
2007                 idx = (int)(intptr_t)inode->priv;
2008                 tidq = etmq->traceid_queues[idx];
2009
2010                 /* Ignore return value */
2011                 cs_etm__process_traceid_queue(etmq, tidq);
2012
2013                 /*
2014                  * Generate an instruction sample with the remaining
2015                  * branchstack entries.
2016                  */
2017                 cs_etm__flush(etmq, tidq);
2018         }
2019 }
2020
2021 static int cs_etm__run_decoder(struct cs_etm_queue *etmq)
2022 {
2023         int err = 0;
2024         struct cs_etm_traceid_queue *tidq;
2025
2026         tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID);
2027         if (!tidq)
2028                 return -EINVAL;
2029
2030         /* Go through each buffer in the queue and decode them one by one */
2031         while (1) {
2032                 err = cs_etm__get_data_block(etmq);
2033                 if (err <= 0)
2034                         return err;
2035
2036                 /* Run trace decoder until buffer consumed or end of trace */
2037                 do {
2038                         err = cs_etm__decode_data_block(etmq);
2039                         if (err)
2040                                 return err;
2041
2042                         /*
2043                          * Process each packet in this chunk, nothing to do if
2044                          * an error occurs other than hoping the next one will
2045                          * be better.
2046                          */
2047                         err = cs_etm__process_traceid_queue(etmq, tidq);
2048
2049                 } while (etmq->buf_len);
2050
2051                 if (err == 0)
2052                         /* Flush any remaining branch stack entries */
2053                         err = cs_etm__end_block(etmq, tidq);
2054         }
2055
2056         return err;
2057 }
2058
2059 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
2060                                            pid_t tid)
2061 {
2062         unsigned int i;
2063         struct auxtrace_queues *queues = &etm->queues;
2064
2065         for (i = 0; i < queues->nr_queues; i++) {
2066                 struct auxtrace_queue *queue = &etm->queues.queue_array[i];
2067                 struct cs_etm_queue *etmq = queue->priv;
2068                 struct cs_etm_traceid_queue *tidq;
2069
2070                 if (!etmq)
2071                         continue;
2072
2073                 tidq = cs_etm__etmq_get_traceid_queue(etmq,
2074                                                 CS_ETM_PER_THREAD_TRACEID);
2075
2076                 if (!tidq)
2077                         continue;
2078
2079                 if ((tid == -1) || (tidq->tid == tid)) {
2080                         cs_etm__set_pid_tid_cpu(etm, tidq);
2081                         cs_etm__run_decoder(etmq);
2082                 }
2083         }
2084
2085         return 0;
2086 }
2087
2088 static int cs_etm__process_queues(struct cs_etm_auxtrace *etm)
2089 {
2090         int ret = 0;
2091         unsigned int cs_queue_nr, queue_nr;
2092         u8 trace_chan_id;
2093         u64 timestamp;
2094         struct auxtrace_queue *queue;
2095         struct cs_etm_queue *etmq;
2096         struct cs_etm_traceid_queue *tidq;
2097
2098         while (1) {
2099                 if (!etm->heap.heap_cnt)
2100                         goto out;
2101
2102                 /* Take the entry at the top of the min heap */
2103                 cs_queue_nr = etm->heap.heap_array[0].queue_nr;
2104                 queue_nr = TO_QUEUE_NR(cs_queue_nr);
2105                 trace_chan_id = TO_TRACE_CHAN_ID(cs_queue_nr);
2106                 queue = &etm->queues.queue_array[queue_nr];
2107                 etmq = queue->priv;
2108
2109                 /*
2110                  * Remove the top entry from the heap since we are about
2111                  * to process it.
2112                  */
2113                 auxtrace_heap__pop(&etm->heap);
2114
2115                 tidq  = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
2116                 if (!tidq) {
2117                         /*
2118                          * No traceID queue has been allocated for this traceID,
2119                          * which means something somewhere went very wrong.  No
2120                          * other choice than simply exit.
2121                          */
2122                         ret = -EINVAL;
2123                         goto out;
2124                 }
2125
2126                 /*
2127                  * Packets associated with this timestamp are already in
2128                  * the etmq's traceID queue, so process them.
2129                  */
2130                 ret = cs_etm__process_traceid_queue(etmq, tidq);
2131                 if (ret < 0)
2132                         goto out;
2133
2134                 /*
2135                  * Packets for this timestamp have been processed, time to
2136                  * move on to the next timestamp, fetching a new auxtrace_buffer
2137                  * if need be.
2138                  */
2139 refetch:
2140                 ret = cs_etm__get_data_block(etmq);
2141                 if (ret < 0)
2142                         goto out;
2143
2144                 /*
2145                  * No more auxtrace_buffers to process in this etmq, simply
2146                  * move on to another entry in the auxtrace_heap.
2147                  */
2148                 if (!ret)
2149                         continue;
2150
2151                 ret = cs_etm__decode_data_block(etmq);
2152                 if (ret)
2153                         goto out;
2154
2155                 timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
2156
2157                 if (!timestamp) {
2158                         /*
2159                          * Function cs_etm__decode_data_block() returns when
2160                          * there is no more traces to decode in the current
2161                          * auxtrace_buffer OR when a timestamp has been
2162                          * encountered on any of the traceID queues.  Since we
2163                          * did not get a timestamp, there is no more traces to
2164                          * process in this auxtrace_buffer.  As such empty and
2165                          * flush all traceID queues.
2166                          */
2167                         cs_etm__clear_all_traceid_queues(etmq);
2168
2169                         /* Fetch another auxtrace_buffer for this etmq */
2170                         goto refetch;
2171                 }
2172
2173                 /*
2174                  * Add to the min heap the timestamp for packets that have
2175                  * just been decoded.  They will be processed and synthesized
2176                  * during the next call to cs_etm__process_traceid_queue() for
2177                  * this queue/traceID.
2178                  */
2179                 cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id);
2180                 ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, timestamp);
2181         }
2182
2183 out:
2184         return ret;
2185 }
2186
2187 static int cs_etm__process_itrace_start(struct cs_etm_auxtrace *etm,
2188                                         union perf_event *event)
2189 {
2190         struct thread *th;
2191
2192         if (etm->timeless_decoding)
2193                 return 0;
2194
2195         /*
2196          * Add the tid/pid to the log so that we can get a match when
2197          * we get a contextID from the decoder.
2198          */
2199         th = machine__findnew_thread(etm->machine,
2200                                      event->itrace_start.pid,
2201                                      event->itrace_start.tid);
2202         if (!th)
2203                 return -ENOMEM;
2204
2205         thread__put(th);
2206
2207         return 0;
2208 }
2209
2210 static int cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace *etm,
2211                                            union perf_event *event)
2212 {
2213         struct thread *th;
2214         bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
2215
2216         /*
2217          * Context switch in per-thread mode are irrelevant since perf
2218          * will start/stop tracing as the process is scheduled.
2219          */
2220         if (etm->timeless_decoding)
2221                 return 0;
2222
2223         /*
2224          * SWITCH_IN events carry the next process to be switched out while
2225          * SWITCH_OUT events carry the process to be switched in.  As such
2226          * we don't care about IN events.
2227          */
2228         if (!out)
2229                 return 0;
2230
2231         /*
2232          * Add the tid/pid to the log so that we can get a match when
2233          * we get a contextID from the decoder.
2234          */
2235         th = machine__findnew_thread(etm->machine,
2236                                      event->context_switch.next_prev_pid,
2237                                      event->context_switch.next_prev_tid);
2238         if (!th)
2239                 return -ENOMEM;
2240
2241         thread__put(th);
2242
2243         return 0;
2244 }
2245
2246 static int cs_etm__process_event(struct perf_session *session,
2247                                  union perf_event *event,
2248                                  struct perf_sample *sample,
2249                                  struct perf_tool *tool)
2250 {
2251         int err = 0;
2252         u64 timestamp;
2253         struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2254                                                    struct cs_etm_auxtrace,
2255                                                    auxtrace);
2256
2257         if (dump_trace)
2258                 return 0;
2259
2260         if (!tool->ordered_events) {
2261                 pr_err("CoreSight ETM Trace requires ordered events\n");
2262                 return -EINVAL;
2263         }
2264
2265         if (sample->time && (sample->time != (u64) -1))
2266                 timestamp = sample->time;
2267         else
2268                 timestamp = 0;
2269
2270         if (timestamp || etm->timeless_decoding) {
2271                 err = cs_etm__update_queues(etm);
2272                 if (err)
2273                         return err;
2274         }
2275
2276         if (etm->timeless_decoding &&
2277             event->header.type == PERF_RECORD_EXIT)
2278                 return cs_etm__process_timeless_queues(etm,
2279                                                        event->fork.tid);
2280
2281         if (event->header.type == PERF_RECORD_ITRACE_START)
2282                 return cs_etm__process_itrace_start(etm, event);
2283         else if (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
2284                 return cs_etm__process_switch_cpu_wide(etm, event);
2285
2286         if (!etm->timeless_decoding &&
2287             event->header.type == PERF_RECORD_AUX)
2288                 return cs_etm__process_queues(etm);
2289
2290         return 0;
2291 }
2292
2293 static int cs_etm__process_auxtrace_event(struct perf_session *session,
2294                                           union perf_event *event,
2295                                           struct perf_tool *tool __maybe_unused)
2296 {
2297         struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2298                                                    struct cs_etm_auxtrace,
2299                                                    auxtrace);
2300         if (!etm->data_queued) {
2301                 struct auxtrace_buffer *buffer;
2302                 off_t  data_offset;
2303                 int fd = perf_data__fd(session->data);
2304                 bool is_pipe = perf_data__is_pipe(session->data);
2305                 int err;
2306
2307                 if (is_pipe)
2308                         data_offset = 0;
2309                 else {
2310                         data_offset = lseek(fd, 0, SEEK_CUR);
2311                         if (data_offset == -1)
2312                                 return -errno;
2313                 }
2314
2315                 err = auxtrace_queues__add_event(&etm->queues, session,
2316                                                  event, data_offset, &buffer);
2317                 if (err)
2318                         return err;
2319
2320                 if (dump_trace)
2321                         if (auxtrace_buffer__get_data(buffer, fd)) {
2322                                 cs_etm__dump_event(etm, buffer);
2323                                 auxtrace_buffer__put_data(buffer);
2324                         }
2325         }
2326
2327         return 0;
2328 }
2329
2330 static bool cs_etm__is_timeless_decoding(struct cs_etm_auxtrace *etm)
2331 {
2332         struct evsel *evsel;
2333         struct evlist *evlist = etm->session->evlist;
2334         bool timeless_decoding = true;
2335
2336         /*
2337          * Circle through the list of event and complain if we find one
2338          * with the time bit set.
2339          */
2340         evlist__for_each_entry(evlist, evsel) {
2341                 if ((evsel->core.attr.sample_type & PERF_SAMPLE_TIME))
2342                         timeless_decoding = false;
2343         }
2344
2345         return timeless_decoding;
2346 }
2347
2348 static const char * const cs_etm_global_header_fmts[] = {
2349         [CS_HEADER_VERSION_0]   = "     Header version                 %llx\n",
2350         [CS_PMU_TYPE_CPUS]      = "     PMU type/num cpus              %llx\n",
2351         [CS_ETM_SNAPSHOT]       = "     Snapshot                       %llx\n",
2352 };
2353
2354 static const char * const cs_etm_priv_fmts[] = {
2355         [CS_ETM_MAGIC]          = "     Magic number                   %llx\n",
2356         [CS_ETM_CPU]            = "     CPU                            %lld\n",
2357         [CS_ETM_ETMCR]          = "     ETMCR                          %llx\n",
2358         [CS_ETM_ETMTRACEIDR]    = "     ETMTRACEIDR                    %llx\n",
2359         [CS_ETM_ETMCCER]        = "     ETMCCER                        %llx\n",
2360         [CS_ETM_ETMIDR]         = "     ETMIDR                         %llx\n",
2361 };
2362
2363 static const char * const cs_etmv4_priv_fmts[] = {
2364         [CS_ETM_MAGIC]          = "     Magic number                   %llx\n",
2365         [CS_ETM_CPU]            = "     CPU                            %lld\n",
2366         [CS_ETMV4_TRCCONFIGR]   = "     TRCCONFIGR                     %llx\n",
2367         [CS_ETMV4_TRCTRACEIDR]  = "     TRCTRACEIDR                    %llx\n",
2368         [CS_ETMV4_TRCIDR0]      = "     TRCIDR0                        %llx\n",
2369         [CS_ETMV4_TRCIDR1]      = "     TRCIDR1                        %llx\n",
2370         [CS_ETMV4_TRCIDR2]      = "     TRCIDR2                        %llx\n",
2371         [CS_ETMV4_TRCIDR8]      = "     TRCIDR8                        %llx\n",
2372         [CS_ETMV4_TRCAUTHSTATUS] = "    TRCAUTHSTATUS                  %llx\n",
2373 };
2374
2375 static void cs_etm__print_auxtrace_info(__u64 *val, int num)
2376 {
2377         int i, j, cpu = 0;
2378
2379         for (i = 0; i < CS_HEADER_VERSION_0_MAX; i++)
2380                 fprintf(stdout, cs_etm_global_header_fmts[i], val[i]);
2381
2382         for (i = CS_HEADER_VERSION_0_MAX; cpu < num; cpu++) {
2383                 if (val[i] == __perf_cs_etmv3_magic)
2384                         for (j = 0; j < CS_ETM_PRIV_MAX; j++, i++)
2385                                 fprintf(stdout, cs_etm_priv_fmts[j], val[i]);
2386                 else if (val[i] == __perf_cs_etmv4_magic)
2387                         for (j = 0; j < CS_ETMV4_PRIV_MAX; j++, i++)
2388                                 fprintf(stdout, cs_etmv4_priv_fmts[j], val[i]);
2389                 else
2390                         /* failure.. return */
2391                         return;
2392         }
2393 }
2394
2395 int cs_etm__process_auxtrace_info(union perf_event *event,
2396                                   struct perf_session *session)
2397 {
2398         struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
2399         struct cs_etm_auxtrace *etm = NULL;
2400         struct int_node *inode;
2401         unsigned int pmu_type;
2402         int event_header_size = sizeof(struct perf_event_header);
2403         int info_header_size;
2404         int total_size = auxtrace_info->header.size;
2405         int priv_size = 0;
2406         int num_cpu;
2407         int err = 0, idx = -1;
2408         int i, j, k;
2409         u64 *ptr, *hdr = NULL;
2410         u64 **metadata = NULL;
2411
2412         /*
2413          * sizeof(auxtrace_info_event::type) +
2414          * sizeof(auxtrace_info_event::reserved) == 8
2415          */
2416         info_header_size = 8;
2417
2418         if (total_size < (event_header_size + info_header_size))
2419                 return -EINVAL;
2420
2421         priv_size = total_size - event_header_size - info_header_size;
2422
2423         /* First the global part */
2424         ptr = (u64 *) auxtrace_info->priv;
2425
2426         /* Look for version '0' of the header */
2427         if (ptr[0] != 0)
2428                 return -EINVAL;
2429
2430         hdr = zalloc(sizeof(*hdr) * CS_HEADER_VERSION_0_MAX);
2431         if (!hdr)
2432                 return -ENOMEM;
2433
2434         /* Extract header information - see cs-etm.h for format */
2435         for (i = 0; i < CS_HEADER_VERSION_0_MAX; i++)
2436                 hdr[i] = ptr[i];
2437         num_cpu = hdr[CS_PMU_TYPE_CPUS] & 0xffffffff;
2438         pmu_type = (unsigned int) ((hdr[CS_PMU_TYPE_CPUS] >> 32) &
2439                                     0xffffffff);
2440
2441         /*
2442          * Create an RB tree for traceID-metadata tuple.  Since the conversion
2443          * has to be made for each packet that gets decoded, optimizing access
2444          * in anything other than a sequential array is worth doing.
2445          */
2446         traceid_list = intlist__new(NULL);
2447         if (!traceid_list) {
2448                 err = -ENOMEM;
2449                 goto err_free_hdr;
2450         }
2451
2452         metadata = zalloc(sizeof(*metadata) * num_cpu);
2453         if (!metadata) {
2454                 err = -ENOMEM;
2455                 goto err_free_traceid_list;
2456         }
2457
2458         /*
2459          * The metadata is stored in the auxtrace_info section and encodes
2460          * the configuration of the ARM embedded trace macrocell which is
2461          * required by the trace decoder to properly decode the trace due
2462          * to its highly compressed nature.
2463          */
2464         for (j = 0; j < num_cpu; j++) {
2465                 if (ptr[i] == __perf_cs_etmv3_magic) {
2466                         metadata[j] = zalloc(sizeof(*metadata[j]) *
2467                                              CS_ETM_PRIV_MAX);
2468                         if (!metadata[j]) {
2469                                 err = -ENOMEM;
2470                                 goto err_free_metadata;
2471                         }
2472                         for (k = 0; k < CS_ETM_PRIV_MAX; k++)
2473                                 metadata[j][k] = ptr[i + k];
2474
2475                         /* The traceID is our handle */
2476                         idx = metadata[j][CS_ETM_ETMTRACEIDR];
2477                         i += CS_ETM_PRIV_MAX;
2478                 } else if (ptr[i] == __perf_cs_etmv4_magic) {
2479                         metadata[j] = zalloc(sizeof(*metadata[j]) *
2480                                              CS_ETMV4_PRIV_MAX);
2481                         if (!metadata[j]) {
2482                                 err = -ENOMEM;
2483                                 goto err_free_metadata;
2484                         }
2485                         for (k = 0; k < CS_ETMV4_PRIV_MAX; k++)
2486                                 metadata[j][k] = ptr[i + k];
2487
2488                         /* The traceID is our handle */
2489                         idx = metadata[j][CS_ETMV4_TRCTRACEIDR];
2490                         i += CS_ETMV4_PRIV_MAX;
2491                 }
2492
2493                 /* Get an RB node for this CPU */
2494                 inode = intlist__findnew(traceid_list, idx);
2495
2496                 /* Something went wrong, no need to continue */
2497                 if (!inode) {
2498                         err = -ENOMEM;
2499                         goto err_free_metadata;
2500                 }
2501
2502                 /*
2503                  * The node for that CPU should not be taken.
2504                  * Back out if that's the case.
2505                  */
2506                 if (inode->priv) {
2507                         err = -EINVAL;
2508                         goto err_free_metadata;
2509                 }
2510                 /* All good, associate the traceID with the metadata pointer */
2511                 inode->priv = metadata[j];
2512         }
2513
2514         /*
2515          * Each of CS_HEADER_VERSION_0_MAX, CS_ETM_PRIV_MAX and
2516          * CS_ETMV4_PRIV_MAX mark how many double words are in the
2517          * global metadata, and each cpu's metadata respectively.
2518          * The following tests if the correct number of double words was
2519          * present in the auxtrace info section.
2520          */
2521         if (i * 8 != priv_size) {
2522                 err = -EINVAL;
2523                 goto err_free_metadata;
2524         }
2525
2526         etm = zalloc(sizeof(*etm));
2527
2528         if (!etm) {
2529                 err = -ENOMEM;
2530                 goto err_free_metadata;
2531         }
2532
2533         err = auxtrace_queues__init(&etm->queues);
2534         if (err)
2535                 goto err_free_etm;
2536
2537         etm->session = session;
2538         etm->machine = &session->machines.host;
2539
2540         etm->num_cpu = num_cpu;
2541         etm->pmu_type = pmu_type;
2542         etm->snapshot_mode = (hdr[CS_ETM_SNAPSHOT] != 0);
2543         etm->metadata = metadata;
2544         etm->auxtrace_type = auxtrace_info->type;
2545         etm->timeless_decoding = cs_etm__is_timeless_decoding(etm);
2546
2547         etm->auxtrace.process_event = cs_etm__process_event;
2548         etm->auxtrace.process_auxtrace_event = cs_etm__process_auxtrace_event;
2549         etm->auxtrace.flush_events = cs_etm__flush_events;
2550         etm->auxtrace.free_events = cs_etm__free_events;
2551         etm->auxtrace.free = cs_etm__free;
2552         session->auxtrace = &etm->auxtrace;
2553
2554         etm->unknown_thread = thread__new(999999999, 999999999);
2555         if (!etm->unknown_thread) {
2556                 err = -ENOMEM;
2557                 goto err_free_queues;
2558         }
2559
2560         /*
2561          * Initialize list node so that at thread__zput() we can avoid
2562          * segmentation fault at list_del_init().
2563          */
2564         INIT_LIST_HEAD(&etm->unknown_thread->node);
2565
2566         err = thread__set_comm(etm->unknown_thread, "unknown", 0);
2567         if (err)
2568                 goto err_delete_thread;
2569
2570         if (thread__init_map_groups(etm->unknown_thread, etm->machine)) {
2571                 err = -ENOMEM;
2572                 goto err_delete_thread;
2573         }
2574
2575         if (dump_trace) {
2576                 cs_etm__print_auxtrace_info(auxtrace_info->priv, num_cpu);
2577                 return 0;
2578         }
2579
2580         if (session->itrace_synth_opts->set) {
2581                 etm->synth_opts = *session->itrace_synth_opts;
2582         } else {
2583                 itrace_synth_opts__set_default(&etm->synth_opts,
2584                                 session->itrace_synth_opts->default_no_sample);
2585                 etm->synth_opts.callchain = false;
2586         }
2587
2588         err = cs_etm__synth_events(etm, session);
2589         if (err)
2590                 goto err_delete_thread;
2591
2592         err = auxtrace_queues__process_index(&etm->queues, session);
2593         if (err)
2594                 goto err_delete_thread;
2595
2596         etm->data_queued = etm->queues.populated;
2597
2598         return 0;
2599
2600 err_delete_thread:
2601         thread__zput(etm->unknown_thread);
2602 err_free_queues:
2603         auxtrace_queues__free(&etm->queues);
2604         session->auxtrace = NULL;
2605 err_free_etm:
2606         zfree(&etm);
2607 err_free_metadata:
2608         /* No need to check @metadata[j], free(NULL) is supported */
2609         for (j = 0; j < num_cpu; j++)
2610                 zfree(&metadata[j]);
2611         zfree(&metadata);
2612 err_free_traceid_list:
2613         intlist__delete(traceid_list);
2614 err_free_hdr:
2615         zfree(&hdr);
2616
2617         return err;
2618 }